From 8750f0b884bc282336a3da61adfa930ba324ce7e Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Jun 2025 19:41:21 -0700 Subject: [PATCH 1/3] Add FileRoute for serving files (#20198) Co-authored-by: Jarred-Sumner <709451+Jarred-Sumner@users.noreply.github.com> Co-authored-by: Dylan Conway Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com> Co-authored-by: Ciro Spaciari --- cmake/sources/ZigSources.txt | 2 + packages/bun-uws/src/AsyncSocket.h | 2 +- packages/bun-uws/src/HttpContext.h | 6 +- packages/bun-uws/src/HttpResponse.h | 4 +- packages/bun-uws/src/LoopData.h | 1 - src/bun.js/api/server.zig | 13 + src/bun.js/api/server/FileRoute.zig | 586 +++++++++++++++++++ src/bun.js/bindings/WTF.zig | 15 + src/bun.js/bindings/webcore/HTTPParsers.cpp | 30 + src/deps/libuwsockets.cpp | 20 + src/deps/uws/Request.zig | 11 + src/deps/uws/Response.zig | 34 ++ src/fs.zig | 1 + src/fs/stat_hash.zig | 49 ++ src/http.zig | 13 + src/io/PipeReader.zig | 24 +- test/internal/ban-words.test.ts | 2 +- test/js/bun/http/bun-serve-file.test.ts | 592 ++++++++++++++++++++ 18 files changed, 1387 insertions(+), 18 deletions(-) create mode 100644 src/bun.js/api/server/FileRoute.zig create mode 100644 src/fs/stat_hash.zig create mode 100644 test/js/bun/http/bun-serve-file.test.ts diff --git a/cmake/sources/ZigSources.txt b/cmake/sources/ZigSources.txt index a5b7749608..76d942b049 100644 --- a/cmake/sources/ZigSources.txt +++ b/cmake/sources/ZigSources.txt @@ -56,6 +56,7 @@ src/bun.js/api/JSBundler.zig src/bun.js/api/JSTranspiler.zig src/bun.js/api/server.zig src/bun.js/api/server/AnyRequestContext.zig +src/bun.js/api/server/FileRoute.zig src/bun.js/api/server/HTMLBundle.zig src/bun.js/api/server/HTTPStatusText.zig src/bun.js/api/server/InspectorBunFrontendDevServerAgent.zig @@ -473,6 +474,7 @@ src/fd.zig src/feature_flags.zig src/fmt.zig src/fs.zig +src/fs/stat_hash.zig src/futex.zig src/generated_perf_trace_events.zig src/generated_versions_list.zig diff --git a/packages/bun-uws/src/AsyncSocket.h b/packages/bun-uws/src/AsyncSocket.h index 941cccd668..81bce9d313 100644 --- a/packages/bun-uws/src/AsyncSocket.h +++ b/packages/bun-uws/src/AsyncSocket.h @@ -260,7 +260,7 @@ public: * since written < buffer_len is very likely to be true */ if(written < max_flush_len) { - [[likely]] + [[likely]] /* Cannot write more at this time, return what we've written so far */ return total_written; } diff --git a/packages/bun-uws/src/HttpContext.h b/packages/bun-uws/src/HttpContext.h index 052b56f880..e0910a08b5 100644 --- a/packages/bun-uws/src/HttpContext.h +++ b/packages/bun-uws/src/HttpContext.h @@ -456,10 +456,9 @@ private: size_t bufferedAmount = asyncSocket->getBufferedAmount(); if (bufferedAmount > 0) { /* Try to flush pending data from the socket's buffer to the network */ - bufferedAmount -= asyncSocket->flush(); - + asyncSocket->flush(); /* Check if there's still data waiting to be sent after flush attempt */ - if (bufferedAmount > 0) { + if (asyncSocket->getBufferedAmount() > 0) { /* Socket buffer is not completely empty yet * - Reset the timeout to prevent premature connection closure * - This allows time for another writable event or new request @@ -498,6 +497,7 @@ private: if (httpResponseData->state & HttpResponseData::HTTP_CONNECTION_CLOSE) { if ((httpResponseData->state & HttpResponseData::HTTP_RESPONSE_PENDING) == 0) { if (asyncSocket->getBufferedAmount() == 0) { + asyncSocket->shutdown(); /* We need to force close after sending FIN since we want to hinder * clients from keeping to send their huge data */ diff --git a/packages/bun-uws/src/HttpResponse.h b/packages/bun-uws/src/HttpResponse.h index 279e9b9cbe..8a2cafe868 100644 --- a/packages/bun-uws/src/HttpResponse.h +++ b/packages/bun-uws/src/HttpResponse.h @@ -112,7 +112,7 @@ public: * one party must tell the other one so. * * This check also serves to limit writing the header only once. */ - if ((httpResponseData->state & HttpResponseData::HTTP_CONNECTION_CLOSE) == 0) { + if ((httpResponseData->state & HttpResponseData::HTTP_CONNECTION_CLOSE) == 0 && !(httpResponseData->state & (HttpResponseData::HTTP_WRITE_CALLED))) { writeHeader("Connection", "close"); } @@ -132,7 +132,6 @@ public: /* Terminating 0 chunk */ Super::write("0\r\n\r\n", 5); - httpResponseData->markDone(); /* We need to check if we should close this socket here now */ @@ -586,7 +585,6 @@ public: if (writtenPtr) { *writtenPtr = total_written; } - /* If we did not fail the write, accept more */ return !has_failed; } diff --git a/packages/bun-uws/src/LoopData.h b/packages/bun-uws/src/LoopData.h index 96e69eec25..52dfc48437 100644 --- a/packages/bun-uws/src/LoopData.h +++ b/packages/bun-uws/src/LoopData.h @@ -118,7 +118,6 @@ public: time_t now = time(0); struct tm tstruct = {}; #ifdef _WIN32 - /* Micro, fucking soft never follows spec. */ gmtime_s(&tstruct, &now); #else gmtime_r(&now, &tstruct); diff --git a/src/bun.js/api/server.zig b/src/bun.js/api/server.zig index 3ca6c5a891..81c3c5c9d0 100644 --- a/src/bun.js/api/server.zig +++ b/src/bun.js/api/server.zig @@ -61,6 +61,7 @@ pub fn writeStatus(comptime ssl: bool, resp_ptr: ?*uws.NewApp(ssl).Response, sta // TODO: rename to StaticBlobRoute? the html bundle is sometimes a static route pub const StaticRoute = @import("./server/StaticRoute.zig"); +pub const FileRoute = @import("./server/FileRoute.zig"); const HTMLBundle = JSC.API.HTMLBundle; @@ -68,6 +69,8 @@ pub const AnyRoute = union(enum) { /// Serve a static file /// "/robots.txt": new Response(...), static: *StaticRoute, + /// Serve a file from disk + file: *FileRoute, /// Bundle an HTML import /// import html from "./index.html"; /// "/": html, @@ -82,6 +85,7 @@ pub const AnyRoute = union(enum) { pub fn memoryCost(this: AnyRoute) usize { return switch (this) { .static => |static_route| static_route.memoryCost(), + .file => |file_route| file_route.memoryCost(), .html => |html_bundle_route| html_bundle_route.data.memoryCost(), .framework_router => @sizeOf(bun.bake.Framework.FileSystemRouterType), }; @@ -90,6 +94,7 @@ pub const AnyRoute = union(enum) { pub fn setServer(this: AnyRoute, server: ?AnyServer) void { switch (this) { .static => |static_route| static_route.server = server, + .file => |file_route| file_route.server = server, .html => |html_bundle_route| html_bundle_route.server = server, .framework_router => {}, // DevServer contains .server field } @@ -98,6 +103,7 @@ pub const AnyRoute = union(enum) { pub fn deref(this: AnyRoute) void { switch (this) { .static => |static_route| static_route.deref(), + .file => |file_route| file_route.deref(), .html => |html_bundle_route| html_bundle_route.deref(), .framework_router => {}, // not reference counted } @@ -106,6 +112,7 @@ pub const AnyRoute = union(enum) { pub fn ref(this: AnyRoute) void { switch (this) { .static => |static_route| static_route.ref(), + .file => |file_route| file_route.ref(), .html => |html_bundle_route| html_bundle_route.ref(), .framework_router => {}, // not reference counted } @@ -182,6 +189,9 @@ pub const AnyRoute = union(enum) { } } + if (try FileRoute.fromJS(global, argument)) |file_route| { + return .{ .file = file_route }; + } return .{ .static = try StaticRoute.fromJS(global, argument) orelse return null }; } }; @@ -2511,6 +2521,9 @@ pub fn NewServer(protocol_enum: enum { http, https }, development_kind: enum { d .static => |static_route| { ServerConfig.applyStaticRoute(any_server, ssl_enabled, app, *StaticRoute, static_route, entry.path, entry.method); }, + .file => |file_route| { + ServerConfig.applyStaticRoute(any_server, ssl_enabled, app, *FileRoute, file_route, entry.path, entry.method); + }, .html => |html_bundle_route| { ServerConfig.applyStaticRoute(any_server, ssl_enabled, app, *HTMLBundle.Route, html_bundle_route.data, entry.path, entry.method); if (dev_server) |dev| { diff --git a/src/bun.js/api/server/FileRoute.zig b/src/bun.js/api/server/FileRoute.zig new file mode 100644 index 0000000000..0399ba7f47 --- /dev/null +++ b/src/bun.js/api/server/FileRoute.zig @@ -0,0 +1,586 @@ +const FileRoute = @This(); + +ref_count: RefCount, +server: ?AnyServer = null, +blob: Blob, +headers: Headers = .{ .allocator = bun.default_allocator }, +status_code: u16, +stat_hash: bun.fs.StatHash = .{}, +has_last_modified_header: bool, +has_content_length_header: bool, + +pub const InitOptions = struct { + server: ?AnyServer, + status_code: u16 = 200, +}; + +pub fn lastModifiedDate(this: *const FileRoute) ?u64 { + if (this.has_last_modified_header) { + if (this.headers.get("last-modified")) |last_modified| { + var string = bun.String.init(last_modified); + defer string.deref(); + const date_f64 = bun.String.parseDate(&string, bun.JSC.VirtualMachine.get().global); + if (!std.math.isNan(date_f64) and std.math.isFinite(date_f64)) { + return @intFromFloat(date_f64); + } + } + } + + if (this.stat_hash.last_modified_u64 > 0) { + return this.stat_hash.last_modified_u64; + } + + return null; +} + +pub fn initFromBlob(blob: Blob, opts: InitOptions) *FileRoute { + const headers = Headers.from(null, bun.default_allocator, .{ .body = &.{ .Blob = blob } }) catch bun.outOfMemory(); + return bun.new(FileRoute, .{ + .ref_count = .init(), + .server = opts.server, + .blob = blob, + .headers = headers, + .status_code = opts.status_code, + }); +} + +fn deinit(this: *FileRoute) void { + this.blob.deinit(); + this.headers.deinit(); + bun.destroy(this); +} + +pub fn memoryCost(this: *const FileRoute) usize { + return @sizeOf(FileRoute) + this.headers.memoryCost() + this.blob.reported_estimated_size; +} + +pub fn fromJS(globalThis: *JSC.JSGlobalObject, argument: JSC.JSValue) bun.JSError!?*FileRoute { + if (argument.as(JSC.WebCore.Response)) |response| { + response.body.value.toBlobIfPossible(); + if (response.body.value == .Blob and response.body.value.Blob.needsToReadFile()) { + if (response.body.value.Blob.store.?.data.file.pathlike == .fd) { + return globalThis.throwTODO("Support serving files from a file descriptor. Please pass a path instead."); + } + + var blob = response.body.value.use(); + + blob.globalThis = globalThis; + blob.allocator = null; + response.body.value = .{ .Blob = blob.dupe() }; + const headers = Headers.from(response.init.headers, bun.default_allocator, .{ .body = &.{ .Blob = blob } }) catch bun.outOfMemory(); + + return bun.new(FileRoute, .{ + .ref_count = .init(), + .server = null, + .blob = blob, + .headers = headers, + .has_last_modified_header = headers.get("last-modified") != null, + .has_content_length_header = headers.get("content-length") != null, + .status_code = response.statusCode(), + }); + } + } + if (argument.as(Blob)) |blob| { + if (blob.needsToReadFile()) { + var b = blob.dupe(); + b.globalThis = globalThis; + b.allocator = null; + return bun.new(FileRoute, .{ + .ref_count = .init(), + .server = null, + .blob = b, + .headers = Headers.from(null, bun.default_allocator, .{ .body = &.{ .Blob = b } }) catch bun.outOfMemory(), + .has_content_length_header = false, + .has_last_modified_header = false, + .status_code = 200, + }); + } + } + return null; +} + +fn writeHeaders(this: *FileRoute, resp: AnyResponse) void { + const entries = this.headers.entries.slice(); + const names = entries.items(.name); + const values = entries.items(.value); + const buf = this.headers.buf.items; + + switch (resp) { + inline .SSL, .TCP => |s| { + for (names, values) |name, value| { + s.writeHeader(name.slice(buf), value.slice(buf)); + } + }, + } + + if (!this.has_last_modified_header) { + if (this.stat_hash.lastModified()) |last_modified| { + resp.writeHeader("last-modified", last_modified); + } + } + + if (this.has_content_length_header) { + resp.markWroteContentLengthHeader(); + } +} + +fn writeStatusCode(_: *FileRoute, status: u16, resp: AnyResponse) void { + switch (resp) { + .SSL => |r| writeStatus(true, r, status), + .TCP => |r| writeStatus(false, r, status), + } +} + +pub fn onHEADRequest(this: *FileRoute, req: *uws.Request, resp: AnyResponse) void { + bun.debugAssert(this.server != null); + + this.on(req, resp, .HEAD); +} + +pub fn onRequest(this: *FileRoute, req: *uws.Request, resp: AnyResponse) void { + this.on(req, resp, bun.http.Method.find(req.method()) orelse .GET); +} + +pub fn on(this: *FileRoute, req: *uws.Request, resp: AnyResponse, method: bun.http.Method) void { + bun.debugAssert(this.server != null); + this.ref(); + if (this.server) |server| { + server.onPendingRequest(); + resp.timeout(server.config().idleTimeout); + } + const path = this.blob.store.?.getPath() orelse { + req.setYield(true); + this.deref(); + return; + }; + + const open_flags = bun.O.RDONLY | bun.O.CLOEXEC | bun.O.NONBLOCK; + + const fd_result = brk: { + if (bun.Environment.isWindows) { + var path_buffer: bun.PathBuffer = undefined; + @memcpy(path_buffer[0..path.len], path); + path_buffer[path.len] = 0; + break :brk bun.sys.open( + path_buffer[0..path.len :0], + open_flags, + 0, + ); + } + break :brk bun.sys.openA( + path, + open_flags, + 0, + ); + }; + + if (fd_result == .err) { + req.setYield(true); + this.deref(); + return; + } + + const fd = fd_result.result; + + const input_if_modified_since_date: ?u64 = req.dateForHeader("if-modified-since"); + + const can_serve_file: bool, const size: u64, const file_type: bun.io.FileType, const pollable: bool = brk: { + const stat = switch (bun.sys.fstat(fd)) { + .result => |s| s, + .err => break :brk .{ false, 0, undefined, false }, + }; + + const stat_size: u64 = @intCast(@max(stat.size, 0)); + const _size: u64 = @min(stat_size, @as(u64, this.blob.size)); + + if (bun.S.ISDIR(@intCast(stat.mode))) { + break :brk .{ false, 0, undefined, false }; + } + + this.stat_hash.hash(stat, path); + + if (bun.S.ISFIFO(@intCast(stat.mode)) or bun.S.ISCHR(@intCast(stat.mode))) { + break :brk .{ true, _size, .pipe, true }; + } + + if (bun.S.ISSOCK(@intCast(stat.mode))) { + break :brk .{ true, _size, .socket, true }; + } + + break :brk .{ true, _size, .file, false }; + }; + + if (!can_serve_file) { + bun.Async.Closer.close(fd, if (bun.Environment.isWindows) bun.windows.libuv.Loop.get()); + req.setYield(true); + this.deref(); + return; + } + + const status_code: u16 = brk: { + // Unlike If-Unmodified-Since, If-Modified-Since can only be used with a + // GET or HEAD. When used in combination with If-None-Match, it is + // ignored, unless the server doesn't support If-None-Match. + if (input_if_modified_since_date) |requested_if_modified_since| { + if (method == .HEAD or method == .GET) { + if (this.lastModifiedDate()) |actual_last_modified_at| { + if (actual_last_modified_at <= requested_if_modified_since) { + break :brk 304; + } + } + } + } + + if (size == 0 and file_type == .file and this.status_code == 200) { + break :brk 204; + } + + break :brk this.status_code; + }; + + req.setYield(false); + + this.writeStatusCode(status_code, resp); + resp.writeMark(); + this.writeHeaders(resp); + + switch (status_code) { + 204, 205, 304, 307, 308 => { + resp.endWithoutBody(resp.shouldCloseConnection()); + this.deref(); + return; + }, + else => {}, + } + + if (file_type == .file and !resp.state().hasWrittenContentLengthHeader()) { + resp.writeHeaderInt("content-length", size); + resp.markWroteContentLengthHeader(); + } + + if (method == .HEAD) { + resp.endWithoutBody(resp.shouldCloseConnection()); + this.deref(); + return; + } + + const transfer = StreamTransfer.create(fd, resp, this, pollable, file_type != .file, file_type); + transfer.start( + if (file_type == .file) this.blob.offset else 0, + if (file_type == .file and this.blob.size > 0) @intCast(size) else null, + ); +} + +fn onResponseComplete(this: *FileRoute, resp: AnyResponse) void { + resp.clearAborted(); + resp.clearOnWritable(); + resp.clearTimeout(); + if (this.server) |server| { + server.onStaticRequestComplete(); + } + this.deref(); +} + +const std = @import("std"); +const bun = @import("bun"); +const JSC = bun.JSC; +const uws = bun.uws; +const Headers = bun.http.Headers; +const AnyServer = JSC.API.AnyServer; +const Blob = JSC.WebCore.Blob; +const writeStatus = @import("../server.zig").writeStatus; +const AnyResponse = uws.AnyResponse; +const Async = bun.Async; +const FileType = bun.io.FileType; +const Output = bun.Output; + +const StreamTransfer = struct { + reader: bun.io.BufferedReader = bun.io.BufferedReader.init(StreamTransfer), + fd: bun.FileDescriptor, + resp: AnyResponse, + route: *FileRoute, + + defer_deinit: ?*bool = null, + max_size: ?u64 = null, + + state: packed struct(u8) { + waiting_for_readable: bool = false, + waiting_for_writable: bool = false, + has_ended_response: bool = false, + has_reader_closed: bool = false, + _: u4 = 0, + } = .{}, + const log = Output.scoped(.StreamTransfer, false); + + pub fn create( + fd: bun.FileDescriptor, + resp: AnyResponse, + route: *FileRoute, + pollable: bool, + nonblocking: bool, + file_type: FileType, + ) *StreamTransfer { + var t = bun.new(StreamTransfer, .{ + .fd = fd, + .resp = resp, + .route = route, + }); + t.reader.flags.close_handle = true; + t.reader.flags.pollable = pollable; + t.reader.flags.nonblocking = nonblocking; + if (comptime bun.Environment.isPosix) { + if (file_type == .socket) { + t.reader.flags.socket = true; + } + } + t.reader.setParent(t); + return t; + } + + fn start(this: *StreamTransfer, start_offset: usize, size: ?usize) void { + log("start", .{}); + + var scope: DeinitScope = undefined; + scope.enter(this); + defer scope.exit(); + + this.state.waiting_for_readable = true; + this.state.waiting_for_writable = true; + this.max_size = size; + + switch (if (start_offset > 0) + this.reader.startFileOffset(this.fd, this.reader.flags.pollable, start_offset) + else + this.reader.start(this.fd, this.reader.flags.pollable)) { + .err => { + this.finish(); + return; + }, + .result => {}, + } + + this.reader.updateRef(true); + + if (bun.Environment.isPosix) { + if (this.reader.handle.getPoll()) |poll| { + if (this.reader.flags.nonblocking) { + poll.flags.insert(.nonblocking); + } + + switch (this.reader.getFileType()) { + .socket => poll.flags.insert(.socket), + .nonblocking_pipe, .pipe => poll.flags.insert(.fifo), + .file => {}, + } + } + } + // the socket maybe open for some time before so we reset the timeout here + if (this.route.server) |server| { + this.resp.timeout(server.config().idleTimeout); + } + this.reader.read(); + + if (!scope.deinit_called) { + // This clones some data so we could avoid that if we're already done. + this.resp.onAborted(*StreamTransfer, onAborted, this); + } + } + + pub fn onReadChunk(this: *StreamTransfer, chunk_: []const u8, state_: bun.io.ReadState) bool { + log("onReadChunk", .{}); + + var scope: DeinitScope = undefined; + scope.enter(this); + defer scope.exit(); + + if (this.state.has_ended_response) { + this.state.waiting_for_readable = false; + return false; + } + + const chunk, const state = brk: { + if (this.max_size) |*max_size| { + const chunk = chunk_[0..@min(chunk_.len, max_size.*)]; + max_size.* -|= chunk.len; + if (state_ != .eof and max_size.* == 0) { + break :brk .{ chunk, .eof }; + } + + break :brk .{ chunk_, state_ }; + } + + break :brk .{ chunk_, state_ }; + }; + + if (state == .eof and !this.state.waiting_for_writable) { + this.state.waiting_for_readable = false; + this.state.has_ended_response = true; + const resp = this.resp; + const route = this.route; + route.onResponseComplete(resp); + resp.end(chunk, resp.shouldCloseConnection()); + log("end: {}", .{chunk.len}); + return false; + } + + if (this.route.server) |server| { + this.resp.timeout(server.config().idleTimeout); + } + + switch (this.resp.write(chunk)) { + .backpressure => { + this.resp.onWritable(*StreamTransfer, onWritable, this); + this.reader.pause(); + this.resp.markNeedsMore(); + this.state.waiting_for_writable = true; + this.state.waiting_for_readable = false; + return false; + }, + .want_more => { + this.state.waiting_for_readable = true; + this.state.waiting_for_writable = false; + + if (state == .eof) { + this.state.waiting_for_readable = false; + return false; + } + + if (bun.Environment.isWindows) + this.reader.unpause(); + + return true; + }, + } + } + + pub fn onReaderDone(this: *StreamTransfer) void { + log("onReaderDone", .{}); + this.state.waiting_for_readable = false; + this.state.has_reader_closed = true; + + var scope: DeinitScope = undefined; + scope.enter(this); + defer scope.exit(); + + this.finish(); + } + + pub fn onReaderError(this: *StreamTransfer, err: bun.sys.Error) void { + log("onReaderError {any}", .{err}); + this.state.waiting_for_readable = false; + + var scope: DeinitScope = undefined; + scope.enter(this); + defer scope.exit(); + + this.finish(); + } + + pub fn eventLoop(this: *StreamTransfer) JSC.EventLoopHandle { + return JSC.EventLoopHandle.init(this.route.server.?.vm().eventLoop()); + } + + pub fn loop(this: *StreamTransfer) *Async.Loop { + return this.eventLoop().loop(); + } + + fn onWritable(this: *StreamTransfer, _: u64, _: AnyResponse) bool { + log("onWritable", .{}); + + var scope: DeinitScope = undefined; + scope.enter(this); + defer scope.exit(); + + if (this.reader.isDone()) { + @branchHint(.unlikely); + log("finish inside onWritable", .{}); + this.finish(); + return true; + } + + // reset the socket timeout before reading more data + if (this.route.server) |server| { + this.resp.timeout(server.config().idleTimeout); + } + + this.state.waiting_for_writable = false; + this.state.waiting_for_readable = true; + this.reader.read(); + return true; + } + + fn finish(this: *StreamTransfer) void { + log("finish", .{}); + this.resp.clearOnWritable(); + this.resp.clearAborted(); + this.resp.clearTimeout(); + + if (!this.state.has_ended_response) { + this.state.has_ended_response = true; + this.state.waiting_for_writable = false; + const resp = this.resp; + const route = this.route; + route.onResponseComplete(resp); + log("endWithoutBody", .{}); + resp.endWithoutBody(resp.shouldCloseConnection()); + } + + if (!this.state.has_reader_closed) { + this.reader.close(); + return; + } + + this.deinit(); + } + + fn onAborted(this: *StreamTransfer, _: AnyResponse) void { + log("onAborted", .{}); + var scope: DeinitScope = undefined; + scope.enter(this); + defer scope.exit(); + + this.finish(); + } + + fn deinit(this: *StreamTransfer) void { + if (this.defer_deinit) |defer_deinit| { + defer_deinit.* = true; + log("deinit deferred", .{}); + return; + } + + log("deinit", .{}); + this.reader.deinit(); + bun.destroy(this); + } +}; + +const DeinitScope = struct { + stream: *StreamTransfer, + prev_defer_deinit: ?*bool, + deinit_called: bool = false, + + /// This has to be an instance method to avoid a use-after-stack. + pub fn enter(this: *DeinitScope, stream: *StreamTransfer) void { + this.stream = stream; + this.deinit_called = false; + this.prev_defer_deinit = this.stream.defer_deinit; + if (this.prev_defer_deinit == null) { + this.stream.defer_deinit = &this.deinit_called; + } + } + + pub fn exit(this: *DeinitScope) void { + if (this.prev_defer_deinit == null and &this.deinit_called == this.stream.defer_deinit) { + this.stream.defer_deinit = this.prev_defer_deinit; + + if (this.deinit_called) { + this.stream.deinit(); + } + } + } +}; + +const RefCount = bun.ptr.RefCount(@This(), "ref_count", deinit, .{}); +pub const ref = RefCount.ref; +pub const deref = RefCount.deref; diff --git a/src/bun.js/bindings/WTF.zig b/src/bun.js/bindings/WTF.zig index 7f2d212f83..e6374bb789 100644 --- a/src/bun.js/bindings/WTF.zig +++ b/src/bun.js/bindings/WTF.zig @@ -17,4 +17,19 @@ pub const WTF = struct { return error.InvalidCharacter; return res; } + + extern fn Bun__writeHTTPDate(buffer: *[32]u8, length: usize, timestampMs: u64) c_int; + + pub fn writeHTTPDate(buffer: *[32]u8, timestampMs: u64) []u8 { + if (timestampMs == 0) { + return buffer[0..0]; + } + + const res = Bun__writeHTTPDate(buffer, 32, timestampMs); + if (res < 1) { + return buffer[0..0]; + } + + return buffer[0..@intCast(res)]; + } }; diff --git a/src/bun.js/bindings/webcore/HTTPParsers.cpp b/src/bun.js/bindings/webcore/HTTPParsers.cpp index 1eb0a3a530..53d18ffc13 100644 --- a/src/bun.js/bindings/webcore/HTTPParsers.cpp +++ b/src/bun.js/bindings/webcore/HTTPParsers.cpp @@ -987,4 +987,34 @@ CrossOriginResourcePolicy parseCrossOriginResourcePolicyHeader(StringView header return CrossOriginResourcePolicy::Invalid; } +extern "C" int Bun__writeHTTPDate(char* buffer, size_t length, uint64_t timestampMs) +{ + if (timestampMs == 0) { + return 0; + } + + time_t timestamp = timestampMs / 1000; + struct tm tstruct = {}; +#ifdef _WIN32 + gmtime_s(&tstruct, ×tamp); +#else + gmtime_r(×tamp, &tstruct); +#endif + static const char wday_name[][4] = { + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" + }; + static const char mon_name[][4] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" + }; + return snprintf(buffer, length, "%.3s, %.2u %.3s %.4u %.2u:%.2u:%.2u GMT", + wday_name[tstruct.tm_wday], + tstruct.tm_mday % 99, + mon_name[tstruct.tm_mon], + (1900 + tstruct.tm_year) % 9999, + tstruct.tm_hour % 99, + tstruct.tm_min % 99, + tstruct.tm_sec % 99); +} + } diff --git a/src/deps/libuwsockets.cpp b/src/deps/libuwsockets.cpp index ed349c230c..ac7c3a347d 100644 --- a/src/deps/libuwsockets.cpp +++ b/src/deps/libuwsockets.cpp @@ -1212,6 +1212,26 @@ extern "C" } } + void uws_res_mark_wrote_content_length_header(int ssl, uws_res_r res) { + if (ssl) { + uWS::HttpResponse *uwsRes = (uWS::HttpResponse *)res; + uwsRes->getHttpResponseData()->state |= uWS::HttpResponseData::HTTP_WROTE_CONTENT_LENGTH_HEADER; + } else { + uWS::HttpResponse *uwsRes = (uWS::HttpResponse *)res; + uwsRes->getHttpResponseData()->state |= uWS::HttpResponseData::HTTP_WROTE_CONTENT_LENGTH_HEADER; + } + } + + void uws_res_write_mark(int ssl, uws_res_r res) { + if (ssl) { + uWS::HttpResponse *uwsRes = (uWS::HttpResponse *)res; + uwsRes->writeMark(); + } else { + uWS::HttpResponse *uwsRes = (uWS::HttpResponse *)res; + uwsRes->writeMark(); + } + } + void uws_res_write_header(int ssl, uws_res_r res, const char *key, size_t key_length, const char *value, size_t value_length) diff --git a/src/deps/uws/Request.zig b/src/deps/uws/Request.zig index b0b40080c3..1f614e037a 100644 --- a/src/deps/uws/Request.zig +++ b/src/deps/uws/Request.zig @@ -25,6 +25,17 @@ pub const Request = opaque { if (len == 0) return null; return ptr[0..len]; } + pub fn dateForHeader(req: *Request, name: []const u8) ?u64 { + const value = header(req, name); + if (value == null) return null; + var string = bun.String.init(value.?); + defer string.deref(); + const date_f64 = bun.String.parseDate(&string, bun.JSC.VirtualMachine.get().global); + if (!std.math.isNan(date_f64) and std.math.isFinite(date_f64)) { + return @intFromFloat(date_f64); + } + return null; + } pub fn query(req: *Request, name: []const u8) []const u8 { var ptr: [*]const u8 = undefined; return ptr[0..c.uws_req_get_query(req, name.ptr, name.len, &ptr)]; diff --git a/src/deps/uws/Response.zig b/src/deps/uws/Response.zig index 3ac7211bf6..800b211e90 100644 --- a/src/deps/uws/Response.zig +++ b/src/deps/uws/Response.zig @@ -104,6 +104,14 @@ pub fn NewResponse(ssl_flag: i32) type { return c.uws_res_has_responded(ssl_flag, res.downcast()); } + pub fn markWroteContentLengthHeader(res: *Response) void { + c.uws_res_mark_wrote_content_length_header(ssl_flag, res.downcast()); + } + + pub fn writeMark(res: *Response) void { + c.uws_res_write_mark(ssl_flag, res.downcast()); + } + pub fn getNativeHandle(res: *Response) bun.FileDescriptor { if (comptime Environment.isWindows) { // on windows uSockets exposes SOCKET @@ -306,6 +314,30 @@ pub const AnyResponse = union(enum) { SSL: *uws.NewApp(true).Response, TCP: *uws.NewApp(false).Response, + pub fn markNeedsMore(this: AnyResponse) void { + return switch (this) { + inline else => |resp| resp.markNeedsMore(), + }; + } + + pub fn markWroteContentLengthHeader(this: AnyResponse) void { + return switch (this) { + inline else => |resp| resp.markWroteContentLengthHeader(), + }; + } + + pub fn writeMark(this: AnyResponse) void { + return switch (this) { + inline else => |resp| resp.writeMark(), + }; + } + + pub fn endSendFile(this: AnyResponse, write_offset: u64, close_connection: bool) void { + return switch (this) { + inline else => |resp| resp.endSendFile(write_offset, close_connection), + }; + } + pub fn socket(this: AnyResponse) *c.uws_res { return switch (this) { inline else => |resp| resp.downcast(), @@ -576,6 +608,8 @@ pub const uws_res = c.uws_res; const c = struct { pub const uws_res = opaque {}; + pub extern fn uws_res_mark_wrote_content_length_header(ssl: i32, res: *c.uws_res) void; + pub extern fn uws_res_write_mark(ssl: i32, res: *c.uws_res) void; pub extern fn us_socket_mark_needs_more_not_ssl(socket: ?*c.uws_res) void; pub extern fn uws_res_state(ssl: c_int, res: *const c.uws_res) State; pub extern fn uws_res_get_remote_address_info(res: *c.uws_res, dest: *[*]const u8, port: *i32, is_ipv6: *bool) usize; diff --git a/src/fs.zig b/src/fs.zig index b503edf413..c3a8533a24 100644 --- a/src/fs.zig +++ b/src/fs.zig @@ -1974,3 +1974,4 @@ pub const Path = struct { // defer std.posix.close(opened); // } +pub const StatHash = @import("./fs/stat_hash.zig"); diff --git a/src/fs/stat_hash.zig b/src/fs/stat_hash.zig new file mode 100644 index 0000000000..3670de2275 --- /dev/null +++ b/src/fs/stat_hash.zig @@ -0,0 +1,49 @@ +value: u64 = 0, + +last_modified_u64: u64 = 0, +last_modified_buffer: [32]u8 = undefined, +last_modified_buffer_len: u8 = 0, + +// TODO: add etag support here! + +pub fn hash(this: *@This(), stat: bun.Stat, path: []const u8) void { + var stat_hasher = std.hash.XxHash64.init(42); + stat_hasher.update(std.mem.asBytes(&stat.size)); + stat_hasher.update(std.mem.asBytes(&stat.mode)); + stat_hasher.update(std.mem.asBytes(&stat.mtime())); + stat_hasher.update(std.mem.asBytes(&stat.ino)); + stat_hasher.update(path); + + const prev = this.value; + this.value = stat_hasher.final(); + + if (prev != this.value and bun.S.ISREG(@intCast(stat.mode))) { + const mtime_timespec = stat.mtime(); + // Clamp negative values to 0 to avoid timestamp overflow issues on Windows + const mtime = bun.timespec{ + .nsec = @intCast(@max(mtime_timespec.nsec, 0)), + .sec = @intCast(@max(mtime_timespec.sec, 0)), + }; + if (mtime.ms() > 0) { + this.last_modified_buffer_len = @intCast(bun.JSC.wtf.writeHTTPDate(&this.last_modified_buffer, mtime.msUnsigned()).len); + this.last_modified_u64 = mtime.msUnsigned(); + } else { + this.last_modified_buffer_len = 0; + this.last_modified_u64 = 0; + } + } else if (!bun.S.ISREG(@intCast(stat.mode))) { + this.last_modified_buffer_len = 0; + this.last_modified_u64 = 0; + } +} + +pub fn lastModified(this: *const @This()) ?[]const u8 { + if (this.last_modified_buffer_len == 0) { + return null; + } + + return this.last_modified_buffer[0..this.last_modified_buffer_len]; +} + +const bun = @import("bun"); +const std = @import("std"); diff --git a/src/http.zig b/src/http.zig index 2086eb370c..3084a08d67 100644 --- a/src/http.zig +++ b/src/http.zig @@ -4794,6 +4794,19 @@ pub const Headers = struct { }; } + pub fn get(this: *const Headers, name: []const u8) ?[]const u8 { + const entries = this.entries.slice(); + const names = entries.items(.name); + const values = entries.items(.value); + for (names, 0..) |name_ptr, i| { + if (bun.strings.eqlCaseInsensitiveASCII(this.asStr(name_ptr), name, true)) { + return this.asStr(values[i]); + } + } + + return null; + } + pub fn append(this: *Headers, name: []const u8, value: []const u8) !void { var offset: u32 = @truncate(this.buf.items.len); try this.buf.ensureUnusedCapacity(this.allocator, name.len + value.len); diff --git a/src/io/PipeReader.zig b/src/io/PipeReader.zig index a1906866df..321b686a98 100644 --- a/src/io/PipeReader.zig +++ b/src/io/PipeReader.zig @@ -147,7 +147,7 @@ const PosixBufferedReader = struct { this.handle = .{ .fd = fd }; } - fn getFileType(this: *const PosixBufferedReader) FileType { + pub fn getFileType(this: *const PosixBufferedReader) FileType { const flags = this.flags; if (flags.socket) { return .socket; @@ -183,7 +183,6 @@ const PosixBufferedReader = struct { // No-op on posix. pub fn pause(this: *PosixBufferedReader) void { _ = this; // autofix - } pub fn takeBuffer(this: *PosixBufferedReader) std.ArrayList(u8) { @@ -443,7 +442,8 @@ const PosixBufferedReader = struct { if (bytes_read == 0) { // EOF - finished and closed pipe parent.closeWithoutReporting(); - parent.done(); + if (!parent.flags.is_done) + parent.done(); return; } @@ -474,7 +474,8 @@ const PosixBufferedReader = struct { if (bytes_read == 0) { parent.closeWithoutReporting(); - parent.done(); + if (!parent.flags.is_done) + parent.done(); return; } @@ -531,7 +532,8 @@ const PosixBufferedReader = struct { parent.closeWithoutReporting(); if (stack_buffer[0 .. stack_buffer.len - stack_buffer_head.len].len > 0) _ = parent.vtable.onReadChunk(stack_buffer[0 .. stack_buffer.len - stack_buffer_head.len], .eof); - parent.done(); + if (!parent.flags.is_done) + parent.done(); return; } @@ -590,7 +592,8 @@ const PosixBufferedReader = struct { if (bytes_read == 0) { parent.closeWithoutReporting(); _ = drainChunk(parent, resizable_buffer.items, .eof); - parent.done(); + if (!parent.flags.is_done) + parent.done(); return; } }, @@ -625,7 +628,8 @@ const PosixBufferedReader = struct { if (bytes_read == 0) { parent.closeWithoutReporting(); _ = drainChunk(parent, resizable_buffer.items, .eof); - parent.done(); + if (!parent.flags.is_done) + parent.done(); return; } @@ -891,11 +895,11 @@ pub const WindowsBufferedReader = struct { MaxBuf.removeFromPipereader(&this.maxbuf); this.buffer().deinit(); const source = this.source orelse return; + this.source = null; if (!source.isClosed()) { // closeImpl will take care of freeing the source this.closeImpl(false); } - this.source = null; } pub fn setRawMode(this: *WindowsBufferedReader, value: bool) bun.JSC.Maybe(void) { @@ -1056,9 +1060,9 @@ pub const WindowsBufferedReader = struct { switch (source) { .sync_file, .file => |file| { if (!this.flags.is_paused) { + this.flags.is_paused = true; // always cancel the current one file.fs.cancel(); - this.flags.is_paused = true; } // always use close_fs here because we can have a operation in progress file.close_fs.data = file; @@ -1066,6 +1070,7 @@ pub const WindowsBufferedReader = struct { }, .pipe => |pipe| { pipe.data = pipe; + this.flags.is_paused = true; pipe.close(onPipeClose); }, .tty => |tty| { @@ -1075,6 +1080,7 @@ pub const WindowsBufferedReader = struct { } tty.data = tty; + this.flags.is_paused = true; tty.close(onTTYClose); }, } diff --git a/test/internal/ban-words.test.ts b/test/internal/ban-words.test.ts index fb482177a9..65ab1f4b3b 100644 --- a/test/internal/ban-words.test.ts +++ b/test/internal/ban-words.test.ts @@ -32,7 +32,7 @@ const words: Record "== alloc.ptr": { reason: "The std.mem.Allocator context pointer can be undefined, which makes this comparison undefined behavior" }, "!= alloc.ptr": { reason: "The std.mem.Allocator context pointer can be undefined, which makes this comparison undefined behavior" }, - [String.raw`: [a-zA-Z0-9_\.\*\?\[\]\(\)]+ = undefined,`]: { reason: "Do not default a struct field to undefined", limit: 241, regex: true }, + [String.raw`: [a-zA-Z0-9_\.\*\?\[\]\(\)]+ = undefined,`]: { reason: "Do not default a struct field to undefined", limit: 242, regex: true }, "usingnamespace": { reason: "Zig 0.15 will remove `usingnamespace`" }, "catch unreachable": { reason: "For out-of-memory, prefer 'catch bun.outOfMemory()'", limit: 1854 }, diff --git a/test/js/bun/http/bun-serve-file.test.ts b/test/js/bun/http/bun-serve-file.test.ts new file mode 100644 index 0000000000..3d49662bf5 --- /dev/null +++ b/test/js/bun/http/bun-serve-file.test.ts @@ -0,0 +1,592 @@ +import type { Server } from "bun"; +import { afterAll, beforeAll, describe, expect, it, mock, test } from "bun:test"; +import { isWindows, rmScope, tempDirWithFiles } from "harness"; +import { unlinkSync } from "node:fs"; +import { join } from "node:path"; + +const LARGE_SIZE = 1024 * 1024 * 8; +const files = { + "hello.txt": "Hello, World!", + "empty.txt": "", + "binary.bin": Buffer.from([0x00, 0x01, 0x02, 0x03, 0xff, 0xfe, 0xfd]), + "large.txt": Buffer.alloc(LARGE_SIZE, "bun").toString(), + "unicode.txt": "Hello 世界 🌍 émojis", + "json.json": JSON.stringify({ message: "test", number: 42 }), + "nested/file.txt": "nested content", + "special chars & symbols.txt": "special file content", + "will-be-deleted.txt": "will be deleted", + "partial.txt": "0123456789ABCDEF", +}; + +describe("Bun.file in serve routes", () => { + let server: Server; + let tempDir: string; + let handler = mock(req => { + return new Response(`fallback: ${req.url}`, { + headers: { + "Content-Type": "text/plain", + }, + }); + }); + + beforeAll(async () => { + tempDir = tempDirWithFiles("bun-serve-file-test-", files); + + const routes = { + "/hello.txt": { + GET: new Response(Bun.file(join(tempDir, "hello.txt"))), + HEAD: new Response(Bun.file(join(tempDir, "hello.txt"))), + }, + "/empty.txt": new Response(Bun.file(join(tempDir, "empty.txt"))), + "/empty-400.txt": new Response(Bun.file(join(tempDir, "empty.txt")), { + status: 400, + }), + "/binary.bin": new Response(Bun.file(join(tempDir, "binary.bin"))), + "/large.txt": new Response(Bun.file(join(tempDir, "large.txt"))), + "/unicode.txt": new Response(Bun.file(join(tempDir, "unicode.txt"))), + "/json.json": new Response(Bun.file(join(tempDir, "json.json"))), + "/nested/file.txt": new Response(Bun.file(join(tempDir, "nested", "file.txt"))), + "/special-chars.txt": new Response(Bun.file(join(tempDir, "special chars & symbols.txt"))), + "/nonexistent.txt": new Response(Bun.file(join(tempDir, "does-not-exist.txt"))), + "/with-headers.txt": new Response(Bun.file(join(tempDir, "hello.txt")), { + headers: { + "X-Custom-Header": "custom-value", + "Cache-Control": "max-age=3600", + }, + }), + "/with-status.txt": new Response(Bun.file(join(tempDir, "hello.txt")), { + status: 201, + statusText: "Created", + }), + "/will-be-deleted.txt": new Response(Bun.file(join(tempDir, "will-be-deleted.txt"))), + "/custom-last-modified.txt": new Response(Bun.file(join(tempDir, "hello.txt")), { + headers: { + "Last-Modified": "Wed, 21 Oct 2015 07:28:00 GMT", + }, + }), + "/partial.txt": new Response(Bun.file(join(tempDir, "partial.txt"))), + "/partial-slice.txt": new Response(Bun.file(join(tempDir, "partial.txt")).slice(5, 10)), + "/fd-not-supported.txt": (() => { + // This would test file descriptors, but they're not supported yet + return new Response(Bun.file(join(tempDir, "hello.txt"))); + })(), + } as const; + + server = Bun.serve({ + routes: routes, + port: 0, + fetch: handler, + }); + server.unref(); + + unlinkSync(join(tempDir, "will-be-deleted.txt")); + }); + + afterAll(() => { + server?.stop(true); + using _ = rmScope(tempDir); + }); + + describe("Basic file serving", () => { + it("serves text file", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url)); + expect(res.status).toBe(200); + expect(await res.text()).toBe("Hello, World!"); + const headers = res.headers.toJSON(); + if (!new Date(headers["last-modified"]!).getTime()) { + throw new Error("Last-Modified header is not a valid date"); + } + + if (!new Date(headers["date"]!).getTime()) { + throw new Error("Date header is not a valid date"); + } + + delete headers.date; + delete headers["last-modified"]; + + // Snapshot the headers so a test fails if we change the headers later. + expect(headers).toMatchInlineSnapshot(` + { + "content-length": "13", + "content-type": "text/plain;charset=utf-8", + } + `); + }); + + it("serves empty file", async () => { + const res = await fetch(new URL(`/empty.txt`, server.url)); + expect(res.status).toBe(204); + expect(await res.text()).toBe(""); + // A server MUST NOT send a Content-Length header field in any response + // with a status code of 1xx (Informational) or 204 (No Content). A server + // MUST NOT send a Content-Length header field in any 2xx (Successful) + // response to a CONNECT request (Section 9.3.6). + expect(res.headers.get("Content-Length")).toBeNull(); + + const headers = res.headers.toJSON(); + delete headers.date; + delete headers["last-modified"]; + + expect(headers).toMatchInlineSnapshot(` + { + "content-type": "text/plain;charset=utf-8", + } + `); + }); + + it("serves empty file with custom status code", async () => { + const res = await fetch(new URL(`/empty-400.txt`, server.url)); + expect(res.status).toBe(400); + expect(await res.text()).toBe(""); + expect(res.headers.get("Content-Length")).toBe("0"); + }); + + it("serves binary file", async () => { + const res = await fetch(new URL(`/binary.bin`, server.url)); + expect(res.status).toBe(200); + const bytes = await res.bytes(); + expect(bytes).toEqual(new Uint8Array([0x00, 0x01, 0x02, 0x03, 0xff, 0xfe, 0xfd])); + expect(res.headers.get("Content-Type")).toMatch(/application\/octet-stream/); + }); + + it("serves large file", async () => { + const res = await fetch(new URL(`/large.txt`, server.url)); + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).toHaveLength(LARGE_SIZE); + + if (files["large.txt"] !== text) { + console.log("Expected length:", files["large.txt"].length); + console.log("Actual length:", text.length); + console.log("First 100 chars expected:", files["large.txt"].slice(0, 100)); + console.log("First 100 chars actual:", text.slice(0, 100)); + console.log("Last 100 chars expected:", files["large.txt"].slice(-100)); + console.log("Last 100 chars actual:", text.slice(-100)); + + // Find first difference + for (let i = 0; i < Math.min(files["large.txt"].length, text.length); i++) { + if (files["large.txt"][i] !== text[i]) { + console.log(`First difference at index ${i}:`); + console.log(`Expected: "${files["large.txt"][i]}" (code: ${files["large.txt"].charCodeAt(i)})`); + console.log(`Actual: "${text[i]}" (code: ${text.charCodeAt(i)})`); + console.log(`Context around difference: "${files["large.txt"].slice(Math.max(0, i - 10), i + 10)}"`); + console.log(`Actual context: "${text.slice(Math.max(0, i - 10), i + 10)}"`); + break; + } + } + throw new Error("large.txt is not the same"); + } + + expect(res.headers.get("Content-Length")).toBe(LARGE_SIZE.toString()); + + const headers = res.headers.toJSON(); + delete headers.date; + delete headers["last-modified"]; + + expect(headers).toMatchInlineSnapshot(` + { + "content-length": "${LARGE_SIZE}", + "content-type": "text/plain;charset=utf-8", + } + `); + }); + + it("serves unicode file", async () => { + const res = await fetch(new URL(`/unicode.txt`, server.url)); + expect(res.status).toBe(200); + expect(await res.text()).toBe("Hello 世界 🌍 émojis"); + + const headers = res.headers.toJSON(); + delete headers.date; + delete headers["last-modified"]; + + expect(headers).toMatchInlineSnapshot(` + { + "content-length": "25", + "content-type": "text/plain;charset=utf-8", + } + `); + }); + + it("serves JSON file with correct content type", async () => { + const res = await fetch(new URL(`/json.json`, server.url)); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ message: "test", number: 42 }); + expect(res.headers.get("Content-Type")).toMatch(/application\/json/); + }); + + it("serves nested file", async () => { + const res = await fetch(new URL(`/nested/file.txt`, server.url)); + expect(res.status).toBe(200); + expect(await res.text()).toBe("nested content"); + }); + + it("serves file with special characters in name", async () => { + const res = await fetch(new URL(`/special-chars.txt`, server.url)); + expect(res.status).toBe(200); + expect(await res.text()).toBe("special file content"); + }); + }); + + describe("HTTP methods", () => { + it("supports HEAD requests", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url), { method: "HEAD" }); + expect(res.status).toBe(200); + expect(await res.text()).toBe(""); + expect(res.headers.get("Content-Length")).toBe("13"); // "Hello, World!" length + expect(res.headers.get("Content-Type")).toMatch(/text\/plain/); + }); + + it("supports GET requests", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url), { method: "GET" }); + expect(res.status).toBe(200); + expect(await res.text()).toBe("Hello, World!"); + }); + }); + + describe("Custom headers and status", () => { + it("preserves custom headers", async () => { + const res = await fetch(new URL(`/with-headers.txt`, server.url)); + expect(res.status).toBe(200); + expect(await res.text()).toBe("Hello, World!"); + expect(res.headers.get("X-Custom-Header")).toBe("custom-value"); + expect(res.headers.get("Cache-Control")).toBe("max-age=3600"); + }); + + it("preserves custom status", async () => { + const res = await fetch(new URL(`/with-status.txt`, server.url)); + expect(res.status).toBe(201); + expect(res.statusText).toBe("Created"); + expect(await res.text()).toBe("Hello, World!"); + }); + }); + + describe("Error handling", () => { + it("handles nonexistent files gracefully", async () => { + const previousCallCount = handler.mock.calls.length; + const res = await fetch(new URL(`/nonexistent.txt`, server.url)); + + // Should fall back to the handler since file doesn't exist + expect(res.status).toBe(200); + expect(await res.text()).toBe(`fallback: ${server.url}nonexistent.txt`); + expect(handler.mock.calls.length).toBe(previousCallCount + 1); + }); + }); + + describe.todo("Range requests", () => { + it("supports partial content requests", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url), { + headers: { + "Range": "bytes=0-4", + }, + }); + + if (res.status === 206) { + expect(await res.text()).toBe("Hello"); + expect(res.headers.get("Content-Range")).toMatch(/bytes 0-4\/13/); + expect(res.headers.get("Accept-Ranges")).toBe("bytes"); + } else { + // If range requests aren't supported, should return full content + expect(res.status).toBe(200); + expect(await res.text()).toBe("Hello, World!"); + } + }); + + it("handles invalid range requests", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url), { + headers: { + "Range": "bytes=20-30", // Beyond file size + }, + }); + + // Should either return 416 Range Not Satisfiable or 200 with full content + expect([200, 416]).toContain(res.status); + }); + }); + + describe("Conditional requests", () => { + describe.each(["GET", "HEAD"])("%s", method => { + it(`handles If-Modified-Since with future date (304)`, async () => { + // First request to get Last-Modified + const res1 = await fetch(new URL(`/hello.txt`, server.url)); + const lastModified = res1.headers.get("Last-Modified"); + expect(lastModified).not.toBeEmpty(); + + // If-Modified-Since is AFTER the file's last modified date (future) + // Should return 304 because file hasn't been modified since that future date + const res2 = await fetch(new URL(`/hello.txt`, server.url), { + method, + headers: { + "If-Modified-Since": new Date(Date.parse(lastModified!) + 10000).toISOString(), + }, + }); + + expect(res2.status).toBe(304); + expect(await res2.text()).toBe(""); + }); + + it(`handles If-Modified-Since with past date (200)`, async () => { + // If-Modified-Since is way in the past + // Should return 200 because file has been modified since then + const res = await fetch(new URL(`/hello.txt`, server.url), { + method, + headers: { + "If-Modified-Since": new Date(Date.now() - 1000000).toISOString(), + }, + }); + + expect(res.status).toBe(200); + }); + }); + + it("ignores If-Modified-Since for non-GET/HEAD requests", async () => { + const res1 = await fetch(new URL(`/hello.txt`, server.url)); + const lastModified = res1.headers.get("Last-Modified"); + + const res2 = await fetch(new URL(`/hello.txt`, server.url), { + method: "POST", + headers: { + "If-Modified-Since": new Date(Date.parse(lastModified!) + 10000).toISOString(), + }, + }); + + // Should not return 304 for POST + expect(res2.status).not.toBe(304); + }); + + it.todo("handles ETag", async () => { + const res1 = await fetch(new URL(`/hello.txt`, server.url)); + const etag = res1.headers.get("ETag"); + + const res2 = await fetch(new URL(`/hello.txt`, server.url), { + headers: { + "If-None-Match": etag!, + }, + }); + + expect(res2.status).toBe(304); + expect(await res2.text()).toBe(""); + }); + }); + + describe("Stress testing", () => { + test.each(["hello.txt", "large.txt"])( + "concurrent requests for %s", + async filename => { + const batchSize = isWindows ? 8 : 32; + const iterations = isWindows ? 2 : 5; + + async function iterate() { + const promises = Array.from({ length: batchSize }, () => + fetch(`${server.url}${filename}`).then(res => { + expect(res.status).toBe(200); + return res.text(); + }), + ); + + const results = await Promise.all(promises); + + // Verify all responses are identical + const expected = results[0]; + results.forEach(result => { + expect(result).toBe(expected); + }); + } + + for (let i = 0; i < iterations; i++) { + await iterate(); + Bun.gc(); + } + }, + 30000, + ); + + it("memory usage stays reasonable", async () => { + Bun.gc(true); + const baseline = (process.memoryUsage.rss() / 1024 / 1024) | 0; + + // Make many requests to large file + for (let i = 0; i < 50; i++) { + const res = await fetch(new URL(`/large.txt`, server.url)); + expect(res.status).toBe(200); + await res.text(); // Consume the response + } + + Bun.gc(true); + const final = (process.memoryUsage.rss() / 1024 / 1024) | 0; + const delta = final - baseline; + + expect(delta).toBeLessThan(100); // Should not leak significant memory + }, 30000); + + it("deleted file goes to handler", async () => { + const previousCallCount = handler.mock.calls.length; + const res = await fetch(new URL(`/will-be-deleted.txt`, server.url)); + expect(res.status).toBe(200); + expect(await res.text()).toBe(`fallback: ${server.url}will-be-deleted.txt`); + expect(handler.mock.calls.length).toBe(previousCallCount + 1); + }); + }); + + describe("Handler fallback", () => { + it("falls back to handler for unmatched routes", async () => { + const previousCallCount = handler.mock.calls.length; + const res = await fetch(new URL(`/not-in-routes.txt`, server.url)); + + expect(res.status).toBe(200); + expect(await res.text()).toBe(`fallback: ${server.url}not-in-routes.txt`); + expect(handler.mock.calls.length).toBe(previousCallCount + 1); + }); + + it("does not call handler for matched file routes", async () => { + const previousCallCount = handler.mock.calls.length; + const res = await fetch(new URL(`/hello.txt`, server.url)); + + expect(res.status).toBe(200); + expect(await res.text()).toBe("Hello, World!"); + expect(handler.mock.calls.length).toBe(previousCallCount); + }); + }); + + describe("Last-Modified header handling", () => { + it("automatically adds Last-Modified header", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url)); + const lastModified = res.headers.get("Last-Modified"); + expect(lastModified).not.toBeNull(); + expect(lastModified).toMatch(/^[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/); + }); + + it("respects custom Last-Modified header", async () => { + const res = await fetch(new URL(`/custom-last-modified.txt`, server.url)); + expect(res.headers.get("Last-Modified")).toBe("Wed, 21 Oct 2015 07:28:00 GMT"); + }); + + it("uses custom Last-Modified for If-Modified-Since checks", async () => { + // Request with If-Modified-Since after custom date + const res1 = await fetch(new URL(`/custom-last-modified.txt`, server.url), { + headers: { + "If-Modified-Since": "Thu, 22 Oct 2015 07:28:00 GMT", + }, + }); + expect(res1.status).toBe(304); + + // Request with If-Modified-Since before custom date + const res2 = await fetch(new URL(`/custom-last-modified.txt`, server.url), { + headers: { + "If-Modified-Since": "Tue, 20 Oct 2015 07:28:00 GMT", + }, + }); + expect(res2.status).toBe(200); + }); + }); + + describe("File slicing", () => { + it("serves complete file", async () => { + const res = await fetch(new URL(`/partial.txt`, server.url)); + expect(res.status).toBe(200); + expect(await res.text()).toBe("0123456789ABCDEF"); + expect(res.headers.get("Content-Length")).toBe("16"); + }); + + it("serves sliced file", async () => { + const res = await fetch(new URL(`/partial-slice.txt`, server.url)); + expect(res.status).toBe(200); + expect(await res.text()).toBe("56789"); + expect(res.headers.get("Content-Length")).toBe("5"); + }); + }); + + describe("Special status codes", () => { + it("returns 204 for empty files with 200 status", async () => { + const res = await fetch(new URL(`/empty.txt`, server.url)); + expect(res.status).toBe(204); + expect(await res.text()).toBe(""); + }); + + it("preserves custom status for empty files", async () => { + const res = await fetch(new URL(`/empty-400.txt`, server.url)); + expect(res.status).toBe(400); + expect(await res.text()).toBe(""); + }); + + it("returns appropriate status for 304 responses", async () => { + const res1 = await fetch(new URL(`/hello.txt`, server.url)); + const lastModified = res1.headers.get("Last-Modified"); + + const res2 = await fetch(new URL(`/hello.txt`, server.url), { + headers: { + "If-Modified-Since": new Date(Date.parse(lastModified!) + 10000).toISOString(), + }, + }); + + expect(res2.status).toBe(304); + expect(res2.headers.get("Content-Length")).toBeNull(); + expect(await res2.text()).toBe(""); + }); + }); + + describe("Streaming and file types", () => { + it("sets Content-Length for regular files", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url)); + expect(res.headers.get("Content-Length")).toBe("13"); + }); + + it("handles HEAD requests with proper headers", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url), { method: "HEAD" }); + expect(res.status).toBe(200); + expect(res.headers.get("Content-Length")).toBe("13"); + expect(res.headers.get("Content-Type")).toMatch(/text\/plain/); + expect(res.headers.get("Last-Modified")).not.toBeNull(); + expect(await res.text()).toBe(""); + }); + + it("handles abort/cancellation gracefully", async () => { + const controller = new AbortController(); + const promise = fetch(new URL(`/large.txt`, server.url), { + signal: controller.signal, + }); + + // Abort immediately + controller.abort(); + + await expect(promise).rejects.toThrow(/abort/i); + }); + }); + + describe("File not found handling", () => { + it("falls back to handler when file doesn't exist", async () => { + const previousCallCount = handler.mock.calls.length; + const res = await fetch(new URL(`/nonexistent.txt`, server.url)); + + expect(res.status).toBe(200); + expect(await res.text()).toBe(`fallback: ${server.url}nonexistent.txt`); + expect(handler.mock.calls.length).toBe(previousCallCount + 1); + }); + + it("falls back to handler when file is deleted after route creation", async () => { + const previousCallCount = handler.mock.calls.length; + const res = await fetch(new URL(`/will-be-deleted.txt`, server.url)); + + expect(res.status).toBe(200); + expect(await res.text()).toBe(`fallback: ${server.url}will-be-deleted.txt`); + expect(handler.mock.calls.length).toBe(previousCallCount + 1); + }); + }); + + describe("Content-Type detection", () => { + it("detects text/plain for .txt files", async () => { + const res = await fetch(new URL(`/hello.txt`, server.url)); + expect(res.headers.get("Content-Type")).toMatch(/text\/plain/); + }); + + it("detects application/json for .json files", async () => { + const res = await fetch(new URL(`/json.json`, server.url)); + expect(res.headers.get("Content-Type")).toMatch(/application\/json/); + }); + + it("detects application/octet-stream for binary files", async () => { + const res = await fetch(new URL(`/binary.bin`, server.url)); + expect(res.headers.get("Content-Type")).toMatch(/application\/octet-stream/); + }); + }); +}); From 6ebad50543bf2c4107d4b4c2ecc062a15a94b71e Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 10 Jun 2025 21:26:00 -0700 Subject: [PATCH 2/3] Introduce ahead of time bundling for HTML imports with `bun build` (#20265) Co-authored-by: Jarred-Sumner <709451+Jarred-Sumner@users.noreply.github.com> Co-authored-by: Dylan Conway Co-authored-by: dylan-conway <35280289+dylan-conway@users.noreply.github.com> --- cmake/sources/ZigSources.txt | 1 + src/OutputFile.zig | 15 +- src/bake/DevServer.zig | 6 +- src/bun.js/ResolveMessage.zig | 1 + src/bun.js/RuntimeTranspilerCache.zig | 3 +- src/bundler/Chunk.zig | 51 ++- src/bundler/Graph.zig | 42 +-- src/bundler/HTMLImportManifest.zig | 239 ++++++++++++ src/bundler/LinkerContext.zig | 52 +++ src/bundler/ParseTask.zig | 29 +- src/bundler/ThreadPool.zig | 39 +- src/bundler/bundle_v2.zig | 323 +++++++++++----- src/bundler/linker_context/computeChunks.zig | 53 ++- src/defines-table.zig | 12 + src/deps/uws/BodyReaderMixin.zig | 30 +- src/import_record.zig | 6 +- src/js_parser.zig | 4 +- src/options.zig | 22 ++ src/runtime.js | 2 + src/runtime.zig | 2 + test/bundler/bundler_edgecase.test.ts | 4 +- test/bundler/bundler_html.test.ts | 2 +- test/bundler/html-import-manifest.test.ts | 370 +++++++++++++++++++ test/cli/install/bun-workspaces.test.ts | 3 +- test/internal/ban-words.test.ts | 6 +- 25 files changed, 1117 insertions(+), 200 deletions(-) create mode 100644 src/bundler/HTMLImportManifest.zig create mode 100644 test/bundler/html-import-manifest.test.ts diff --git a/cmake/sources/ZigSources.txt b/cmake/sources/ZigSources.txt index 76d942b049..4ce90b84c6 100644 --- a/cmake/sources/ZigSources.txt +++ b/cmake/sources/ZigSources.txt @@ -258,6 +258,7 @@ src/bundler/Chunk.zig src/bundler/DeferredBatchTask.zig src/bundler/entry_points.zig src/bundler/Graph.zig +src/bundler/HTMLImportManifest.zig src/bundler/linker_context/computeChunks.zig src/bundler/linker_context/computeCrossChunkDependencies.zig src/bundler/linker_context/convertStmtsForChunk.zig diff --git a/src/OutputFile.zig b/src/OutputFile.zig index 00345593e8..db5290d047 100644 --- a/src/OutputFile.zig +++ b/src/OutputFile.zig @@ -21,6 +21,7 @@ side: ?bun.bake.Side, /// entrypoint like sourcemaps and bytecode entry_point_index: ?u32, referenced_css_files: []const Index = &.{}, +source_index: Index.Optional = .none, pub const Index = bun.GenericIndex(u32, OutputFile); @@ -62,11 +63,19 @@ pub const FileOperation = struct { } }; -pub const Kind = @typeInfo(Value).Union.tag_type.?; +pub const Kind = enum { + move, + copy, + noop, + buffer, + pending, + saved, +}; + // TODO: document how and why all variants of this union(enum) are used, // specifically .move and .copy; the new bundler has to load files in memory // in order to hash them, so i think it uses .buffer for those -pub const Value = union(enum) { +pub const Value = union(Kind) { move: FileOperation, copy: FileOperation, noop: u0, @@ -177,6 +186,7 @@ pub const Options = struct { source_map_index: ?u32 = null, bytecode_index: ?u32 = null, output_path: string, + source_index: Index.Optional = .none, size: ?usize = null, input_path: []const u8 = "", display_size: u32 = 0, @@ -205,6 +215,7 @@ pub fn init(options: Options) OutputFile { .input_loader = options.input_loader, .src_path = Fs.Path.init(options.input_path), .dest_path = options.output_path, + .source_index = options.source_index, .size = options.size orelse switch (options.data) { .buffer => |buf| buf.data.len, .file => |file| file.size, diff --git a/src/bake/DevServer.zig b/src/bake/DevServer.zig index f44c1dcddc..9d749e6918 100644 --- a/src/bake/DevServer.zig +++ b/src/bake/DevServer.zig @@ -4858,11 +4858,7 @@ pub fn IncrementalGraph(side: bake.Side) type { // Additionally, clear the cached entry of the file from the path to // source index map. const hash = bun.hash(abs_path); - for ([_]*bun.bundle_v2.PathToSourceIndexMap{ - &bv2.graph.path_to_source_index_map, - &bv2.graph.client_path_to_source_index_map, - &bv2.graph.ssr_path_to_source_index_map, - }) |map| { + for (&bv2.graph.build_graphs.values) |*map| { _ = map.remove(hash); } } diff --git a/src/bun.js/ResolveMessage.zig b/src/bun.js/ResolveMessage.zig index e433fb1cb2..0dbd055f67 100644 --- a/src/bun.js/ResolveMessage.zig +++ b/src/bun.js/ResolveMessage.zig @@ -45,6 +45,7 @@ pub const ResolveMessage = struct { else break :brk "ERR_MODULE_NOT_FOUND", + .html_manifest, .entry_point_run, .entry_point_build, .at, diff --git a/src/bun.js/RuntimeTranspilerCache.zig b/src/bun.js/RuntimeTranspilerCache.zig index fbbe31565a..dbab5c9958 100644 --- a/src/bun.js/RuntimeTranspilerCache.zig +++ b/src/bun.js/RuntimeTranspilerCache.zig @@ -10,7 +10,8 @@ /// Version 11: Fix \uFFFF printing regression /// Version 12: "use strict"; makes it CommonJS if we otherwise don't know which one to pick. /// Version 13: Hoist `import.meta.require` definition, see #15738 -const expected_version = 13; +/// Version 14: Updated global defines table list. +const expected_version = 14; const bun = @import("bun"); const std = @import("std"); diff --git a/src/bundler/Chunk.zig b/src/bundler/Chunk.zig index 97388a8f5c..f9aace07f7 100644 --- a/src/bundler/Chunk.zig +++ b/src/bundler/Chunk.zig @@ -118,6 +118,20 @@ pub const Chunk = struct { shifts: []sourcemap.SourceMapShifts, }; + pub fn getSize(this: *const IntermediateOutput) usize { + return switch (this.*) { + .pieces => |pieces| brk: { + var total: usize = 0; + for (pieces.slice()) |piece| { + total += piece.data_len; + } + break :brk total; + }, + .joiner => |*joiner| joiner.len, + .empty => 0, + }; + } + pub fn code( this: *IntermediateOutput, allocator_to_use: ?std.mem.Allocator, @@ -128,7 +142,7 @@ pub const Chunk = struct { chunks: []Chunk, display_size: ?*usize, enable_source_map_shifts: bool, - ) !CodeResult { + ) bun.OOM!CodeResult { return switch (enable_source_map_shifts) { inline else => |source_map_shifts| this.codeWithSourceMapShifts( allocator_to_use, @@ -153,7 +167,7 @@ pub const Chunk = struct { chunks: []Chunk, display_size: ?*usize, comptime enable_source_map_shifts: bool, - ) !CodeResult { + ) bun.OOM!CodeResult { const additional_files = graph.input_files.items(.additional_files); const unique_key_for_additional_files = graph.input_files.items(.unique_key_for_additional_file); switch (this.*) { @@ -180,7 +194,7 @@ pub const Chunk = struct { count += piece.data_len; switch (piece.query.kind) { - .chunk, .asset, .scb => { + .chunk, .asset, .scb, .html_import => { const index = piece.query.index; const file_path = switch (piece.query.kind) { .asset => brk: { @@ -195,6 +209,15 @@ pub const Chunk = struct { }, .chunk => chunks[index].final_rel_path, .scb => chunks[entry_point_chunks_for_scb[index]].final_rel_path, + .html_import => { + count += std.fmt.count("{}", .{HTMLImportManifest.formatEscapedJSON(.{ + .index = index, + .graph = graph, + .chunks = chunks, + .linker_graph = linker_graph, + })}); + continue; + }, .none => unreachable, }; @@ -239,7 +262,7 @@ pub const Chunk = struct { remain = remain[data.len..]; switch (piece.query.kind) { - .asset, .chunk, .scb => { + .asset, .chunk, .scb, .html_import => { const index = piece.query.index; const file_path = switch (piece.query.kind) { .asset => brk: { @@ -272,6 +295,19 @@ pub const Chunk = struct { break :brk piece_chunk.final_rel_path; }, + .html_import => { + var fixed_buffer_stream = std.io.fixedBufferStream(remain); + const writer = fixed_buffer_stream.writer(); + + HTMLImportManifest.writeEscapedJSON(index, graph, linker_graph, chunks, writer) catch unreachable; + remain = remain[fixed_buffer_stream.pos..]; + + if (enable_source_map_shifts) { + shift.before.advance(chunk.unique_key); + shifts.appendAssumeCapacity(shift); + } + continue; + }, else => unreachable, }; @@ -385,10 +421,10 @@ pub const Chunk = struct { } pub const Query = packed struct(u32) { - index: u30, + index: u29, kind: Kind, - pub const Kind = enum(u2) { + pub const Kind = enum(u3) { /// The last piece in an array uses this to indicate it is just data none, /// Given a source index, print the asset's output @@ -397,6 +433,8 @@ pub const Chunk = struct { chunk, /// Given a server component boundary index, print the chunk's output path scb, + /// Given an HTML import index, print the manifest + html_import, }; pub const none: Query = .{ .index = 0, .kind = .none }; @@ -618,3 +656,4 @@ const CrossChunkImport = bundler.CrossChunkImport; const CompileResult = bundler.CompileResult; const cheapPrefixNormalizer = bundler.cheapPrefixNormalizer; const LinkerContext = bundler.LinkerContext; +const HTMLImportManifest = @import("./HTMLImportManifest.zig"); diff --git a/src/bundler/Graph.zig b/src/bundler/Graph.zig index cb4e9cebe1..7035951caa 100644 --- a/src/bundler/Graph.zig +++ b/src/bundler/Graph.zig @@ -1,4 +1,4 @@ -pub const Graph = @This(); +const Graph = @This(); pool: *ThreadPool, heap: ThreadlocalArena = .{}, @@ -34,32 +34,24 @@ pending_items: u32 = 0, /// tasks will be run, and the count is "moved" back to `pending_items` deferred_pending: u32 = 0, -/// Maps a hashed path string to a source index, if it exists in the compilation. -/// Instead of accessing this directly, consider using BundleV2.pathToSourceIndexMap -path_to_source_index_map: PathToSourceIndexMap = .{}, -/// When using server components, a completely separate file listing is -/// required to avoid incorrect inlining of defines and dependencies on -/// other files. This is relevant for files shared between server and client -/// and have no "use " directive, and must be duplicated. -/// -/// To make linking easier, this second graph contains indices into the -/// same `.ast` and `.input_files` arrays. -client_path_to_source_index_map: PathToSourceIndexMap = .{}, -/// When using server components with React, there is an additional module -/// graph which is used to contain SSR-versions of all client components; -/// the SSR graph. The difference between the SSR graph and the server -/// graph is that this one does not apply '--conditions react-server' -/// -/// In Bun's React Framework, it includes SSR versions of 'react' and -/// 'react-dom' (an export condition is used to provide a different -/// implementation for RSC, which is potentially how they implement -/// server-only features such as async components). -ssr_path_to_source_index_map: PathToSourceIndexMap = .{}, +/// A map of build targets to their corresponding module graphs. +build_graphs: std.EnumArray(options.Target, PathToSourceIndexMap) = .initFill(.{}), /// When Server Components is enabled, this holds a list of all boundary /// files. This happens for all files with a "use " directive. server_component_boundaries: ServerComponentBoundary.List = .{}, +/// Track HTML imports from server-side code +/// Each entry represents a server file importing an HTML file that needs a client build +/// +/// OutputPiece.Kind.HTMLManifest corresponds to indices into the array. +html_imports: struct { + /// Source index of the server file doing the import + server_source_indices: BabyList(Index.Int) = .{}, + /// Source index of the HTML file being imported + html_source_indices: BabyList(Index.Int) = .{}, +} = .{}, + estimated_file_loader_count: usize = 0, /// For Bake, a count of the CSS asts is used to make precise @@ -82,11 +74,15 @@ pub const InputFile = struct { is_plugin_file: bool = false, }; +pub inline fn pathToSourceIndexMap(this: *Graph, target: options.Target) *PathToSourceIndexMap { + return this.build_graphs.getPtr(target); +} + /// Schedule a task to be run on the JS thread which resolves the promise of /// each `.defer()` called in an onLoad plugin. /// /// Returns true if there were more tasks queued. -pub fn drainDeferredTasks(this: *@This(), transpiler: *BundleV2) bool { +pub fn drainDeferredTasks(this: *Graph, transpiler: *BundleV2) bool { transpiler.thread_lock.assertLocked(); if (this.deferred_pending > 0) { diff --git a/src/bundler/HTMLImportManifest.zig b/src/bundler/HTMLImportManifest.zig new file mode 100644 index 0000000000..58a199cdea --- /dev/null +++ b/src/bundler/HTMLImportManifest.zig @@ -0,0 +1,239 @@ +//! HTMLImportManifest generates JSON manifests for HTML imports in Bun's bundler. +//! +//! When you import an HTML file in JavaScript: +//! ```javascript +//! import index from "./index.html"; +//! console.log(index); +//! ``` +//! +//! Bun transforms this into a call to `__jsonParse()` with a JSON manifest containing +//! metadata about all the files generated from the HTML import: +//! +//! ```javascript +//! var src_default = __jsonParse( +//! '{"index":"./index.html","files":[{"input":"index.html","path":"./index-f2me3qnf.js","loader":"js","isEntry":true,"headers":{"etag": "eet6gn75","content-type": "text/javascript;charset=utf-8"}},{"input":"index.html","path":"./index.html","loader":"html","isEntry":true,"headers":{"etag": "r9njjakd","content-type": "text/html;charset=utf-8"}},{"input":"index.html","path":"./index-gysa5fmk.css","loader":"css","isEntry":true,"headers":{"etag": "50zb7x61","content-type": "text/css;charset=utf-8"}},{"input":"logo.svg","path":"./logo-kygw735p.svg","loader":"file","isEntry":false,"headers":{"etag": "kygw735p","content-type": "application/octet-stream"}},{"input":"react.svg","path":"./react-ck11dneg.svg","loader":"file","isEntry":false,"headers":{"etag": "ck11dneg","content-type": "application/octet-stream"}}]}' +//! ); +//! ``` +//! +//! The manifest JSON structure contains: +//! - `index`: The original HTML file path +//! - `files`: Array of all generated files with metadata: +//! - `input`: Original source file path +//! - `path`: Generated output file path (with content hash) +//! - `loader`: File type/loader used (js, css, html, file, etc.) +//! - `isEntry`: Whether this file is an entry point +//! - `headers`: HTTP headers including ETag and Content-Type +//! +//! This enables applications to: +//! 1. Know all files generated from an HTML import +//! 2. Get proper MIME types and ETags for serving files +//! 3. Implement proper caching strategies +//! 4. Handle assets referenced by the HTML file +//! +//! The manifest is generated during the linking phase and serialized as a JSON string +//! that gets embedded directly into the JavaScript output. +const HTMLImportManifest = @This(); + +index: u32, +graph: *const Graph, +chunks: []Chunk, +linker_graph: *const LinkerGraph, + +pub fn format(this: HTMLImportManifest, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) bun.OOM!void { + return write(this.index, this.graph, this.linker_graph, this.chunks, writer) catch |err| switch (err) { + // We use std.fmt.count for this + error.NoSpaceLeft => unreachable, + error.OutOfMemory => return error.OutOfMemory, + else => unreachable, + }; +} + +fn writeEntryItem( + writer: anytype, + input: []const u8, + path: []const u8, + hash: u64, + loader: options.Loader, + kind: bun.JSC.API.BuildArtifact.OutputKind, +) !void { + try writer.writeAll("{"); + + if (input.len > 0) { + try writer.writeAll("\"input\":"); + try bun.js_printer.writeJSONString(input, @TypeOf(writer), writer, .utf8); + try writer.writeAll(","); + } + + try writer.writeAll("\"path\":"); + try bun.js_printer.writeJSONString(path, @TypeOf(writer), writer, .utf8); + + try writer.writeAll(",\"loader\":\""); + try writer.writeAll(@tagName(loader)); + try writer.writeAll("\",\"isEntry\":"); + try writer.writeAll(if (kind == .@"entry-point") "true" else "false"); + try writer.writeAll(",\"headers\":{"); + + if (hash > 0) { + var base64_buf: [bun.base64.encodeLenFromSize(@sizeOf(@TypeOf(hash))) + 2]u8 = undefined; + const base64 = base64_buf[0..bun.base64.encodeURLSafe(&base64_buf, &std.mem.toBytes(hash))]; + try writer.print( + \\"etag":"{s}", + , .{base64}); + } + + try writer.print( + \\"content-type":"{s}" + , .{ + // Valid mime types are valid headers, which do not need to be escaped in JSON. + loader.toMimeType(&.{ + path, + }).value, + }); + + try writer.writeAll("}}"); +} + +// Extremely unfortunate, but necessary due to E.String not accepting pre-rescaped input and this happening at the very end. +pub fn writeEscapedJSON(index: u32, graph: *const Graph, linker_graph: *const LinkerGraph, chunks: []const Chunk, writer: anytype) !void { + var stack = std.heap.stackFallback(4096, bun.default_allocator); + const allocator = stack.get(); + var bytes = std.ArrayList(u8).init(allocator); + defer bytes.deinit(); + try write(index, graph, linker_graph, chunks, bytes.writer()); + try bun.js_printer.writePreQuotedString(bytes.items, @TypeOf(writer), writer, '"', false, true, .utf8); +} + +fn escapedJSONFormatter(this: HTMLImportManifest, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) bun.OOM!void { + return writeEscapedJSON(this.index, this.graph, this.linker_graph, this.chunks, writer) catch |err| switch (err) { + // We use std.fmt.count for this + error.NoSpaceLeft => unreachable, + error.OutOfMemory => return error.OutOfMemory, + else => unreachable, + }; +} + +pub fn formatEscapedJSON(this: HTMLImportManifest) std.fmt.Formatter(escapedJSONFormatter) { + return std.fmt.Formatter(escapedJSONFormatter){ .data = this }; +} + +pub fn write(index: u32, graph: *const Graph, linker_graph: *const LinkerGraph, chunks: []const Chunk, writer: anytype) !void { + const browser_source_index = graph.html_imports.html_source_indices.slice()[index]; + const server_source_index = graph.html_imports.server_source_indices.slice()[index]; + const sources: []const bun.logger.Source = graph.input_files.items(.source); + const bv2: *const BundleV2 = @alignCast(@fieldParentPtr("graph", graph)); + var entry_point_bits = try bun.bit_set.AutoBitSet.initEmpty(bun.default_allocator, graph.entry_points.items.len); + defer entry_point_bits.deinit(bun.default_allocator); + + const root_dir = if (bv2.transpiler.options.root_dir.len > 0) bv2.transpiler.options.root_dir else bun.fs.FileSystem.instance.top_level_dir; + + try writer.writeAll("{"); + + for (chunks) |*ch| { + if (ch.entry_point.source_index == browser_source_index and ch.entry_point.is_entry_point) { + entry_point_bits.set(ch.entry_point.entry_point_id); + + if (ch.content == .html) { + try writer.writeAll("\"index\":"); + try bun.js_printer.writeJSONString(ch.final_rel_path, @TypeOf(writer), writer, .utf8); + try writer.writeAll(","); + } + } + } + + // Start the files array + + try writer.writeAll("\"files\":["); + + var first = true; + + const additional_output_files = graph.additional_output_files.items; + const file_entry_bits: []const AutoBitSet = linker_graph.files.items(.entry_bits); + var already_visited_output_file = try bun.bit_set.AutoBitSet.initEmpty(bun.default_allocator, additional_output_files.len); + defer already_visited_output_file.deinit(bun.default_allocator); + + // Write all chunks that have files associated with this entry point. + for (chunks) |*ch| { + if (ch.entryBits().hasIntersection(&entry_point_bits)) { + if (!first) try writer.writeAll(","); + first = false; + + try writeEntryItem( + writer, + brk: { + if (!ch.entry_point.is_entry_point) break :brk ""; + var path_for_key = bun.path.relativeNormalized( + root_dir, + sources[ch.entry_point.source_index].path.text, + .posix, + false, + ); + if (path_for_key.len > 2 and strings.eqlComptime(path_for_key[0..2], "./")) { + path_for_key = path_for_key[2..]; + } + + break :brk path_for_key; + }, + ch.final_rel_path, + ch.isolated_hash, + ch.content.loader(), + if (ch.entry_point.is_entry_point) + .@"entry-point" + else + .chunk, + ); + } + } + + for (additional_output_files, 0..) |*output_file, i| { + // Only print the file once. + if (already_visited_output_file.isSet(i)) continue; + + if (output_file.source_index.unwrap()) |source_index| { + if (source_index.get() == server_source_index) continue; + const bits: *const AutoBitSet = &file_entry_bits[source_index.get()]; + + if (bits.hasIntersection(&entry_point_bits)) { + already_visited_output_file.set(i); + if (!first) try writer.writeAll(","); + first = false; + + var path_for_key = bun.path.relativeNormalized( + root_dir, + sources[source_index.get()].path.text, + .posix, + false, + ); + if (path_for_key.len > 2 and strings.eqlComptime(path_for_key[0..2], "./")) { + path_for_key = path_for_key[2..]; + } + + try writeEntryItem( + writer, + path_for_key, + output_file.dest_path, + output_file.hash, + output_file.loader, + output_file.output_kind, + ); + } + } + } + + try writer.writeAll("]}"); +} + +const bun = @import("bun"); +const strings = bun.strings; +const default_allocator = bun.default_allocator; + +const std = @import("std"); +const options = @import("../options.zig"); + +const Loader = options.Loader; +const AutoBitSet = bun.bit_set.AutoBitSet; +const bundler = bun.bundle_v2; +const BundleV2 = bundler.BundleV2; +const Graph = bundler.Graph; +const LinkerGraph = bundler.LinkerGraph; + +const Chunk = bundler.Chunk; diff --git a/src/bundler/LinkerContext.zig b/src/bundler/LinkerContext.zig index 1a2d4f3246..89f238947d 100644 --- a/src/bundler/LinkerContext.zig +++ b/src/bundler/LinkerContext.zig @@ -291,6 +291,50 @@ pub const LinkerContext = struct { _ = this.pending_task_count.fetchSub(1, .monotonic); } + fn processHtmlImportFiles(this: *LinkerContext) void { + const server_source_indices = &this.parse_graph.html_imports.server_source_indices; + const html_source_indices = &this.parse_graph.html_imports.html_source_indices; + if (server_source_indices.len > 0) { + const input_files: []const Logger.Source = this.parse_graph.input_files.items(.source); + const map = this.parse_graph.pathToSourceIndexMap(.browser); + const parts: []const BabyList(js_ast.Part) = this.graph.ast.items(.parts); + const actual_ref = this.graph.runtimeFunction("__jsonParse"); + + for (server_source_indices.slice()) |html_import| { + const source = &input_files[html_import]; + const source_index = map.get(source.path.hashKey()) orelse { + @panic("Assertion failed: HTML import file not found in pathToSourceIndexMap"); + }; + + html_source_indices.push(this.graph.allocator, source_index) catch bun.outOfMemory(); + + // S.LazyExport is a call to __jsonParse. + const original_ref = parts[html_import] + .at(1) + .stmts[0] + .data + .s_lazy_export + .e_call + .target + .data + .e_import_identifier + .ref; + + // Make the __jsonParse in that file point to the __jsonParse in the runtime chunk. + this.graph.symbols.get(original_ref).?.link = actual_ref; + + // When --splitting is enabled, we have to make sure we import the __jsonParse function. + this.graph.generateSymbolImportAndUse( + html_import, + Index.part(1).get(), + actual_ref, + 1, + Index.runtime, + ) catch bun.outOfMemory(); + } + } + } + pub noinline fn link( this: *LinkerContext, bundle: *BundleV2, @@ -309,6 +353,8 @@ pub const LinkerContext = struct { this.computeDataForSourceMap(@as([]Index.Int, @ptrCast(reachable))); } + this.processHtmlImportFiles(); + if (comptime FeatureFlags.help_catch_memory_issues) { this.checkForMemoryCorruption(); } @@ -2355,6 +2401,7 @@ pub const LinkerContext = struct { 'A' => .asset, 'C' => .chunk, 'S' => .scb, + 'H' => .html_import, else => { if (bun.Environment.isDebug) bun.Output.debugWarn("Invalid output piece boundary", .{}); @@ -2385,6 +2432,11 @@ pub const LinkerContext = struct { bun.Output.debugWarn("Invalid output piece boundary", .{}); break; }, + .html_import => if (index >= c.parse_graph.html_imports.server_source_indices.len) { + if (bun.Environment.isDebug) + bun.Output.debugWarn("Invalid output piece boundary", .{}); + break; + }, else => unreachable, } diff --git a/src/bundler/ParseTask.zig b/src/bundler/ParseTask.zig index 9f1acebf52..c42df528f3 100644 --- a/src/bundler/ParseTask.zig +++ b/src/bundler/ParseTask.zig @@ -1047,22 +1047,12 @@ fn getSourceCode( const allocator = this.allocator; var data = this.data; - var transpiler = &data.transpiler; + const transpiler = &data.transpiler; errdefer transpiler.resetStore(); const resolver: *Resolver = &transpiler.resolver; var file_path = task.path; var loader = task.loader orelse file_path.loader(&transpiler.options.loaders) orelse options.Loader.file; - // Do not process files as HTML if any of the following are true: - // - building for node or bun.js - // - // We allow non-entrypoints to import HTML so that people could - // potentially use an onLoad plugin that returns HTML. - if (task.known_target != .browser) { - loader = loader.disableHTML(); - task.loader = loader; - } - var contents_came_from_plugin: bool = false; return try getCodeForParseTask(task, log, transpiler, resolver, allocator, &file_path, &loader, &contents_came_from_plugin); } @@ -1076,22 +1066,11 @@ fn runWithSourceCode( ) anyerror!Result.Success { const allocator = this.allocator; - var data = this.data; - var transpiler = &data.transpiler; + var transpiler = this.transpilerForTarget(task.known_target); errdefer transpiler.resetStore(); var resolver: *Resolver = &transpiler.resolver; const file_path = &task.path; - var loader = task.loader orelse file_path.loader(&transpiler.options.loaders) orelse options.Loader.file; - - // Do not process files as HTML if any of the following are true: - // - building for node or bun.js - // - // We allow non-entrypoints to import HTML so that people could - // potentially use an onLoad plugin that returns HTML. - if (task.known_target != .browser) { - loader = loader.disableHTML(); - task.loader = loader; - } + const loader = task.loader orelse file_path.loader(&transpiler.options.loaders) orelse options.Loader.file; // WARNING: Do not change the variant of `task.contents_or_fd` from // `.fd` to `.contents` (or back) after this point! @@ -1154,7 +1133,7 @@ fn runWithSourceCode( ((transpiler.options.server_components or transpiler.options.dev_server != null) and task.known_target == .browser)) { - transpiler = this.ctx.client_transpiler; + transpiler = this.ctx.client_transpiler.?; resolver = &transpiler.resolver; bun.assert(transpiler.options.target == .browser); } diff --git a/src/bundler/ThreadPool.zig b/src/bundler/ThreadPool.zig index 23017f9e93..1ba0f27fe8 100644 --- a/src/bundler/ThreadPool.zig +++ b/src/bundler/ThreadPool.zig @@ -218,6 +218,8 @@ pub const ThreadPool = struct { estimated_input_lines_of_code: usize = 0, macro_context: js_ast.Macro.MacroContext, transpiler: Transpiler = undefined, + other_transpiler: Transpiler = undefined, + has_loaded_other_transpiler: bool = false, }; pub fn init(worker: *Worker, v2: *BundleV2) void { @@ -233,7 +235,7 @@ pub const ThreadPool = struct { this.heap = ThreadlocalArena.init() catch unreachable; this.allocator = this.heap.allocator(); - var allocator = this.allocator; + const allocator = this.allocator; this.ast_memory_allocator = .{ .allocator = this.allocator }; this.ast_memory_allocator.reset(); @@ -245,21 +247,38 @@ pub const ThreadPool = struct { }; this.data.log.* = Logger.Log.init(allocator); this.ctx = ctx; - this.data.transpiler = ctx.transpiler.*; - this.data.transpiler.setLog(this.data.log); - this.data.transpiler.setAllocator(allocator); - this.data.transpiler.linker.resolver = &this.data.transpiler.resolver; - this.data.transpiler.macro_context = js_ast.Macro.MacroContext.init(&this.data.transpiler); - this.data.macro_context = this.data.transpiler.macro_context.?; this.temporary_arena = bun.ArenaAllocator.init(this.allocator); this.stmt_list = LinkerContext.StmtList.init(this.allocator); + this.initializeTranspiler(&this.data.transpiler, ctx.transpiler, allocator); - const CacheSet = @import("../cache.zig"); - - this.data.transpiler.resolver.caches = CacheSet.Set.init(this.allocator); debug("Worker.create()", .{}); } + fn initializeTranspiler(this: *Worker, transpiler: *Transpiler, from: *Transpiler, allocator: std.mem.Allocator) void { + transpiler.* = from.*; + transpiler.setLog(this.data.log); + transpiler.setAllocator(allocator); + transpiler.linker.resolver = &transpiler.resolver; + transpiler.macro_context = js_ast.Macro.MacroContext.init(transpiler); + this.data.macro_context = transpiler.macro_context.?; + const CacheSet = @import("../cache.zig"); + transpiler.resolver.caches = CacheSet.Set.init(allocator); + } + + pub fn transpilerForTarget(this: *Worker, target: bun.options.Target) *Transpiler { + if (target == .browser and this.data.transpiler.options.target != target) { + if (!this.data.has_loaded_other_transpiler) { + this.data.has_loaded_other_transpiler = true; + this.initializeTranspiler(&this.data.other_transpiler, this.ctx.client_transpiler.?, this.allocator); + } + + bun.debugAssert(this.data.other_transpiler.options.target == target); + return &this.data.other_transpiler; + } + + return &this.data.transpiler; + } + pub fn run(this: *Worker, ctx: *BundleV2) void { if (!this.has_created) { this.create(ctx); diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index 60e9fe2199..9c7c076fdc 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -106,7 +106,7 @@ pub const BundleV2 = struct { transpiler: *Transpiler, /// When Server Component is enabled, this is used for the client bundles /// and `transpiler` is used for the server bundles. - client_transpiler: *Transpiler, + client_transpiler: ?*Transpiler, /// See bake.Framework.ServerComponents.separate_ssr_graph ssr_transpiler: *Transpiler, /// When Bun Bake is used, the resolved framework is passed here @@ -167,17 +167,69 @@ pub const BundleV2 = struct { } } + fn ensureClientTranspiler(this: *BundleV2) void { + if (this.client_transpiler == null) { + _ = this.initializeClientTranspiler() catch bun.outOfMemory(); + } + } + + fn initializeClientTranspiler(this: *BundleV2) !*Transpiler { + @branchHint(.cold); + const allocator = this.graph.allocator; + + const this_transpiler = this.transpiler; + const client_transpiler = try allocator.create(Transpiler); + const defines = this_transpiler.options.transform_options.define; + client_transpiler.* = this_transpiler.*; + client_transpiler.options = this_transpiler.options; + + client_transpiler.options.target = .browser; + client_transpiler.options.main_fields = options.Target.DefaultMainFields.get(options.Target.browser); + client_transpiler.options.conditions = try options.ESMConditions.init(allocator, options.Target.browser.defaultConditions()); + client_transpiler.options.define = try options.Define.init( + allocator, + if (defines) |user_defines| + try options.Define.Data.fromInput(try options.stringHashMapFromArrays( + options.defines.RawDefines, + allocator, + user_defines.keys, + user_defines.values, + ), this_transpiler.options.transform_options.drop, this_transpiler.log, allocator) + else + null, + null, + this_transpiler.options.define.drop_debugger, + ); + + client_transpiler.setLog(this_transpiler.log); + client_transpiler.setAllocator(allocator); + client_transpiler.linker.resolver = &client_transpiler.resolver; + client_transpiler.macro_context = js_ast.Macro.MacroContext.init(client_transpiler); + const CacheSet = @import("../cache.zig"); + client_transpiler.resolver.caches = CacheSet.Set.init(allocator); + client_transpiler.resolver.opts = client_transpiler.options; + + this.client_transpiler = client_transpiler; + return client_transpiler; + } + /// Most of the time, accessing .transpiler directly is OK. This is only /// needed when it is important to distinct between client and server /// /// Note that .log, .allocator, and other things are shared /// between the three transpiler configurations - pub inline fn transpilerForTarget(this: *BundleV2, target: options.Target) *Transpiler { - return if (!this.transpiler.options.server_components and this.linker.dev_server == null) - this.transpiler - else switch (target) { + pub inline fn transpilerForTarget(noalias this: *BundleV2, target: options.Target) *Transpiler { + if (!this.transpiler.options.server_components and this.linker.dev_server == null) { + if (target == .browser and this.transpiler.options.target.isServerSide()) { + return this.client_transpiler orelse this.initializeClientTranspiler() catch bun.outOfMemory(); + } + + return this.transpiler; + } + + return switch (target) { else => this.transpiler, - .browser => this.client_transpiler, + .browser => this.client_transpiler.?, .bake_server_components_ssr => this.ssr_transpiler, }; } @@ -192,15 +244,8 @@ pub const BundleV2 = struct { return this.transpiler.log; } - /// Same semantics as bundlerForTarget for `path_to_source_index_map` pub inline fn pathToSourceIndexMap(this: *BundleV2, target: options.Target) *PathToSourceIndexMap { - return if (!this.transpiler.options.server_components) - &this.graph.path_to_source_index_map - else switch (target) { - else => &this.graph.path_to_source_index_map, - .browser => &this.graph.client_path_to_source_index_map, - .bake_server_components_ssr => &this.graph.ssr_path_to_source_index_map, - }; + return this.graph.pathToSourceIndexMap(target); } const ReachableFileVisitor = struct { @@ -340,7 +385,7 @@ pub const BundleV2 = struct { .all_import_records = this.graph.ast.items(.import_records), .all_loaders = this.graph.input_files.items(.loader), .all_urls_for_css = all_urls_for_css, - .redirect_map = this.graph.path_to_source_index_map, + .redirect_map = this.pathToSourceIndexMap(this.transpiler.options.target).*, .dynamic_import_entry_points = &this.dynamic_import_entry_points, .scb_bitset = scb_bitset, .scb_list = if (scb_bitset != null) @@ -556,14 +601,14 @@ pub const BundleV2 = struct { const entry = this.pathToSourceIndexMap(target).getOrPut(this.graph.allocator, path.hashKey()) catch bun.outOfMemory(); if (!entry.found_existing) { path.* = this.pathWithPrettyInitialized(path.*, target) catch bun.outOfMemory(); - const loader: Loader = (brk: { + const loader: Loader = brk: { const record: *ImportRecord = &this.graph.ast.items(.import_records)[import_record.importer_source_index].slice()[import_record.import_record_index]; if (record.loader) |out_loader| { break :brk out_loader; } break :brk path.loader(&transpiler.options.loaders) orelse options.Loader.file; // HTML is only allowed at the entry point. - }).disableHTML(); + }; const idx = this.enqueueParseTask( &resolve_result, &.{ @@ -581,9 +626,9 @@ pub const BundleV2 = struct { // It makes sense to separate these for JS because the target affects DCE if (this.transpiler.options.server_components and !loader.isJavaScriptLike()) { const a, const b = switch (target) { - else => .{ &this.graph.client_path_to_source_index_map, &this.graph.ssr_path_to_source_index_map }, - .browser => .{ &this.graph.path_to_source_index_map, &this.graph.ssr_path_to_source_index_map }, - .bake_server_components_ssr => .{ &this.graph.path_to_source_index_map, &this.graph.client_path_to_source_index_map }, + else => .{ this.pathToSourceIndexMap(.browser), this.pathToSourceIndexMap(.bake_server_components_ssr) }, + .browser => .{ this.pathToSourceIndexMap(this.transpiler.options.target), this.pathToSourceIndexMap(.bake_server_components_ssr) }, + .bake_server_components_ssr => .{ this.pathToSourceIndexMap(this.transpiler.options.target), this.pathToSourceIndexMap(.browser) }, }; a.put(this.graph.allocator, entry.key_ptr.*, entry.value_ptr.*) catch bun.outOfMemory(); if (this.framework.?.server_components.?.separate_ssr_graph) @@ -675,9 +720,9 @@ pub const BundleV2 = struct { } this.incrementScanCounter(); const source_index = Index.source(this.graph.input_files.len); + const loader = brk: { const loader = path.loader(&this.transpiler.options.loaders) orelse .file; - if (target != .browser) break :brk loader.disableHTML(); break :brk loader; }; @@ -746,7 +791,7 @@ pub const BundleV2 = struct { this.* = .{ .transpiler = transpiler, - .client_transpiler = transpiler, + .client_transpiler = null, .ssr_transpiler = transpiler, .framework = null, .graph = .{ @@ -775,7 +820,7 @@ pub const BundleV2 = struct { this.linker.framework = &this.framework.?; this.plugins = bo.plugins; if (transpiler.options.server_components) { - bun.assert(this.client_transpiler.options.server_components); + bun.assert(this.client_transpiler.?.options.server_components); if (bo.framework.server_components.?.separate_ssr_graph) bun.assert(this.ssr_transpiler.options.server_components); } @@ -882,7 +927,7 @@ pub const BundleV2 = struct { // try this.graph.entry_points.append(allocator, Index.runtime); try this.graph.ast.append(bun.default_allocator, JSAst.empty); - try this.graph.path_to_source_index_map.put(this.graph.allocator, bun.hash("bun:wrap"), Index.runtime.get()); + try this.pathToSourceIndexMap(this.transpiler.options.target).put(this.graph.allocator, bun.hash("bun:wrap"), Index.runtime.get()); var runtime_parse_task = try this.graph.allocator.create(ParseTask); runtime_parse_task.* = rt.parse_task; runtime_parse_task.ctx = this; @@ -912,7 +957,6 @@ pub const BundleV2 = struct { try this.graph.entry_points.ensureUnusedCapacity(this.graph.allocator, num_entry_points); try this.graph.input_files.ensureUnusedCapacity(this.graph.allocator, num_entry_points); - try this.graph.path_to_source_index_map.ensureUnusedCapacity(this.graph.allocator, @intCast(num_entry_points)); switch (variant) { .normal => { @@ -920,7 +964,27 @@ pub const BundleV2 = struct { const resolved = this.transpiler.resolveEntryPoint(entry_point) catch continue; - _ = try this.enqueueEntryItem(null, resolved, true, this.transpiler.options.target); + _ = try this.enqueueEntryItem( + null, + resolved, + true, + brk: { + const main_target = this.transpiler.options.target; + + if (main_target.isServerSide()) { + if (resolved.pathConst()) |path| { + if (path.loader(&this.transpiler.options.loaders)) |loader| { + if (loader == .html) { + this.ensureClientTranspiler(); + break :brk .browser; + } + } + } + } + + break :brk main_target; + }, + ); } }, .dev_server => { @@ -1127,15 +1191,13 @@ pub const BundleV2 = struct { pub fn enqueueParseTask( this: *BundleV2, - resolve_result: *const _resolver.Result, + noalias resolve_result: *const _resolver.Result, source: *const Logger.Source, - loader_: Loader, + loader: Loader, known_target: options.Target, ) OOM!Index.Int { const source_index = Index.init(@as(u32, @intCast(this.graph.ast.len))); this.graph.ast.append(bun.default_allocator, JSAst.empty) catch unreachable; - // Only enable HTML loader when it's an entry point. - const loader = loader_.disableHTML(); this.graph.input_files.append(bun.default_allocator, .{ .source = source.*, @@ -1447,7 +1509,8 @@ pub const BundleV2 = struct { const file_allocators = this.graph.input_files.items(.allocator); const unique_key_for_additional_files = this.graph.input_files.items(.unique_key_for_additional_file); const content_hashes_for_additional_files = this.graph.input_files.items(.content_hash_for_additional_file); - const sources = this.graph.input_files.items(.source); + const sources: []const Logger.Source = this.graph.input_files.items(.source); + const targets: []const options.Target = this.graph.ast.items(.target); var additional_output_files = std.ArrayList(options.OutputFile).init(this.transpiler.allocator); const additional_files: []BabyList(AdditionalFile) = this.graph.input_files.items(.additional_files); @@ -1457,34 +1520,47 @@ pub const BundleV2 = struct { const index = reachable_source.get(); const key = unique_key_for_additional_files[index]; if (key.len > 0) { - var template = PathTemplate.asset; + var template = if (this.graph.html_imports.server_source_indices.len > 0 and this.transpiler.options.asset_naming.len == 0) + PathTemplate.assetWithTarget + else + PathTemplate.asset; + if (this.transpiler.options.asset_naming.len > 0) template.data = this.transpiler.options.asset_naming; const source = &sources[index]; - var pathname = source.path.name; - // TODO: outbase - pathname = Fs.PathName.init(bun.path.relativePlatform(this.transpiler.options.root_dir, source.path.text, .loose, false)); + const output_path = brk: { + var pathname = source.path.name; - template.placeholder.name = pathname.base; - template.placeholder.dir = pathname.dir; - template.placeholder.ext = pathname.ext; - if (template.placeholder.ext.len > 0 and template.placeholder.ext[0] == '.') - template.placeholder.ext = template.placeholder.ext[1..]; + // TODO: outbase + pathname = Fs.PathName.init(bun.path.relativePlatform(this.transpiler.options.root_dir, source.path.text, .loose, false)); - if (template.needs(.hash)) { - template.placeholder.hash = content_hashes_for_additional_files[index]; - } + template.placeholder.name = pathname.base; + template.placeholder.dir = pathname.dir; + template.placeholder.ext = pathname.ext; + if (template.placeholder.ext.len > 0 and template.placeholder.ext[0] == '.') + template.placeholder.ext = template.placeholder.ext[1..]; + + if (template.needs(.hash)) { + template.placeholder.hash = content_hashes_for_additional_files[index]; + } + + if (template.needs(.target)) { + template.placeholder.target = @tagName(targets[index]); + } + break :brk std.fmt.allocPrint(bun.default_allocator, "{}", .{template}) catch bun.outOfMemory(); + }; const loader = loaders[index]; additional_output_files.append(options.OutputFile.init(.{ + .source_index = .init(index), .data = .{ .buffer = .{ .data = source.contents, .allocator = file_allocators[index], } }, .size = source.contents.len, - .output_path = std.fmt.allocPrint(bun.default_allocator, "{}", .{template}) catch bun.outOfMemory(), + .output_path = output_path, .input_path = bun.default_allocator.dupe(u8, source.path.text) catch bun.outOfMemory(), .input_loader = .file, .output_kind = .asset, @@ -2066,7 +2142,6 @@ pub const BundleV2 = struct { task.io_task.node.next = null; this.incrementScanCounter(); - // Handle onLoad plugins if (!this.enqueueOnLoadPluginIfNeeded(task)) { if (loader.shouldCopyForBundling()) { var additional_files: *BabyList(AdditionalFile) = &this.graph.input_files.items(.additional_files)[source_index.get()]; @@ -2728,7 +2803,7 @@ pub const BundleV2 = struct { continue; } - const transpiler, const bake_graph: bake.Graph, const target = + const transpiler: *Transpiler, const bake_graph: bake.Graph, const target: options.Target = if (import_record.tag == .bake_resolve_to_ssr_graph) brk: { if (this.framework == null) { this.logForResolutionFailures(source.path.text, .ssr).addErrorFmt( @@ -2766,7 +2841,7 @@ pub const BundleV2 = struct { }; var had_busted_dir_cache = false; - var resolve_result = inner: while (true) break transpiler.resolver.resolveWithFramework( + var resolve_result: _resolver.Result = inner: while (true) break transpiler.resolver.resolveWithFramework( source_dir, import_record.path.text, import_record.kind, @@ -2972,15 +3047,24 @@ pub const BundleV2 = struct { const hash_key = path.hashKey(); + const import_record_loader = import_record.loader orelse path.loader(&transpiler.options.loaders) orelse .file; + import_record.loader = import_record_loader; + + const is_html_entrypoint = import_record_loader == .html and target.isServerSide() and this.transpiler.options.dev_server == null; + if (this.pathToSourceIndexMap(target).get(hash_key)) |id| { if (this.transpiler.options.dev_server != null and loader != .html) { import_record.path = this.graph.input_files.items(.source)[id].path; } else { - import_record.source_index = Index.init(id); + import_record.source_index = .init(id); } continue; } + if (is_html_entrypoint) { + import_record.kind = .html_manifest; + } + const resolve_entry = resolve_queue.getOrPut(hash_key) catch bun.outOfMemory(); if (resolve_entry.found_existing) { import_record.path = resolve_entry.value_ptr.*.path; @@ -3001,11 +3085,15 @@ pub const BundleV2 = struct { import_record.path = path.*; debug("created ParseTask: {s}", .{path.text}); - const resolve_task = bun.default_allocator.create(ParseTask) catch bun.outOfMemory(); resolve_task.* = ParseTask.init(&resolve_result, Index.invalid, this); resolve_task.secondary_path_for_commonjs_interop = secondary_path_to_copy; - resolve_task.known_target = target; + + resolve_task.known_target = if (import_record.kind == .html_manifest) + .browser + else + target; + resolve_task.jsx = resolve_result.jsx; resolve_task.jsx.development = switch (transpiler.options.force_node_env) { .development => true, @@ -3013,24 +3101,13 @@ pub const BundleV2 = struct { .unspecified => transpiler.options.jsx.development, }; - // Figure out the loader. - { - if (import_record.loader) |l| { - resolve_task.loader = l; - } - - if (resolve_task.loader == null) { - resolve_task.loader = path.loader(&this.transpiler.options.loaders); - resolve_task.tree_shaking = this.transpiler.options.tree_shaking; - } - - // HTML must be an entry point. - if (resolve_task.loader) |*l| { - l.* = l.disableHTML(); - } - } - + resolve_task.loader = import_record_loader; + resolve_task.tree_shaking = transpiler.options.tree_shaking; resolve_entry.value_ptr.* = resolve_task; + + if (is_html_entrypoint) { + this.generateServerHTMLModule(path, target, import_record, hash_key) catch unreachable; + } } if (last_error) |err| { @@ -3050,6 +3127,58 @@ pub const BundleV2 = struct { return resolve_queue; } + fn generateServerHTMLModule(this: *BundleV2, path: *const Fs.Path, target: options.Target, import_record: *ImportRecord, hash_key: u64) !void { + // 1. Create the ast right here + // 2. Create a separate "virutal" module that becomes the manifest later on. + // 3. Add it to the graph + const graph = &this.graph; + const empty_html_file_source: Logger.Source = .{ + .path = path.*, + .index = Index.source(graph.input_files.len), + .contents = "", + }; + var js_parser_options = bun.js_parser.Parser.Options.init(this.transpilerForTarget(target).options.jsx, .html); + js_parser_options.bundle = true; + + const unique_key = try std.fmt.allocPrint(graph.allocator, "{any}H{d:0>8}", .{ + bun.fmt.hexIntLower(this.unique_key), + graph.html_imports.server_source_indices.len, + }); + + const transpiler = this.transpilerForTarget(target); + + const ast_for_html_entrypoint = JSAst.init((try bun.js_parser.newLazyExportAST( + graph.allocator, + transpiler.options.define, + js_parser_options, + transpiler.log, + Expr.init( + E.String, + E.String{ + .data = unique_key, + }, + Logger.Loc.Empty, + ), + &empty_html_file_source, + + // We replace this runtime API call's ref later via .link on the Symbol. + "__jsonParse", + )).?); + + var fake_input_file = Graph.InputFile{ + .source = empty_html_file_source, + .side_effects = .no_side_effects__pure_data, + }; + + try graph.input_files.append(graph.allocator, fake_input_file); + try graph.ast.append(graph.allocator, ast_for_html_entrypoint); + + import_record.source_index = fake_input_file.source.index; + try this.pathToSourceIndexMap(target).put(graph.allocator, hash_key, fake_input_file.source.index.get()); + try graph.html_imports.server_source_indices.push(graph.allocator, fake_input_file.source.index.get()); + this.ensureClientTranspiler(); + } + const ResolveQueue = std.AutoArrayHashMap(u64, *ParseTask); pub fn onNotifyDefer(this: *BundleV2) void { @@ -3064,24 +3193,23 @@ pub const BundleV2 = struct { pub fn onParseTaskComplete(parse_result: *ParseTask.Result, this: *BundleV2) void { const trace = bun.perf.trace("Bundler.onParseTaskComplete"); + const graph = &this.graph; defer trace.end(); if (parse_result.external.function != null) { const source = switch (parse_result.value) { inline .empty, .err => |data| data.source_index.get(), .success => |val| val.source.index.get(), }; - const loader: Loader = this.graph.input_files.items(.loader)[source]; + const loader: Loader = graph.input_files.items(.loader)[source]; if (!loader.shouldCopyForBundling()) { this.finalizers.append(bun.default_allocator, parse_result.external) catch bun.outOfMemory(); } else { - this.graph.input_files.items(.allocator)[source] = ExternalFreeFunctionAllocator.create(parse_result.external.function.?, parse_result.external.ctx.?); + graph.input_files.items(.allocator)[source] = ExternalFreeFunctionAllocator.create(parse_result.external.function.?, parse_result.external.ctx.?); } } defer bun.default_allocator.destroy(parse_result); - const graph = &this.graph; - var diff: i32 = -1; defer { logScanCounter("in parse task .pending_items += {d} = {d}\n", .{ diff, @as(i32, @intCast(graph.pending_items)) + diff }); @@ -3090,7 +3218,7 @@ pub const BundleV2 = struct { this.onAfterDecrementScanCounter(); } - var resolve_queue = ResolveQueue.init(this.graph.allocator); + var resolve_queue = ResolveQueue.init(graph.allocator); defer resolve_queue.deinit(); var process_log = true; @@ -3172,17 +3300,23 @@ pub const BundleV2 = struct { var iter = resolve_queue.iterator(); const path_to_source_index_map = this.pathToSourceIndexMap(result.ast.target); + const original_target = result.ast.target; while (iter.next()) |entry| { const hash = entry.key_ptr.*; - const value = entry.value_ptr.*; + const value: *ParseTask = entry.value_ptr.*; - var existing = path_to_source_index_map.getOrPut(graph.allocator, hash) catch unreachable; + const loader = value.loader orelse value.path.loader(&this.transpiler.options.loaders) orelse options.Loader.file; + + const is_html_entrypoint = loader == .html and original_target.isServerSide() and this.transpiler.options.dev_server == null; + + const map = if (is_html_entrypoint) this.pathToSourceIndexMap(.browser) else path_to_source_index_map; + var existing = map.getOrPut(graph.allocator, hash) catch unreachable; // If the same file is imported and required, and those point to different files // Automatically rewrite it to the secondary one if (value.secondary_path_for_commonjs_interop) |secondary_path| { const secondary_hash = secondary_path.hashKey(); - if (path_to_source_index_map.get(secondary_hash)) |secondary| { + if (map.get(secondary_hash)) |secondary| { existing.found_existing = true; existing.value_ptr.* = secondary; } @@ -3195,21 +3329,24 @@ pub const BundleV2 = struct { .side_effects = value.side_effects, }; - const loader = new_task.loader orelse new_input_file.source.path.loader(&this.transpiler.options.loaders) orelse options.Loader.file; - new_input_file.source.index = Index.source(graph.input_files.len); new_input_file.source.path = new_task.path; // We need to ensure the loader is set or else importstar_ts/ReExportTypeOnlyFileES6 will fail. new_input_file.loader = loader; - - existing.value_ptr.* = new_input_file.source.index.get(); new_task.source_index = new_input_file.source.index; - new_task.ctx = this; + existing.value_ptr.* = new_task.source_index.get(); + + diff += 1; + graph.input_files.append(bun.default_allocator, new_input_file) catch unreachable; graph.ast.append(bun.default_allocator, JSAst.empty) catch unreachable; - diff += 1; + + if (is_html_entrypoint) { + this.ensureClientTranspiler(); + this.graph.entry_points.append(this.graph.allocator, new_input_file.source.index) catch unreachable; + } if (this.enqueueOnLoadPluginIfNeeded(new_task)) { continue; @@ -3217,20 +3354,16 @@ pub const BundleV2 = struct { if (loader.shouldCopyForBundling()) { var additional_files: *BabyList(AdditionalFile) = &graph.input_files.items(.additional_files)[result.source.index.get()]; - additional_files.push(this.graph.allocator, .{ .source_index = new_task.source_index.get() }) catch unreachable; + additional_files.push(graph.allocator, .{ .source_index = new_task.source_index.get() }) catch unreachable; new_input_file.side_effects = _resolver.SideEffects.no_side_effects__pure_data; graph.estimated_file_loader_count += 1; } graph.pool.schedule(new_task); } else { - const loader = value.loader orelse - graph.input_files.items(.source)[existing.value_ptr.*].path.loader(&this.transpiler.options.loaders) orelse - options.Loader.file; - if (loader.shouldCopyForBundling()) { var additional_files: *BabyList(AdditionalFile) = &graph.input_files.items(.additional_files)[result.source.index.get()]; - additional_files.push(this.graph.allocator, .{ .source_index = existing.value_ptr.* }) catch unreachable; + additional_files.push(graph.allocator, .{ .source_index = existing.value_ptr.* }) catch unreachable; graph.estimated_file_loader_count += 1; } @@ -3238,9 +3371,9 @@ pub const BundleV2 = struct { } } - var import_records = result.ast.import_records.clone(this.graph.allocator) catch unreachable; + var import_records = result.ast.import_records.clone(graph.allocator) catch unreachable; - const input_file_loaders = this.graph.input_files.items(.loader); + const input_file_loaders = graph.input_files.items(.loader); const save_import_record_source_index = this.transpiler.options.dev_server == null or result.loader == .html or result.loader.isCSS(); @@ -3255,11 +3388,11 @@ pub const BundleV2 = struct { } var list = pending_entry.value.list(); - list.deinit(this.graph.allocator); + list.deinit(graph.allocator); } if (result.ast.css != null) { - this.graph.css_file_count += 1; + graph.css_file_count += 1; } for (import_records.slice(), 0..) |*record, i| { @@ -3325,7 +3458,7 @@ pub const BundleV2 = struct { break :brk .{ server_index, Index.invalid.get() }; }; - this.graph.path_to_source_index_map.put( + graph.pathToSourceIndexMap(result.ast.target).put( graph.allocator, result.source.path.hashKey(), reference_source_index, @@ -3350,7 +3483,7 @@ pub const BundleV2 = struct { dev_server.handleParseTaskFailure( err.err, err.target.bakeGraph(), - this.graph.input_files.items(.source)[err.source_index.get()].path.text, + graph.input_files.items(.source)[err.source_index.get()].path.text, &err.log, this, ) catch bun.outOfMemory(); @@ -3368,7 +3501,7 @@ pub const BundleV2 = struct { } if (Environment.allow_assert and this.transpiler.options.dev_server != null) { - bun.assert(this.graph.ast.items(.parts)[err.source_index.get()].len == 0); + bun.assert(graph.ast.items(.parts)[err.source_index.get()].len == 0); } }, } @@ -4051,4 +4184,4 @@ pub const ThreadPool = @import("ThreadPool.zig").ThreadPool; pub const ParseTask = @import("ParseTask.zig").ParseTask; pub const LinkerContext = @import("LinkerContext.zig").LinkerContext; pub const LinkerGraph = @import("LinkerGraph.zig").LinkerGraph; -pub const Graph = @import("Graph.zig").Graph; +pub const Graph = @import("Graph.zig"); diff --git a/src/bundler/linker_context/computeChunks.zig b/src/bundler/linker_context/computeChunks.zig index 6aef6b0f0c..2bd7b79d2f 100644 --- a/src/bundler/linker_context/computeChunks.zig +++ b/src/bundler/linker_context/computeChunks.zig @@ -316,11 +316,20 @@ pub noinline fn computeChunks( if (chunk.entry_point.is_entry_point and (chunk.content == .html or (kinds[chunk.entry_point.source_index] == .user_specified and !chunk.has_html_chunk))) { - chunk.template = PathTemplate.file; - if (this.resolver.opts.entry_naming.len > 0) - chunk.template.data = this.resolver.opts.entry_naming; + // Use fileWithTarget template if there are HTML imports and user hasn't manually set naming + if (this.parse_graph.html_imports.server_source_indices.len > 0 and this.resolver.opts.entry_naming.len == 0) { + chunk.template = PathTemplate.fileWithTarget; + } else { + chunk.template = PathTemplate.file; + if (this.resolver.opts.entry_naming.len > 0) + chunk.template.data = this.resolver.opts.entry_naming; + } } else { - chunk.template = PathTemplate.chunk; + if (this.parse_graph.html_imports.server_source_indices.len > 0 and this.resolver.opts.chunk_naming.len == 0) { + chunk.template = PathTemplate.chunkWithTarget; + } else { + chunk.template = PathTemplate.chunk; + } if (this.resolver.opts.chunk_naming.len > 0) chunk.template.data = this.resolver.opts.chunk_naming; } @@ -329,20 +338,34 @@ pub noinline fn computeChunks( chunk.template.placeholder.name = pathname.base; chunk.template.placeholder.ext = chunk.content.ext(); - // this if check is a specific fix for `bun build hi.ts --external '*'`, without leading `./` - const dir_path = if (pathname.dir.len > 0) pathname.dir else "."; - - var real_path_buf: bun.PathBuffer = undefined; - const dir = dir: { - var dir = std.fs.cwd().openDir(dir_path, .{}) catch { - break :dir bun.path.normalizeBuf(dir_path, &real_path_buf, .auto); + if (chunk.template.needs(.target)) { + // Determine the target from the AST of the entry point source + const ast_targets = this.graph.ast.items(.target); + const chunk_target = ast_targets[chunk.entry_point.source_index]; + chunk.template.placeholder.target = switch (chunk_target) { + .browser => "browser", + .bun => "bun", + .node => "node", + .bun_macro => "macro", + .bake_server_components_ssr => "ssr", }; - defer dir.close(); + } - break :dir try bun.FD.fromStdDir(dir).getFdPath(&real_path_buf); - }; + if (chunk.template.needs(.dir)) { + // this if check is a specific fix for `bun build hi.ts --external '*'`, without leading `./` + const dir_path = if (pathname.dir.len > 0) pathname.dir else "."; + var real_path_buf: bun.PathBuffer = undefined; + const dir = dir: { + var dir = bun.sys.openatA(.cwd(), dir_path, bun.O.PATH | bun.O.DIRECTORY, 0).unwrap() catch { + break :dir bun.path.normalizeBuf(dir_path, &real_path_buf, .auto); + }; + defer dir.close(); - chunk.template.placeholder.dir = try resolve_path.relativeAlloc(this.allocator, this.resolver.opts.root_dir, dir); + break :dir try dir.getFdPath(&real_path_buf); + }; + + chunk.template.placeholder.dir = try resolve_path.relativeAlloc(this.allocator, this.resolver.opts.root_dir, dir); + } } return chunks; diff --git a/src/defines-table.zig b/src/defines-table.zig index 5aca93d2bc..feab03ffcc 100644 --- a/src/defines-table.zig +++ b/src/defines-table.zig @@ -19,6 +19,13 @@ const defines = @import("./defines.zig"); // these functions has any side effects. It only says something about // referencing these function without calling them. pub const GlobalDefinesKey = [_][]const string{ + + // Array: Static methods + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array#Static_methods + &[_]string{ "Array", "from" }, + &[_]string{ "Array", "of" }, + &[_]string{ "Array", "isArray" }, + // Object: Static methods // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object#Static_methods &[_]string{ "Object", "assign" }, @@ -33,6 +40,8 @@ pub const GlobalDefinesKey = [_][]const string{ &[_]string{ "Object", "getOwnPropertyNames" }, &[_]string{ "Object", "getOwnPropertySymbols" }, &[_]string{ "Object", "getPrototypeOf" }, + &[_]string{ "Object", "groupBy" }, + &[_]string{ "Object", "hasOwn" }, &[_]string{ "Object", "is" }, &[_]string{ "Object", "isExtensible" }, &[_]string{ "Object", "isFrozen" }, @@ -146,6 +155,9 @@ pub const GlobalDefinesKey = [_][]const string{ &[_]string{ "Reflect", "set" }, &[_]string{ "Reflect", "setPrototypeOf" }, + &[_]string{ "JSON", "parse" }, + &[_]string{ "JSON", "stringify" }, + // Console method references are assumed to have no side effects // https://developer.mozilla.org/en-US/docs/Web/API/console &[_]string{ "console", "assert" }, diff --git a/src/deps/uws/BodyReaderMixin.zig b/src/deps/uws/BodyReaderMixin.zig index a9aacae716..b6ff1d3d8f 100644 --- a/src/deps/uws/BodyReaderMixin.zig +++ b/src/deps/uws/BodyReaderMixin.zig @@ -31,7 +31,7 @@ pub fn BodyReaderMixin( }; } fn onAborted(mixin: *Mixin, _: Response) void { - mixin.body.deinit(); + mixin.body.clearAndFree(); onError(@fieldParentPtr(field, mixin)); } }; @@ -41,32 +41,48 @@ pub fn BodyReaderMixin( fn onData(ctx: *@This(), resp: uws.AnyResponse, chunk: []const u8, last: bool) !void { if (last) { - var body = ctx.body; // stack copy so onBody can free everything - resp.clearAborted(); + // Free everything after + var body = ctx.body; + defer body.deinit(); + ctx.body = .init(ctx.body.allocator); resp.clearOnData(); if (body.items.len > 0) { try body.appendSlice(chunk); - try onBody(@fieldParentPtr(field, ctx), ctx.body.items, resp); + try onBody(@fieldParentPtr(field, ctx), body.items, resp); } else { try onBody(@fieldParentPtr(field, ctx), chunk, resp); } - body.deinit(); } else { try ctx.body.appendSlice(chunk); } } fn onOOM(ctx: *@This(), r: uws.AnyResponse) void { + var body = ctx.body; + ctx.body = .init(ctx.body.allocator); + body.deinit(); + r.clearAborted(); + r.clearOnData(); + r.clearOnWritable(); + r.writeStatus("500 Internal Server Error"); r.endWithoutBody(false); - ctx.body.deinit(); + onError(@fieldParentPtr(field, ctx)); } fn onInvalid(ctx: *@This(), r: uws.AnyResponse) void { + var body = ctx.body; + ctx.body = .init(body.allocator); + body.deinit(); + + r.clearAborted(); + r.clearOnData(); + r.clearOnWritable(); + r.writeStatus("400 Bad Request"); r.endWithoutBody(false); - ctx.body.deinit(); + onError(@fieldParentPtr(field, ctx)); } }; diff --git a/src/import_record.zig b/src/import_record.zig index e9a86e3078..f5c5b43195 100644 --- a/src/import_record.zig +++ b/src/import_record.zig @@ -27,7 +27,9 @@ pub const ImportKind = enum(u8) { /// A CSS "composes" property composes = 9, - internal = 10, + html_manifest = 10, + + internal = 11, pub const Label = std.EnumArray(ImportKind, []const u8); pub const all_labels: Label = brk: { @@ -45,6 +47,7 @@ pub const ImportKind = enum(u8) { labels.set(ImportKind.url, "url-token"); labels.set(ImportKind.composes, "composes"); labels.set(ImportKind.internal, "internal"); + labels.set(ImportKind.html_manifest, "html_manifest"); break :brk labels; }; @@ -60,6 +63,7 @@ pub const ImportKind = enum(u8) { labels.set(ImportKind.url, "url()"); labels.set(ImportKind.internal, ""); labels.set(ImportKind.composes, "composes"); + labels.set(ImportKind.html_manifest, "HTML import"); break :brk labels; }; diff --git a/src/js_parser.zig b/src/js_parser.zig index 821a9c66f5..6261b6fa77 100644 --- a/src/js_parser.zig +++ b/src/js_parser.zig @@ -3080,7 +3080,7 @@ pub const Parser = struct { if (runtime_api_call.len > 0) { var args = try p.allocator.alloc(Expr, 1); args[0] = expr; - final_expr = try p.callRuntime(expr.loc, runtime_api_call, args); + final_expr = p.callRuntime(expr.loc, runtime_api_call, args); } const ns_export_part = js_ast.Part{ @@ -3092,7 +3092,7 @@ pub const Parser = struct { .data = .{ .s_lazy_export = brk: { const data = try p.allocator.create(Expr.Data); - data.* = expr.data; + data.* = final_expr.data; break :brk data; }, }, diff --git a/src/options.zig b/src/options.zig index d161e41f06..674ab709d5 100644 --- a/src/options.zig +++ b/src/options.zig @@ -2541,6 +2541,7 @@ pub const PathTemplate = struct { try writer.print("{any}", .{bun.fmt.truncatedHash32(hash)}); } }, + .target => try writeReplacingSlashesOnWindows(writer, self.placeholder.target), } remain = remain[end_len + 1 ..]; } @@ -2553,12 +2554,14 @@ pub const PathTemplate = struct { name: []const u8 = "", ext: []const u8 = "", hash: ?u64 = null, + target: []const u8 = "", pub const map = bun.ComptimeStringMap(std.meta.FieldEnum(Placeholder), .{ .{ "dir", .dir }, .{ "name", .name }, .{ "ext", .ext }, .{ "hash", .hash }, + .{ "target", .target }, }); }; @@ -2571,13 +2574,32 @@ pub const PathTemplate = struct { }, }; + pub const chunkWithTarget = PathTemplate{ + .data = "[dir]/[target]/chunk-[hash].[ext]", + .placeholder = .{ + .name = "chunk", + .ext = "js", + .dir = "", + }, + }; + pub const file = PathTemplate{ .data = "[dir]/[name].[ext]", .placeholder = .{}, }; + pub const fileWithTarget = PathTemplate{ + .data = "[dir]/[target]/[name].[ext]", + .placeholder = .{}, + }; + pub const asset = PathTemplate{ .data = "./[name]-[hash].[ext]", .placeholder = .{}, }; + + pub const assetWithTarget = PathTemplate{ + .data = "[dir]/[target]/[name]-[hash].[ext]", + .placeholder = .{}, + }; }; diff --git a/src/runtime.js b/src/runtime.js index b311e9f15f..2848d4d3b3 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -173,3 +173,5 @@ export var __esm = (fn, res) => () => (fn && (res = fn((fn = 0))), res); // This is used for JSX inlining with React. export var $$typeof = /* @__PURE__ */ Symbol.for("react.element"); + +export var __jsonParse = /* @__PURE__ */ a => JSON.parse(a); diff --git a/src/runtime.zig b/src/runtime.zig index b44382c492..7854881b1c 100644 --- a/src/runtime.zig +++ b/src/runtime.zig @@ -315,6 +315,7 @@ pub const Runtime = struct { @"$$typeof": ?Ref = null, __using: ?Ref = null, __callDispose: ?Ref = null, + __jsonParse: ?Ref = null, pub const all = [_][]const u8{ "__name", @@ -330,6 +331,7 @@ pub const Runtime = struct { "$$typeof", "__using", "__callDispose", + "__jsonParse", }; const all_sorted: [all.len]string = brk: { @setEvalBranchQuota(1000000); diff --git a/test/bundler/bundler_edgecase.test.ts b/test/bundler/bundler_edgecase.test.ts index 07a340653e..0623c0f3ad 100644 --- a/test/bundler/bundler_edgecase.test.ts +++ b/test/bundler/bundler_edgecase.test.ts @@ -2025,7 +2025,7 @@ describe("bundler", () => { itBundled("edgecase/NoOutWithTwoFiles", { files: { "/entry.ts": ` - import index from './index.html' + import index from './index.html' with { type: 'file' } console.log(index); `, "/index.html": ` @@ -2051,7 +2051,7 @@ describe("bundler", () => { itBundled("edgecase/OutWithTwoFiles", { files: { "/entry.ts": ` - import index from './index.html' + import index from './index.html' with { type: 'file' } console.log(index); `, "/index.html": ` diff --git a/test/bundler/bundler_html.test.ts b/test/bundler/bundler_html.test.ts index f1e8b9a54a..f69e512381 100644 --- a/test/bundler/bundler_html.test.ts +++ b/test/bundler/bundler_html.test.ts @@ -485,7 +485,7 @@ export const largeModule = { outdir: "out/", files: { "/in/entry.js": ` -import htmlContent from './template.html'; +import htmlContent from './template.html' with { type: 'file' }; console.log('Loaded HTML:', htmlContent);`, "/in/template.html": ` diff --git a/test/bundler/html-import-manifest.test.ts b/test/bundler/html-import-manifest.test.ts new file mode 100644 index 0000000000..99fe0eef2d --- /dev/null +++ b/test/bundler/html-import-manifest.test.ts @@ -0,0 +1,370 @@ +import { describe, expect } from "bun:test"; +import { itBundled } from "./expectBundled"; + +describe("bundler", () => { + // Test HTML import manifest with enhanced metadata + itBundled("html-import/manifest-with-metadata", { + outdir: "out/", + files: { + "/server.js": ` +import html from "./client.html"; + +if (!html.files.find(a => a.path === html.index)) { + throw new Error("Bad file"); +} + +console.log(JSON.stringify(html, null, 2)); + +`, + "/client.html": ` + + + + + + + +

Client HTML

+ +`, + "/styles.css": ` +body { + background-color: #f0f0f0; + margin: 0; + padding: 20px; +} +h1 { + color: #333; +}`, + "/client.js": ` +import favicon from './favicon.png'; +console.log("Client script loaded"); +window.addEventListener('DOMContentLoaded', () => { + console.log('DOM ready'); +}); +console.log(favicon); +`, + "/favicon.png": Buffer.from([ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, // PNG header + 0x00, + 0x00, + 0x00, + 0x0d, + 0x49, + 0x48, + 0x44, + 0x52, // IHDR chunk + 0x00, + 0x00, + 0x00, + 0x10, + 0x00, + 0x00, + 0x00, + 0x10, // 16x16 + 0x08, + 0x02, + 0x00, + 0x00, + 0x00, + 0x90, + 0x91, + 0x68, // 8-bit RGB + 0x36, + 0x00, + 0x00, + 0x00, + 0x00, + 0x49, + 0x45, + 0x4e, // IEND chunk + 0x44, + 0xae, + 0x42, + 0x60, + 0x82, + ]), + }, + entryPoints: ["/server.js"], + target: "bun", + + run: { + validate({ stdout, stderr }) { + expect(stdout).toMatchInlineSnapshot(` + "{ + "index": "./client.html", + "files": [ + { + "input": "client.html", + "path": "./client-5y90hwq3.js", + "loader": "js", + "isEntry": true, + "headers": { + "etag": "xGxKikG0dN0", + "content-type": "text/javascript;charset=utf-8" + } + }, + { + "input": "client.html", + "path": "./client.html", + "loader": "html", + "isEntry": true, + "headers": { + "etag": "hZ3u5t2Rmuo", + "content-type": "text/html;charset=utf-8" + } + }, + { + "input": "client.html", + "path": "./client-0z58sk45.css", + "loader": "css", + "isEntry": true, + "headers": { + "etag": "0k_h5oYVQlA", + "content-type": "text/css;charset=utf-8" + } + }, + { + "input": "favicon.png", + "path": "./favicon-wjepk3hq.png", + "loader": "file", + "isEntry": false, + "headers": { + "etag": "fFLOVvPDEZc", + "content-type": "image/png" + } + } + ] + } + " + `); + }, + }, + }); + + // Test manifest with multiple HTML imports + itBundled("html-import/multiple-manifests", { + outdir: "out/", + files: { + "/server.js": ` +import homeHtml from "./home.html"; +import aboutHtml from "./about.html"; +console.log("Home manifest:", homeHtml); +console.log("About manifest:", aboutHtml); +`, + "/home.html": ` + + + + + + + +

Home Page

+ +`, + "/about.html": ` + + + + + + + +

About Page

+ +`, + "/home.css": "body { background: #fff; }", + "/home.js": "console.log('Home page');", + "/about.css": "body { background: #f0f0f0; }", + "/about.js": "console.log('About page');", + }, + entryPoints: ["/server.js"], + target: "bun", + + onAfterBundle(api) { + const serverCode = api.readFile("out/server.js"); + + // The manifests are embedded as escaped JSON strings in __jsonParse calls + const manifestMatches = [...serverCode.matchAll(/__jsonParse\("(.+?)"\)/gs)]; + expect(manifestMatches.length).toBe(2); + let manifests = []; + for (const match of manifestMatches) { + // The captured group contains the escaped JSON string + const escapedJson = match[1]; + // Parse the escaped JSON string + const manifest = JSON.parse(JSON.parse('"' + escapedJson + '"')); + manifests.push(manifest); + expect(manifest.index).toBeDefined(); + expect(manifest.files).toBeDefined(); + expect(Array.isArray(manifest.files)).toBe(true); + + // Each manifest should have HTML, JS, and CSS + const loaders = manifest.files.map((f: any) => f.loader); + expect(loaders).toContain("html"); + expect(loaders).toContain("js"); + expect(loaders).toContain("css"); + + // All files should have enhanced metadata + for (const file of manifest.files) { + expect(file).toHaveProperty("headers"); + expect(file).toHaveProperty("isEntry"); + expect(file.headers).toHaveProperty("etag"); + expect(file.headers).toHaveProperty("content-type"); + } + } + + expect(manifests).toMatchInlineSnapshot(` + [ + { + "files": [ + { + "headers": { + "content-type": "text/javascript;charset=utf-8", + "etag": "DLJP98vzFzQ", + }, + "input": "home.html", + "isEntry": true, + "loader": "js", + "path": "./home-5f8tg1jd.js", + }, + { + "headers": { + "content-type": "text/html;charset=utf-8", + "etag": "_Qy4EtlcGvs", + }, + "input": "home.html", + "isEntry": true, + "loader": "html", + "path": "./home.html", + }, + { + "headers": { + "content-type": "text/css;charset=utf-8", + "etag": "6qg2qb7a2qo", + }, + "input": "home.html", + "isEntry": true, + "loader": "css", + "path": "./home-5pdcqqze.css", + }, + ], + "index": "./home.html", + }, + { + "files": [ + { + "headers": { + "content-type": "text/javascript;charset=utf-8", + "etag": "t8rrkgPylZo", + }, + "input": "about.html", + "isEntry": true, + "loader": "js", + "path": "./about-e59abjgr.js", + }, + { + "headers": { + "content-type": "text/html;charset=utf-8", + "etag": "igL7YEH9e0I", + }, + "input": "about.html", + "isEntry": true, + "loader": "html", + "path": "./about.html", + }, + { + "headers": { + "content-type": "text/css;charset=utf-8", + "etag": "DE8kdBXWhVg", + }, + "input": "about.html", + "isEntry": true, + "loader": "css", + "path": "./about-7apjgk42.css", + }, + ], + "index": "./about.html", + }, + ] + `); + }, + }); + + // Test that import with {type: 'file'} still works as a file import + itBundled("html-import/with-type-file-attribute", { + outdir: "out/", + files: { + "/entry.js": ` +import htmlUrl from "./page.html" with { type: 'file' }; +import htmlManifest from "./index.html"; + +// Test that htmlUrl is a string (file path) +if (typeof htmlUrl !== 'string') { + throw new Error("Expected htmlUrl to be a string, got " + typeof htmlUrl); +} + +// Test that htmlManifest is an object with expected properties +if (typeof htmlManifest !== 'object' || !htmlManifest.index || !Array.isArray(htmlManifest.files)) { + throw new Error("Expected htmlManifest to be an object with index and files array"); +} + +console.log("✓ File import returned URL:", htmlUrl); +console.log("✓ HTML import returned manifest with", htmlManifest.files.length, "files"); +console.log("✓ Both import types work correctly"); +`, + "/page.html": ` + + + + Page imported as file + + +

This HTML is imported with type: 'file'

+ +`, + "/index.html": ` + + + + + + +

Test Page

+ +`, + "/styles.css": `body { background: #fff; }`, + }, + entryPoints: ["/entry.js"], + target: "bun", + + run: { + validate({ stdout }) { + expect(stdout).toContain("✓ File import returned URL:"); + expect(stdout).toContain("✓ HTML import returned manifest with"); + expect(stdout).toContain("✓ Both import types work correctly"); + }, + }, + + onAfterBundle(api) { + // Check that the generated code correctly handles both import types + const entryCode = api.readFile("out/entry.js"); + + // Should have a file import for page.html + expect(entryCode).toContain('var page_default = "./page-'); + expect(entryCode).toContain('.html";'); + + // Should have a manifest import for index.html + expect(entryCode).toContain('__jsonParse("'); + expect(entryCode).toContain('\\\"index\\\":\\\"./index.html\\\"'); + expect(entryCode).toContain('\\\"files\\\":['); + }, + }); +}); diff --git a/test/cli/install/bun-workspaces.test.ts b/test/cli/install/bun-workspaces.test.ts index a249d5e32b..9be8ad1e57 100644 --- a/test/cli/install/bun-workspaces.test.ts +++ b/test/cli/install/bun-workspaces.test.ts @@ -1,8 +1,7 @@ import { file, spawn, write } from "bun"; import { install_test_helpers } from "bun:internal-for-testing"; -import { afterAll, beforeAll, beforeEach, describe, expect, test, afterEach } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; import { mkdirSync, rmSync, writeFileSync } from "fs"; -import { readlink } from "fs/promises"; import { cp, exists, mkdir, rm } from "fs/promises"; import { assertManifestsPopulated, diff --git a/test/internal/ban-words.test.ts b/test/internal/ban-words.test.ts index 65ab1f4b3b..d8ba199180 100644 --- a/test/internal/ban-words.test.ts +++ b/test/internal/ban-words.test.ts @@ -32,12 +32,12 @@ const words: Record "== alloc.ptr": { reason: "The std.mem.Allocator context pointer can be undefined, which makes this comparison undefined behavior" }, "!= alloc.ptr": { reason: "The std.mem.Allocator context pointer can be undefined, which makes this comparison undefined behavior" }, - [String.raw`: [a-zA-Z0-9_\.\*\?\[\]\(\)]+ = undefined,`]: { reason: "Do not default a struct field to undefined", limit: 242, regex: true }, + [String.raw`: [a-zA-Z0-9_\.\*\?\[\]\(\)]+ = undefined,`]: { reason: "Do not default a struct field to undefined", limit: 243, regex: true }, "usingnamespace": { reason: "Zig 0.15 will remove `usingnamespace`" }, - "catch unreachable": { reason: "For out-of-memory, prefer 'catch bun.outOfMemory()'", limit: 1854 }, + "catch unreachable": { reason: "For out-of-memory, prefer 'catch bun.outOfMemory()'", limit: 1857 }, "std.fs.Dir": { reason: "Prefer bun.sys + bun.FD instead of std.fs", limit: 180 }, - "std.fs.cwd": { reason: "Prefer bun.FD.cwd()", limit: 103 }, + "std.fs.cwd": { reason: "Prefer bun.FD.cwd()", limit: 102 }, "std.fs.File": { reason: "Prefer bun.sys + bun.FD instead of std.fs", limit: 62 }, ".stdFile()": { reason: "Prefer bun.sys + bun.FD instead of std.fs.File. Zig hides 'errno' when Bun wants to match libuv", limit: 18 }, ".stdDir()": { reason: "Prefer bun.sys + bun.FD instead of std.fs.File. Zig hides 'errno' when Bun wants to match libuv", limit: 48 }, From 7a069d7214c3ff8c147527045802b44a244e3331 Mon Sep 17 00:00:00 2001 From: pfg Date: Wed, 11 Jun 2025 16:00:58 -0700 Subject: [PATCH 3/3] Add back zls binary after zig upgrade (#20327) --- cmake/tools/SetupZig.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/tools/SetupZig.cmake b/cmake/tools/SetupZig.cmake index 781e7e2d06..5515015e59 100644 --- a/cmake/tools/SetupZig.cmake +++ b/cmake/tools/SetupZig.cmake @@ -20,7 +20,7 @@ else() unsupported(CMAKE_SYSTEM_NAME) endif() -set(ZIG_COMMIT "41f20cec62e9e933c1f1430533ee95837fcd00a8") +set(ZIG_COMMIT "0a0120fa92cd7f6ab244865688b351df634f0707") optionx(ZIG_TARGET STRING "The zig target to use" DEFAULT ${DEFAULT_ZIG_TARGET}) if(CMAKE_BUILD_TYPE STREQUAL "Release")