diff --git a/src/StandaloneModuleGraph.zig b/src/StandaloneModuleGraph.zig index 649fd5e7ed..c355936369 100644 --- a/src/StandaloneModuleGraph.zig +++ b/src/StandaloneModuleGraph.zig @@ -233,7 +233,7 @@ pub const StandaloneModuleGraph = struct { }); stored.external_source_names = file_names; - stored.underlying_provider = .{ .data = @truncate(@intFromPtr(data)) }; + stored.underlying_provider = .{ .data = @truncate(@intFromPtr(data)), .load_hint = .none }; stored.is_standalone_module_graph = true; const parsed = stored.new(); // allocate this on the heap diff --git a/src/bake/DevServer.zig b/src/bake/DevServer.zig index 33c6e29d85..574830db31 100644 --- a/src/bake/DevServer.zig +++ b/src/bake/DevServer.zig @@ -69,6 +69,10 @@ html_router: HTMLRouter, /// Assets are accessible via `/_bun/asset/` /// This store is not thread safe. assets: Assets, +/// Similar to `assets`, specialized for the additional needs of source mappings. +source_maps: SourceMapStore, +// /// Allocations that require a reference count. +// ref_strings: RefString.Store, /// All bundling failures are stored until a file is saved and rebuilt. /// They are stored in the wire format the HMR runtime expects so that /// serialization only happens once. @@ -108,7 +112,6 @@ bundler_options: bake.SplitBundlerOptions, server_transpiler: Transpiler, client_transpiler: Transpiler, ssr_transpiler: Transpiler, -watcher_thread_resolver: bun.resolver.Resolver, /// The log used by all `server_transpiler`, `client_transpiler` and `ssr_transpiler`. /// Note that it is rarely correct to write messages into it. Instead, associate /// messages with the IncrementalGraph file or Route using `SerializedFailure` @@ -378,6 +381,7 @@ pub fn init(options: Options) bun.JSOOM!*DevServer { .files = .empty, .refs = .empty, }, + .source_maps = .empty, .log = .init(allocator), .plugin_state = .unknown, .bundling_failures = .{}, @@ -393,7 +397,6 @@ pub fn init(options: Options) bun.JSOOM!*DevServer { .server_transpiler = undefined, .client_transpiler = undefined, .ssr_transpiler = undefined, - .watcher_thread_resolver = undefined, .bun_watcher = undefined, .configuration_hash_key = undefined, .router = undefined, @@ -446,9 +449,6 @@ pub fn init(options: Options) bun.JSOOM!*DevServer { dev.ssr_transpiler.resolver.watcher = dev.bun_watcher.getResolveWatcher(); } - dev.watcher_thread_resolver = dev.server_transpiler.resolver; - dev.watcher_thread_resolver.watcher = null; - assert(dev.server_transpiler.resolver.opts.target != .browser); assert(dev.client_transpiler.resolver.opts.target == .browser); @@ -736,7 +736,6 @@ pub fn memoryCost(dev: *DevServer) usize { .server_register_update_callback = {}, .deferred_request_pool = {}, .assume_perfect_incremental_bundling = {}, - .watcher_thread_resolver = {}, // pointers that are not considered a part of DevServer .vm = {}, @@ -767,6 +766,14 @@ pub fn memoryCost(dev: *DevServer) usize { .assets = { cost += dev.assets.memoryCost(); }, + .source_maps = { + cost += memoryCostArrayHashMap(dev.source_maps.entries); + for (dev.source_maps.entries.values()) |entry| { + cost += entry.response.memoryCost(); + cost += memoryCostSlice(entry.sourceContents()); + cost += memoryCostSlice(entry.file_paths); // do not re-count contents + } + }, .incremental_result = { cost += memoryCostArrayList(dev.incremental_result.client_components_added); cost += memoryCostArrayList(dev.incremental_result.html_routes_soft_affected); @@ -870,6 +877,7 @@ fn scanInitialRoutes(dev: *DevServer) !void { } /// Returns true if a catch-all handler was attached. +// TODO: rename to setRoutes to match server.zig pub fn attachRoutes(dev: *DevServer, server: anytype) !bool { // TODO: all paths here must be prefixed with publicPath if set. dev.server = bun.JSC.API.AnyServer.from(server); @@ -879,6 +887,7 @@ pub fn attachRoutes(dev: *DevServer, server: anytype) !bool { app.get(client_prefix ++ "/:route", *DevServer, dev, wrapGenericRequestHandler(onJsRequest, is_ssl)); app.get(asset_prefix ++ "/:asset", *DevServer, dev, wrapGenericRequestHandler(onAssetRequest, is_ssl)); app.get(internal_prefix ++ "/src/*", *DevServer, dev, wrapGenericRequestHandler(onSrcRequest, is_ssl)); + app.post(internal_prefix ++ "/report_error", *DevServer, dev, wrapGenericRequestHandler(ErrorReportRequest.run, is_ssl)); app.any(internal_prefix, *DevServer, dev, wrapGenericRequestHandler(onNotFound, is_ssl)); @@ -923,24 +932,43 @@ fn notFound(resp: anytype) void { } fn onNotFoundCorked(resp: anytype) void { - resp.writeHeaderInt("Content-Length", 0); resp.writeStatus("404 Not Found"); - resp.endWithoutBody(false); + resp.end("Not Found", false); +} + +fn onOutdatedJSCorked(resp: anytype) void { + // Send a payload to instantly reload the page. This only happens when the + // client bundle is invalidated while the page is loading, aka when you + // perform many file updates that cannot be hot-updated. + resp.writeStatus("200 OK"); + resp.writeHeader("Content-Type", MimeType.javascript.value); + resp.end( + \\try{location.reload()}catch(_){} + \\addEventListener("DOMContentLoaded",function(event){location.reload()}) + , false); } fn onJsRequest(dev: *DevServer, req: *Request, resp: AnyResponse) void { const route_id = req.parameter(0); - if (!bun.strings.hasSuffixComptime(route_id, ".js")) + const is_map = bun.strings.hasSuffixComptime(route_id, ".js.map"); + if (!is_map and !bun.strings.hasSuffixComptime(route_id, ".js")) return notFound(resp); - const min_len = "-00000000FFFFFFFF.js".len; + const min_len = "00000000FFFFFFFF.js".len + (if (is_map) ".map".len else 0); if (route_id.len < min_len) return notFound(resp); - const hex = route_id[route_id.len - min_len + 1 ..][0 .. @sizeOf(u64) * 2]; + const hex = route_id[route_id.len - min_len ..][0 .. @sizeOf(u64) * 2]; if (hex.len != @sizeOf(u64) * 2) return notFound(resp); const id = parseHexToInt(u64, hex) orelse return notFound(resp); + if (is_map) { + const entry = dev.source_maps.entries.getPtr(id) orelse + return notFound(resp); + entry.response.onRequest(req, resp); + return; + } + const route_bundle_index: RouteBundle.Index = .init(@intCast(id & 0xFFFFFFFF)); const generation: u32 = @intCast(id >> 32); @@ -951,8 +979,7 @@ fn onJsRequest(dev: *DevServer, req: *Request, resp: AnyResponse) void { if (route_bundle.client_script_generation != generation or route_bundle.server_state != .loaded) { - bun.Output.debugWarn("TODO: Outdated JS Payload", .{}); - return notFound(resp); + return resp.corked(onOutdatedJSCorked, .{resp}); } dev.onJsRequestWithBundle(route_bundle_index, resp, bun.http.Method.which(req.method()) orelse .POST); @@ -1009,7 +1036,7 @@ inline fn redirectHandler(comptime path: []const u8, comptime is_ssl: bool) fn ( _ = req; resp.writeStatus("302 Found"); resp.writeHeader("Location", path); - resp.endWithoutBody(false); + resp.end("Redirecting...", false); } }.handle; } @@ -1023,7 +1050,6 @@ fn onIncrementalVisualizerCorked(resp: anytype) void { @embedFile("incremental_visualizer.html") else bun.runtimeEmbedFile(.src_eager, "bake/incremental_visualizer.html"); - resp.writeHeaderInt("Content-Length", code.len); resp.end(code, false); } @@ -1080,7 +1106,7 @@ fn ensureRouteIsBundled( }, .err => { // TODO: render plugin error page - resp.endWithoutBody(true); + resp.end("Plugin Error", false); return; }, .loaded => {}, @@ -1137,12 +1163,11 @@ fn ensureRouteIsBundled( continue :sw .loaded; }, .evaluation_failure => { - resp.corked(sendSerializedFailures, .{ - dev, + try dev.sendSerializedFailures( resp, (&(dev.routeBundlePtr(route_bundle_index).data.framework.evaluate_failure.?))[0..1], .evaluation, - }); + ); }, .loaded => switch (kind) { .server_handler => dev.onFrameworkRequestWithBundle(route_bundle_index, .{ .stack = req }, resp), @@ -1202,12 +1227,11 @@ fn checkRouteFailures( return .rebuild; } - resp.corked(sendSerializedFailures, .{ - dev, + try dev.sendSerializedFailures( resp, dev.incremental_result.failures_added.items, .bundler, - }); + ); return .stop; } else { // Failures are unreachable by this route, so it is OK to load. @@ -1426,6 +1450,7 @@ fn getJavaScriptCodeForHTMLFile( try w.writeAll(" "); try bun.js_printer.writeJSONString(input_file_sources[index.get()].path.pretty, @TypeOf(w), w, .utf8); try w.writeAll("(m) {\n "); + try w.writeAll(" return Promise.all([\n"); for (import_records[index.get()].slice()) |import| { if (import.source_index.isValid()) { if (!loaders[import.source_index.get()].isJavaScriptLike()) @@ -1435,10 +1460,11 @@ fn getJavaScriptCodeForHTMLFile( continue; } - try w.writeAll(" m.dynamicImport("); + try w.writeAll(" m.dynamicImport("); try bun.js_printer.writeJSONString(import.path.pretty, @TypeOf(w), w, .utf8); - try w.writeAll(");\n "); + try w.writeAll("),\n "); } + try w.writeAll(" ]);\n "); try w.writeAll("},\n"); // Avoid-recloning if it is was moved to the hap @@ -1768,24 +1794,12 @@ fn generateClientBundle(dev: *DevServer, route_bundle: *RouteBundle) bun.OOM![]u .html => |html| html.bundled_file, }; - const hash = hash: { - var source_map_hash: bun.bundle_v2.ContentHasher.Hash = .init(0x4b10); // arbitrarily different seed than what .initial_response uses - const keys = dev.client_graph.bundled_files.keys(); - for (dev.client_graph.current_chunk_parts.items) |part| { - source_map_hash.update(keys[part.get()]); - source_map_hash.update(dev.client_graph.source_maps.items[part.get()].vlq_chunk.slice()); - } - break :hash source_map_hash.final(); - }; // Insert the source map - if (try dev.assets.putOrIncrementRefCount(hash, 1)) |static_route_ptr| { - // TODO: this asset is never unreferenced - const source_map = try dev.client_graph.takeSourceMap(.initial_response, sfa, dev.allocator); - errdefer dev.allocator.free(source_map); - static_route_ptr.* = StaticRoute.initFromAnyBlob(&.fromOwnedSlice(dev.allocator, source_map), .{ - .server = dev.server.?, - .mime_type = &.json, - }); + const source_map_id = @as(u64, route_bundle.client_script_generation) << 32; + if (try dev.source_maps.putOrIncrementRefCount(source_map_id, 1)) |entry| { + var arena = std.heap.ArenaAllocator.init(sfa); + defer arena.deinit(); + try dev.client_graph.takeSourceMap(.initial_response, arena.allocator(), dev.allocator, entry); } const client_bundle = dev.client_graph.takeJSBundle(.{ @@ -1795,12 +1809,9 @@ fn generateClientBundle(dev: *DevServer, route_bundle: *RouteBundle) bun.OOM![]u else "", .react_refresh_entry_point = react_fast_refresh_id, - .source_map_id = hash, + .source_map_id = source_map_id, }); - const source_map = try dev.client_graph.takeSourceMap(.initial_response, sfa, dev.allocator); - dev.allocator.free(source_map); - return client_bundle; } @@ -1993,6 +2004,7 @@ pub fn finalizeBundle( .gts = undefined, }; + const quoted_source_contents = bv2.linker.graph.files.items(.quoted_source_contents); // Pass 1, update the graph's nodes, resolving every bundler source // index into its `IncrementalGraph(...).FileIndex` for ( @@ -2007,10 +2019,24 @@ pub fn finalizeBundle( bun.assert(!part_range.source_index.isRuntime()); break :brk .empty; }; + // TODO: investigate why linker.files is not indexed by linker's index + // const linker_index = bv2.linker.graph.stable_source_indices[index.get()]; + // const quoted_contents = quoted_source_contents[linker_index]; + const quoted_contents = quoted_source_contents[part_range.source_index.get()]; switch (targets[part_range.source_index.get()].bakeGraph()) { - .server => try dev.server_graph.receiveChunk(&ctx, index, .{ .js = compile_result.code() }, source_map, false), - .ssr => try dev.server_graph.receiveChunk(&ctx, index, .{ .js = compile_result.code() }, source_map, true), - .client => try dev.client_graph.receiveChunk(&ctx, index, .{ .js = compile_result.code() }, source_map, false), + inline else => |graph| try (switch (graph) { + .client => dev.client_graph, + else => dev.server_graph, + }).receiveChunk( + &ctx, + index, + .{ .js = .{ + .code = compile_result.code(), + .source_map = source_map, + .quoted_contents = .initOwned(quoted_contents, bun.default_allocator), + } }, + graph == .ssr, + ), } } @@ -2061,7 +2087,7 @@ pub fn finalizeBundle( } } - try dev.client_graph.receiveChunk(&ctx, index, .{ .css = hash }, null, false); + try dev.client_graph.receiveChunk(&ctx, index, .{ .css = hash }, false); // If imported on server, there needs to be a server-side file entry // so that edges can be attached. When a file is only imported on @@ -2087,8 +2113,11 @@ pub fn finalizeBundle( try dev.client_graph.receiveChunk( &ctx, index, - .{ .js = generated_js }, - null, // HTML chunk does not have a source map. + .{ .js = .{ + .code = generated_js, + .source_map = .empty, + .quoted_contents = comptime .initNeverFree(""), + } }, false, ); const client_index = ctx.getCachedIndex(.client, index).*.unwrap() orelse @panic("unresolved index"); @@ -2372,18 +2401,13 @@ pub fn finalizeBundle( const keys = dev.client_graph.bundled_files.keys(); for (dev.client_graph.current_chunk_parts.items) |part| { source_map_hash.update(keys[part.get()]); - source_map_hash.update(dev.client_graph.source_maps.items[part.get()].vlq_chunk.slice()); + source_map_hash.update(dev.client_graph.source_maps.items[part.get()].vlq()); } - break :hash source_map_hash.final(); + // Set the bottom bit. This ensures that the resource can never be confused for a route bundle. + break :hash source_map_hash.final() | 1; }; - // Insert the source map - if (try dev.assets.putOrIncrementRefCount(hash, hot_update_subscribers)) |static_route_ptr| { - const source_map = try dev.client_graph.takeSourceMap(.hmr_chunk, bv2.graph.allocator, dev.allocator); - errdefer dev.allocator.free(source_map); - static_route_ptr.* = StaticRoute.initFromAnyBlob(&.fromOwnedSlice(dev.allocator, source_map), .{ - .server = dev.server.?, - .mime_type = &.json, - }); + if (try dev.source_maps.putOrIncrementRefCount(hash, 1)) |entry| { + try dev.client_graph.takeSourceMap(.hmr_chunk, bv2.graph.allocator, dev.allocator, entry); } // Build and send the source chunk try dev.client_graph.takeJSBundleToList(&hot_update_payload, .{ @@ -2418,12 +2442,11 @@ pub fn finalizeBundle( .bundled_html_page => |ram| ram.response, }; - resp.corked(sendSerializedFailures, .{ - dev, + try dev.sendSerializedFailures( resp, dev.bundling_failures.keys(), .bundler, - }); + ); } return; } @@ -2437,8 +2460,8 @@ pub fn finalizeBundle( Output.enableBuffering(); } - if (Environment.isDebug) { - Output.prettyErrorln("DevServer: {}, RSS: {}", .{ + if (debug.isVisible()) { + Output.prettyErrorln("DevServer: {}, RSS: {}", .{ bun.fmt.size(dev.memoryCost(), .{}), bun.fmt.size(bun.sys.selfProcessMemoryUsage() orelse 0, .{}), }); @@ -2450,8 +2473,8 @@ pub fn finalizeBundle( } } else { dev.bundles_since_last_error = 0; - if (Environment.isDebug) { - Output.prettyErrorln("DevServer: {}, RSS: {}", .{ + if (debug.isVisible()) { + Output.prettyErrorln("DevServer: {}, RSS: {}", .{ bun.fmt.size(dev.memoryCost(), .{}), bun.fmt.size(bun.sys.selfProcessMemoryUsage() orelse 0, .{}), }); @@ -2763,24 +2786,11 @@ fn sendSerializedFailures( resp: AnyResponse, failures: []const SerializedFailure, kind: ErrorPageKind, -) void { - switch (resp) { - inline else => |r| sendSerializedFailuresInner(dev, r, failures, kind), - } -} +) !void { + var buf: std.ArrayList(u8) = try .initCapacity(bun.default_allocator, 2048); + errdefer buf.deinit(); -fn sendSerializedFailuresInner( - dev: *DevServer, - resp: anytype, - failures: []const SerializedFailure, - kind: ErrorPageKind, -) void { - // TODO: write to Blob and serve that - resp.writeStatus("500 Internal Server Error"); - resp.writeHeader("Content-Type", MimeType.html.value); - - // TODO: what to do about return values here? - _ = resp.write(switch (kind) { + try buf.appendSlice(switch (kind) { inline else => |k| std.fmt.comptimePrint( \\ \\ @@ -2801,29 +2811,38 @@ fn sendSerializedFailuresInner( ), }); - var sfb = std.heap.stackFallback(65536, dev.allocator); - var arena_state = std.heap.ArenaAllocator.init(sfb.get()); - defer arena_state.deinit(); - for (failures) |fail| { const len = bun.base64.encodeLen(fail.data); - const buf = arena_state.allocator().alloc(u8, len) catch bun.outOfMemory(); - const encoded = buf[0..bun.base64.encode(buf, fail.data)]; - _ = resp.write(encoded); - _ = arena_state.reset(.retain_capacity); + try buf.ensureUnusedCapacity(len); + const start = buf.items.len; + buf.items.len += len; + const to_write_into = buf.items[start..]; + + var encoded = to_write_into[0..bun.base64.encode(to_write_into, fail.data)]; + while (encoded.len > 0 and encoded[encoded.len - 1] == '=') { + encoded.len -= 1; + } + + buf.items.len = start + encoded.len; } - const pre = "\"),c=>c.charCodeAt(0));"; + const pre = "\"),c=>c.charCodeAt(0));let config={bun:\"" ++ bun.Global.package_json_version_with_canary ++ "\"};"; const post = ""; if (Environment.codegen_embed) { - _ = resp.end(pre ++ @embedFile("bake-codegen/bake.error.js") ++ post, false); + try buf.appendSlice(pre ++ @embedFile("bake-codegen/bake.error.js") ++ post); } else { - _ = resp.write(pre); - _ = resp.write(bun.runtimeEmbedFile(.codegen_eager, "bake.error.js")); - _ = resp.end(post, false); + try buf.appendSlice(pre); + try buf.appendSlice(bun.runtimeEmbedFile(.codegen_eager, "bake.error.js")); + try buf.appendSlice(post); } + + StaticRoute.sendBlobThenDeinit(resp, &.fromArrayList(buf), .{ + .mime_type = &.html, + .server = dev.server.?, + .status_code = 500, + }); } fn sendBuiltInNotFound(resp: anytype) void { @@ -3076,46 +3095,51 @@ pub fn IncrementalGraph(side: bake.Side) type { }, }; + fn freeFileContent(g: *IncrementalGraph(.client), index: FileIndex, key: []const u8, file: File, css: enum { unref_css, ignore_css }) void { + switch (file.flags.kind) { + .js, .asset => { + bun.default_allocator.free(file.jsCode()); + const map = &g.source_maps.items[index.get()]; + bun.default_allocator.free(map.vlq()); + if (map.quoted_contents_flags.is_owned) { + map.quotedContentsCowString().deinit(bun.default_allocator); + } + }, + .css => if (css == .unref_css) { + g.owner().assets.unrefByPath(key); + }, + .unknown => {}, + } + } + /// Packed source mapping data pub const PackedMap = struct { - /// Allocated by default_allocator. Access with `.slice()` + /// Allocated by default_allocator. Access with `.vlq()` /// This is stored to allow lazy construction of source map files. - /// Aligned to 4 bytes to reduce struct size/padding. - vlq_chunk: struct { - ptr_top: u32, - ptr_bottom: u32, - len: u32, - - pub fn init(data: []const u8) @This() { - if (@inComptime() and data.len == 0) { - return .{ .ptr_top = 0, .ptr_bottom = 0xffffffff, .len = 0 }; - } - return .{ - .ptr_top = @intCast(@intFromPtr(data.ptr) >> 32), - .ptr_bottom = @intCast(@intFromPtr(data.ptr) & 0xffffffff), - .len = @intCast(data.len), - }; - } - - pub fn slice(chunk: @This()) []const u8 { - return @as([*]const u8, @ptrFromInt( - (@as(usize, chunk.ptr_top) << 32) + chunk.ptr_bottom, - ))[0..chunk.len]; - } + vlq_ptr: [*]u8, + vlq_len: u32, + /// The bundler runs quoting on multiple threads, so it only makes + /// sense to preserve that effort for concatenation and + /// re-concatenation. + quoted_contents_ptr: [*]const u8, + quoted_contents_flags: packed struct(u32) { + len: u31, + is_owned: bool, }, - /// This field is overloaded depending on if the source map data is + /// This field is overloaded depending on if the VLQ data is /// present or not (.len != 0). extra: extern union { /// Used to track the last state of the source map chunk. This - /// is used when concatenating chunks. The generated column - /// is not tracked because it is always zero (all chunks end - /// in a newline because minification is off), and the generated - /// line is recomputed on demand + /// is used when concatenating chunks. The generated column is + /// not tracked because it is always zero (all chunks end in a + /// newline because minification is off), and the generated line + /// is recomputed on demand and is different per concatenation. end_state: extern struct { original_line: i32, original_column: i32, }, empty: extern struct { + /// Number of lines to skip. Computed on demand. line_count: bun.GenericIndex(u32, u8).Optional, // Cannot use RouteBundle.Index because `extern union` above :( html_bundle_route_index: bun.GenericIndex(u32, u8).Optional, @@ -3123,28 +3147,51 @@ pub fn IncrementalGraph(side: bake.Side) type { }, pub const empty: PackedMap = .{ - .vlq_chunk = .init(""), + .vlq_ptr = &[0]u8{}, + .vlq_len = 0, + .quoted_contents_ptr = &[0]u8{}, + .quoted_contents_flags = .{ + .len = 0, + .is_owned = false, + }, .extra = .{ .empty = .{ .line_count = .none, .html_bundle_route_index = .none, } }, }; - pub fn fromSourceMap(source_map: SourceMap.Chunk) PackedMap { - return if (source_map.buffer.list.items.len > 0) .{ - .vlq_chunk = .init(source_map.buffer.list.items), - .extra = .{ - .end_state = .{ - .original_line = source_map.end_state.original_line, - .original_column = source_map.end_state.original_column, - }, + pub fn vlq(self: @This()) []u8 { + return self.vlq_ptr[0..self.vlq_len]; + } + + pub fn quotedContentsCowString(self: @This()) bun.CowString { + return bun.CowString.initUnchecked(self.quoted_contents_ptr[0..self.quoted_contents_flags.len], self.quoted_contents_flags.is_owned); + } + + pub fn quotedContents(self: @This()) []const u8 { + return self.quoted_contents_ptr[0..self.quoted_contents_flags.len]; + } + + pub fn fromNonEmptySourceMap(source_map: SourceMap.Chunk, quoted_contents: bun.CowString) !PackedMap { + assert(source_map.buffer.list.items.len > 0); + return .{ + .vlq_ptr = source_map.buffer.list.items.ptr, + .vlq_len = @intCast(source_map.buffer.list.items.len), + .quoted_contents_ptr = quoted_contents.ptr, + .quoted_contents_flags = .{ + .len = @intCast(quoted_contents.flags.len), + .is_owned = quoted_contents.flags.is_owned, }, - } else .empty; + .extra = .{ .end_state = .{ + .original_line = source_map.end_state.original_line, + .original_column = source_map.end_state.original_column, + } }, + }; } comptime { - assert(@sizeOf(@This()) == @sizeOf(u32) * 5); - assert(@alignOf(@This()) == @alignOf(u32)); + assert(@sizeOf(@This()) == @sizeOf(usize) * 4); + assert(@alignOf(@This()) == @alignOf(usize)); } }; @@ -3223,10 +3270,13 @@ pub fn IncrementalGraph(side: bake.Side) type { ctx: *HotUpdateContext, index: bun.JSAst.Index, content: union(enum) { - js: []const u8, + js: struct { + code: []const u8, + source_map: SourceMap.Chunk, + quoted_contents: bun.CowString, + }, css: u64, }, - source_map: ?SourceMap.Chunk, is_ssr_graph: bool, ) !void { const dev = g.owner(); @@ -3238,7 +3288,7 @@ pub fn IncrementalGraph(side: bake.Side) type { if (Environment.allow_assert) { switch (content) { .css => {}, - .js => |js| if (bun.strings.isAllWhitespace(js)) { + .js => |js| if (bun.strings.isAllWhitespace(js.code)) { // Should at least contain the function wrapper bun.Output.panic("Empty chunk is impossible: {s} {s}", .{ key, @@ -3253,7 +3303,7 @@ pub fn IncrementalGraph(side: bake.Side) type { // Dump to filesystem if enabled if (bun.FeatureFlags.bake_debugging_features and content == .js) if (dev.dump_dir) |dump_dir| { - dumpBundleForChunk(dev, dump_dir, side, key, content.js, true, is_ssr_graph); + dumpBundleForChunk(dev, dump_dir, side, key, content.js.code, true, is_ssr_graph); }; const gop = try g.bundled_files.getOrPut(dev.allocator, key); @@ -3263,8 +3313,9 @@ pub fn IncrementalGraph(side: bake.Side) type { gop.key_ptr.* = try bun.default_allocator.dupe(u8, key); try g.first_dep.append(dev.allocator, .none); try g.first_import.append(dev.allocator, .none); - if (side == .client) - try g.source_maps.append(dev.allocator, if (source_map) |map| .fromSourceMap(map) else .empty); + if (side == .client) { + try g.source_maps.append(dev.allocator, .empty); + } } if (g.stale_files.bit_length > gop.index) { @@ -3288,15 +3339,7 @@ pub fn IncrementalGraph(side: bake.Side) type { }; if (gop.found_existing) { // Free the original content - switch (gop.value_ptr.flags.kind) { - .js, .asset => { - bun.default_allocator.free(gop.value_ptr.jsCode()); - bun.default_allocator.free(g.source_maps.items[file_index.get()].vlq_chunk.slice()); - if (source_map) |map| g.source_maps.items[file_index.get()] = .fromSourceMap(map); - }, - .css => {}, // do not need to unref css as it has been replaced already - .unknown => {}, - } + g.freeFileContent(file_index, key, gop.value_ptr.*, .ignore_css); // Free a failure if it exists if (gop.value_ptr.flags.failed) { @@ -3318,10 +3361,17 @@ pub fn IncrementalGraph(side: bake.Side) type { switch (content) { .css => |css| gop.value_ptr.* = .initCSS(css, flags), .js => |js| { - gop.value_ptr.* = .initJavaScript(js, flags); + gop.value_ptr.* = .initJavaScript(js.code, flags); + + // Append source map + if (js.source_map.buffer.len() > 0) { + g.source_maps.items[file_index.get()] = try .fromNonEmptySourceMap(js.source_map, js.quoted_contents); + } else { + js.quoted_contents.deinit(bun.default_allocator); + } // Track JavaScript chunks for concatenation try g.current_chunk_parts.append(dev.allocator, file_index); - g.current_chunk_len += js.len; + g.current_chunk_len += js.code.len; }, } }, @@ -3383,8 +3433,13 @@ pub fn IncrementalGraph(side: bake.Side) type { } } if (content == .js) { - try g.current_chunk_parts.append(dev.allocator, content.js); - g.current_chunk_len += content.js.len; + try g.current_chunk_parts.append(dev.allocator, content.js.code); + g.current_chunk_len += content.js.code.len; + content.js.quoted_contents.deinit(bun.default_allocator); + if (content.js.source_map.buffer.len() > 0) { + var vlq = content.js.source_map.buffer; + vlq.deinit(); + } } }, } @@ -3975,16 +4030,8 @@ pub fn IncrementalGraph(side: bake.Side) type { .kind = .unknown, }; if (gop.found_existing) { - const source_map = &g.source_maps.items[file_index.get()]; - switch (gop.value_ptr.flags.kind) { - .js, .asset => g.owner().allocator.free(gop.value_ptr.jsCode()), - .css => g.owner().assets.unrefByPath(gop.key_ptr.*), - .unknown => {}, - } - if (source_map.vlq_chunk.len > 0) { - g.owner().allocator.free(source_map.vlq_chunk.slice()); - source_map.* = .empty; - } + g.freeFileContent(file_index, gop.key_ptr.*, gop.value_ptr.*, .unref_css); + flags.is_html_route = flags.is_html_route or gop.value_ptr.flags.is_html_route; flags.failed = gop.value_ptr.flags.failed; flags.is_special_framework_file = gop.value_ptr.flags.is_special_framework_file; @@ -4063,8 +4110,6 @@ pub fn IncrementalGraph(side: bake.Side) type { gop.key_ptr.* = try bun.default_allocator.dupe(u8, abs_path); try g.first_dep.append(g.owner().allocator, .none); try g.first_import.append(g.owner().allocator, .none); - if (side == .client) - try g.source_maps.append(g.owner().allocator, .empty); } switch (side) { @@ -4141,16 +4186,7 @@ pub fn IncrementalGraph(side: bake.Side) type { .kind = .unknown, }; if (found_existing) { - const source_map = &g.source_maps.items[file_index.get()]; - switch (gop.value_ptr.flags.kind) { - .js, .asset => g.owner().allocator.free(gop.value_ptr.jsCode()), - .css => g.owner().assets.unrefByPath(gop.key_ptr.*), - .unknown => {}, - } - if (source_map.vlq_chunk.len > 0) { - g.owner().allocator.free(source_map.vlq_chunk.slice()); - source_map.* = .empty; - } + g.freeFileContent(file_index, gop.key_ptr.*, gop.value_ptr.*, .unref_css); flags.is_html_route = gop.value_ptr.flags.is_html_route; flags.is_special_framework_file = gop.value_ptr.flags.is_special_framework_file; flags.is_hmr_root = gop.value_ptr.flags.is_hmr_root; @@ -4377,6 +4413,7 @@ pub fn IncrementalGraph(side: bake.Side) type { } else { try w.writeAll("null"); } + try w.writeAll(",\n bun: \"" ++ bun.Global.package_json_version_with_canary ++ "\""); try w.writeAll(",\n version: \""); try w.writeAll(&g.owner().configuration_hash_key); try w.writeAll("\""); @@ -4395,7 +4432,7 @@ pub fn IncrementalGraph(side: bake.Side) type { } try w.writeAll("})"); if (side == .client) if (options.source_map_id) |source_map_id| { - try w.writeAll("\n//# sourceMappingURL=" ++ asset_prefix ++ "/"); + try w.writeAll("\n//# sourceMappingURL=" ++ client_prefix ++ "/"); try w.writeAll(&std.fmt.bytesToHex(std.mem.asBytes(&source_map_id), .lower)); try w.writeAll(".js.map\n"); }; @@ -4436,8 +4473,14 @@ pub fn IncrementalGraph(side: bake.Side) type { }; } - /// Uses `arena` as a temporary allocator, returning a string owned by `gpa` - pub fn takeSourceMap(g: *@This(), kind: ChunkKind, arena: std.mem.Allocator, gpa: Allocator) bun.OOM![]u8 { + pub const SourceMapGeneration = struct { + json: []u8, + mappings: bun.StringPointer, + file_paths: [][]const u8, + }; + + /// Uses `arena` as a temporary allocator, fills in all fields of `out` except ref_count + pub fn takeSourceMap(g: *@This(), kind: ChunkKind, arena: std.mem.Allocator, gpa: Allocator, out: *SourceMapStore.Entry) bun.OOM!void { if (side == .server) @compileError("not implemented"); const paths = g.bundled_files.keys(); @@ -4452,18 +4495,17 @@ pub fn IncrementalGraph(side: bake.Side) type { }; j.pushStatic( - \\{"version":3,"sources":[ + \\{"version":3,"sources":["bun://Bun/Bun HMR Runtime" ); + // This buffer is temporary, holding the quoted source paths, joined with commas. var source_map_strings = std.ArrayList(u8).init(arena); defer source_map_strings.deinit(); - var needs_comma = false; + + var path_count: usize = 0; for (g.current_chunk_parts.items) |entry| { - if (source_maps[entry.get()].vlq_chunk.len == 0) - continue; - if (needs_comma) - try source_map_strings.appendSlice(","); - needs_comma = true; + path_count += 1; + try source_map_strings.appendSlice(","); const path = paths[entry.get()]; if (std.fs.path.isAbsolute(path)) { const is_windows_drive_path = Environment.isWindows and bun.path.isSepAny(path[0]); @@ -4479,41 +4521,78 @@ pub fn IncrementalGraph(side: bake.Side) type { else path, // invalid but must not crash &source_map_strings, - ) catch |err| { - switch (err) { - error.IncompleteUTF8 => @panic("Unexpected: source file with incomplete UTF-8 as file path"), - error.OutOfMemory => |e| return e, - } + ) catch |err| switch (err) { + error.IncompleteUTF8 => @panic("Unexpected: asset with incomplete UTF-8 as file path"), + error.OutOfMemory => |e| return e, }; } else { // posix paths always start with '/' // -> file:///path/to/file.js // windows drive letter paths have the extra slash added // -> file:///C:/path/to/file.js - bun.strings.percentEncodeWrite(path, &source_map_strings) catch |err| { - switch (err) { - error.IncompleteUTF8 => @panic("Unexpected: source file with incomplete UTF-8 as file path"), - error.OutOfMemory => |e| return e, - } + bun.strings.percentEncodeWrite(path, &source_map_strings) catch |err| switch (err) { + error.IncompleteUTF8 => @panic("Unexpected: asset with incomplete UTF-8 as file path"), + error.OutOfMemory => |e| return e, }; } try source_map_strings.appendSlice("\""); } else { try source_map_strings.appendSlice("\"bun://"); - bun.strings.percentEncodeWrite(path, &source_map_strings) catch |err| { - switch (err) { - error.IncompleteUTF8 => @panic("Unexpected: source file with incomplete UTF-8 as file path"), - error.OutOfMemory => |e| return e, - } + bun.strings.percentEncodeWrite(path, &source_map_strings) catch |err| switch (err) { + error.IncompleteUTF8 => @panic("Unexpected: asset with incomplete UTF-8 as file path"), + error.OutOfMemory => |e| return e, }; try source_map_strings.appendSlice("\""); } } + var file_paths = try ArrayListUnmanaged([]const u8).initCapacity(gpa, path_count); + errdefer file_paths.deinit(gpa); + var source_contents = try ArrayListUnmanaged(bun.StringPointer).initCapacity(gpa, path_count); + errdefer source_contents.deinit(gpa); + try source_contents.ensureTotalCapacity(gpa, path_count); j.pushStatic(source_map_strings.items); j.pushStatic( - \\],"names":[],"mappings":" + \\],"sourcesContent":["// (Bun's internal HMR runtime is minified)" + ); + for (g.current_chunk_parts.items) |file_index| { + const map = &source_maps[file_index.get()]; + // For empty chunks, put a blank entry. This allows HTML + // files to get their stack remapped, despite having no + // actual mappings. + if (map.vlq_len == 0) { + const ptr: bun.StringPointer = .{ + .offset = @intCast(j.len + ",\"".len), + .length = 0, + }; + j.pushStatic(",\"\""); + file_paths.appendAssumeCapacity(paths[file_index.get()]); + source_contents.appendAssumeCapacity(ptr); + continue; + } + j.pushStatic(","); + const quoted_slice = map.quotedContents(); + // Store the location of the source file. Since it is going + // to be stored regardless for use by the served source map. + // These 8 bytes per file allow remapping sources without + // reading from disk, as well as ensuring that remaps to + // this exact sourcemap can print the previous state of + // the code when it was modified. + bun.assert(quoted_slice[0] == '"'); + bun.assert(quoted_slice[quoted_slice.len - 1] == '"'); + const ptr: bun.StringPointer = .{ + .offset = @intCast(j.len + "\"".len), + .length = @intCast(quoted_slice.len - "\"\"".len), + }; + j.pushStatic(quoted_slice); + file_paths.appendAssumeCapacity(paths[file_index.get()]); + source_contents.appendAssumeCapacity(ptr); + } + // This first mapping makes the bytes from line 0 column 0 to the next mapping + j.pushStatic( + \\],"names":[],"mappings":"AAAA ); + const mappings_start = j.len - 4; var prev_end_state: SourceMap.SourceMapState = .{ .generated_line = 0, .generated_column = 0, @@ -4525,10 +4604,10 @@ pub fn IncrementalGraph(side: bake.Side) type { // +2 because the magic fairy in my dreams said it would align the source maps. var lines_between: u32 = runtime.line_count + 2; - var non_empty_source_index: i32 = 0; - for (g.current_chunk_parts.items) |entry| { + // Join all of the mappings together. + for (g.current_chunk_parts.items, 1..) |entry, source_index| { const source_map = &source_maps[entry.get()]; - if (source_map.vlq_chunk.len == 0) { + if (source_map.vlq_len == 0) { if (source_map.extra.empty.line_count.unwrap()) |line_count| { lines_between += line_count.get(); } else { @@ -4536,11 +4615,59 @@ pub fn IncrementalGraph(side: bake.Side) type { source_map.extra.empty.line_count = .init(count); lines_between += count; } + // TODO: consider reviving this code. it generates a valid + // map according to Source Map Visualization, but it does + // not map correctly in Chrome or Bun. This code tries to + // add a mapping for each source line of unmapped code, so + // at the very minimum the file name can be retrieved. + // In practice, Bun may not need this as the only chunks + // with this behavior are HTML and empty JS files. + + // if (lines_between > 0) { + // j.pushStatic(try bun.strings.repeatingAlloc(arena, lines_between, ';')); + // lines_between = 0; + // } + // if (source_map.extra.empty.line_count.unwrap()) |lc| { + // lines_between = lc.get(); + // } else { + // const count: u32 = @intCast(bun.strings.countChar(files[entry.get()].jsCode(), '\n')); + // source_map.extra.empty.line_count = .init(count); + // lines_between = count; + // } + // // For empty chunks, put a blank entry for HTML. This will + // // consist of a single mapping per line to cover the file + // assert(lines_between > 1); // there is one line to open the function, and one line `},` to close it + // const repeating_mapping = ";AAAA"; // VLQ [0, 0, 0, 0] to repeat a column zero same source index mapping + // const original_line = SourceMap.encodeVLQWithLookupTable(-prev_end_state.original_line); + // const original_column = SourceMap.encodeVLQWithLookupTable(-prev_end_state.original_column); + // var list = std.ArrayList(u8).init(arena); + // const first = "AC"; // VLQ [0, 1], generated column and generated source index + // try list.ensureTotalCapacityPrecise( + // @as(usize, (lines_between + 1) * repeating_mapping.len) + + // first.len + + // @as(usize, original_line.len) + + // @as(usize, original_column.len), + // ); + // list.appendSliceAssumeCapacity(first); + // list.appendSliceAssumeCapacity(original_line.slice()); + // list.appendSliceAssumeCapacity(original_column.slice()); + // for (0..lines_between + 1) |_| { + // list.appendSliceAssumeCapacity(repeating_mapping); + // } + // j.pushStatic(list.items); + // prev_end_state = .{ + // .source_index = @intCast(source_index), + // .generated_line = 0, + // .generated_column = 0, + // .original_line = 0, + // .original_column = 0, + // }; + // lines_between = 1; continue; } const start_state: SourceMap.SourceMapState = .{ - .source_index = non_empty_source_index, + .source_index = @intCast(source_index), .generated_line = @intCast(lines_between), .generated_column = 0, .original_line = 0, @@ -4553,34 +4680,46 @@ pub fn IncrementalGraph(side: bake.Side) type { arena, prev_end_state, start_state, - source_map.vlq_chunk.slice(), + source_map.vlq(), ); prev_end_state = .{ - .source_index = non_empty_source_index, + .source_index = @intCast(source_index), .generated_line = 0, .generated_column = 0, .original_line = source_map.extra.end_state.original_line, .original_column = source_map.extra.end_state.original_column, }; - - non_empty_source_index += 1; } + const mappings_len = j.len - mappings_start; - const slice = try j.doneWithEnd(gpa, "\"}"); + const json_bytes = try j.doneWithEnd(gpa, "\"}"); + errdefer @compileError("last try should be the final alloc"); if (bun.FeatureFlags.bake_debugging_features) if (g.owner().dump_dir) |dump_dir| { const rel_path_escaped = "latest_chunk.js.map"; dumpBundle(dump_dir, switch (side) { .client => .client, .server => .server, - }, rel_path_escaped, slice, false) catch |err| { + }, rel_path_escaped, json_bytes, false) catch |err| { bun.handleErrorReturnTrace(err, @errorReturnTrace()); Output.warn("Could not dump bundle: {}", .{err}); }; }; - return slice; + out.* = .{ + .ref_count = out.ref_count, + .response = .initFromAnyBlob(&.fromOwnedSlice(gpa, json_bytes), .{ + .server = g.owner().server, + .mime_type = &.json, + }), + .mappings_data = .{ + .offset = @intCast(mappings_start), + .length = @intCast(mappings_len), + }, + .file_paths = file_paths.items, + .source_contents = source_contents.items.ptr, + }; } fn disconnectAndDeleteFile(g: *@This(), file_index: FileIndex) void { @@ -4775,14 +4914,11 @@ fn initGraphTraceState(dev: *const DevServer, sfa: Allocator, extra_client_bits: /// When a file fails to import a relative path, directory watchers are added so /// that when a matching file is created, the dependencies can be rebuilt. This /// handles HMR cases where a user writes an import before creating the file, -/// or moves files around. +/// or moves files around. This structure is not thread-safe. /// /// This structure manages those watchers, including releasing them once /// import resolution failures are solved. const DirectoryWatchStore = struct { - /// This guards all store state - lock: Mutex, - /// List of active watchers. Can be re-ordered on removal watches: bun.StringArrayHashMapUnmanaged(Entry), dependencies: ArrayListUnmanaged(Dep), @@ -4790,7 +4926,6 @@ const DirectoryWatchStore = struct { dependencies_free_list: ArrayListUnmanaged(Dep.Index), const empty: DirectoryWatchStore = .{ - .lock = .{}, .watches = .{}, .dependencies = .{}, .dependencies_free_list = .{}, @@ -4806,9 +4941,6 @@ const DirectoryWatchStore = struct { specifier: []const u8, renderer: bake.Graph, ) bun.OOM!void { - store.lock.lock(); - defer store.lock.unlock(); - // When it does not resolve to a file path, there is // nothing to track. Bake does not watch node_modules. if (!(bun.strings.startsWith(specifier, "./") or @@ -5060,8 +5192,8 @@ pub const SerializedFailure = struct { } pub const Packed = packed struct(u32) { - kind: enum(u2) { none, route, client, server }, data: u30, + kind: enum(u2) { none, route, client, server }, pub fn decode(owner: Packed) Owner { return switch (owner.kind) { @@ -5071,6 +5203,10 @@ pub const SerializedFailure = struct { .route => .{ .route = RouteBundle.Index.init(owner.data) }, }; } + + comptime { + assert(@as(u32, @bitCast(Packed{ .kind = .none, .data = 1 })) == 1); + } }; }; @@ -5495,19 +5631,9 @@ pub const IncomingMessageId = enum(u8) { /// Subscribe to an event channel. Payload is a sequence of chars available /// in HmrTopic. subscribe = 's', - // /// Subscribe to `.route_manifest` events. No payload. - // subscribe_route_manifest = 'r', - // /// Emit a hot update for a file without actually changing its on-disk - // /// content. This can be used by an editor extension to stream contents in - // /// IDE to reflect in the browser. This is gated to only work on localhost - // /// socket connections. - // virtual_file_change = 'w', /// Emitted on client-side navigations. /// Rest of payload is a UTF-8 string. set_url = 'n', - /// Emit a message from the browser. Payload is opaque bytes that DevServer - /// does not care about. In practice, the payload is a JSON object. - browser_message = 'm', /// Invalid data _, @@ -5518,7 +5644,6 @@ const HmrTopic = enum(u8) { errors = 'e', browser_error = 'E', visualizer = 'v', - // route_manifest = 'r', /// Invalid data _, @@ -5621,7 +5746,7 @@ const HmrSocket = struct { var response: [5]u8 = .{MessageId.set_url_response.char()} ++ std.mem.toBytes(rbi.get()); _ = ws.send(&response, .binary, false, true); }, - else => ws.close(), + _ => ws.close(), } } @@ -5736,10 +5861,11 @@ pub const HotReloadEvent = struct { owner: *DevServer, /// Initialized in WatcherAtomics.watcherReleaseAndSubmitEvent concurrent_task: JSC.ConcurrentTask, - /// The watcher is not able to peek into the incremental graph to know what - /// files to invalidate, so the watch events are de-duplicated and passed - /// along. - files: bun.StringArrayHashMapUnmanaged(Watcher.Event.Op), + /// The watcher is not able to peek into IncrementalGraph to know what files + /// to invalidate, so the watch events are de-duplicated and passed along. + files: bun.StringArrayHashMapUnmanaged(void), + /// Directories are watched so that resolution failures can be solved. + dirs: bun.StringArrayHashMapUnmanaged(void), /// Initialized by the WatcherAtomics.watcherAcquireEvent timer: std.time.Timer, /// This event may be referenced by either DevServer or Watcher thread. @@ -5751,23 +5877,22 @@ pub const HotReloadEvent = struct { .owner = owner, .concurrent_task = undefined, .files = .empty, + .dirs = .empty, .timer = undefined, .contention_indicator = .init(0), }; } - pub fn append( - event: *HotReloadEvent, - allocator: Allocator, - file_path: []const u8, - op: Watcher.Event.Op, - ) void { - const gop = event.files.getOrPut(allocator, file_path) catch bun.outOfMemory(); - if (gop.found_existing) { - gop.value_ptr.* = gop.value_ptr.merge(op); - } else { - gop.value_ptr.* = op; - } + pub fn isEmpty(ev: *const HotReloadEvent) bool { + return (ev.files.count() + ev.dirs.count()) == 0; + } + + pub fn appendFile(event: *HotReloadEvent, allocator: Allocator, file_path: []const u8) void { + _ = event.files.getOrPut(allocator, file_path) catch bun.outOfMemory(); + } + + pub fn appendDir(event: *HotReloadEvent, allocator: Allocator, file_path: []const u8) void { + _ = event.dirs.getOrPut(allocator, file_path) catch bun.outOfMemory(); } /// Invalidates items in IncrementalGraph, appending all new items to `entry_points` @@ -5775,15 +5900,59 @@ pub const HotReloadEvent = struct { event: *HotReloadEvent, dev: *DevServer, entry_points: *EntryPointList, - alloc: Allocator, + temp_alloc: Allocator, ) void { - const changed_file_paths = event.files.keys(); - dev.graph_safety_lock.lock(); defer dev.graph_safety_lock.unlock(); + // First handle directories, because this may mutate `event.files` + for (event.dirs.keys()) |changed_dir_with_slash| { + const changed_dir = bun.strings.withoutTrailingSlashWindowsPath(changed_dir_with_slash); + + // Bust resolution cache, but since Bun does not watch all + // directories in a codebase, this only targets the following resolutions + _ = dev.server_transpiler.resolver.bustDirCache(changed_dir); + + // if a directory watch exists for resolution + // failures, check those now. + if (dev.directory_watchers.watches.getIndex(bun.strings.withoutTrailingSlashWindowsPath(changed_dir))) |watcher_index| { + const entry = &dev.directory_watchers.watches.values()[watcher_index]; + var new_chain: DirectoryWatchStore.Dep.Index.Optional = .none; + var it: ?DirectoryWatchStore.Dep.Index = entry.first_dep; + + while (it) |index| { + const dep = &dev.directory_watchers.dependencies.items[index.get()]; + it = dep.next.unwrap(); + + if ((dev.server_transpiler.resolver.resolve( + bun.path.dirname(dep.source_file_path, .auto), + dep.specifier, + .stmt, + ) catch null) != null) { + // the resolution result is not preserved as safely + // transferring it into BundleV2 is too complicated. the + // resolution is cached, anyways. + event.appendFile(dev.allocator, dep.source_file_path); + dev.directory_watchers.freeDependencyIndex(dev.allocator, index) catch bun.outOfMemory(); + } else { + // rebuild a new linked list for unaffected files + dep.next = new_chain; + new_chain = index.toOptional(); + } + } + + if (new_chain.unwrap()) |new_first_dep| { + entry.first_dep = new_first_dep; + } else { + // without any files to depend on this watcher is freed + dev.directory_watchers.freeEntry(watcher_index); + } + } + } + + const changed_file_paths = event.files.keys(); inline for (.{ &dev.server_graph, &dev.client_graph }) |g| { - g.invalidate(changed_file_paths, entry_points, alloc) catch bun.outOfMemory(); + g.invalidate(changed_file_paths, entry_points, temp_alloc) catch bun.outOfMemory(); } if (dev.has_tailwind_plugin_hack) |*map| { @@ -5791,7 +5960,7 @@ pub const HotReloadEvent = struct { const file = dev.client_graph.bundled_files.get(abs_path) orelse continue; if (file.flags.kind == .css) - entry_points.appendCss(alloc, abs_path) catch bun.outOfMemory(); + entry_points.appendCss(temp_alloc, abs_path) catch bun.outOfMemory(); } } @@ -5891,7 +6060,7 @@ const WatcherAtomics = struct { switch (ev.contention_indicator.swap(1, .seq_cst)) { 0 => { // New event, initialize the timer if it is empty. - if (ev.files.count() == 0) + if (ev.isEmpty()) ev.timer = std.time.Timer.start() catch unreachable; }, 1 => { @@ -5918,7 +6087,7 @@ const WatcherAtomics = struct { state.watcher_has_event.unlock(); ev.owner.bun_watcher.thread_lock.assertLocked(); - if (ev.files.count() > 0) { + if (!ev.isEmpty()) { @branchHint(.likely); // There are files to be processed, increment this count first. const prev_count = state.watcher_events_emitted.fetchAdd(1, .seq_cst); @@ -6023,8 +6192,6 @@ pub fn onFileUpdate(dev: *DevServer, events: []Watcher.Event, changed_files: []? defer dev.bun_watcher.flushEvictions(); - // TODO: alot of code is missing - // TODO: story for busting resolution cache smartly? for (events) |event| { // TODO: why does this out of bounds when you delete every file in the directory? if (event.index >= file_paths.len) continue; @@ -6039,52 +6206,14 @@ pub fn onFileUpdate(dev: *DevServer, events: []Watcher.Event, changed_files: []? switch (kind) { .file => { if (event.op.delete or event.op.rename) { + // TODO: audit this line heavily dev.bun_watcher.removeAtIndex(event.index, 0, &.{}, .file); } - ev.append(dev.allocator, file_path, event.op); + ev.appendFile(dev.allocator, file_path); }, .directory => { - // bust the directory cache since this directory has changed - _ = dev.server_transpiler.resolver.bustDirCache(bun.strings.withoutTrailingSlashWindowsPath(file_path)); - - // if a directory watch exists for resolution - // failures, check those now. - dev.directory_watchers.lock.lock(); - defer dev.directory_watchers.lock.unlock(); - if (dev.directory_watchers.watches.getIndex(bun.strings.withoutTrailingSlashWindowsPath(file_path))) |watcher_index| { - const entry = &dev.directory_watchers.watches.values()[watcher_index]; - var new_chain: DirectoryWatchStore.Dep.Index.Optional = .none; - var it: ?DirectoryWatchStore.Dep.Index = entry.first_dep; - - while (it) |index| { - const dep = &dev.directory_watchers.dependencies.items[index.get()]; - it = dep.next.unwrap(); - - if ((dev.watcher_thread_resolver.resolve( - bun.path.dirname(dep.source_file_path, .auto), - dep.specifier, - .stmt, - ) catch null) != null) { - // the resolution result is not preserved as safely - // transferring it into BundleV2 is too complicated. the - // resolution is cached, anyways. - ev.append(dev.allocator, dep.source_file_path, .{ .write = true }); - dev.directory_watchers.freeDependencyIndex(dev.allocator, index) catch bun.outOfMemory(); - } else { - // rebuild a new linked list for unaffected files - dep.next = new_chain; - new_chain = index.toOptional(); - } - } - - if (new_chain.unwrap()) |new_first_dep| { - entry.first_dep = new_first_dep; - } else { - // without any files to depend on this watcher is freed - dev.directory_watchers.freeEntry(watcher_index); - } - } + ev.appendDir(dev.allocator, file_path); }, } } @@ -6247,8 +6376,6 @@ pub const EntryPointList = struct { ssr: bool = false, /// When this is set, also set .client = true css: bool = false, - // /// Indicates the file might have been deleted. - // potentially_deleted: bool = false, unused: enum(u4) { unused = 0 } = .unused, }; @@ -6346,7 +6473,7 @@ pub const Assets = struct { files: AutoArrayHashMapUnmanaged(u64, *StaticRoute), /// Indexed by the same index of `files`. The value is never `0`. refs: ArrayListUnmanaged(u32), - + /// When mutating `files`'s keys, the map must be reindexed to function. needs_reindex: bool = false, pub const EntryIndex = bun.GenericIndex(u30, Assets); @@ -6362,8 +6489,8 @@ pub const Assets = struct { null; } - // / When an asset is overwritten, it receives a new URL to get around browser auto-caching. - /// The old URL is immediately invalidated. + /// When an asset is overwritten, it receives a new URL to get around browser caching. + /// The old URL is immediately revoked. pub fn replacePath( assets: *Assets, /// not allocated @@ -6490,6 +6617,172 @@ pub const Assets = struct { } }; +/// Storage for source maps on `/_bun/client/{id}.js.map` +/// +/// All source maps are referenced counted, so that when a bundle is unloaded, +/// the unreachable source map URLs are revoked, and their memory freed. +pub const SourceMapStore = struct { + entries: AutoArrayHashMapUnmanaged(u64, Entry), + + pub const empty: SourceMapStore = .{ .entries = .empty }; + + pub const Entry = struct { + /// Sum of: + /// - How many active pages could try to request this source map? + /// - For route bundle client scripts, 1 until invalidation. + ref_count: u32, + /// Notes: + /// - Bundler produces VLQ mappings, and there is not a well written + /// implementation to go from Mapping.List to VLQ. + /// - All mapping info needs to be somewhere, since watch updates would + /// taint IncrementalGraph state, leading to broken old sourcemaps. + /// - The "mappings" field, being VLQ, cannot contain escape sequences, + /// therefore it can still be extracted without a JSON parser. + /// - VLQ encoding is smaller than Mapping.List + /// + /// Since server-side error remapping is semi-rare, the parsed source + /// maps are released as soon as the error is finished printing, with + /// currently no re-use. + response: *StaticRoute, + /// Pointer into `response.blob` to where `mappings` is stored. + mappings_data: bun.StringPointer, + /// Outer slice is allocated. Inner slices are shared keys in + /// IncrementalGraph. Indexes are off by one because of the HMR + /// Runtime, which is not present in this list. + file_paths: [][]const u8, + /// Only entries from plugins are persisted after a bundle. Like + /// `file_paths`, indexes are off by one. A pointer into `response.blob` + /// to JSON encoded source contents, not including the quotes. + // TODO: reduce the size of this slice + source_contents: [*]bun.StringPointer, + + pub fn sourceContents(self: @This()) []const bun.StringPointer { + return self.source_contents[0..self.file_paths.len]; + } + + fn deinit(entry: *Entry, dev: *DevServer) void { + dev.allocator.free(entry.file_paths); + dev.allocator.free(entry.source_contents); + + // should not have been removed or unreferenced. + assert(entry.ref_count > 0); + assert(entry.file_paths.ptr == entry.file_paths.ptr); + } + }; + + /// If an *Entry is returned, caller must initialize it with the source map. + pub fn putOrIncrementRefCount(store: *SourceMapStore, source_map_id: u64, ref_count: u32) !?*Entry { + const gop = try store.entries.getOrPut(bun.default_allocator, source_map_id); + if (!gop.found_existing) { + gop.value_ptr.* = .{ + .ref_count = ref_count, + .source_contents = undefined, + .response = undefined, + .mappings_data = undefined, + .file_paths = undefined, + }; + return gop.value_ptr; + } else { + gop.value_ptr.*.ref_count += ref_count; + return null; + } + } + + const GetResult = struct { + index: bun.GenericIndex(u32, Entry), + mappings: SourceMap.Mapping.List, + file_paths: []const []const u8, + source_contents: []const bun.StringPointer, + bytes: []const u8, + + pub fn deinit(self: *@This()) void { + self.mappings.deinit(bun.default_allocator); + // file paths and source contents are borrowed + } + }; + + pub fn getParsedSourceMap(store: *SourceMapStore, source_map_id: u64) ?GetResult { + const index = store.entries.getIndex(source_map_id) orelse + return null; // source map was collected. + const entry = &store.entries.values()[index]; + + switch (SourceMap.Mapping.parse( + bun.default_allocator, + entry.mappings_data.slice(entry.response.blob.slice()), + null, + @intCast(entry.file_paths.len), + 0, // unused + )) { + .fail => |fail| { + Output.debugWarn("Failed to re-parse source map: {s}", .{fail.msg}); + return null; + }, + .success => |psm| { + // TODO: consider storing mappings for, say, a minute. this + // means having a loop in client code would re-use the + // allocation. if this is deemed a bad idea, replace this + // comment with an explanation why. + // entry.parsed_mappings = psm.mappings; + return .{ + .index = .init(@intCast(index)), + .mappings = psm.mappings, + .file_paths = entry.file_paths, + .source_contents = entry.source_contents[0..entry.file_paths.len], + .bytes = entry.response.blob.slice(), + }; + }, + } + } +}; + +// NOTE: not used but keeping around in case a use case is found soon +// /// Instead of a pointer to `struct { data: []const u8, ref_count: u32 }`, all +// /// reference counts are in a shared map. This is used by strings shared between +// /// source maps and the incremental graph. Not thread-safe. +// /// +// /// Transfer from default allocator to RefString with `dev.ref_strings.register(slice)` +// /// +// /// Prefer `CowString` (maybe allocated or borrowed) or `[]const u8` (known lifetime) over this structure. +// const RefString = struct { +// /// Allocated by `bun.default_allocator`, free with `.unref()` +// data: []const u8, + +// pub fn deref(str: RefString, store: *Store) void { +// const index = store.strings.getIndex(str.ptr) orelse unreachable; +// const slice = store.strings.entries.slice(); +// const ref_count = &slice.items(.value)[index]; +// if (ref_count.* == 1) { +// store.strings.swapRemoveAt(index); +// bun.default_allocator.free(str.data); +// } else { +// ref_count.* -= 1; +// } +// } + +// pub fn dupeRef(str: RefString, store: *Store) RefString { +// const ref_count = store.strings.getPtr(str.ptr) orelse unreachable; +// ref_count.* += 1; +// return str; +// } + +// pub const Store = struct { +// /// Key -> Data. Value -> Reference count +// strings: AutoArrayHashMapUnmanaged([*]u8, u32), + +// pub const empty: Store = .{ .strings = .empty }; + +// pub fn register(store: *Store, data: []u8) !RefString { +// const gop = try store.strings.getOrPut(bun.default_allocator, data.ptr); +// if (gop.found_existing) { +// gop.value_ptr.* += 1; +// } else { +// gop.value_ptr = 1; +// } +// return .{ .data = data }; +// } +// }; +// }; + pub fn onPluginsResolved(dev: *DevServer, plugins: ?*Plugin) !void { dev.bundler_options.plugin = plugins; dev.plugin_state = .loaded; @@ -6506,6 +6799,363 @@ pub fn onPluginsRejected(dev: *DevServer) !void { // TODO: allow recovery from this state } +/// Fetched when a client-side error happens. This performs two actions +/// - Logs the remapped stack trace to the console. +/// - Replies with the remapped stack trace. +/// Payload: +/// - `u32`: Responding message ID (echoed back) +/// - `u32`: Length of message +/// - `[n]u8`: Message +/// - `u32`: Length of error name +/// - `[n]u8`: Error name +/// - `u32`: Number of stack frames. For each +/// - `u32`: Line number (0 for unavailable) +/// - `u32`: Column number (0 for unavailable) +/// - `u32`: Length of file name (0 for unavailable) +/// - `[n]u8`: File name +/// - `u32`: Length of function name (0 for unavailable) +/// - `[n]u8`: Function name +const ErrorReportRequest = struct { + dev: *DevServer, + body: uws.BodyReaderMixin(@This(), "body", runWithBody, finalize), + + fn run(dev: *DevServer, _: *Request, resp: anytype) void { + const ctx = bun.new(ErrorReportRequest, .{ + .dev = dev, + .body = .init(dev.allocator), + }); + ctx.dev.server.?.onPendingRequest(); + ctx.body.readBody(resp); + } + + fn finalize(ctx: *ErrorReportRequest) void { + ctx.dev.server.?.onStaticRequestComplete(); + bun.destroy(ctx); + } + + fn runWithBody(ctx: *ErrorReportRequest, body: []const u8, r: AnyResponse) !void { + var s = std.io.fixedBufferStream(body); + const reader = s.reader(); + + var sfa = std.heap.stackFallback(131072, ctx.dev.allocator); + const temp_alloc = sfa.get(); + var arena = std.heap.ArenaAllocator.init(temp_alloc); + defer arena.deinit(); + + // Read payload, assemble ZigException + const name = try readString32(reader, temp_alloc); + defer temp_alloc.free(name); + const message = try readString32(reader, temp_alloc); + defer temp_alloc.free(message); + const browser_url = try readString32(reader, temp_alloc); + defer temp_alloc.free(browser_url); + var frames: ArrayListUnmanaged(JSC.ZigStackFrame) = .empty; + defer frames.deinit(temp_alloc); + const stack_count = @min(try reader.readInt(u32, .little), 255); // does not support more than 255 + try frames.ensureTotalCapacity(temp_alloc, stack_count); + for (0..stack_count) |_| { + const line = try reader.readInt(i32, .little); + const column = try reader.readInt(i32, .little); + const function_name = try readString32(reader, temp_alloc); + const file_name = try readString32(reader, temp_alloc); + frames.appendAssumeCapacity(.{ + .function_name = .init(function_name), + .source_url = .init(file_name), + .position = if (line > 0) .{ + .line = .fromOneBased(line + 1), + .column = .fromOneBased(@max(1, column)), + .line_start_byte = 0, + } else .{ + .line = .invalid, + .column = .invalid, + .line_start_byte = 0, + }, + .code_type = .None, + .remapped = false, + }); + } + + const runtime_name = "Bun HMR Runtime"; + + // All files that DevServer could provide a source map fit the pattern: + // `/_bun/client/