diff --git a/src/NullableAllocator.zig b/src/NullableAllocator.zig new file mode 100644 index 0000000000..3c0fb5f366 --- /dev/null +++ b/src/NullableAllocator.zig @@ -0,0 +1,30 @@ +//! A nullable allocator the same size as `std.mem.Allocator`. +const std = @import("std"); +const NullableAllocator = @This(); + +ptr: *anyopaque = undefined, +// Utilize the null pointer optimization on the vtable instead of +// the regular ptr because some allocator implementations might tag their +// `ptr` property. +vtable: ?*const std.mem.Allocator.VTable = null, + +pub inline fn init(allocator: ?std.mem.Allocator) NullableAllocator { + return if (allocator) |a| .{ + .ptr = a.ptr, + .vtable = a.vtable, + } else .{}; +} + +pub inline fn isNull(this: NullableAllocator) bool { + return this.vtable == null; +} + +pub inline fn get(this: NullableAllocator) ?std.mem.Allocator { + return if (this.vtable) |vt| std.mem.Allocator{ .ptr = this.ptr, .vtable = vt } else null; +} + +comptime { + if (@sizeOf(NullableAllocator) != @sizeOf(std.mem.Allocator)) { + @compileError("Expected the sizes to be the same."); + } +} diff --git a/src/StringJoiner.zig b/src/StringJoiner.zig new file mode 100644 index 0000000000..a1385b998f --- /dev/null +++ b/src/StringJoiner.zig @@ -0,0 +1,163 @@ +//! Rope-like data structure for joining many small strings into one big string. +//! Implemented as a linked list of potentially-owned slices and a length. +const std = @import("std"); +const default_allocator = bun.default_allocator; +const bun = @import("root").bun; +const string = bun.string; +const Allocator = std.mem.Allocator; +const NullableAllocator = bun.NullableAllocator; +const StringJoiner = @This(); +const assert = bun.assert; + +/// Temporary allocator used for nodes and duplicated strings. +/// It is recommended to use a stack-fallback allocator for this. +allocator: Allocator, + +/// Total length of all nodes +len: usize = 0, + +head: ?*Node = null, +tail: ?*Node = null, + +/// Avoid an extra pass over the list when joining +watcher: Watcher = .{}, + +const Node = struct { + allocator: NullableAllocator = .{}, + slice: []const u8 = "", + next: ?*Node = null, + + pub fn init(joiner_alloc: Allocator, slice: []const u8, slice_alloc: ?Allocator) *Node { + const node = joiner_alloc.create(Node) catch bun.outOfMemory(); + node.* = .{ + .slice = slice, + .allocator = NullableAllocator.init(slice_alloc), + }; + return node; + } + + pub fn deinit(node: *Node, joiner_alloc: Allocator) void { + if (node.allocator.get()) |alloc| + alloc.free(node.slice); + joiner_alloc.destroy(node); + } +}; + +pub const Watcher = struct { + input: []const u8 = "", + estimated_count: u32 = 0, + needs_newline: bool = false, +}; + +/// `data` is expected to live until `.done` is called +pub fn pushStatic(this: *StringJoiner, data: []const u8) void { + this.push(data, null); +} + +/// `data` is cloned +pub fn pushCloned(this: *StringJoiner, data: []const u8) void { + this.push( + this.allocator.dupe(u8, data) catch bun.outOfMemory(), + this.allocator, + ); +} + +pub fn push(this: *StringJoiner, data: []const u8, allocator: ?Allocator) void { + this.len += data.len; + + const new_tail = Node.init(this.allocator, data, allocator); + + if (data.len > 0) { + this.watcher.estimated_count += @intFromBool( + this.watcher.input.len > 0 and + bun.strings.contains(data, this.watcher.input), + ); + this.watcher.needs_newline = data[data.len - 1] != '\n'; + } + + if (this.tail) |current_tail| { + current_tail.next = new_tail; + } else { + assert(this.head == null); + this.head = new_tail; + } + this.tail = new_tail; +} + +/// This deinits the string joiner on success, the new string is owned by `allocator` +pub fn done(this: *StringJoiner, allocator: Allocator) ![]u8 { + var current: ?*Node = this.head orelse { + assert(this.tail == null); + assert(this.len == 0); + return &.{}; + }; + + const slice = try allocator.alloc(u8, this.len); + + var remaining = slice; + while (current) |node| { + @memcpy(remaining[0..node.slice.len], node.slice); + remaining = remaining[node.slice.len..]; + + const prev = node; + current = node.next; + prev.deinit(this.allocator); + } + + bun.assert(remaining.len == 0); + + return slice; +} + +/// Same as `.done`, but appends extra slice `end` +pub fn doneWithEnd(this: *StringJoiner, allocator: Allocator, end: []const u8) ![]u8 { + var current: ?*Node = this.head orelse { + assert(this.tail == null); + assert(this.len == 0); + + if (end.len > 0) { + return allocator.dupe(u8, end); + } + + return &.{}; + }; + + const slice = try allocator.alloc(u8, this.len + end.len); + + var remaining = slice; + while (current) |node| { + @memcpy(remaining[0..node.slice.len], node.slice); + remaining = remaining[node.slice.len..]; + + const prev = node; + current = node.next; + prev.deinit(this.allocator); + } + + bun.assert(remaining.len == end.len); + @memcpy(remaining, end); + + return slice; +} + +pub fn lastByte(this: *const StringJoiner) u8 { + const slice = (this.tail orelse return 0).slice; + return if (slice.len > 0) slice[slice.len - 1] else 0; +} + +pub fn ensureNewlineAtEnd(this: *StringJoiner) void { + if (this.watcher.needs_newline) { + this.watcher.needs_newline = false; + this.pushStatic("\n"); + } +} + +pub fn contains(this: *const StringJoiner, slice: string) bool { + var el = this.head; + while (el) |node| { + el = node.next; + if (bun.strings.contains(node.slice, slice)) return true; + } + + return false; +} diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index 3e0808b58a..cc6e60d8f7 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -16,7 +16,7 @@ const JSC = bun.JSC; const Shimmer = JSC.Shimmer; const ConsoleObject = JSC.ConsoleObject; const FFI = @import("./FFI.zig"); -const NullableAllocator = @import("../../nullable_allocator.zig").NullableAllocator; +const NullableAllocator = bun.NullableAllocator; const MutableString = bun.MutableString; const JestPrettyFormat = @import("../test/pretty_format.zig").JestPrettyFormat; const String = bun.String; diff --git a/src/bun.js/webcore/blob.zig b/src/bun.js/webcore/blob.zig index 059f24f8cd..e948620548 100644 --- a/src/bun.js/webcore/blob.zig +++ b/src/bun.js/webcore/blob.zig @@ -29,13 +29,13 @@ const JSPromise = JSC.JSPromise; const JSValue = JSC.JSValue; const JSError = JSC.JSError; const JSGlobalObject = JSC.JSGlobalObject; -const NullableAllocator = @import("../../nullable_allocator.zig").NullableAllocator; +const NullableAllocator = bun.NullableAllocator; const VirtualMachine = JSC.VirtualMachine; const Task = JSC.Task; const JSPrinter = bun.js_printer; const picohttp = bun.picohttp; -const StringJoiner = @import("../../string_joiner.zig"); +const StringJoiner = bun.StringJoiner; const uws = bun.uws; const invalid_fd = bun.invalid_fd; @@ -225,33 +225,31 @@ pub const Blob = struct { const joiner = &this.joiner; const boundary = this.boundary; - joiner.append("--", 0, null); - joiner.append(boundary, 0, null); - joiner.append("\r\n", 0, null); + joiner.pushStatic("--"); + joiner.pushStatic(boundary); // note: "static" here means "outlives the joiner" + joiner.pushStatic("\r\n"); - joiner.append("Content-Disposition: form-data; name=\"", 0, null); + joiner.pushStatic("Content-Disposition: form-data; name=\""); const name_slice = name.toSlice(allocator); - joiner.append(name_slice.slice(), 0, name_slice.allocator.get()); - name_slice.deinit(); + joiner.push(name_slice.slice(), name_slice.allocator.get()); switch (entry) { .string => |value| { - joiner.append("\"\r\n\r\n", 0, null); + joiner.pushStatic("\"\r\n\r\n"); const value_slice = value.toSlice(allocator); - joiner.append(value_slice.slice(), 0, value_slice.allocator.get()); + joiner.push(value_slice.slice(), value_slice.allocator.get()); }, .file => |value| { - joiner.append("\"; filename=\"", 0, null); + joiner.pushStatic("\"; filename=\""); const filename_slice = value.filename.toSlice(allocator); - joiner.append(filename_slice.slice(), 0, filename_slice.allocator.get()); - filename_slice.deinit(); - joiner.append("\"\r\n", 0, null); + joiner.push(filename_slice.slice(), filename_slice.allocator.get()); + joiner.pushStatic("\"\r\n"); const blob = value.blob; const content_type = if (blob.content_type.len > 0) blob.content_type else "application/octet-stream"; - joiner.append("Content-Type: ", 0, null); - joiner.append(content_type, 0, null); - joiner.append("\r\n\r\n", 0, null); + joiner.pushStatic("Content-Type: "); + joiner.pushStatic(content_type); + joiner.pushStatic("\r\n\r\n"); if (blob.store) |store| { blob.resolveSize(); @@ -277,19 +275,19 @@ pub const Blob = struct { this.failed = true; }, .result => |result| { - joiner.append(result.slice(), 0, result.buffer.allocator); + joiner.push(result.slice(), result.buffer.allocator); }, } }, .bytes => |_| { - joiner.append(blob.sharedView(), 0, null); + joiner.pushStatic(blob.sharedView()); }, } } }, } - joiner.append("\r\n", 0, null); + joiner.pushStatic("\r\n"); } }; @@ -532,7 +530,7 @@ pub const Blob = struct { var context = FormDataContext{ .allocator = allocator, - .joiner = StringJoiner{ .use_pool = false, .node_allocator = stack_mem_all }, + .joiner = .{ .allocator = stack_mem_all }, .boundary = boundary, .globalThis = globalThis, }; @@ -542,9 +540,9 @@ pub const Blob = struct { return Blob.initEmpty(globalThis); } - context.joiner.append("--", 0, null); - context.joiner.append(boundary, 0, null); - context.joiner.append("--\r\n", 0, null); + context.joiner.pushStatic("--"); + context.joiner.pushStatic(boundary); + context.joiner.pushStatic("--\r\n"); const store = Blob.Store.init(context.joiner.done(allocator) catch unreachable, allocator) catch unreachable; var blob = Blob.initWithStore(store, globalThis); @@ -4066,7 +4064,7 @@ pub const Blob = struct { var stack_allocator = std.heap.stackFallback(1024, bun.default_allocator); const stack_mem_all = stack_allocator.get(); var stack: std.ArrayList(JSValue) = std.ArrayList(JSValue).init(stack_mem_all); - var joiner = StringJoiner{ .use_pool = false, .node_allocator = stack_mem_all }; + var joiner = StringJoiner{ .allocator = stack_mem_all }; var could_have_non_ascii = false; defer if (stack_allocator.fixed_buffer_allocator.end_index >= 1024) stack.deinit(); @@ -4081,11 +4079,7 @@ pub const Blob = struct { var sliced = current.toSlice(global, bun.default_allocator); const allocator = sliced.allocator.get(); could_have_non_ascii = could_have_non_ascii or allocator != null; - joiner.append( - sliced.slice(), - 0, - allocator, - ); + joiner.push(sliced.slice(), allocator); }, .Array, .DerivedArray => { @@ -4111,11 +4105,7 @@ pub const Blob = struct { var sliced = item.toSlice(global, bun.default_allocator); const allocator = sliced.allocator.get(); could_have_non_ascii = could_have_non_ascii or allocator != null; - joiner.append( - sliced.slice(), - 0, - allocator, - ); + joiner.push(sliced.slice(), allocator); continue; }, JSC.JSValue.JSType.ArrayBuffer, @@ -4134,7 +4124,7 @@ pub const Blob = struct { => { could_have_non_ascii = true; var buf = item.asArrayBuffer(global).?; - joiner.append(buf.byteSlice(), 0, null); + joiner.pushStatic(buf.byteSlice()); continue; }, .Array, .DerivedArray => { @@ -4146,16 +4136,12 @@ pub const Blob = struct { .DOMWrapper => { if (item.as(Blob)) |blob| { could_have_non_ascii = could_have_non_ascii or !(blob.is_all_ascii orelse false); - joiner.append(blob.sharedView(), 0, null); + joiner.pushStatic(blob.sharedView()); continue; } else if (current.toSliceClone(global)) |sliced| { const allocator = sliced.allocator.get(); could_have_non_ascii = could_have_non_ascii or allocator != null; - joiner.append( - sliced.slice(), - 0, - allocator, - ); + joiner.push(sliced.slice(), allocator); } }, else => {}, @@ -4169,15 +4155,11 @@ pub const Blob = struct { .DOMWrapper => { if (current.as(Blob)) |blob| { could_have_non_ascii = could_have_non_ascii or !(blob.is_all_ascii orelse false); - joiner.append(blob.sharedView(), 0, null); + joiner.pushStatic(blob.sharedView()); } else if (current.toSliceClone(global)) |sliced| { const allocator = sliced.allocator.get(); could_have_non_ascii = could_have_non_ascii or allocator != null; - joiner.append( - sliced.slice(), - 0, - allocator, - ); + joiner.push(sliced.slice(), allocator); } }, @@ -4196,7 +4178,7 @@ pub const Blob = struct { JSC.JSValue.JSType.DataView, => { var buf = current.asArrayBuffer(global).?; - joiner.append(buf.slice(), 0, null); + joiner.pushStatic(buf.slice()); could_have_non_ascii = true; }, @@ -4204,11 +4186,7 @@ pub const Blob = struct { var sliced = current.toSlice(global, bun.default_allocator); const allocator = sliced.allocator.get(); could_have_non_ascii = could_have_non_ascii or allocator != null; - joiner.append( - sliced.slice(), - 0, - allocator, - ); + joiner.push(sliced.slice(), allocator); }, } current = stack.popOrNull() orelse break; diff --git a/src/bun.js/webcore/body.zig b/src/bun.js/webcore/body.zig index cebbcedc8c..27193bb750 100644 --- a/src/bun.js/webcore/body.zig +++ b/src/bun.js/webcore/body.zig @@ -31,13 +31,13 @@ const JSPromise = JSC.JSPromise; const JSValue = JSC.JSValue; const JSError = JSC.JSError; const JSGlobalObject = JSC.JSGlobalObject; -const NullableAllocator = @import("../../nullable_allocator.zig").NullableAllocator; +const NullableAllocator = bun.NullableAllocator; const VirtualMachine = JSC.VirtualMachine; const Task = JSC.Task; const JSPrinter = bun.js_printer; const picohttp = bun.picohttp; -const StringJoiner = @import("../../string_joiner.zig"); +const StringJoiner = bun.StringJoiner; const uws = bun.uws; const Blob = JSC.WebCore.Blob; diff --git a/src/bun.js/webcore/request.zig b/src/bun.js/webcore/request.zig index 117f0a6e97..2ea98d4f91 100644 --- a/src/bun.js/webcore/request.zig +++ b/src/bun.js/webcore/request.zig @@ -32,13 +32,13 @@ const JSPromise = JSC.JSPromise; const JSValue = JSC.JSValue; const JSError = JSC.JSError; const JSGlobalObject = JSC.JSGlobalObject; -const NullableAllocator = @import("../../nullable_allocator.zig").NullableAllocator; +const NullableAllocator = bun.NullableAllocator; const VirtualMachine = JSC.VirtualMachine; const Task = JSC.Task; const JSPrinter = bun.js_printer; const picohttp = bun.picohttp; -const StringJoiner = @import("../../string_joiner.zig"); +const StringJoiner = bun.StringJoiner; const uws = bun.uws; const InlineBlob = JSC.WebCore.InlineBlob; diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig index 9a3373982f..c252419889 100644 --- a/src/bun.js/webcore/response.zig +++ b/src/bun.js/webcore/response.zig @@ -32,14 +32,14 @@ const JSPromise = JSC.JSPromise; const JSValue = JSC.JSValue; const JSError = JSC.JSError; const JSGlobalObject = JSC.JSGlobalObject; -const NullableAllocator = @import("../../nullable_allocator.zig").NullableAllocator; +const NullableAllocator = bun.NullableAllocator; const DataURL = @import("../../resolver/data_url.zig").DataURL; const VirtualMachine = JSC.VirtualMachine; const Task = JSC.Task; const JSPrinter = bun.js_printer; const picohttp = bun.picohttp; -const StringJoiner = @import("../../string_joiner.zig"); +const StringJoiner = bun.StringJoiner; const uws = bun.uws; const Mutex = @import("../../lock.zig").Lock; diff --git a/src/bun.js/webcore/streams.zig b/src/bun.js/webcore/streams.zig index 4d0d7911cb..e79c6a07be 100644 --- a/src/bun.js/webcore/streams.zig +++ b/src/bun.js/webcore/streams.zig @@ -37,7 +37,7 @@ const VirtualMachine = JSC.VirtualMachine; const Task = JSC.Task; const JSPrinter = bun.js_printer; const picohttp = bun.picohttp; -const StringJoiner = @import("../../string_joiner.zig"); +const StringJoiner = bun.StringJoiner; const uws = bun.uws; const Blob = JSC.WebCore.Blob; const Response = JSC.WebCore.Response; diff --git a/src/bun.zig b/src/bun.zig index 68e3a7e92a..6a31ebe356 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1465,8 +1465,9 @@ pub const fast_debug_build_mode = fast_debug_build_cmd != .None and Environment.isDebug; pub const MultiArrayList = @import("./multi_array_list.zig").MultiArrayList; +pub const StringJoiner = @import("./StringJoiner.zig"); +pub const NullableAllocator = @import("./NullableAllocator.zig"); -pub const Joiner = @import("./string_joiner.zig"); pub const renamer = @import("./renamer.zig"); pub const sourcemap = struct { pub usingnamespace @import("./sourcemap/sourcemap.zig"); diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index b350fe8fed..8f4b4456d7 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -65,7 +65,7 @@ const js_printer = @import("../js_printer.zig"); const js_ast = @import("../js_ast.zig"); const linker = @import("../linker.zig"); const sourcemap = bun.sourcemap; -const Joiner = bun.Joiner; +const StringJoiner = bun.StringJoiner; const base64 = bun.base64; const Ref = @import("../ast/base.zig").Ref; const Define = @import("../defines.zig").Define; @@ -6786,9 +6786,8 @@ const LinkerContext = struct { break :brk CompileResult.empty; }; - var j = bun.Joiner{ - .use_pool = false, - .node_allocator = worker.allocator, + var j = StringJoiner{ + .allocator = worker.allocator, .watcher = .{ .input = chunk.unique_key, }, @@ -6808,8 +6807,8 @@ const LinkerContext = struct { const hashbang = c.graph.ast.items(.hashbang)[chunk.entry_point.source_index]; if (hashbang.len > 0) { - j.push(hashbang); - j.push("\n"); + j.pushStatic(hashbang); + j.pushStatic("\n"); line_offset.advance(hashbang); line_offset.advance("\n"); newline_before_comment = true; @@ -6817,7 +6816,7 @@ const LinkerContext = struct { } if (is_bun) { - j.push("// @bun\n"); + j.pushStatic("// @bun\n"); line_offset.advance("// @bun\n"); } } @@ -6831,7 +6830,7 @@ const LinkerContext = struct { if (cross_chunk_prefix.len > 0) { newline_before_comment = true; line_offset.advance(cross_chunk_prefix); - j.append(cross_chunk_prefix, 0, bun.default_allocator); + j.push(cross_chunk_prefix, bun.default_allocator); } // Concatenate the generated JavaScript chunks together @@ -6853,7 +6852,7 @@ const LinkerContext = struct { if (c.options.mode == .bundle and !c.options.minify_whitespace and source_index != prev_filename_comment and compile_result.code().len > 0) { prev_filename_comment = source_index; if (newline_before_comment) { - j.push("\n"); + j.pushStatic("\n"); line_offset.advance("\n"); } @@ -6875,25 +6874,25 @@ const LinkerContext = struct { switch (comment_type) { .multiline => { - j.push("/* "); + j.pushStatic("/* "); line_offset.advance("/* "); }, .single => { - j.push("// "); + j.pushStatic("// "); line_offset.advance("// "); }, } - j.push(pretty); + j.pushStatic(pretty); line_offset.advance(pretty); switch (comment_type) { .multiline => { - j.push(" */\n"); + j.pushStatic(" */\n"); line_offset.advance(" */\n"); }, .single => { - j.push("\n"); + j.pushStatic("\n"); line_offset.advance("\n"); }, } @@ -6902,10 +6901,10 @@ const LinkerContext = struct { if (is_runtime) { line_offset.advance(compile_result.code()); - j.append(compile_result.code(), 0, bun.default_allocator); + j.push(compile_result.code(), bun.default_allocator); } else { const generated_offset = line_offset; - j.append(compile_result.code(), 0, bun.default_allocator); + j.push(compile_result.code(), bun.default_allocator); if (compile_result.source_map_chunk()) |source_map_chunk| { line_offset.reset(); @@ -6930,16 +6929,16 @@ const LinkerContext = struct { // Stick the entry point tail at the end of the file. Deliberately don't // include any source mapping information for this because it's automatically // generated and doesn't correspond to a location in the input file. - j.append(tail_code, 0, bun.default_allocator); + j.push(tail_code, bun.default_allocator); } // Put the cross-chunk suffix inside the IIFE if (cross_chunk_suffix.len > 0) { if (newline_before_comment) { - j.push("\n"); + j.pushStatic("\n"); } - j.append(cross_chunk_suffix, 0, bun.default_allocator); + j.push(cross_chunk_suffix, bun.default_allocator); } if (c.options.output_format == .iife) { @@ -6950,7 +6949,7 @@ const LinkerContext = struct { else without_newline; - j.push(with_newline); + j.pushStatic(with_newline); } j.ensureNewlineAtEnd(); @@ -6992,9 +6991,8 @@ const LinkerContext = struct { const trace = tracer(@src(), "generateSourceMapForChunk"); defer trace.end(); - var j = Joiner{ - .node_allocator = worker.allocator, - .use_pool = false, + var j = StringJoiner{ + .allocator = worker.allocator, }; const sources = c.parse_graph.input_files.items(.source); @@ -7005,7 +7003,7 @@ const LinkerContext = struct { var next_source_index: u32 = 0; const source_indices = results.items(.source_index); - j.push("{\n \"version\": 3,\n \"sources\": ["); + j.pushStatic("{\n \"version\": 3,\n \"sources\": ["); if (source_indices.len > 0) { { var path = sources[source_indices[0]].path; @@ -7017,7 +7015,7 @@ const LinkerContext = struct { var quote_buf = try MutableString.init(worker.allocator, path.pretty.len + 2); quote_buf = try js_printer.quoteForJSON(path.pretty, quote_buf, false); - j.push(quote_buf.list.items); + j.pushStatic(quote_buf.list.items); // freed by arena } if (source_indices.len > 1) { for (source_indices[1..]) |index| { @@ -7031,24 +7029,24 @@ const LinkerContext = struct { var quote_buf = try MutableString.init(worker.allocator, path.pretty.len + ", ".len + 2); quote_buf.appendAssumeCapacity(", "); quote_buf = try js_printer.quoteForJSON(path.pretty, quote_buf, false); - j.push(quote_buf.list.items); + j.pushStatic(quote_buf.list.items); // freed by arena } } } - j.push("],\n \"sourcesContent\": ["); + j.pushStatic("],\n \"sourcesContent\": ["); if (source_indices.len > 0) { - j.push("\n "); - j.push(quoted_source_map_contents[source_indices[0]]); + j.pushStatic("\n "); + j.pushStatic(quoted_source_map_contents[source_indices[0]]); if (source_indices.len > 1) { for (source_indices[1..]) |index| { - j.push(",\n "); - j.push(quoted_source_map_contents[index]); + j.pushStatic(",\n "); + j.pushStatic(quoted_source_map_contents[index]); } } } - j.push("\n ],\n \"mappings\": \""); + j.pushStatic("\n ],\n \"mappings\": \""); const mapping_start = j.len; var prev_end_state = sourcemap.SourceMapState{}; @@ -7086,14 +7084,18 @@ const LinkerContext = struct { const mapping_end = j.len; if (comptime FeatureFlags.source_map_debug_id) { - j.push("\",\n \"debugId\": \""); - j.push(try std.fmt.allocPrint(worker.allocator, "{}", .{bun.sourcemap.DebugIDFormatter{ .id = isolated_hash }})); - j.push("\",\n \"names\": []\n}"); + j.pushStatic("\",\n \"debugId\": \""); + j.push( + try std.fmt.allocPrint(worker.allocator, "{}", .{bun.sourcemap.DebugIDFormatter{ .id = isolated_hash }}), + worker.allocator, + ); + j.pushStatic("\",\n \"names\": []\n}"); } else { - j.push("\",\n \"names\": []\n}"); + j.pushStatic("\",\n \"names\": []\n}"); } const done = try j.done(worker.allocator); + bun.assert(done[0] == '{'); var pieces = sourcemap.SourceMapPieces.init(worker.allocator); if (can_have_shifts) { @@ -7174,7 +7176,7 @@ const LinkerContext = struct { } else { var el = chunk.intermediate_output.joiner.head; while (el) |e| : (el = e.next) { - hasher.write(e.data.slice); + hasher.write(e.slice); } } @@ -8932,7 +8934,7 @@ const LinkerContext = struct { } const SubstituteChunkFinalPathResult = struct { - j: Joiner, + j: StringJoiner, shifts: []sourcemap.SourceMapShifts, }; @@ -10813,7 +10815,7 @@ const LinkerContext = struct { pub fn breakOutputIntoPieces( c: *LinkerContext, allocator: std.mem.Allocator, - j: *bun.Joiner, + j: *StringJoiner, count: u32, ) !Chunk.IntermediateOutput { const trace = tracer(@src(), "breakOutputIntoPieces"); @@ -11123,7 +11125,7 @@ pub const Chunk = struct { /// If the chunk doesn't have any references to other chunks, then /// `joiner` contains the contents of the chunk. This is more efficient /// because it avoids doing a join operation twice. - joiner: bun.Joiner, + joiner: StringJoiner, empty: void, diff --git a/src/crash_handler.zig b/src/crash_handler.zig index 7cd0b83e54..bf68337c98 100644 --- a/src/crash_handler.zig +++ b/src/crash_handler.zig @@ -835,9 +835,9 @@ const Platform = enum(u8) { /// '1' - original. uses 7 char hash with VLQ encoded stack-frames /// '2' - same as '1' but this build is known to be a canary build const version_char = if (bun.Environment.is_canary) - "1" + "2" else - "2"; + "1"; const git_sha = if (bun.Environment.git_sha.len > 0) bun.Environment.git_sha[0..7] else "unknown"; diff --git a/src/nullable_allocator.zig b/src/nullable_allocator.zig deleted file mode 100644 index 8af65d6bbf..0000000000 --- a/src/nullable_allocator.zig +++ /dev/null @@ -1,31 +0,0 @@ -const std = @import("std"); - -/// A nullable allocator the same size as `std.mem.Allocator`. -pub const NullableAllocator = struct { - ptr: *anyopaque = undefined, - // Utilize the null pointer optimization on the vtable instead of - // the regular ptr because some allocator implementations might tag their - // `ptr` property. - vtable: ?*const std.mem.Allocator.VTable = null, - - pub inline fn init(a: std.mem.Allocator) @This() { - return .{ - .ptr = a.ptr, - .vtable = a.vtable, - }; - } - - pub inline fn isNull(this: @This()) bool { - return this.vtable == null; - } - - pub inline fn get(this: @This()) ?std.mem.Allocator { - return if (this.vtable) |vt| std.mem.Allocator{ .ptr = this.ptr, .vtable = vt } else null; - } -}; - -comptime { - if (@sizeOf(NullableAllocator) != @sizeOf(std.mem.Allocator)) { - @compileError("Expected the sizes to be the same."); - } -} diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index ab221f53c8..e46c136eae 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -11,7 +11,7 @@ const BabyList = JSAst.BabyList; const Logger = bun.logger; const strings = bun.strings; const MutableString = bun.MutableString; -const Joiner = @import("../string_joiner.zig"); +const StringJoiner = bun.StringJoiner; const JSPrinter = bun.js_printer; const URL = bun.URL; const FileSystem = bun.fs.FileSystem; @@ -886,9 +886,14 @@ pub const SourceMapPieces = struct { var current: usize = 0; var generated = LineColumnOffset{}; var prev_shift_column_delta: i32 = 0; - var j = Joiner{}; - j.push(this.prefix.items); + // the joiner's node allocator contains string join nodes as well as some vlq encodings + // it doesnt contain json payloads or source code, so 16kb is probably going to cover + // most applications. + var sfb = std.heap.stackFallback(16384, bun.default_allocator); + var j = StringJoiner{ .allocator = sfb.get() }; + + j.pushStatic(this.prefix.items); const mappings = this.mappings.items; while (current < mappings.len) { @@ -938,23 +943,24 @@ pub const SourceMapPieces = struct { continue; } - j.push(mappings[start_of_run..potential_end_of_run]); + j.pushStatic(mappings[start_of_run..potential_end_of_run]); assert(shift.before.lines == shift.after.lines); const shift_column_delta = shift.after.columns - shift.before.columns; const vlq_value = decode_result.value + shift_column_delta - prev_shift_column_delta; const encode = encodeVLQ(vlq_value); - j.push(encode.bytes[0..encode.len]); + j.pushCloned(encode.bytes[0..encode.len]); prev_shift_column_delta = shift_column_delta; start_of_run = potential_start_of_run; } - j.push(mappings[start_of_run..]); - j.push(this.suffix.items); + j.pushStatic(mappings[start_of_run..]); - return try j.done(allocator); + const str = try j.doneWithEnd(allocator, this.suffix.items); + bun.assert(str[0] == '{'); // invalid json + return str; } }; @@ -967,18 +973,18 @@ pub const SourceMapPieces = struct { // After all chunks are computed, they are joined together in a second pass. // This rewrites the first mapping in each chunk to be relative to the end // state of the previous chunk. -pub fn appendSourceMapChunk(j: *Joiner, allocator: std.mem.Allocator, prev_end_state_: SourceMapState, start_state_: SourceMapState, source_map_: bun.string) !void { +pub fn appendSourceMapChunk(j: *StringJoiner, allocator: std.mem.Allocator, prev_end_state_: SourceMapState, start_state_: SourceMapState, source_map_: bun.string) !void { var prev_end_state = prev_end_state_; var start_state = start_state_; // Handle line breaks in between this mapping and the previous one if (start_state.generated_line > 0) { - j.append(try strings.repeatingAlloc(allocator, @as(usize, @intCast(start_state.generated_line)), ';'), 0, allocator); + j.push(try strings.repeatingAlloc(allocator, @intCast(start_state.generated_line), ';'), allocator); prev_end_state.generated_column = 0; } var source_map = source_map_; if (strings.indexOfNotChar(source_map, ';')) |semicolons| { - j.push(source_map[0..semicolons]); + j.pushStatic(source_map[0..semicolons]); source_map = source_map[semicolons..]; prev_end_state.generated_column = 0; start_state.generated_column = 0; @@ -1009,19 +1015,18 @@ pub fn appendSourceMapChunk(j: *Joiner, allocator: std.mem.Allocator, prev_end_s start_state.original_line += original_line_.value; start_state.original_column += original_column_.value; - j.append( + j.push( appendMappingToBuffer( MutableString.initEmpty(allocator), j.lastByte(), prev_end_state, start_state, ).list.items, - 0, allocator, ); // Then append everything after that without modification. - j.push(source_map); + j.pushStatic(source_map); } const vlq_lookup_table: [256]VLQ = brk: { @@ -1455,9 +1460,8 @@ pub fn appendMappingToBuffer(buffer_: MutableString, last_byte: u8, prev_state: buffer.appendCharAssumeCapacity(','); } - comptime var i: usize = 0; - inline while (i < vlq.len) : (i += 1) { - buffer.appendAssumeCapacity(vlq[i].bytes[0..vlq[i].len]); + inline for (vlq) |item| { + buffer.appendAssumeCapacity(item.bytes[0..item.len]); } return buffer; diff --git a/src/string_immutable.zig b/src/string_immutable.zig index 74a6c65f78..2f53139b01 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -5,7 +5,6 @@ const string = bun.string; const stringZ = bun.stringZ; const CodePoint = bun.CodePoint; const bun = @import("root").bun; -pub const joiner = @import("./string_joiner.zig"); const log = bun.Output.scoped(.STR, true); const js_lexer = @import("./js_lexer.zig"); const grapheme = @import("./grapheme.zig"); diff --git a/src/string_joiner.zig b/src/string_joiner.zig deleted file mode 100644 index 61a8eebb95..0000000000 --- a/src/string_joiner.zig +++ /dev/null @@ -1,165 +0,0 @@ -/// Rope-like data structure for joining many small strings into one big string. -const std = @import("std"); -const default_allocator = bun.default_allocator; -const bun = @import("root").bun; -const string = bun.string; -const Allocator = std.mem.Allocator; -const ObjectPool = @import("./pool.zig").ObjectPool; -const Joiner = @This(); - -const Joinable = struct { - offset: u31 = 0, - needs_deinit: bool = false, - allocator: Allocator = undefined, - slice: []const u8 = "", - - pub const Pool = ObjectPool(Joinable, null, true, 4); -}; - -len: usize = 0, -use_pool: bool = true, -node_allocator: Allocator = undefined, - -head: ?*Joinable.Pool.Node = null, -tail: ?*Joinable.Pool.Node = null, - -/// Avoid an extra pass over the list when joining -watcher: Watcher = .{}, - -pub const Watcher = struct { - input: []const u8 = "", - estimated_count: u32 = 0, - needs_newline: bool = false, -}; - -pub fn done(this: *Joiner, allocator: Allocator) ![]u8 { - if (this.head == null) { - const out: []u8 = &[_]u8{}; - return out; - } - - var slice = try allocator.alloc(u8, this.len); - var remaining = slice; - var el_ = this.head; - while (el_) |join| { - const to_join = join.data.slice[join.data.offset..]; - @memcpy(remaining[0..to_join.len], to_join); - - remaining = remaining[@min(remaining.len, to_join.len)..]; - - var prev = join; - el_ = join.next; - if (prev.data.needs_deinit) { - prev.data.allocator.free(prev.data.slice); - prev.data = Joinable{}; - } - - if (this.use_pool) prev.release(); - } - - return slice[0 .. slice.len - remaining.len]; -} - -pub fn doneWithEnd(this: *Joiner, allocator: Allocator, end: []const u8) ![]u8 { - if (this.head == null and end.len == 0) { - return &[_]u8{}; - } - - if (this.head == null) { - var slice = try allocator.alloc(u8, end.len); - @memcpy(slice[0..end.len], end); - - return slice; - } - - var slice = try allocator.alloc(u8, this.len + end.len); - var remaining = slice; - var el_ = this.head; - while (el_) |join| { - const to_join = join.data.slice[join.data.offset..]; - @memcpy(remaining[0..to_join.len], to_join); - - remaining = remaining[@min(remaining.len, to_join.len)..]; - - var prev = join; - el_ = join.next; - if (prev.data.needs_deinit) { - prev.data.allocator.free(prev.data.slice); - prev.data = Joinable{}; - } - - if (this.use_pool) prev.release(); - } - - @memcpy(remaining[0..end.len], end); - - remaining = remaining[@min(remaining.len, end.len)..]; - - return slice[0 .. slice.len - remaining.len]; -} - -pub fn lastByte(this: *const Joiner) u8 { - if (this.tail) |tail| { - const slice = tail.data.slice[tail.data.offset..]; - return if (slice.len > 0) slice[slice.len - 1] else 0; - } - - return 0; -} - -pub fn push(this: *Joiner, slice: string) void { - this.append(slice, 0, null); -} - -pub fn ensureNewlineAtEnd(this: *Joiner) void { - if (this.watcher.needs_newline) { - this.watcher.needs_newline = false; - this.push("\n"); - } -} - -pub fn append(this: *Joiner, slice: string, offset: u32, allocator: ?Allocator) void { - const data = slice[offset..]; - this.len += @as(u32, @truncate(data.len)); - - const new_tail = if (this.use_pool) - Joinable.Pool.get(default_allocator) - else - (this.node_allocator.create(Joinable.Pool.Node) catch unreachable); - - this.watcher.estimated_count += @intFromBool( - this.watcher.input.len > 0 and - bun.strings.contains(data, this.watcher.input), - ); - - this.watcher.needs_newline = this.watcher.input.len > 0 and data.len > 0 and - data[data.len - 1] != '\n'; - - new_tail.* = .{ - .allocator = default_allocator, - .data = Joinable{ - .offset = @as(u31, @truncate(offset)), - .allocator = allocator orelse undefined, - .needs_deinit = allocator != null, - .slice = slice, - }, - }; - - var tail = this.tail orelse { - this.tail = new_tail; - this.head = new_tail; - return; - }; - tail.next = new_tail; - this.tail = new_tail; -} - -pub fn contains(this: *const Joiner, slice: string) bool { - var el = this.head; - while (el) |node| { - el = node.next; - if (bun.strings.contains(node.data.slice[node.data.offset..], slice)) return true; - } - - return false; -} diff --git a/test/bun.lockb b/test/bun.lockb index edfce7c97c..6d3168fc91 100755 Binary files a/test/bun.lockb and b/test/bun.lockb differ diff --git a/test/bundler/bundler_edgecase.test.ts b/test/bundler/bundler_edgecase.test.ts index 6b0d4883e2..2676f55703 100644 --- a/test/bundler/bundler_edgecase.test.ts +++ b/test/bundler/bundler_edgecase.test.ts @@ -381,13 +381,13 @@ describe("bundler", () => { }, }); itBundled("edgecase/RequireUnknownExtension", { - todo: true, files: { "/entry.js": /* js */ ` require('./x.aaaa') `, "/x.aaaa": `x`, }, + outdir: "/out", }); itBundled("edgecase/PackageJSONDefaultConditionRequire", { files: { @@ -1056,6 +1056,30 @@ describe("bundler", () => { }, target: "bun", }); + itBundled("edgecase/EmitInvalidSourceMap1", { + files: { + "/src/index.ts": /* ts */ ` + const y = await import("./second.mts"); + import * as z from "./third.mts"; + const v = await import("./third.mts"); + console.log(z, v, y); + `, + "/src/second.mts": /* ts */ ` + export default "swag"; + `, + "/src/third.mts": /* ts */ ` + export default "bun"; + `, + }, + outdir: "/out", + target: "bun", + sourceMap: "external", + minifySyntax: true, + minifyIdentifiers: true, + minifyWhitespace: true, + splitting: true, + }); + // TODO(@paperdave): test every case of this. I had already tested it manually, but it may break later const requireTranspilationListESM = [ // input, output:bun, output:node diff --git a/test/bundler/expectBundled.ts b/test/bundler/expectBundled.ts index f405addbb3..a1b26b388c 100644 --- a/test/bundler/expectBundled.ts +++ b/test/bundler/expectBundled.ts @@ -1,7 +1,7 @@ /** * See `./expectBundled.md` for how this works. */ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, readdirSync } from "fs"; import path from "path"; import { bunEnv, bunExe } from "harness"; import { tmpdir } from "os"; @@ -10,6 +10,7 @@ import { BuildConfig, BunPlugin, fileURLToPath } from "bun"; import type { Matchers } from "bun:test"; import { PluginBuilder } from "bun"; import * as esbuild from "esbuild"; +import { SourceMapConsumer } from "source-map"; /** Dedent module does a bit too much with their stuff. we will be much simpler */ function dedent(str: string | TemplateStringsArray, ...args: any[]) { @@ -1247,6 +1248,24 @@ for (const [key, blob] of build.outputs) { } } + // Check that all source maps are valid JSON + if (opts.sourceMap === "external" && outdir) { + for (const file of readdirSync(outdir, { recursive: true })) { + if (file.endsWith(".map")) { + const parsed = await Bun.file(path.join(outdir, file)).json(); + await SourceMapConsumer.with(parsed, null, async map => { + map.eachMapping(m => { + expect(m.source).toBeDefined(); + expect(m.generatedLine).toBeGreaterThanOrEqual(0); + expect(m.generatedColumn).toBeGreaterThanOrEqual(0); + expect(m.originalLine).toBeGreaterThanOrEqual(0); + expect(m.originalColumn).toBeGreaterThanOrEqual(0); + }); + }); + } + } + } + // Runtime checks! if (run) { const runs = Array.isArray(run) ? run : [run]; @@ -1411,23 +1430,12 @@ export function itBundled( try { expectBundled(id, opts, true); } catch (error) { - // it.todo(id, () => { - // throw error; - // }); return ref; } } if (opts.todo && !FILTER) { it.todo(id, () => expectBundled(id, opts as any)); - // it(id, async () => { - // try { - // await expectBundled(id, opts as any); - // } catch (error) { - // return; - // } - // throw new Error(`Expected test to fail but it passed.`); - // }); } else { it(id, () => expectBundled(id, opts as any)); } diff --git a/test/package.json b/test/package.json index ab6737fde3..9695e7cd3e 100644 --- a/test/package.json +++ b/test/package.json @@ -45,6 +45,7 @@ "sinon": "6.0.0", "socket.io": "4.7.1", "socket.io-client": "4.7.1", + "source-map": "0.7.4", "st": "3.0.0", "string-width": "7.0.0", "stripe": "15.4.0",