From 226f42e04a52a1c0727220e14ad94dcd2f47e3af Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 10 Jul 2024 21:57:40 -0700 Subject: [PATCH] Rewrite js_ast.NewBaseStore (#12388) Co-authored-by: dave caruso --- src/bun.js/api/JSTranspiler.zig | 3 +- src/bun.js/module_loader.zig | 2 - src/bun_js.zig | 12 +- src/bundler.zig | 13 +- src/bundler/bundle_v2.zig | 4 +- src/cli.zig | 4 +- src/cli/bunx_command.zig | 4 +- src/cli/create_command.zig | 4 +- src/cli/filter_arg.zig | 5 +- src/cli/test_command.zig | 4 +- src/cli/upgrade_command.zig | 4 +- src/install/install.zig | 195 ++++++--------- src/install/lockfile.zig | 5 +- src/js_ast.zig | 423 ++++++++++++++------------------ src/string_immutable.zig | 11 +- src/sys.zig | 2 +- transpiler-2.mjs | 7 + 17 files changed, 314 insertions(+), 388 deletions(-) create mode 100644 transpiler-2.mjs diff --git a/src/bun.js/api/JSTranspiler.zig b/src/bun.js/api/JSTranspiler.zig index 4c9b78f640..04a2a2d475 100644 --- a/src/bun.js/api/JSTranspiler.zig +++ b/src/bun.js/api/JSTranspiler.zig @@ -140,10 +140,9 @@ pub const TransformTask = struct { .allocator = allocator, }; ast_memory_allocator.reset(); + JSAst.Stmt.Data.Store.memory_allocator = ast_memory_allocator; JSAst.Expr.Data.Store.memory_allocator = ast_memory_allocator; - JSAst.Stmt.Data.Store.create(bun.default_allocator); - JSAst.Expr.Data.Store.create(bun.default_allocator); defer { JSAst.Stmt.Data.Store.reset(); diff --git a/src/bun.js/module_loader.zig b/src/bun.js/module_loader.zig index bb770bbdd9..6bfaf54550 100644 --- a/src/bun.js/module_loader.zig +++ b/src/bun.js/module_loader.zig @@ -808,8 +808,6 @@ pub const ModuleLoader = struct { // This is all the state used by the printer to print the module parse_result: ParseResult, - // stmt_blocks: []*js_ast.Stmt.Data.Store.All.Block = &[_]*js_ast.Stmt.Data.Store.All.Block{}, - // expr_blocks: []*js_ast.Expr.Data.Store.All.Block = &[_]*js_ast.Expr.Data.Store.All.Block{}, promise: JSC.Strong = .{}, path: Fs.Path, specifier: string = "", diff --git a/src/bun_js.zig b/src/bun_js.zig index 1568fa5777..91f97accf3 100644 --- a/src/bun_js.zig +++ b/src/bun_js.zig @@ -49,8 +49,8 @@ pub const Run = struct { const graph_ptr = try bun.default_allocator.create(bun.StandaloneModuleGraph); graph_ptr.* = graph; - js_ast.Expr.Data.Store.create(default_allocator); - js_ast.Stmt.Data.Store.create(default_allocator); + js_ast.Expr.Data.Store.create(); + js_ast.Stmt.Data.Store.create(); var arena = try Arena.init(); if (!ctx.debug.loaded_bunfig) { @@ -145,17 +145,17 @@ pub const Run = struct { try bun.CLI.Arguments.loadConfigPath(ctx.allocator, true, "bunfig.toml", ctx, .RunCommand); } + // The shell does not need to initialize JSC. + // JSC initialization costs 1-3ms. We skip this if we know it's a shell script. if (strings.endsWithComptime(entry_path, ".sh")) { const exit_code = try bootBunShell(ctx, entry_path); Global.exitWide(exit_code); return; } - // The shell does not need to initialize JSC. - // JSC initialization costs 1-3ms bun.JSC.initialize(); - js_ast.Expr.Data.Store.create(default_allocator); - js_ast.Stmt.Data.Store.create(default_allocator); + js_ast.Expr.Data.Store.create(); + js_ast.Stmt.Data.Store.create(); var arena = try Arena.init(); run = .{ diff --git a/src/bundler.zig b/src/bundler.zig index 56b4506ac5..c34b5629df 100644 --- a/src/bundler.zig +++ b/src/bundler.zig @@ -439,11 +439,10 @@ pub const Bundler = struct { opts: Api.TransformOptions, env_loader_: ?*DotEnv.Loader, ) !Bundler { - js_ast.Expr.Data.Store.create(allocator); - js_ast.Stmt.Data.Store.create(allocator); - const fs = try Fs.FileSystem.init( - opts.absolute_working_dir, - ); + js_ast.Expr.Data.Store.create(); + js_ast.Stmt.Data.Store.create(); + + const fs = try Fs.FileSystem.init(opts.absolute_working_dir); const bundle_options = try options.BundleOptions.fromApi( allocator, fs, @@ -576,8 +575,8 @@ pub const Bundler = struct { this.options.jsx.setProduction(this.env.isProduction()); - js_ast.Expr.Data.Store.create(this.allocator); - js_ast.Stmt.Data.Store.create(this.allocator); + js_ast.Expr.Data.Store.create(); + js_ast.Stmt.Data.Store.create(); defer js_ast.Expr.Data.Store.reset(); defer js_ast.Stmt.Data.Store.reset(); diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index 9ba7a53337..4bb974b65e 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -6756,8 +6756,8 @@ const LinkerContext = struct { const c = ctx.c; bun.assert(chunk.content == .javascript); - js_ast.Expr.Data.Store.create(bun.default_allocator); - js_ast.Stmt.Data.Store.create(bun.default_allocator); + js_ast.Expr.Data.Store.create(); + js_ast.Stmt.Data.Store.create(); defer chunk.renamer.deinit(bun.default_allocator); diff --git a/src/cli.zig b/src/cli.zig index ee78ee52e6..43bffb5e36 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -279,8 +279,8 @@ pub const Arguments = struct { Global.exit(1); }; - js_ast.Stmt.Data.Store.create(allocator); - js_ast.Expr.Data.Store.create(allocator); + js_ast.Stmt.Data.Store.create(); + js_ast.Expr.Data.Store.create(); defer { js_ast.Stmt.Data.Store.reset(); js_ast.Expr.Data.Store.reset(); diff --git a/src/cli/bunx_command.zig b/src/cli/bunx_command.zig index bb0fc13a0f..c0bb2fa7f7 100644 --- a/src/cli/bunx_command.zig +++ b/src/cli/bunx_command.zig @@ -79,8 +79,8 @@ pub const BunxCommand = struct { const package_json_contents = package_json_read.bytes.items; const source = bun.logger.Source.initPathString(bun.span(subpath_z), package_json_contents); - bun.JSAst.Expr.Data.Store.create(default_allocator); - bun.JSAst.Stmt.Data.Store.create(default_allocator); + bun.JSAst.Expr.Data.Store.create(); + bun.JSAst.Stmt.Data.Store.create(); const expr = try bun.JSON.ParsePackageJSONUTF8(&source, bundler.log, bundler.allocator); diff --git a/src/cli/create_command.zig b/src/cli/create_command.zig index e6f00e0550..aaf2f44cf4 100644 --- a/src/cli/create_command.zig +++ b/src/cli/create_command.zig @@ -50,8 +50,8 @@ pub var initialized_store = false; pub fn initializeStore() void { if (initialized_store) return; initialized_store = true; - js_ast.Expr.Data.Store.create(default_allocator); - js_ast.Stmt.Data.Store.create(default_allocator); + js_ast.Expr.Data.Store.create(); + js_ast.Stmt.Data.Store.create(); } const skip_dirs = &[_]bun.OSPathSlice{ diff --git a/src/cli/filter_arg.zig b/src/cli/filter_arg.zig index 4ac81ae076..d1b67fa6dd 100644 --- a/src/cli/filter_arg.zig +++ b/src/cli/filter_arg.zig @@ -37,9 +37,8 @@ fn globIgnoreFn(val: []const u8) bool { const GlobWalker = Glob.GlobWalker_(globIgnoreFn, Glob.DirEntryAccessor, false); pub fn getCandidatePackagePatterns(allocator: std.mem.Allocator, log: *bun.logger.Log, out_patterns: *std.ArrayList([]u8), workdir_: []const u8, root_buf: *bun.PathBuffer) ![]const u8 { - bun.JSAst.Expr.Data.Store.create(bun.default_allocator); - bun.JSAst.Stmt.Data.Store.create(bun.default_allocator); - + bun.JSAst.Expr.Data.Store.create(); + bun.JSAst.Stmt.Data.Store.create(); defer { bun.JSAst.Expr.Data.Store.reset(); bun.JSAst.Stmt.Data.Store.reset(); diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig index d12e877767..ebf9ef5b06 100644 --- a/src/cli/test_command.zig +++ b/src/cli/test_command.zig @@ -782,8 +782,8 @@ pub const TestCommand = struct { reporter.jest.callback = &reporter.callback; jest.Jest.runner = &reporter.jest; reporter.jest.test_options = &ctx.test_options; - js_ast.Expr.Data.Store.create(default_allocator); - js_ast.Stmt.Data.Store.create(default_allocator); + js_ast.Expr.Data.Store.create(); + js_ast.Stmt.Data.Store.create(); var vm = try JSC.VirtualMachine.init( .{ .allocator = ctx.allocator, diff --git a/src/cli/upgrade_command.zig b/src/cli/upgrade_command.zig index eeaac8b226..2f731d6701 100644 --- a/src/cli/upgrade_command.zig +++ b/src/cli/upgrade_command.zig @@ -45,8 +45,8 @@ pub var initialized_store = false; pub fn initializeStore() void { if (initialized_store) return; initialized_store = true; - js_ast.Expr.Data.Store.create(default_allocator); - js_ast.Stmt.Data.Store.create(default_allocator); + js_ast.Expr.Data.Store.create(); + js_ast.Stmt.Data.Store.create(); } pub const Version = struct { diff --git a/src/install/install.zig b/src/install/install.zig index 953c7fc762..00e4a06480 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -95,8 +95,8 @@ pub fn initializeStore() void { } initialized_store = true; - JSAst.Expr.Data.Store.create(default_allocator); - JSAst.Stmt.Data.Store.create(default_allocator); + JSAst.Expr.Data.Store.create(); + JSAst.Stmt.Data.Store.create(); } /// The default store we use pre-allocates around 16 MB of memory per thread @@ -2787,7 +2787,7 @@ pub const PackageManager = struct { if (comptime opts.init_reset_store) initializeStore(); - const json_result = json_parser.ParsePackageJSONUTF8WithOpts( + const json = json_parser.ParsePackageJSONUTF8WithOpts( &source, log, allocator, @@ -2798,12 +2798,13 @@ pub const PackageManager = struct { .always_decode_escape_sequences = opts.always_decode_escape_sequences, .guess_indentation = opts.guess_indentation, }, - ); - - const json = json_result catch |err| return .{ .parse_err = err }; + ) catch |err| { + bun.handleErrorReturnTrace(err, @errorReturnTrace()); + return .{ .parse_err = err }; + }; entry.value_ptr.* = .{ - .root = json.root.deepClone(allocator) catch bun.outOfMemory(), + .root = json.root.deepClone(bun.default_allocator) catch bun.outOfMemory(), .source = source, .indentation = json.indentation, }; @@ -7571,6 +7572,12 @@ pub const PackageManager = struct { current_package_json: *Expr, options: EditOptions, ) !void { + // using data store is going to result in undefined memory issues as + // the store is cleared in some workspace situations. the solution + // is to always avoid the store + Expr.Disabler.disable(); + defer Expr.Disabler.enable(); + const allocator = manager.allocator; inline for (dependency_groups) |group| { @@ -7624,13 +7631,9 @@ pub const PackageManager = struct { else allocator.dupe(u8, "latest") catch bun.outOfMemory(); - dep.value = Expr.init( - E.String, - E.String{ - .data = temp_version, - }, - logger.Loc.Empty, - ).clone(allocator) catch bun.outOfMemory(); + dep.value = Expr.allocate(allocator, E.String, .{ + .data = temp_version, + }, logger.Loc.Empty); } } } else { @@ -7701,29 +7704,21 @@ pub const PackageManager = struct { // negative because the real package might have a scope // e.g. "dep": "npm:@foo/bar@1.2.3" if (strings.lastIndexOfChar(dep_literal, '@')) |at_index| { - dep.value = try Expr.init( - E.String, - E.String{ - .data = try std.fmt.allocPrint(allocator, "{s}@{s}", .{ - dep_literal[0..at_index], - new_version, - }), - }, - logger.Loc.Empty, - ).clone(allocator); + dep.value = Expr.allocate(allocator, E.String, .{ + .data = try std.fmt.allocPrint(allocator, "{s}@{s}", .{ + dep_literal[0..at_index], + new_version, + }), + }, logger.Loc.Empty); break :updated; } // fallthrough and replace entire version. } - dep.value = try Expr.init( - E.String, - E.String{ - .data = new_version, - }, - logger.Loc.Empty, - ).clone(allocator); + dep.value = Expr.allocate(allocator, E.String, .{ + .data = new_version, + }, logger.Loc.Empty); break :updated; } } @@ -7744,6 +7739,12 @@ pub const PackageManager = struct { dependency_list: string, options: EditOptions, ) !void { + // using data store is going to result in undefined memory issues as + // the store is cleared in some workspace situations. the solution + // is to always avoid the store + Expr.Disabler.disable(); + defer Expr.Disabler.enable(); + const allocator = manager.allocator; var remaining = updates.len; var replacing: usize = 0; @@ -7885,13 +7886,9 @@ pub const PackageManager = struct { while (i > 0) { i -= 1; if (deps[i].data == .e_missing) { - deps[i] = try Expr.init( - E.String, - E.String{ - .data = package_name, - }, - logger.Loc.Empty, - ).clone(allocator); + deps[i] = Expr.allocate(allocator, E.String, .{ + .data = package_name, + }, logger.Loc.Empty); break; } } @@ -7951,27 +7948,19 @@ pub const PackageManager = struct { } if (new_dependencies[k].key == null) { - new_dependencies[k].key = try JSAst.Expr.init( - JSAst.E.String, - JSAst.E.String{ - .data = try allocator.dupe(u8, if (request.is_aliased) - request.name - else if (request.resolved_name.isEmpty()) - request.version.literal.slice(request.version_buf) - else - request.resolved_name.slice(request.version_buf)), - }, - logger.Loc.Empty, - ).clone(allocator); + new_dependencies[k].key = JSAst.Expr.allocate(allocator, JSAst.E.String, .{ + .data = try allocator.dupe(u8, if (request.is_aliased) + request.name + else if (request.resolved_name.isEmpty()) + request.version.literal.slice(request.version_buf) + else + request.resolved_name.slice(request.version_buf)), + }, logger.Loc.Empty); - new_dependencies[k].value = try JSAst.Expr.init( - JSAst.E.String, - JSAst.E.String{ - // we set it later - .data = "", - }, - logger.Loc.Empty, - ).clone(allocator); + new_dependencies[k].value = JSAst.Expr.allocate(allocator, JSAst.E.String, .{ + // we set it later + .data = "", + }, logger.Loc.Empty); request.e_string = new_dependencies[k].value.?.data.e_string; @@ -7990,13 +7979,9 @@ pub const PackageManager = struct { } } - break :brk JSAst.Expr.init( - JSAst.E.Object, - JSAst.E.Object{ - .properties = JSAst.G.Property.List.init(new_dependencies), - }, - logger.Loc.Empty, - ); + break :brk JSAst.Expr.allocate(allocator, JSAst.E.Object, .{ + .properties = JSAst.G.Property.List.init(new_dependencies), + }, logger.Loc.Empty); }; dependencies_object.data.e_object.properties = JSAst.G.Property.List.init(new_dependencies); @@ -8016,13 +8001,9 @@ pub const PackageManager = struct { } } - break :brk Expr.init( - E.Array, - E.Array{ - .items = JSAst.ExprNodeList.init(new_trusted_deps), - }, - logger.Loc.Empty, - ); + break :brk Expr.allocate(allocator, E.Array, .{ + .items = JSAst.ExprNodeList.init(new_trusted_deps), + }, logger.Loc.Empty); }; if (options.add_trusted_dependencies and trusted_dependencies_to_add > 0) { @@ -8035,81 +8016,55 @@ pub const PackageManager = struct { if (current_package_json.data != .e_object or current_package_json.data.e_object.properties.len == 0) { var root_properties = try allocator.alloc(JSAst.G.Property, if (options.add_trusted_dependencies) 2 else 1); root_properties[0] = JSAst.G.Property{ - .key = JSAst.Expr.init( - JSAst.E.String, - JSAst.E.String{ - .data = dependency_list, - }, - logger.Loc.Empty, - ), + .key = JSAst.Expr.allocate(allocator, JSAst.E.String, .{ + .data = dependency_list, + }, logger.Loc.Empty), .value = dependencies_object, }; if (options.add_trusted_dependencies) { root_properties[1] = JSAst.G.Property{ - .key = Expr.init( - E.String, - E.String{ - .data = trusted_dependencies_string, - }, - logger.Loc.Empty, - ), + .key = Expr.allocate(allocator, E.String, .{ + .data = trusted_dependencies_string, + }, logger.Loc.Empty), .value = trusted_dependencies_array, }; } - current_package_json.* = JSAst.Expr.init( - JSAst.E.Object, - JSAst.E.Object{ .properties = JSAst.G.Property.List.init(root_properties) }, - logger.Loc.Empty, - ); + current_package_json.* = JSAst.Expr.allocate(allocator, JSAst.E.Object, .{ + .properties = JSAst.G.Property.List.init(root_properties), + }, logger.Loc.Empty); } else { if (needs_new_dependency_list and needs_new_trusted_dependencies_list) { var root_properties = try allocator.alloc(G.Property, current_package_json.data.e_object.properties.len + 2); @memcpy(root_properties[0..current_package_json.data.e_object.properties.len], current_package_json.data.e_object.properties.slice()); root_properties[root_properties.len - 2] = .{ - .key = Expr.init(E.String, E.String{ + .key = Expr.allocate(allocator, E.String, E.String{ .data = dependency_list, }, logger.Loc.Empty), .value = dependencies_object, }; root_properties[root_properties.len - 1] = .{ - .key = Expr.init( - E.String, - E.String{ - .data = trusted_dependencies_string, - }, - logger.Loc.Empty, - ), + .key = Expr.allocate(allocator, E.String, .{ + .data = trusted_dependencies_string, + }, logger.Loc.Empty), .value = trusted_dependencies_array, }; - current_package_json.* = Expr.init( - E.Object, - E.Object{ - .properties = G.Property.List.init(root_properties), - }, - logger.Loc.Empty, - ); + current_package_json.* = Expr.allocate(allocator, E.Object, .{ + .properties = G.Property.List.init(root_properties), + }, logger.Loc.Empty); } else if (needs_new_dependency_list or needs_new_trusted_dependencies_list) { var root_properties = try allocator.alloc(JSAst.G.Property, current_package_json.data.e_object.properties.len + 1); @memcpy(root_properties[0..current_package_json.data.e_object.properties.len], current_package_json.data.e_object.properties.slice()); root_properties[root_properties.len - 1] = .{ - .key = JSAst.Expr.init( - JSAst.E.String, - JSAst.E.String{ - .data = if (needs_new_dependency_list) dependency_list else trusted_dependencies_string, - }, - logger.Loc.Empty, - ), + .key = JSAst.Expr.allocate(allocator, JSAst.E.String, .{ + .data = if (needs_new_dependency_list) dependency_list else trusted_dependencies_string, + }, logger.Loc.Empty), .value = if (needs_new_dependency_list) dependencies_object else trusted_dependencies_array, }; - current_package_json.* = JSAst.Expr.init( - JSAst.E.Object, - JSAst.E.Object{ - .properties = JSAst.G.Property.List.init(root_properties), - }, - logger.Loc.Empty, - ); + current_package_json.* = JSAst.Expr.allocate(allocator, JSAst.E.Object, .{ + .properties = JSAst.G.Property.List.init(root_properties), + }, logger.Loc.Empty); } } } diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index d0e7115a6e..4b53876e6c 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -3848,7 +3848,7 @@ pub const Package = extern struct { var workspace = Package{}; - const json = PackageManager.instance.workspace_package_json_cache.getWithSource(allocator, log, source, .{}).unwrap() catch break :brk false; + const json = PackageManager.instance.workspace_package_json_cache.getWithSource(bun.default_allocator, log, source, .{}).unwrap() catch break :brk false; try workspace.parseWithJSON( to_lockfile, @@ -4396,6 +4396,7 @@ pub const Package = extern struct { abs_package_json_path, log, ) catch |err| { + bun.handleErrorReturnTrace(err, @errorReturnTrace()); switch (err) { error.EISNOTDIR, error.EISDIR, error.EACCESS, error.EPERM, error.ENOENT, error.FileNotFound => { log.addErrorFmt( @@ -4531,6 +4532,8 @@ pub const Package = extern struct { abs_package_json_path, log, ) catch |err| { + bun.handleErrorReturnTrace(err, @errorReturnTrace()); + const entry_base: []const u8 = Path.basename(matched_path); switch (err) { error.FileNotFound, error.PermissionDenied => continue, diff --git a/src/js_ast.zig b/src/js_ast.zig index 068a750032..24cac20345 100644 --- a/src/js_ast.zig +++ b/src/js_ast.zig @@ -36,189 +36,165 @@ const MimeType = bun.http.MimeType; /// although it may contain no statements if there is nothing to export. pub const namespace_export_part_index = 0; -pub fn NewBaseStore(comptime Union: anytype, comptime count: usize) type { - var max_size = 0; - var max_align = 1; - for (Union) |kind| { - max_size = @max(@sizeOf(kind), max_size); - max_align = if (@sizeOf(kind) == 0) max_align else @max(@alignOf(kind), max_align); - } +/// This "Store" is a specialized memory allocation strategy very similar to an +/// arena, used for allocating expression and statement nodes during JavaScript +/// parsing and visiting. Allocations are grouped into large blocks, where each +/// block is treated as a fixed-buffer allocator. When a block runs out of +/// space, a new one is created; all blocks are joined as a linked list. +/// +/// Similarly to an arena, you can call .reset() to reset state, reusing memory +/// across operations. +pub fn NewStore(comptime types: []const type, comptime count: usize) type { + const largest_size, const largest_align = brk: { + var largest_size = 0; + var largest_align = 1; + for (types) |T| { + if (@sizeOf(T) == 0) { + @compileError("NewStore does not support 0 size type: " ++ @typeName(T)); + } + largest_size = @max(@sizeOf(T), largest_size); + largest_align = @max(@alignOf(T), largest_align); + } + break :brk .{ largest_size, largest_align }; + }; - const UnionValueType = [max_size]u8; - const SizeType = std.math.IntFittingRange(0, (count + 1)); - const MaxAlign = max_align; + const backing_allocator = bun.default_allocator; + + const log = Output.scoped(.Store, false); return struct { - const Allocator = std.mem.Allocator; - const Self = @This(); - pub const WithBase = struct { - head: Block = Block{}, - store: Self, - }; + const Store = @This(); + + current: *Block, + debug_lock: std.debug.SafetyLock = .{}, pub const Block = struct { - used: SizeType = 0, - items: [count]UnionValueType align(MaxAlign) = undefined, + pub const size = largest_size * count * 2; + pub const Size = std.math.IntFittingRange(0, size + largest_size); - pub inline fn isFull(block: *const Block) bool { - return block.used >= @as(SizeType, count); - } + buffer: [size]u8 align(largest_align) = undefined, + bytes_used: Size = 0, + next: ?*Block = null, - pub fn append(block: *Block, comptime ValueType: type, value: ValueType) *UnionValueType { - if (comptime Environment.allow_assert) bun.assert(block.used < count); - const index = block.used; - block.items[index][0..value.len].* = value.*; - block.used +|= 1; - return &block.items[index]; + pub fn tryAlloc(block: *Block, comptime T: type) ?*T { + const start = std.mem.alignForward(usize, block.bytes_used, @alignOf(T)); + if (start + @sizeOf(T) > block.buffer.len) return null; + defer block.bytes_used = @intCast(start + @sizeOf(T)); + + // it's simpler to use @ptrCast, but as a sanity check, we also + // try to compute the slice. Zig will report an out of bounds + // panic if the null detection logic above is wrong + if (Environment.isDebug) { + _ = block.buffer[block.bytes_used..][0..@sizeOf(T)]; + } + + return @alignCast(@ptrCast(&block.buffer[start])); } }; - const Overflow = struct { - const max = 4096 * 3; - const UsedSize = std.math.IntFittingRange(0, max + 1); - used: UsedSize = 0, - allocated: UsedSize = 0, - allocator: Allocator = default_allocator, - ptrs: [max]*Block = undefined, - - pub fn tail(this: *Overflow) *Block { - if (this.ptrs[this.used].isFull()) { - this.used +%= 1; - if (this.allocated > this.used) { - this.ptrs[this.used].used = 0; - } - } - - if (this.allocated <= this.used) { - var new_ptrs = this.allocator.alloc(Block, 2) catch unreachable; - new_ptrs[0] = Block{}; - new_ptrs[1] = Block{}; - this.ptrs[this.allocated] = &new_ptrs[0]; - this.ptrs[this.allocated + 1] = &new_ptrs[1]; - this.allocated +%= 2; - } - - return this.ptrs[this.used]; - } - - pub inline fn slice(this: *Overflow) []*Block { - return this.ptrs[0..this.used]; - } + const PreAlloc = struct { + metadata: Store, + first_block: Block, }; - overflow: Overflow = Overflow{}, + pub fn firstBlock(store: *Store) *Block { + return &@as(*PreAlloc, @fieldParentPtr("metadata", store)).first_block; + } - pub threadlocal var _self: ?*Self = null; + pub fn init() *Store { + log("init", .{}); + const prealloc = backing_allocator.create(PreAlloc) catch bun.outOfMemory(); - pub fn reclaim() []*Block { - var overflow = &_self.?.overflow; + prealloc.first_block.bytes_used = 0; + prealloc.first_block.next = null; - if (overflow.used == 0) { - if (overflow.allocated == 0 or overflow.ptrs[0].used == 0) { - return &.{}; + prealloc.metadata = .{ + .current = &prealloc.first_block, + }; + + return &prealloc.metadata; + } + + pub fn deinit(store: *Store) void { + log("deinit", .{}); + var it = store.firstBlock().next; // do not free `store.head` + while (it) |next| { + if (Environment.isDebug) + @memset(next.buffer, undefined); + it = next.next; + backing_allocator.destroy(next); + } + + const prealloc: PreAlloc = @fieldParentPtr("metadata", store); + bun.assert(&prealloc.first_block == store.head); + backing_allocator.destroy(prealloc); + } + + pub fn reset(store: *Store) void { + store.debug_lock.assertUnlocked(); + log("reset", .{}); + + if (Environment.isDebug) { + var it: ?*Block = store.firstBlock(); + while (it) |next| : (it = next.next) { + next.bytes_used = undefined; + @memset(&next.buffer, undefined); } } - var to_move = overflow.ptrs[0..overflow.allocated][overflow.used..]; + store.current = store.firstBlock(); + store.current.bytes_used = 0; + } - // This returns the list of maxed out blocks - var used_list = overflow.slice(); + fn allocate(store: *Store, comptime T: type) *T { + comptime bun.assert(@sizeOf(T) > 0); // don't allocate! + comptime if (!supportsType(T)) { + @compileError("Store does not know about type: " ++ @typeName(T)); + }; - // The last block may be partially used. - if (overflow.allocated > overflow.used and to_move.len > 0 and to_move.ptr[0].used > 0) { - to_move = to_move[1..]; - used_list.len += 1; + store.debug_lock.assertUnlocked(); + + if (store.current.tryAlloc(T)) |ptr| + return ptr; + + // a new block is needed + const next_block = if (store.current.next) |next| brk: { + next.bytes_used = 0; + break :brk next; + } else brk: { + const new_block = backing_allocator.create(Block) catch + bun.outOfMemory(); + new_block.next = null; + new_block.bytes_used = 0; + store.current.next = new_block; + break :brk new_block; + }; + + store.current = next_block; + + return next_block.tryAlloc(T) orelse + unreachable; // newly initialized blocks must have enough space for at least one + } + + pub inline fn append(store: *Store, comptime T: type, data: T) *T { + const ptr = store.allocate(T); + if (Environment.isDebug) { + log("append({s}) -> 0x{x}", .{ bun.meta.typeName(T), @intFromPtr(ptr) }); } - - const used = overflow.allocator.dupe(*Block, used_list) catch unreachable; - - for (to_move, overflow.ptrs[0..to_move.len]) |b, *out| { - b.* = Block{ - .items = undefined, - .used = 0, - }; - out.* = b; - } - - overflow.allocated = @as(Overflow.UsedSize, @truncate(to_move.len)); - overflow.used = 0; - - return used; + ptr.* = data; + return ptr; } - /// Reset all AST nodes, allowing the memory to be reused for the next parse. - /// Only call this when we're done with ALL AST nodes, or you risk - /// undefined memory bugs. - /// - /// Nested parsing should either use the same store, or call - /// Store.reclaim. - pub fn reset() void { - const blocks = _self.?.overflow.slice(); - for (blocks) |b| { - if (comptime Environment.isDebug) { - // ensure we crash if we use a freed value - const bytes = std.mem.asBytes(&b.items); - @memset(bytes, undefined); - } - b.used = 0; - } - _self.?.overflow.used = 0; + pub fn lock(store: *Store) void { + store.debug_lock.lock(); } - pub fn init(allocator: std.mem.Allocator) *Self { - var base = allocator.create(WithBase) catch unreachable; - base.* = WithBase{ .store = .{ .overflow = Overflow{ .allocator = allocator } } }; - var instance = &base.store; - instance.overflow.ptrs[0] = &base.head; - instance.overflow.allocated = 1; - - _self = instance; - - return _self.?; + pub fn unlock(store: *Store) void { + store.debug_lock.unlock(); } - pub fn onThreadExit(_: *anyopaque) callconv(.C) void { - deinit(); - } - - fn deinit() void { - if (_self) |this| { - _self = null; - const sliced = this.overflow.slice(); - var allocator = this.overflow.allocator; - - if (sliced.len > 1) { - var i: usize = 1; - const end = sliced.len; - while (i < end) { - const ptrs = @as(*[2]Block, @ptrCast(sliced[i])); - allocator.free(ptrs); - i += 2; - } - this.overflow.allocated = 1; - } - var base_store: *WithBase = @fieldParentPtr("store", this); - if (this.overflow.ptrs[0] == &base_store.head) { - allocator.destroy(base_store); - } - } - } - - pub fn append(comptime Disabler: type, comptime ValueType: type, value: ValueType) *ValueType { - Disabler.assert(); - return _self.?._append(ValueType, value); - } - - inline fn _append(self: *Self, comptime ValueType: type, value: ValueType) *ValueType { - const bytes = std.mem.asBytes(&value); - const BytesAsSlice = @TypeOf(bytes); - - var block = self.overflow.tail(); - - return @as( - *ValueType, - @ptrCast(@alignCast(block.append(BytesAsSlice, bytes))), - ); + fn supportsType(T: type) bool { + return std.mem.indexOfScalar(type, types, T) != null; } }; } @@ -3129,7 +3105,7 @@ pub const Stmt = struct { s_lazy_export: Expr.Data, pub const Store = struct { - const Union = [_]type{ + const StoreType = NewStore(&.{ S.Block, S.Break, S.Class, @@ -3157,53 +3133,47 @@ pub const Stmt = struct { S.Switch, S.Throw, S.Try, - S.TypeScript, S.While, S.With, - }; - const All = NewBaseStore(Union, 128); - pub threadlocal var memory_allocator: ?*ASTMemoryAllocator = null; + }, 128); - threadlocal var has_inited = false; + pub threadlocal var instance: ?*StoreType = null; + pub threadlocal var memory_allocator: ?*ASTMemoryAllocator = null; pub threadlocal var disable_reset = false; - pub fn create(allocator: std.mem.Allocator) void { - if (has_inited or memory_allocator != null) { + + pub fn create() void { + if (instance != null or memory_allocator != null) { return; } - has_inited = true; - _ = All.init(allocator); + instance = StoreType.init(); } pub fn reset() void { if (disable_reset or memory_allocator != null) return; - All.reset(); + instance.?.reset(); } pub fn deinit() void { - if (!has_inited or memory_allocator != null) return; - All.deinit(); - has_inited = false; + if (instance == null or memory_allocator != null) return; + instance.?.deinit(); + instance = null; } pub inline fn assert() void { if (comptime Environment.allow_assert) { - if (!has_inited and memory_allocator == null) + if (instance == null and memory_allocator == null) bun.unreachablePanic("Store must be init'd", .{}); } } - pub fn append(comptime ValueType: type, value: anytype) *ValueType { + pub fn append(comptime T: type, value: T) *T { if (memory_allocator) |allocator| { - return allocator.append(ValueType, value); + return allocator.append(T, value); } - return All.append(Disabler, ValueType, value); - } - - pub fn toOwnedSlice() []*Store.All.Block { - if (!has_inited or Store.All._self.?.overflow.used == 0 or disable_reset) return &[_]*Store.All.Block{}; - return Store.All.reclaim(); + Disabler.assert(); + return instance.?.append(T, value); } }; }; @@ -5937,83 +5907,72 @@ pub const Expr = struct { } pub const Store = struct { - const often = 512; - const medium = 256; - const rare = 24; - - const All = NewBaseStore( - &([_]type{ - E.Array, - E.Unary, - E.Binary, - E.Class, - E.New, - E.Function, - E.Call, - E.Dot, - E.Index, - E.Arrow, - E.RegExp, - - E.PrivateIdentifier, - E.JSXElement, - E.Number, - E.BigInt, - E.Object, - E.Spread, - E.String, - E.TemplatePart, - E.Template, - E.Await, - E.Yield, - E.If, - E.Import, - }), - 512, - ); + const StoreType = NewStore(&.{ + E.Array, + E.Arrow, + E.Await, + E.BigInt, + E.Binary, + E.Call, + E.Class, + E.Dot, + E.Function, + E.If, + E.Import, + E.Index, + E.InlinedEnum, + E.JSXElement, + E.New, + E.Number, + E.Object, + E.PrivateIdentifier, + E.RegExp, + E.Spread, + E.String, + E.Template, + E.TemplatePart, + E.Unary, + E.UTF8String, + E.Yield, + }, 512); + pub threadlocal var instance: ?*StoreType = null; pub threadlocal var memory_allocator: ?*ASTMemoryAllocator = null; - - threadlocal var has_inited = false; pub threadlocal var disable_reset = false; - pub fn create(allocator: std.mem.Allocator) void { - if (has_inited or memory_allocator != null) { + + pub fn create() void { + if (instance != null or memory_allocator != null) { return; } - has_inited = true; - _ = All.init(allocator); + instance = StoreType.init(); } pub fn reset() void { if (disable_reset or memory_allocator != null) return; - All.reset(); + instance.?.reset(); } pub fn deinit() void { - if (!has_inited or memory_allocator != null) return; - All.deinit(); - has_inited = false; + if (instance == null or memory_allocator != null) return; + instance.?.deinit(); + instance = null; } pub inline fn assert() void { if (comptime Environment.allow_assert) { - if (!has_inited and memory_allocator == null) + if (instance == null and memory_allocator == null) bun.unreachablePanic("Store must be init'd", .{}); } } - pub fn append(comptime ValueType: type, value: ValueType) *ValueType { + pub fn append(comptime T: type, value: T) *T { if (memory_allocator) |allocator| { - return allocator.append(ValueType, value); + return allocator.append(T, value); } - return All.append(Disabler, ValueType, value); - } - - pub fn toOwnedSlice() []*Store.All.Block { - if (!has_inited or Store.All._self.?.overflow.used == 0 or disable_reset or memory_allocator != null) return &[_]*Store.All.Block{}; - return Store.All.reclaim(); + Disabler.assert(); + return instance.?.append(T, value); } }; diff --git a/src/string_immutable.zig b/src/string_immutable.zig index db2e54696d..f2e4b29b34 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -119,9 +119,15 @@ pub fn indexOfAny(slice: string, comptime str: anytype) ?OptionalUsize { return null; } + pub fn indexOfAny16(self: []const u16, comptime str: anytype) ?OptionalUsize { - for (self, 0..) |c, i| { - inline for (str) |a| { + return indexOfAnyT(u16, self, str); +} + +pub fn indexOfAnyT(comptime T: type, str: []const T, comptime chars: anytype) ?OptionalUsize { + if (T == u8) return indexOfAny(str, chars); + for (str, 0..) |c, i| { + inline for (chars) |a| { if (c == a) { return @as(OptionalUsize, @intCast(i)); } @@ -130,6 +136,7 @@ pub fn indexOfAny16(self: []const u16, comptime str: anytype) ?OptionalUsize { return null; } + pub inline fn containsComptime(self: string, comptime str: string) bool { if (comptime str.len == 0) @compileError("Don't call this with an empty string plz."); diff --git a/src/sys.zig b/src/sys.zig index 1e74deb903..ac93527180 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -791,7 +791,7 @@ pub fn normalizePathWindows( return .{ .result = norm }; } - if (std.mem.indexOfAny(T, path_, &.{ '\\', '/', '.' }) == null) { + if (bun.strings.indexOfAnyT(T, path_, &.{ '\\', '/', '.' }) == null) { if (buf.len < path.len) { return .{ .err = .{ diff --git a/transpiler-2.mjs b/transpiler-2.mjs new file mode 100644 index 0000000000..860fda29e6 --- /dev/null +++ b/transpiler-2.mjs @@ -0,0 +1,7 @@ +import { expect } from "bun:test"; + +const str = 'console.log((b,' + '0,0,0,z,'.repeat(10000) + 'x, 0));'; +const result = new Bun.Transpiler({ minify: true }) + .transformSync(str); +expect(result).toBe('console.log((b,' + 'z,'.repeat(10000) + 'x,0));'); +