From 11070b8e163e33269678e72b11ace39ecc048aec Mon Sep 17 00:00:00 2001 From: Dylan Conway Date: Tue, 3 Jun 2025 23:44:09 -0700 Subject: [PATCH] add support for `"workspaces.nohoist"` and `"workspaces.hoistingLimits"` (#20124) --- src/bun.js/api/glob.zig | 5 +- src/cli/filter_arg.zig | 8 +- src/cli/outdated_command.zig | 6 +- src/cli/pack_command.zig | 10 +- src/glob.zig | 39 +++- src/glob/GlobWalker.zig | 3 - src/glob/match.zig | 39 +--- src/install/install.zig | 17 +- src/install/lockfile.zig | 37 ++++ src/install/lockfile/CatalogMap.zig | 4 + src/install/lockfile/Package.zig | 175 +++++++++++++----- src/install/lockfile/Package/WorkspaceMap.zig | 6 +- src/install/lockfile/Tree.zig | 106 ++++++++++- src/install/lockfile/bun.lock.zig | 67 +++++++ src/install/lockfile/bun.lockb.zig | 49 +++++ src/install/resolvers/folder_resolver.zig | 2 +- src/shell/interpreter.zig | 3 +- src/shell/shell.zig | 3 +- 18 files changed, 453 insertions(+), 126 deletions(-) diff --git a/src/bun.js/api/glob.zig b/src/bun.js/api/glob.zig index 151c7c1f54..c4d2b8d44e 100644 --- a/src/bun.js/api/glob.zig +++ b/src/bun.js/api/glob.zig @@ -1,6 +1,5 @@ const Glob = @This(); -const globImpl = @import("../../glob.zig"); -const GlobWalker = globImpl.BunGlobWalker; +const GlobWalker = bun.glob.BunGlobWalker; const ArgumentsSlice = JSC.CallFrame.ArgumentsSlice; const Syscall = @import("../../sys.zig"); const std = @import("std"); @@ -389,7 +388,7 @@ pub fn match(this: *Glob, globalThis: *JSGlobalObject, callframe: *JSC.CallFrame var str = try str_arg.toSlice(globalThis, arena.allocator()); defer str.deinit(); - return JSC.JSValue.jsBoolean(globImpl.match(arena.allocator(), this.pattern, str.slice()).matches()); + return JSC.JSValue.jsBoolean(bun.glob.match(this.pattern, str.slice()).matches()); } pub fn convertUtf8(codepoints: *std.ArrayList(u32), pattern: []const u8) !void { diff --git a/src/cli/filter_arg.zig b/src/cli/filter_arg.zig index 3c85fe500e..8fd509c74a 100644 --- a/src/cli/filter_arg.zig +++ b/src/cli/filter_arg.zig @@ -5,7 +5,7 @@ const Output = bun.Output; const Global = bun.Global; const strings = bun.strings; const JSON = bun.JSON; -const Glob = @import("../glob.zig"); +const glob = bun.glob; const SKIP_LIST = .{ // skip hidden directories @@ -31,7 +31,7 @@ fn globIgnoreFn(val: []const u8) bool { return false; } -const GlobWalker = Glob.GlobWalker(globIgnoreFn, Glob.walk.DirEntryAccessor, false); +const GlobWalker = glob.GlobWalker(globIgnoreFn, glob.walk.DirEntryAccessor, false); pub fn getCandidatePackagePatterns(allocator: std.mem.Allocator, log: *bun.logger.Log, out_patterns: *std.ArrayList([]u8), workdir_: []const u8, root_buf: *bun.PathBuffer) ![]const u8 { bun.JSAst.Expr.Data.Store.create(); @@ -186,7 +186,7 @@ pub const FilterSet = struct { pub fn matchesPath(self: *const FilterSet, path: []const u8) bool { for (self.filters) |filter| { - if (Glob.walk.matchImpl(self.allocator, filter.pattern, path).matches()) { + if (glob.match(filter.pattern, path).matches()) { return true; } } @@ -199,7 +199,7 @@ pub const FilterSet = struct { .name => name, .path => path, }; - if (Glob.walk.matchImpl(self.allocator, filter.pattern, target).matches()) { + if (glob.match(filter.pattern, target).matches()) { return true; } } diff --git a/src/cli/outdated_command.zig b/src/cli/outdated_command.zig index 83963b1f48..96df1061ba 100644 --- a/src/cli/outdated_command.zig +++ b/src/cli/outdated_command.zig @@ -194,14 +194,14 @@ pub const OutdatedCommand = struct { const abs_res_path = path.joinAbsStringBuf(FileSystem.instance.top_level_dir, &path_buf, &[_]string{res_path}, .posix); - if (!glob.walk.matchImpl(allocator, pattern, strings.withoutTrailingSlash(abs_res_path)).matches()) { + if (!glob.match(pattern, strings.withoutTrailingSlash(abs_res_path)).matches()) { break :matched false; } }, .name => |pattern| { const name = pkg_names[workspace_pkg_id].slice(string_buf); - if (!glob.walk.matchImpl(allocator, pattern, name).matches()) { + if (!glob.match(pattern, name).matches()) { break :matched false; } }, @@ -305,7 +305,7 @@ pub const OutdatedCommand = struct { .path => unreachable, .name => |name_pattern| { if (name_pattern.len == 0) continue; - if (!glob.walk.matchImpl(bun.default_allocator, name_pattern, dep.name.slice(string_buf)).matches()) { + if (!glob.match(name_pattern, dep.name.slice(string_buf)).matches()) { break :match false; } }, diff --git a/src/cli/pack_command.zig b/src/cli/pack_command.zig index 89c2403d35..9f9b4256b8 100644 --- a/src/cli/pack_command.zig +++ b/src/cli/pack_command.zig @@ -325,7 +325,7 @@ pub const PackCommand = struct { // normally the behavior of `index.js` and `**/index.js` are the same, // but includes require `**/` const match_path = if (include.flags.@"leading **/") entry_name else entry_subpath; - switch (glob.walk.matchImpl(allocator, include.glob.slice(), match_path)) { + switch (glob.match(include.glob.slice(), match_path)) { .match => included = true, .negate_no_match, .negate_match => unreachable, else => {}, @@ -342,7 +342,7 @@ pub const PackCommand = struct { const match_path = if (exclude.flags.@"leading **/") entry_name else entry_subpath; // NOTE: These patterns have `!` so `.match` logic is // inverted here - switch (glob.walk.matchImpl(allocator, exclude.glob.slice(), match_path)) { + switch (glob.match(exclude.glob.slice(), match_path)) { .negate_no_match => included = false, else => {}, } @@ -1066,7 +1066,7 @@ pub const PackCommand = struct { // check default ignores that only apply to the root project directory for (root_default_ignore_patterns) |pattern| { - switch (glob.walk.matchImpl(bun.default_allocator, pattern, entry_name)) { + switch (glob.match(pattern, entry_name)) { .match => { // cannot be reversed return .{ @@ -1093,7 +1093,7 @@ pub const PackCommand = struct { for (default_ignore_patterns) |pattern_info| { const pattern, const can_override = pattern_info; - switch (glob.walk.matchImpl(bun.default_allocator, pattern, entry_name)) { + switch (glob.match(pattern, entry_name)) { .match => { if (can_override) { ignored = true; @@ -1135,7 +1135,7 @@ pub const PackCommand = struct { if (pattern.flags.dirs_only and entry.kind != .directory) continue; const match_path = if (pattern.flags.rel_path) rel else entry_name; - switch (glob.walk.matchImpl(bun.default_allocator, pattern.glob.slice(), match_path)) { + switch (glob.match(pattern.glob.slice(), match_path)) { .match => { ignored = true; ignore_pattern = pattern.glob.slice(); diff --git a/src/glob.zig b/src/glob.zig index 5519351638..995d2cacaf 100644 --- a/src/glob.zig +++ b/src/glob.zig @@ -1,8 +1,39 @@ +const std = @import("std"); +pub const match = @import("./glob/match.zig").match; pub const walk = @import("./glob/GlobWalker.zig"); -pub const match_impl = @import("./glob/match.zig"); -pub const match = match_impl.match; -pub const detectGlobSyntax = match_impl.detectGlobSyntax; - pub const GlobWalker = walk.GlobWalker_; pub const BunGlobWalker = GlobWalker(null, walk.SyscallAccessor, false); pub const BunGlobWalkerZ = GlobWalker(null, walk.SyscallAccessor, true); + +/// Returns true if the given string contains glob syntax, +/// excluding those escaped with backslashes +/// TODO: this doesn't play nicely with Windows directory separator and +/// backslashing, should we just require the user to supply posix filepaths? +pub fn detectGlobSyntax(potential_pattern: []const u8) bool { + // Negation only allowed in the beginning of the pattern + if (potential_pattern.len > 0 and potential_pattern[0] == '!') return true; + + // In descending order of how popular the token is + const SPECIAL_SYNTAX: [4]u8 = comptime [_]u8{ '*', '{', '[', '?' }; + + inline for (SPECIAL_SYNTAX) |token| { + var slice = potential_pattern[0..]; + while (slice.len > 0) { + if (std.mem.indexOfScalar(u8, slice, token)) |idx| { + // Check for even number of backslashes preceding the + // token to know that it's not escaped + var i = idx; + var backslash_count: u16 = 0; + + while (i > 0 and potential_pattern[i - 1] == '\\') : (i -= 1) { + backslash_count += 1; + } + + if (backslash_count % 2 == 0) return true; + slice = slice[idx + 1 ..]; + } else break; + } + } + + return false; +} diff --git a/src/glob/GlobWalker.zig b/src/glob/GlobWalker.zig index 2fb0b13f2d..b9f6564641 100644 --- a/src/glob/GlobWalker.zig +++ b/src/glob/GlobWalker.zig @@ -1322,7 +1322,6 @@ pub fn GlobWalker_( fn matchPatternSlow(this: *GlobWalker, pattern_component: *Component, filepath: []const u8) bool { return match( - this.arena.allocator(), pattern_component.patternSlice(this.pattern), filepath, ).matches(); @@ -1684,5 +1683,3 @@ pub fn matchWildcardFilepath(glob: []const u8, path: []const u8) bool { pub fn matchWildcardLiteral(literal: []const u8, path: []const u8) bool { return std.mem.eql(u8, literal, path); } - -pub const matchImpl = match; diff --git a/src/glob/match.zig b/src/glob/match.zig index 95f4817d23..5bebf2b246 100644 --- a/src/glob/match.zig +++ b/src/glob/match.zig @@ -25,8 +25,6 @@ const std = @import("std"); const bun = @import("bun"); -const Allocator = std.mem.Allocator; - /// used in matchBrace to determine the size of the stack buffer used in the stack fallback allocator /// that is created for handling braces /// One such stack buffer is created recursively for each pair of braces @@ -38,7 +36,7 @@ const Brace = struct { }; const BraceStack = std.BoundedArray(Brace, 10); -pub const MatchResult = enum { +const MatchResult = enum { no_match, match, @@ -121,7 +119,7 @@ const Wildcard = struct { /// Used to escape any of the special characters above. // TODO: consider just taking arena and resetting to initial state, // all usages of this function pass in Arena.allocator() -pub fn match(_: Allocator, glob: []const u8, path: []const u8) MatchResult { +pub fn match(glob: []const u8, path: []const u8) MatchResult { var state = State{}; var negated = false; @@ -491,39 +489,6 @@ inline fn skipGlobstars(glob: []const u8, glob_index: *u32) void { glob_index.* -= 2; } -/// Returns true if the given string contains glob syntax, -/// excluding those escaped with backslashes -/// TODO: this doesn't play nicely with Windows directory separator and -/// backslashing, should we just require the user to supply posix filepaths? -pub fn detectGlobSyntax(potential_pattern: []const u8) bool { - // Negation only allowed in the beginning of the pattern - if (potential_pattern.len > 0 and potential_pattern[0] == '!') return true; - - // In descending order of how popular the token is - const SPECIAL_SYNTAX: [4]u8 = comptime [_]u8{ '*', '{', '[', '?' }; - - inline for (SPECIAL_SYNTAX) |token| { - var slice = potential_pattern[0..]; - while (slice.len > 0) { - if (std.mem.indexOfScalar(u8, slice, token)) |idx| { - // Check for even number of backslashes preceding the - // token to know that it's not escaped - var i = idx; - var backslash_count: u16 = 0; - - while (i > 0 and potential_pattern[i - 1] == '\\') : (i -= 1) { - backslash_count += 1; - } - - if (backslash_count % 2 == 0) return true; - slice = slice[idx + 1 ..]; - } else break; - } - } - - return false; -} - const BraceIndex = struct { start: u32 = 0, end: u32 = 0, diff --git a/src/install/install.zig b/src/install/install.zig index aa91191385..83b1445674 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -9386,7 +9386,7 @@ pub const PackageManager = struct { var resolver: void = {}; var package = Lockfile.Package{}; - try package.parseWithJSON(lockfile, manager, manager.allocator, manager.log, package_json_source, json, void, &resolver, Features.folder); + try package.fromJson(lockfile, manager, manager.allocator, manager.log, package_json_source, json, void, &resolver, Features.folder); const name = lockfile.str(&package.name); const actual_package = switch (lockfile.package_index.get(package.name_hash) orelse { @@ -9797,7 +9797,7 @@ pub const PackageManager = struct { var resolver: void = {}; var package = Lockfile.Package{}; - try package.parseWithJSON(lockfile, manager, manager.allocator, manager.log, package_json_source, json, void, &resolver, Features.folder); + try package.fromJson(lockfile, manager, manager.allocator, manager.log, package_json_source, json, void, &resolver, Features.folder); const name = lockfile.str(&package.name); const actual_package = switch (lockfile.package_index.get(package.name_hash) orelse { @@ -12594,8 +12594,13 @@ pub const PackageManager = struct { lockfile.overrides.count(&lockfile, builder); lockfile.catalogs.count(&lockfile, builder); + for (lockfile.nohoist_patterns.items) |pattern| { + builder.count(pattern.slice(lockfile.buffers.string_bytes.items)); + } maybe_root.scripts.count(lockfile.buffers.string_bytes.items, *Lockfile.StringBuilder, builder); + manager.lockfile.hoisting_limits = lockfile.hoisting_limits; + const off = @as(u32, @truncate(manager.lockfile.buffers.dependencies.items.len)); const len = @as(u32, @truncate(new_dependencies.len)); var packages = manager.lockfile.packages.slice(); @@ -12628,6 +12633,12 @@ pub const PackageManager = struct { manager.lockfile.overrides = try lockfile.overrides.clone(manager, &lockfile, manager.lockfile, builder); manager.lockfile.catalogs = try lockfile.catalogs.clone(manager, &lockfile, manager.lockfile, builder); + manager.lockfile.nohoist_patterns.clearRetainingCapacity(); + try manager.lockfile.nohoist_patterns.ensureTotalCapacity(manager.lockfile.allocator, lockfile.nohoist_patterns.items.len); + for (lockfile.nohoist_patterns.items) |pattern| { + manager.lockfile.nohoist_patterns.appendAssumeCapacity(builder.append(String, pattern.slice(lockfile.buffers.string_bytes.items))); + } + manager.lockfile.trusted_dependencies = if (lockfile.trusted_dependencies) |trusted_dependencies| try trusted_dependencies.clone(manager.lockfile.allocator) else @@ -13118,7 +13129,7 @@ pub const PackageManager = struct { }, }; - switch (bun.glob.walk.matchImpl(manager.allocator, pattern, path_or_name)) { + switch (bun.glob.match(pattern, path_or_name)) { .match, .negate_match => install_root_dependencies = true, .negate_no_match => { diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index c328b05263..aece496330 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -25,6 +25,21 @@ trusted_dependencies: ?TrustedDependenciesSet = null, patched_dependencies: PatchedDependenciesMap = .{}, overrides: OverrideMap = .{}, catalogs: CatalogMap = .{}, +nohoist_patterns: std.ArrayListUnmanaged(String) = .{}, + +hoisting_limits: HoistingLimits = .none, + +pub const HoistingLimits = enum(u8) { + none, + workspaces, + dependencies, + + const Map = bun.ComptimeEnumMap(HoistingLimits); + + pub fn fromStr(input: string) ?HoistingLimits { + return Map.get(input); + } +}; pub const Stream = std.io.FixedBufferStream([]u8); pub const default_filename = "bun.lockb"; @@ -335,7 +350,9 @@ pub fn loadFromBytes(this: *Lockfile, pm: ?*PackageManager, buf: []u8, allocator this.workspace_versions = .{}; this.overrides = .{}; this.catalogs = .{}; + this.nohoist_patterns = .{}; this.patched_dependencies = .{}; + this.hoisting_limits = .none; const load_result = Lockfile.Serializer.load(this, &stream, allocator, log, pm) catch |err| { return LoadResult{ .err = .{ .step = .parse_file, .value = err, .lockfile_path = "bun.lockb", .format = .binary } }; @@ -612,6 +629,10 @@ pub fn cleanWithLogger( new.initEmpty( old.allocator, ); + + // important to set this before cloner.flush() + new.hoisting_limits = old.hoisting_limits; + try new.string_pool.ensureTotalCapacity(old.string_pool.capacity()); try new.package_index.ensureTotalCapacity(old.package_index.capacity()); try new.packages.ensureTotalCapacity(old.allocator, old.packages.len); @@ -624,9 +645,16 @@ pub fn cleanWithLogger( var builder = new.stringBuilder(); old.overrides.count(old, &builder); old.catalogs.count(old, &builder); + for (old.nohoist_patterns.items) |pattern| { + builder.count(pattern.slice(old.buffers.string_bytes.items)); + } try builder.allocate(); new.overrides = try old.overrides.clone(manager, old, new, &builder); new.catalogs = try old.catalogs.clone(manager, old, new, &builder); + try new.nohoist_patterns.ensureTotalCapacity(new.allocator, old.nohoist_patterns.items.len); + for (old.nohoist_patterns.items) |pattern| { + new.nohoist_patterns.appendAssumeCapacity(builder.append(String, pattern.slice(old.buffers.string_bytes.items))); + } } // Step 1. Recreate the lockfile with only the packages that are still alive @@ -891,8 +919,10 @@ pub fn hoist( }; try (Tree{}).processSubtree( + .root, Tree.root_dep_id, Tree.invalid_id, + .{}, method, &builder, if (method == .filter) manager.options.log_level, @@ -900,9 +930,13 @@ pub fn hoist( // This goes breadth-first while (builder.queue.readItem()) |item| { + var subpath = item.subpath; + defer subpath.deinit(builder.allocator); try builder.list.items(.tree)[item.tree_id].processSubtree( + item.subtree_kind, item.dependency_id, item.hoist_root_id, + subpath, method, &builder, if (method == .filter) manager.options.log_level, @@ -1207,6 +1241,8 @@ pub fn initEmpty(this: *Lockfile, allocator: Allocator) void { .workspace_versions = .{}, .overrides = .{}, .catalogs = .{}, + .nohoist_patterns = .{}, + .hoisting_limits = .none, .meta_hash = zero_hash, }; } @@ -1606,6 +1642,7 @@ pub fn deinit(this: *Lockfile) void { this.workspace_versions.deinit(this.allocator); this.overrides.deinit(this.allocator); this.catalogs.deinit(this.allocator); + this.nohoist_patterns.deinit(this.allocator); } pub const EqlSorter = struct { diff --git a/src/install/lockfile/CatalogMap.zig b/src/install/lockfile/CatalogMap.zig index a531ffc24e..20ecff0e96 100644 --- a/src/install/lockfile/CatalogMap.zig +++ b/src/install/lockfile/CatalogMap.zig @@ -17,6 +17,10 @@ pub fn get(this: *CatalogMap, lockfile: *const Lockfile, catalog_name: String, d }; } + if (this.groups.count() == 0) { + return null; + } + const group = this.groups.getContext(catalog_name, String.arrayHashContext(lockfile, null)) orelse { return null; }; diff --git a/src/install/lockfile/Package.zig b/src/install/lockfile/Package.zig index e1ed1d1c93..98891a8b2e 100644 --- a/src/install/lockfile/Package.zig +++ b/src/install/lockfile/Package.zig @@ -527,6 +527,8 @@ pub const Package = extern struct { update: u32 = 0, overrides_changed: bool = false, catalogs_changed: bool = false, + nohoist_changed: bool = false, + hoisting_limits_changed: bool = false, // bool for if this dependency should be added to lockfile trusted dependencies. // it is false when the new trusted dependency is coming from the default list. @@ -542,7 +544,11 @@ pub const Package = extern struct { } pub inline fn hasDiffs(this: Summary) bool { - return this.add > 0 or this.remove > 0 or this.update > 0 or this.overrides_changed or this.catalogs_changed or + return this.add > 0 or this.remove > 0 or this.update > 0 or + this.overrides_changed or + this.catalogs_changed or + this.nohoist_changed or + this.hoisting_limits_changed or this.added_trusted_dependencies.count() > 0 or this.removed_trusted_dependencies.count() > 0 or this.patched_dependencies_changed; @@ -592,60 +598,28 @@ pub const Package = extern struct { } } - if (is_root) catalogs: { + if (is_root) { + catalogs: { - // don't sort if lengths are different - if (from_lockfile.catalogs.default.count() != to_lockfile.catalogs.default.count()) { - summary.catalogs_changed = true; - break :catalogs; - } - - if (from_lockfile.catalogs.groups.count() != to_lockfile.catalogs.groups.count()) { - summary.catalogs_changed = true; - break :catalogs; - } - - from_lockfile.catalogs.sort(from_lockfile); - to_lockfile.catalogs.sort(to_lockfile); - - for ( - from_lockfile.catalogs.default.keys(), - from_lockfile.catalogs.default.values(), - to_lockfile.catalogs.default.keys(), - to_lockfile.catalogs.default.values(), - ) |from_dep_name, *from_dep, to_dep_name, *to_dep| { - if (!from_dep_name.eql(to_dep_name, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items)) { + // don't sort if lengths are different + if (from_lockfile.catalogs.default.count() != to_lockfile.catalogs.default.count()) { summary.catalogs_changed = true; break :catalogs; } - if (!from_dep.eql(to_dep, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items)) { - summary.catalogs_changed = true; - break :catalogs; - } - } - - for ( - from_lockfile.catalogs.groups.keys(), - from_lockfile.catalogs.groups.values(), - to_lockfile.catalogs.groups.keys(), - to_lockfile.catalogs.groups.values(), - ) |from_catalog_name, from_catalog_deps, to_catalog_name, to_catalog_deps| { - if (!from_catalog_name.eql(to_catalog_name, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items)) { + if (from_lockfile.catalogs.groups.count() != to_lockfile.catalogs.groups.count()) { summary.catalogs_changed = true; break :catalogs; } - if (from_catalog_deps.count() != to_catalog_deps.count()) { - summary.catalogs_changed = true; - break :catalogs; - } + from_lockfile.catalogs.sort(from_lockfile); + to_lockfile.catalogs.sort(to_lockfile); for ( - from_catalog_deps.keys(), - from_catalog_deps.values(), - to_catalog_deps.keys(), - to_catalog_deps.values(), + from_lockfile.catalogs.default.keys(), + from_lockfile.catalogs.default.values(), + to_lockfile.catalogs.default.keys(), + to_lockfile.catalogs.default.values(), ) |from_dep_name, *from_dep, to_dep_name, *to_dep| { if (!from_dep_name.eql(to_dep_name, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items)) { summary.catalogs_changed = true; @@ -657,6 +631,71 @@ pub const Package = extern struct { break :catalogs; } } + + for ( + from_lockfile.catalogs.groups.keys(), + from_lockfile.catalogs.groups.values(), + to_lockfile.catalogs.groups.keys(), + to_lockfile.catalogs.groups.values(), + ) |from_catalog_name, from_catalog_deps, to_catalog_name, to_catalog_deps| { + if (!from_catalog_name.eql(to_catalog_name, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items)) { + summary.catalogs_changed = true; + break :catalogs; + } + + if (from_catalog_deps.count() != to_catalog_deps.count()) { + summary.catalogs_changed = true; + break :catalogs; + } + + for ( + from_catalog_deps.keys(), + from_catalog_deps.values(), + to_catalog_deps.keys(), + to_catalog_deps.values(), + ) |from_dep_name, *from_dep, to_dep_name, *to_dep| { + if (!from_dep_name.eql(to_dep_name, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items)) { + summary.catalogs_changed = true; + break :catalogs; + } + + if (!from_dep.eql(to_dep, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items)) { + summary.catalogs_changed = true; + break :catalogs; + } + } + } + } + + nohoist: { + if (from_lockfile.nohoist_patterns.items.len != to_lockfile.nohoist_patterns.items.len) { + summary.nohoist_changed = true; + break :nohoist; + } + + const Sorter = String.Sorter(.asc); + var sorter: Sorter = .{ + .lhs_buf = from_lockfile.buffers.string_bytes.items, + .rhs_buf = from_lockfile.buffers.string_bytes.items, + }; + std.sort.pdq(String, from_lockfile.nohoist_patterns.items, sorter, Sorter.lessThan); + sorter = .{ + .lhs_buf = to_lockfile.buffers.string_bytes.items, + .rhs_buf = to_lockfile.buffers.string_bytes.items, + }; + std.sort.pdq(String, to_lockfile.nohoist_patterns.items, sorter, Sorter.lessThan); + + for (from_lockfile.nohoist_patterns.items, to_lockfile.nohoist_patterns.items) |from_pattern, to_pattern| { + if (!from_pattern.eql(to_pattern, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items)) { + summary.nohoist_changed = true; + break :nohoist; + } + } + } + + // hoistingLimits + if (from_lockfile.hoisting_limits != to_lockfile.hoisting_limits) { + summary.hoisting_limits_changed = true; } } @@ -863,7 +902,7 @@ pub const Package = extern struct { const json = pm.workspace_package_json_cache.getWithSource(bun.default_allocator, log, source, .{}).unwrap() catch break :brk false; var resolver: void = {}; - try workspace.parseWithJSON( + try workspace.fromJson( to_lockfile, pm, allocator, @@ -964,7 +1003,7 @@ pub const Package = extern struct { Global.crash(); }; - try package.parseWithJSON( + try package.fromJson( lockfile, pm, allocator, @@ -1265,7 +1304,7 @@ pub const Package = extern struct { return this_dep; } - pub fn parseWithJSON( + pub fn fromJson( package: *Package, lockfile: *Lockfile, pm: *PackageManager, @@ -1576,6 +1615,41 @@ pub const Package = extern struct { if (json.get("workspaces")) |workspaces_expr| { lockfile.catalogs.parseCount(lockfile, workspaces_expr, &string_builder); + + if (workspaces_expr.get("nohoist")) |nohoist_expr| { + switch (nohoist_expr.data) { + .e_array => |nohoist_arr| { + for (nohoist_arr.slice()) |pattern_expr| { + switch (pattern_expr.data) { + .e_string => |pattern_str| { + string_builder.count(pattern_str.slice(allocator)); + }, + else => { + try log.addError(&source, pattern_expr.loc, "Expected a string"); + return error.InvalidPackageJSON; + }, + } + } + }, + else => { + try log.addError(&source, nohoist_expr.loc, "Expected an array of strings"); + return error.InvalidPackageJSON; + }, + } + } + + if (workspaces_expr.get("hoistingLimits")) |hoisting_limits_expr| { + if (!hoisting_limits_expr.isString()) { + try log.addError(&source, hoisting_limits_expr.loc, "Expected one string value of \"none\", \"workspaces\", or \"dependencies\""); + return error.InvalidPackageJSON; + } + + const hoisting_limits_str = hoisting_limits_expr.data.e_string.slice(allocator); + lockfile.hoisting_limits = Lockfile.HoistingLimits.fromStr(hoisting_limits_str) orelse { + try log.addError(&source, hoisting_limits_expr.loc, "Expected one of \"none\", \"workspaces\", or \"dependencies\""); + return error.InvalidPackageJSON; + }; + } } } @@ -1941,6 +2015,13 @@ pub const Package = extern struct { try lockfile.overrides.parseAppend(pm, lockfile, package, log, source, json, &string_builder); if (json.get("workspaces")) |workspaces_expr| { try lockfile.catalogs.parseAppend(pm, lockfile, log, &source, workspaces_expr, &string_builder); + if (workspaces_expr.get("nohoist")) |nohoist_expr| { + lockfile.nohoist_patterns.clearRetainingCapacity(); + try lockfile.nohoist_patterns.ensureTotalCapacity(allocator, nohoist_expr.data.e_array.items.len); + for (nohoist_expr.data.e_array.slice()) |pattern_expr| { + lockfile.nohoist_patterns.appendAssumeCapacity(string_builder.append(String, pattern_expr.data.e_string.slice(allocator))); + } + } } } diff --git a/src/install/lockfile/Package/WorkspaceMap.zig b/src/install/lockfile/Package/WorkspaceMap.zig index c5cda6f5c3..68411f3f57 100644 --- a/src/install/lockfile/Package/WorkspaceMap.zig +++ b/src/install/lockfile/Package/WorkspaceMap.zig @@ -128,7 +128,7 @@ pub fn processNamesArray( if (input_path.len == 0 or input_path.len == 1 and input_path[0] == '.') continue; - if (Glob.detectGlobSyntax(input_path)) { + if (glob.detectGlobSyntax(input_path)) { workspace_globs.append(input_path) catch bun.outOfMemory(); continue; } @@ -373,7 +373,7 @@ fn ignoredWorkspacePaths(path: []const u8) bool { } return false; } -const GlobWalker = Glob.GlobWalker(ignoredWorkspacePaths, Glob.walk.SyscallAccessor, false); +const GlobWalker = glob.GlobWalker(ignoredWorkspacePaths, glob.walk.SyscallAccessor, false); const WorkspaceMap = @This(); const bun = @import("bun"); @@ -389,7 +389,7 @@ const Allocator = std.mem.Allocator; const install = bun.install; const Lockfile = install.Lockfile; const StringBuilder = Lockfile.StringBuilder; -const Glob = bun.glob; +const glob = bun.glob; const stringZ = [:0]const u8; const Path = bun.path; const strings = bun.strings; diff --git a/src/install/lockfile/Tree.zig b/src/install/lockfile/Tree.zig index f20ff9b013..f2b7a18f66 100644 --- a/src/install/lockfile/Tree.zig +++ b/src/install/lockfile/Tree.zig @@ -52,7 +52,7 @@ pub const HoistDependencyResult = union(enum) { hoisted, placement: struct { id: Id, - bundled: bool = false, + is_new_hoist_root: bool = false, }, // replace: struct { // dest_id: Id, @@ -246,6 +246,10 @@ pub fn Builder(comptime method: BuilderMethod) type { install_root_dependencies: if (method == .filter) bool else void, path_buf: []u8, + pub fn hasNohoistPatterns(this: *const @This()) bool { + return this.lockfile.nohoist_patterns.items.len != 0; + } + pub fn maybeReportError(this: *@This(), comptime fmt: string, args: anytype) void { this.log.addErrorFmt(null, logger.Loc.Empty, this.allocator, fmt, args) catch {}; } @@ -316,10 +320,21 @@ pub fn Builder(comptime method: BuilderMethod) type { }; } +const SubtreeKind = enum(u8) { + root, + root_direct_dependency, + root_transitive, + workspace, + workspace_direct_dependency, + workspace_transitive, +}; + pub fn processSubtree( this: *const Tree, + subtree_kind: SubtreeKind, dependency_id: DependencyID, hoist_root_id: Tree.Id, + subpath: std.ArrayListUnmanaged(u8), comptime method: BuilderMethod, builder: *Builder(method), log_level: if (method == .filter) PackageManager.Options.LogLevel else void, @@ -332,9 +347,11 @@ pub fn processSubtree( if (resolution_list.len == 0) return; + const parent_tree_id = this.id; + try builder.list.append(builder.allocator, .{ .tree = .{ - .parent = this.id, + .parent = parent_tree_id, .id = @as(Id, @truncate(builder.list.len)), .dependency_id = dependency_id, }, @@ -387,7 +404,7 @@ pub fn processSubtree( DepSorter.isLessThan, ); - for (builder.sort_buf.items) |dep_id| { + next_dep: for (builder.sort_buf.items) |dep_id| { const pkg_id = builder.resolutions[dep_id]; // Skip unresolved packages, e.g. "peerDependencies" if (pkg_id >= max_package_id) continue; @@ -469,7 +486,7 @@ pub fn processSubtree( }, }; - switch (bun.glob.walk.matchImpl(builder.allocator, pattern, path_or_name)) { + switch (bun.glob.match(pattern, path_or_name)) { .match, .negate_match => match = true, .negate_no_match => { @@ -491,18 +508,53 @@ pub fn processSubtree( } } + const dependency = builder.dependencies[dep_id]; + var dep_subpath: std.ArrayListUnmanaged(u8) = .{}; + if (builder.hasNohoistPatterns()) { + try dep_subpath.ensureTotalCapacity( + builder.allocator, + subpath.items.len + dependency.name.len() + @intFromBool(subpath.items.len != 0), + ); + if (subpath.items.len != 0) { + dep_subpath.appendSliceAssumeCapacity(subpath.items); + dep_subpath.appendAssumeCapacity('/'); + } + dep_subpath.appendSliceAssumeCapacity(dependency.name.slice(builder.buf())); + } + const hoisted: HoistDependencyResult = hoisted: { - const dependency = builder.dependencies[dep_id]; // don't hoist if it's a folder dependency or a bundled dependency. if (dependency.behavior.isBundled()) { - break :hoisted .{ .placement = .{ .id = next.id, .bundled = true } }; + break :hoisted .{ .placement = .{ .id = next.id, .is_new_hoist_root = true } }; } if (pkg_resolutions[pkg_id].tag == .folder) { break :hoisted .{ .placement = .{ .id = next.id } }; } + const is_new_hoist_root = switch (builder.lockfile.hoisting_limits) { + .none => false, + .workspaces => subtree_kind == .root or subtree_kind == .workspace, + + // not only does this keep transitive dependencies within direct dependencies, + // it also keeps workspace dependencies within the workspace. + .dependencies => subtree_kind != .root_transitive and subtree_kind != .workspace_transitive, + }; + + for (builder.lockfile.nohoist_patterns.items) |pattern| { + if (bun.glob.match(pattern.slice(builder.buf()), dep_subpath.items).matches()) { + // prevent hoisting this package. it's dependencies + // are allowed to hoist beyond it unless "hoistingLimits" + // sets this tree as a new hoist root. + break :hoisted .{ .placement = .{ .id = next.id, .is_new_hoist_root = is_new_hoist_root } }; + } + } + + if (is_new_hoist_root) { + break :hoisted .{ .placement = .{ .id = next.id, .is_new_hoist_root = true } }; + } + break :hoisted try next.hoistDependency( true, hoist_root_id, @@ -518,15 +570,48 @@ pub fn processSubtree( switch (hoisted) { .dependency_loop, .hoisted => continue, .placement => |dest| { + if (builder.hasNohoistPatterns() or builder.lockfile.hoisting_limits != .none) { + // Look for cycles. Only done when nohoist patterns or hoisting limits + // are used because they can cause cyclic dependencies to not + // deduplicate, resulting in infinite loops (e.g. can happen easily + // if all hoisting is disabled with '**') + + const skip_root_pkgs = switch (subtree_kind) { + .root, .root_direct_dependency, .root_transitive => false, + .workspace, .workspace_direct_dependency, .workspace_transitive => true, + }; + + // TODO: this isn't totally correct. this handles cycles, but it's + // only looking for the same package higher in the tree + var curr = parent_tree_id; + while (curr != invalid_id and (!skip_root_pkgs or curr != 0)) { + for (dependency_lists[curr].items) |placed_parent_dep_id| { + const placed_parent_pkg_id = builder.resolutions[placed_parent_dep_id]; + if (placed_parent_pkg_id == pkg_id) { + continue :next_dep; + } + } + + curr = trees[curr].parent; + } + } dependency_lists[dest.id].append(builder.allocator, dep_id) catch bun.outOfMemory(); trees[dest.id].dependencies.len += 1; if (builder.resolution_lists[pkg_id].len > 0) { + const next_subtree_kind: SubtreeKind = switch (subtree_kind) { + .root => if (dependency.behavior.isWorkspaceOnly()) .workspace else .root_direct_dependency, + .root_direct_dependency => .root_transitive, + .root_transitive => .root_transitive, + .workspace => .workspace_direct_dependency, + .workspace_direct_dependency => .workspace_transitive, + .workspace_transitive => .workspace_transitive, + }; try builder.queue.writeItem(.{ .tree_id = dest.id, + .subtree_kind = next_subtree_kind, .dependency_id = dep_id, - - // if it's bundled, start a new hoist root - .hoist_root_id = if (dest.bundled) dest.id else hoist_root_id, + .subpath = dep_subpath, + .hoist_root_id = if (dest.is_new_hoist_root) dest.id else hoist_root_id, }); } }, @@ -631,6 +716,9 @@ fn hoistDependency( pub const FillItem = struct { tree_id: Tree.Id, dependency_id: DependencyID, + subpath: std.ArrayListUnmanaged(u8), + + subtree_kind: SubtreeKind, /// If valid, dependencies will not hoist /// beyond this tree if they're in a subtree diff --git a/src/install/lockfile/bun.lock.zig b/src/install/lockfile/bun.lock.zig index 7b782d5f69..dc3b643dbc 100644 --- a/src/install/lockfile/bun.lock.zig +++ b/src/install/lockfile/bun.lock.zig @@ -372,6 +372,38 @@ pub const Stringifier = struct { try writer.writeAll("},\n"); } + if (lockfile.nohoist_patterns.items.len > 0) { + const Sorter = String.Sorter(.asc); + const sorter: Sorter = .{ + .lhs_buf = lockfile.buffers.string_bytes.items, + .rhs_buf = lockfile.buffers.string_bytes.items, + }; + std.sort.pdq(String, lockfile.nohoist_patterns.items, sorter, Sorter.lessThan); + + try writeIndent(writer, indent); + try writer.writeAll( + \\"nohoist": [ + \\ + ); + indent.* += 1; + + for (lockfile.nohoist_patterns.items) |pattern| { + try writeIndent(writer, indent); + try writer.print("{},\n", .{pattern.fmtJson(buf, .{})}); + } + + try decIndent(writer, indent); + try writer.writeAll("],\n"); + } + + if (lockfile.hoisting_limits != .none) { + try writeIndent(writer, indent); + try writer.print( + \\"hoistingLimits": "{s}", + \\ + , .{@tagName(lockfile.hoisting_limits)}); + } + var tree_deps_sort_buf: std.ArrayListUnmanaged(DependencyID) = .{}; defer tree_deps_sort_buf.deinit(allocator); @@ -1002,6 +1034,8 @@ const ParseError = OOM || error{ InvalidOverridesObject, InvalidCatalogObject, InvalidCatalogsObject, + InvalidNohoistArray, + InvalidHoistingLimitsValue, InvalidDependencyName, InvalidDependencyVersion, InvalidPackageResolution, @@ -1394,6 +1428,39 @@ pub fn parseIntoBinaryLockfile( } } + if (root.get("nohoist")) |nohoist_expr| { + if (!nohoist_expr.isArray()) { + try log.addError(source, nohoist_expr.loc, "Expected an array of strings"); + return error.InvalidNohoistArray; + } + + var nohoist_patterns: std.ArrayListUnmanaged(String) = try .initCapacity(allocator, nohoist_expr.data.e_array.items.len); + + for (nohoist_expr.data.e_array.slice()) |pattern_expr| { + if (!pattern_expr.isString()) { + try log.addError(source, pattern_expr.loc, "Expected a string"); + return error.InvalidNohoistArray; + } + + nohoist_patterns.appendAssumeCapacity(try string_buf.append(pattern_expr.data.e_string.slice(allocator))); + } + + lockfile.nohoist_patterns = nohoist_patterns; + } + + if (root.get("hoistingLimits")) |hoisting_limits_expr| { + if (!hoisting_limits_expr.isString()) { + try log.addError(source, hoisting_limits_expr.loc, "Expected a string"); + return error.InvalidHoistingLimitsValue; + } + + const hoisting_limits_str = hoisting_limits_expr.data.e_string.slice(allocator); + lockfile.hoisting_limits = BinaryLockfile.HoistingLimits.fromStr(hoisting_limits_str) orelse { + try log.addError(source, hoisting_limits_expr.loc, "Expected one of \"none\", \"workspaces\", or \"dependencies\""); + return error.InvalidHoistingLimitsValue; + }; + } + const workspaces_obj = root.getObject("workspaces") orelse { try log.addError(source, root.loc, "Missing a workspaces object property"); return error.InvalidWorkspaceObject; diff --git a/src/install/lockfile/bun.lockb.zig b/src/install/lockfile/bun.lockb.zig index 8f546724f4..6af1a927ec 100644 --- a/src/install/lockfile/bun.lockb.zig +++ b/src/install/lockfile/bun.lockb.zig @@ -7,6 +7,8 @@ const has_trusted_dependencies_tag: u64 = @bitCast(@as([8]u8, "tRuStEDd".*)); const has_empty_trusted_dependencies_tag: u64 = @bitCast(@as([8]u8, "eMpTrUsT".*)); const has_overrides_tag: u64 = @bitCast(@as([8]u8, "oVeRriDs".*)); const has_catalogs_tag: u64 = @bitCast(@as([8]u8, "cAtAlOgS".*)); +const has_nohoist_tag: u64 = @bitCast(@as([8]u8, "nO hOiSt".*)); +const has_hoisting_limits_tag: u64 = @bitCast(@as([8]u8, "hOiStLiM".*)); pub fn save(this: *Lockfile, verbose_log: bool, bytes: *std.ArrayList(u8), total_size: *usize, end_pos: *usize) !void { @@ -244,6 +246,24 @@ pub fn save(this: *Lockfile, verbose_log: bool, bytes: *std.ArrayList(u8), total } } + if (this.nohoist_patterns.items.len > 0) { + try writer.writeAll(std.mem.asBytes(&has_nohoist_tag)); + + try Lockfile.Buffers.writeArray( + StreamType, + stream, + @TypeOf(writer), + writer, + []String, + this.nohoist_patterns.items, + ); + } + + if (this.hoisting_limits != .none) { + try writer.writeAll(std.mem.asBytes(&has_hoisting_limits_tag)); + try writer.writeInt(@typeInfo(Lockfile.HoistingLimits).@"enum".tag_type, @intFromEnum(this.hoisting_limits), .little); + } + total_size.* = try stream.getPos(); try writer.writeAll(&alignment_bytes_to_repeat_buffer); @@ -520,6 +540,35 @@ pub fn load( } } + { + const remaining_in_buffer = total_buffer_size -| stream.pos; + + if (remaining_in_buffer > 8 and total_buffer_size <= stream.buffer.len) { + const next_num = try reader.readInt(u64, .little); + if (next_num == has_nohoist_tag) { + lockfile.nohoist_patterns = try Lockfile.Buffers.readArray(stream, allocator, std.ArrayListUnmanaged(String)); + } else { + stream.pos -= 8; + } + } + } + + { + const remaining_in_buffer = total_buffer_size -| stream.pos; + + if (remaining_in_buffer > 8 and total_buffer_size <= stream.buffer.len) { + const next_num = try reader.readInt(u64, .little); + if (next_num == has_hoisting_limits_tag) { + const HoistingLimitsInt = @typeInfo(Lockfile.HoistingLimits).@"enum".tag_type; + lockfile.hoisting_limits = std.meta.intToEnum(Lockfile.HoistingLimits, try reader.readInt(HoistingLimitsInt, .little)) catch { + return error.InvalidLockfile; + }; + } else { + stream.pos -= 8; + } + } + } + lockfile.scratch = Lockfile.Scratch.init(allocator); lockfile.package_index = PackageIndex.Map.initContext(allocator, .{}); lockfile.string_pool = StringPool.init(allocator); diff --git a/src/install/resolvers/folder_resolver.zig b/src/install/resolvers/folder_resolver.zig index 6e5bad2f3f..ee2a0d4ab5 100644 --- a/src/install/resolvers/folder_resolver.zig +++ b/src/install/resolvers/folder_resolver.zig @@ -184,7 +184,7 @@ pub const FolderResolution = union(Tag) { const json = try manager.workspace_package_json_cache.getWithPath(manager.allocator, manager.log, abs, .{}).unwrap(); - try package.parseWithJSON( + try package.fromJson( manager.lockfile, manager, manager.allocator, diff --git a/src/shell/interpreter.zig b/src/shell/interpreter.zig index ed91a760a9..9a7fdb9a08 100644 --- a/src/shell/interpreter.zig +++ b/src/shell/interpreter.zig @@ -30,7 +30,6 @@ const JSGlobalObject = bun.JSC.JSGlobalObject; const which = bun.which; const Braces = @import("./braces.zig"); const Syscall = bun.sys; -const Glob = @import("../glob.zig"); const ResolvePath = bun.path; const TaggedPointerUnion = bun.TaggedPointerUnion; pub const WorkPoolTask = JSC.WorkPoolTask; @@ -46,7 +45,7 @@ const ShellError = shell.ShellError; const ast = shell.AST; const SmolList = shell.SmolList; -const GlobWalker = Glob.BunGlobWalkerZ; +const GlobWalker = bun.glob.BunGlobWalkerZ; const stdin_no = 0; const stdout_no = 1; diff --git a/src/shell/shell.zig b/src/shell/shell.zig index fcc8cfece0..28d745ed39 100644 --- a/src/shell/shell.zig +++ b/src/shell/shell.zig @@ -7,7 +7,6 @@ const JSC = bun.JSC; const JSValue = bun.JSC.JSValue; const JSGlobalObject = bun.JSC.JSGlobalObject; const Syscall = @import("../sys.zig"); -const Glob = @import("../glob.zig"); const CodepointIterator = @import("../string_immutable.zig").UnsignedCodepointIterator; const isAllAscii = @import("../string_immutable.zig").isAllASCII; @@ -25,7 +24,7 @@ pub const IOReader = Interpreter.IOReader; // pub const IOWriter = interpret.IOWriter; // pub const SubprocessMini = subproc.ShellSubprocessMini; -const GlobWalker = Glob.GlobWalker_(null, true); +const GlobWalker = bun.glob.GlobWalker(null, true); // const GlobWalker = Glob.BunGlobWalker; pub const SUBSHELL_TODO_ERROR = "Subshells are not implemented, please open GitHub issue!";