diff --git a/src/bun.js/module_loader.zig b/src/bun.js/module_loader.zig index 8ba486bc7f..0fca8e98e8 100644 --- a/src/bun.js/module_loader.zig +++ b/src/bun.js/module_loader.zig @@ -1016,7 +1016,7 @@ pub const ModuleLoader = struct { continue; } - const package = pm.lockfile.packages.get(package_id); + const package = &pm.lockfile.packages.get(package_id); bun.assert(package.resolution.tag != .root); var name_and_version_hash: ?u64 = null; diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index e11de7c948..478e79a41f 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -961,8 +961,12 @@ pub const BundleV2 = struct { this.bundler.resolver.env_loader = this.bundler.env; this.linker.dev_server = bundler.options.dev_server; + this.linker.options.skip_file_path_comments = this.linker.options.minify_whitespace; if (this.bundler.options.global_cache.isEnabled()) { + // If we allow file path comments, then you get file paths to ~/.bun/install/cache/lodash@4.17.21@@1/... + // which is not a great developer experience, and bad in CI. + this.linker.options.skip_file_path_comments = true; bun.http.HTTPThread.init(&.{}); this.bundler.resolver.package_manager = bun.PackageManager.initWithEventLoop( JSC.EventLoopHandle.init(&this.linker.loop), @@ -2541,15 +2545,17 @@ pub const BundleV2 = struct { continue; } - resolveImportRecordForParseTask(this, pending_auto_install, parse_result, ast, source_dir, source, resolve_queue, &last_error, import_record); + resolveImportRecordForParseTask(this, pending_auto_install, parse_result, ast, source_dir, source, resolve_queue, &last_error, import_record, @truncate(i)); } }, true => { const import_records = ast.import_records.slice(); + pending_auto_install.*.?.pending_import_count = 0; + // only resolve for pending auto installs for (this.pending_auto_installs.items(.import_record_id)[pending_auto_install.*.?.pending_auto_install_index..][0..pending_auto_install.*.?.pending_import_total]) |import_record_id| { const import_record = &import_records[import_record_id]; - resolveImportRecordForParseTask(this, pending_auto_install, parse_result, ast, source_dir, source, resolve_queue, &last_error, import_record); + resolveImportRecordForParseTask(this, pending_auto_install, parse_result, ast, source_dir, source, resolve_queue, &last_error, import_record, import_record_id); } }, } @@ -2619,8 +2625,21 @@ pub const BundleV2 = struct { _ = url; // autofix } - pub fn onResolve(_: *BundleV2) void { + pub fn onResolve(this: *BundleV2) void { debug("onResolve", .{}); + _ = pollPendingAutoInstalls(this); + } + + pub fn onExtract( + this: *BundleV2, + dependency_id: Install.DependencyID, + data: *const Install.ExtractData, + comptime log_level: Install.PackageManager.Options.LogLevel, + ) void { + _ = dependency_id; // autofix + _ = data; // autofix + _ = log_level; // autofix + _ = pollPendingAutoInstalls(this); } pub fn runTasks(this: *BundleV2) void { @@ -2632,8 +2651,8 @@ pub const BundleV2 = struct { *BundleV2, this, .{ - .onExtract = {}, - .onResolve = AutoInstall.onResolve, + .onExtract = AutoInstall.onExtract, + .onResolve = {}, .onPackageManifestError = AutoInstall.onPackageManifestError, .onPackageDownloadError = AutoInstall.onPackageDownloadError, .progress_bar = true, @@ -2646,8 +2665,8 @@ pub const BundleV2 = struct { *BundleV2, this, .{ - .onExtract = {}, - .onResolve = AutoInstall.onResolve, + .onExtract = AutoInstall.onExtract, + .onResolve = {}, .onPackageManifestError = AutoInstall.onPackageManifestError, .onPackageDownloadError = AutoInstall.onPackageDownloadError, }, @@ -2672,6 +2691,7 @@ pub const BundleV2 = struct { pub fn hasAnyPendingAutoInstallsAfterPoll(this: *BundleV2) bool { debug("onPoll", .{}); + _ = pollPendingAutoInstalls(this); runTasks(this); return pollPendingAutoInstalls(this); } @@ -2714,10 +2734,10 @@ pub const BundleV2 = struct { switch (tag) { .resolve => { + any_pending = true; if (package_id == Install.invalid_package_id) { continue; } - any_pending = true; // if we get here, the package has already been resolved. tags[tag_i] = .download; @@ -2735,35 +2755,27 @@ pub const BundleV2 = struct { continue; } - const package = pm.lockfile.packages.get(package_id); + const package = &pm.lockfile.packages.get(package_id); bun.assert(package.resolution.tag != .root); var name_and_version_hash: ?u64 = null; var patchfile_hash: ?u64 = null; - switch (pm.determinePreinstallState(package, pm.lockfile, &name_and_version_hash, &patchfile_hash)) { + switch (pm.determinePreinstallStateAndPossiblyDownloadPackageIntoCache(package, root_id, pm.lockfile, &name_and_version_hash, &patchfile_hash)) { .done => { - // we are only truly done if all the dependencies are done. - const current_tasks = pm.total_tasks; - // so if enqueuing all the dependencies produces no new tasks, we are done. - pm.enqueueDependencyList(package.dependencies); - if (current_tasks == pm.total_tasks) { - tags[tag_i] = .done; - prev_done_count += 1; + tags[tag_i] = .done; + prev_done_count += 1; - const remaining_auto_installs = auto_install.pending_import_count; - const auto_installs_traversed = tag_i - prev_auto_install_index; - const has_visited_all_import_records_for_this_auto_install = auto_installs_traversed == remaining_auto_installs; - if (has_visited_all_import_records_for_this_auto_install and prev_done_count == remaining_auto_installs) { - if (auto_install.parse_task_result != null) { - debug("Auto installs for {s} are done", .{auto_install.source().path.text}); - } - - // Mark as no longer pending - auto_install.pending_import_count = 0; - auto_installs_run_queue.append(auto_install) catch bun.outOfMemory(); + const remaining_auto_installs = auto_install.pending_import_count; + const auto_installs_traversed = (tag_i - prev_auto_install_index) + 1; + const has_visited_all_import_records_for_this_auto_install = auto_installs_traversed == remaining_auto_installs; + if (has_visited_all_import_records_for_this_auto_install and prev_done_count == remaining_auto_installs) { + if (auto_install.parse_task_result != null) { + debug("Auto installs for {s} are done", .{auto_install.source().path.text}); } - } else { - any_pending = true; + + // Mark as no longer pending + auto_install.pending_import_count = 0; + auto_installs_run_queue.append(auto_install) catch bun.outOfMemory(); } }, .extracting => { @@ -2781,11 +2793,13 @@ pub const BundleV2 = struct { if (!any_pending) { pm.endProgressBar(); + } else { + pm.flushDependencyQueue(); } for (auto_installs_run_queue.items) |auto_install_const| { var auto_install: ?*AutoInstall = auto_install_const; - onParseTaskCompleteWithAutoInstall(auto_install.*.parse_task_result.?, this, &auto_install); + onParseTaskCompleteWithAutoInstall(auto_install.?.*.parse_task_result.?, this, &auto_install); } return any_pending; @@ -2806,6 +2820,7 @@ pub const BundleV2 = struct { resolve_queue: *ResolveQueue, last_error: *?anyerror, import_record: *ImportRecord, + import_record_id: u32, ) void { const bundler, const renderer: bake.Graph, const target = if (import_record.tag == .bake_resolve_to_ssr_graph) @@ -2875,6 +2890,7 @@ pub const BundleV2 = struct { var pending = pending_; pending.user_data = pending_auto_install.*.?; + pending.import_record_id = import_record_id; this.pending_auto_installs.append(this.graph.allocator, pending) catch bun.outOfMemory(); return; @@ -5443,6 +5459,7 @@ pub const LinkerContext = struct { minify_whitespace: bool = false, minify_syntax: bool = false, minify_identifiers: bool = false, + skip_file_path_comments: bool = false, banner: []const u8 = "", footer: []const u8 = "", experimental_css: bool = false, @@ -9631,7 +9648,7 @@ pub const LinkerContext = struct { compile_results_for_source_map.setCapacity(worker.allocator, compile_results.len) catch bun.outOfMemory(); const show_comments = c.options.mode == .bundle and - !c.options.minify_whitespace; + !c.options.skip_file_path_comments; const emit_targets_in_commands = show_comments and (if (ctx.c.framework) |fw| fw.server_components != null else false); diff --git a/src/cli.zig b/src/cli.zig index 14756c7856..d0d1f680a1 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -285,7 +285,7 @@ pub const Arguments = struct { clap.parseParam("--experimental-css Enabled experimental CSS bundling") catch unreachable, clap.parseParam("--experimental-css-chunking Chunk CSS files together to reduce duplicated CSS loaded in a browser. Only has an affect when multiple entrypoints import CSS") catch unreachable, clap.parseParam("--no-install Disable auto install in the Bun runtime") catch unreachable, - clap.parseParam("--auto-install Configure auto-install behavior. One of \"auto\" (default, auto-installs when no node_modules), \"fallback\" (missing packages only), \"force\" (always).") catch unreachable, + clap.parseParam("--auto-install ? Configure auto-install behavior. One of \"auto\" (auto-installs when no node_modules), \"fallback\" (missing packages only), \"force\" (always), \"disable\" (default, no auto-installs).") catch unreachable, clap.parseParam("-i Auto-install dependencies during execution. Equivalent to --install=fallback.") catch unreachable, clap.parseParam("--dump-environment-variables") catch unreachable, clap.parseParam("--conditions ... Pass custom conditions to resolve") catch unreachable, @@ -846,6 +846,8 @@ pub const Arguments = struct { Output.errGeneric("Invalid value for --auto-install: \"{s}\". Must be either \"auto\", \"fallback\", \"force\", or \"disable\"\n", .{enum_value}); Global.exit(1); } + } else { + ctx.debug.global_cache = .disable; } if (args.options("--external").len > 0) { diff --git a/src/install/install.zig b/src/install/install.zig index 0eadd3674c..aae420ee34 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -3166,49 +3166,48 @@ pub const PackageManager = struct { if (comptime Environment.allow_assert) bun.assert(this.lockfile.buffers.dependencies.items.len == this.lockfile.buffers.resolutions.items.len); break :brk .{ @truncate(index), false }; }; + _ = is_newly_added; // autofix const initial_pending_task_count = this.pendingTaskCount(); + _ = initial_pending_task_count; // autofix - if (this.lockfile.buffers.resolutions.items[dep_id] == invalid_package_id) { + var resolution_id = this.lockfile.buffers.resolutions.items[dep_id]; + if (resolution_id == invalid_package_id) { this.enqueueDependencyWithMainAndSuccessFn( dep_id, &this.lockfile.buffers.dependencies.items[dep_id], - invalid_package_id, + resolution_id, false, assignRootResolution, failRootResolution, ) catch |err| { return .{ .failure = err }; }; + // We might immediately resolve it. + resolution_id = this.lockfile.buffers.resolutions.items[dep_id]; } - const resolution_id = switch (this.lockfile.buffers.resolutions.items[dep_id]) { - invalid_package_id => brk: { - this.drainDependencyList(); + this.drainDependencyList(); - break :brk this.lockfile.buffers.resolutions.items[dep_id]; - }, - // we managed to synchronously resolve the dependency - else => |pkg_id| pkg_id, - }; - - if (resolution_id == invalid_package_id and (!is_newly_added or this.pendingTaskCount() > initial_pending_task_count)) { - bun.assert(this.pendingTaskCount() > 0); - return .{ .pending = dep_id }; + var out_name_and_version_hash: ?u64 = null; + var out_patchfile_hash: ?u64 = null; + if (resolution_id != invalid_package_id) { + switch (this.determinePreinstallStateAndPossiblyDownloadPackageIntoCache(&this.lockfile.packages.get(resolution_id), dep_id, this.lockfile, &out_name_and_version_hash, &out_patchfile_hash)) { + .extract, .extracting => { + return .{ .pending = dep_id }; + }, + else => { + return .{ + .resolution = .{ + .resolution = this.lockfile.packages.items(.resolution)[resolution_id], + .package_id = resolution_id, + }, + }; + }, + } } - if (resolution_id == invalid_package_id) { - return .{ - .not_found = {}, - }; - } - - return .{ - .resolution = .{ - .resolution = this.lockfile.packages.items(.resolution)[resolution_id], - .package_id = resolution_id, - }, - }; + return .{ .pending = dep_id }; } pub fn globalLinkDir(this: *PackageManager) !std.fs.Dir { @@ -3287,7 +3286,7 @@ pub const PackageManager = struct { pub fn determinePreinstallState( manager: *PackageManager, - this: Package, + this: *const Package, lockfile: *Lockfile, out_name_and_version_hash: *?u64, out_patchfile_hash: *?u64, @@ -3372,6 +3371,94 @@ pub const PackageManager = struct { } } + pub fn determinePreinstallStateAndPossiblyDownloadPackageIntoCache( + manager: *PackageManager, + this: *const Package, + dependency_id: DependencyID, + lockfile: *Lockfile, + out_name_and_version_hash: *?u64, + out_patchfile_hash: *?u64, + ) PreinstallState { + switch (manager.getPreinstallState(this.meta.id)) { + .unknown => switch (manager.determinePreinstallState(this, lockfile, out_name_and_version_hash, out_patchfile_hash)) { + .extract => { + if (this.resolution.tag.canEnqueueInstallTask()) { + switch (this.resolution.tag) { + .git => { + manager.enqueueGitForCheckout( + dependency_id, + "", + &this.resolution, + .{ .root_dependency = dependency_id }, + out_name_and_version_hash.*, + ); + }, + .github => { + const url = manager.allocGitHubURL(&this.resolution.value.github); + defer manager.allocator.free(url); + manager.enqueueTarballForDownload( + dependency_id, + this.meta.id, + url, + .{ .root_dependency = dependency_id }, + out_name_and_version_hash.*, + ); + }, + .local_tarball => { + manager.enqueueTarballForReading( + dependency_id, + "", + &this.resolution, + .{ .root_dependency = dependency_id }, + ); + }, + .remote_tarball => { + manager.enqueueTarballForDownload( + dependency_id, + this.meta.id, + manager.lockfile.str(&this.resolution.value.remote_tarball), + .{ .root_dependency = dependency_id }, + out_name_and_version_hash.*, + ); + }, + .npm => { + if (comptime Environment.isDebug) { + // Very old versions of Bun didn't store the tarball url when it didn't seem necessary + // This caused bugs. We can't assert on it because they could come from old lockfiles + if (this.resolution.value.npm.url.isEmpty()) { + Output.debugWarn("package {s}@{} missing tarball_url", .{ manager.lockfile.str(&this.name), this.resolution.fmt(manager.lockfile.buffers.string_bytes.items, .posix) }); + } + } + + manager.enqueuePackageForDownload( + manager.lockfile.str(&this.name), + dependency_id, + this.meta.id, + this.resolution.value.npm.version, + manager.lockfile.str(&this.resolution.value.npm.url), + .{ .root_dependency = dependency_id }, + out_name_and_version_hash.*, + ); + }, + else => { + if (comptime Environment.allow_assert) { + @panic("unreachable, handled above"); + } + }, + } + + manager.setPreinstallState(this.meta.id, lockfile, .extracting); + return .extracting; + } + + return .extract; + }, + else => |val| return val, + }, + else => |val| return val, + } + } + pub fn scopeForPackageName(this: *const PackageManager, name: string) *const Npm.Registry.Scope { if (name.len == 0 or name[0] != '@') return &this.options.scope; return this.options.registries.getPtr( @@ -4213,7 +4300,7 @@ pub const PackageManager = struct { var patchfile_hash: ?u64 = null; return switch (this.determinePreinstallState( - package, + &package, this.lockfile, &name_and_version_hash, &patchfile_hash, @@ -6210,6 +6297,7 @@ pub const PackageManager = struct { const name = manifest_req.name; if (comptime log_level.showProgress()) { if (!has_updated_this_run) { + manager.startProgressBarIfNone(); manager.setNodeName(manager.downloads_node.?, name.slice(), ProgressStrings.download_emoji, true); has_updated_this_run = true; } @@ -6557,6 +6645,7 @@ pub const PackageManager = struct { if (comptime log_level.showProgress()) { if (!has_updated_this_run) { + manager.startProgressBarIfNone(); manager.setNodeName(manager.downloads_node.?, extract.name.slice(), ProgressStrings.extract_emoji, true); has_updated_this_run = true; } @@ -6624,6 +6713,7 @@ pub const PackageManager = struct { if (comptime log_level.showProgress()) { if (!has_updated_this_run) { + manager.startProgressBarIfNone(); manager.setNodeName(manager.downloads_node.?, manifest.name(), ProgressStrings.download_emoji, true); has_updated_this_run = true; } @@ -6751,6 +6841,7 @@ pub const PackageManager = struct { if (comptime log_level.showProgress()) { if (!has_updated_this_run) { + manager.startProgressBarIfNone(); manager.setNodeName(manager.downloads_node.?, alias, ProgressStrings.extract_emoji, true); has_updated_this_run = true; } @@ -6796,6 +6887,7 @@ pub const PackageManager = struct { if (comptime log_level.showProgress()) { if (!has_updated_this_run) { + manager.startProgressBarIfNone(); manager.setNodeName(manager.downloads_node.?, name, ProgressStrings.download_emoji, true); has_updated_this_run = true; } @@ -6875,6 +6967,7 @@ pub const PackageManager = struct { if (comptime log_level.showProgress()) { if (!has_updated_this_run) { + manager.startProgressBarIfNone(); manager.setNodeName(manager.downloads_node.?, alias.slice(), ProgressStrings.download_emoji, true); has_updated_this_run = true; } @@ -6888,6 +6981,7 @@ pub const PackageManager = struct { if (comptime log_level.showProgress()) { if (@hasField(@TypeOf(callbacks), "progress_bar") and callbacks.progress_bar == true) { const completed_items = manager.total_tasks - manager.pendingTaskCount(); + manager.startProgressBarIfNone(); if (completed_items != manager.downloads_node.?.unprotected_completed_items or has_updated_this_run) { manager.downloads_node.?.setCompletedItems(completed_items); manager.downloads_node.?.setEstimatedTotalItems(manager.total_tasks); diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index 3005a0c548..ae2a788697 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -204,7 +204,7 @@ pub const PatchTask = struct { var out_name_and_version_hash: ?u64 = null; var out_patchfile_hash: ?u64 = null; manager.setPreinstallState(pkg.meta.id, manager.lockfile, .unknown); - switch (manager.determinePreinstallState(pkg, manager.lockfile, &out_name_and_version_hash, &out_patchfile_hash)) { + switch (manager.determinePreinstallState(&pkg, manager.lockfile, &out_name_and_version_hash, &out_patchfile_hash)) { .done => { // patched pkg in folder path, should now be handled by PackageInstall.install() debug("pkg: {s} done", .{pkg.name.slice(manager.lockfile.buffers.string_bytes.items)}); diff --git a/src/install/resolution.zig b/src/install/resolution.zig index 2fecc21aff..42ca8eb78c 100644 --- a/src/install/resolution.zig +++ b/src/install/resolution.zig @@ -342,7 +342,10 @@ pub const Resolution = extern struct { } pub fn canEnqueueInstallTask(this: Tag) bool { - return this == .npm or this == .local_tarball or this == .remote_tarball or this == .git or this == .github; + return switch (this) { + .npm, .local_tarball, .remote_tarball, .git, .github, .gitlab => true, + else => false, + }; } }; };