[bun install] Fix RenameAcrossMountPoints error

src/fs.zig

@@ -82,11 +82,16 @@ pub const FileSystem = struct {
 
     dirname_store: *DirnameStore,
     filename_store: *FilenameStore,
+    inside_home_dir: bool = false,
 
     _tmpdir: ?std.fs.Dir = null,
 
     threadlocal var tmpdir_handle: ?std.fs.Dir = null;
 
+    pub fn setInsideHomeDir(this: *FileSystem, home_dir: string) void {
+        this.inside_home_dir = strings.hasPrefix(this.top_level_dir, home_dir);
+    }
+
     pub fn tmpdir(fs: *FileSystem) std.fs.Dir {
         if (tmpdir_handle == null) {
             tmpdir_handle = fs.fs.openTmpDir() catch unreachable;
@@ -97,7 +102,7 @@ pub const FileSystem = struct {
 
     pub fn tmpname(_: *const FileSystem, extname: string, buf: []u8, hash: u64) ![*:0]u8 {
         // PRNG was...not so random
-        return try std.fmt.bufPrintZ(buf, "{x}{s}", .{ @truncate(u64, @intCast(u128, hash) * @intCast(u128, std.time.nanoTimestamp())), extname });
+        return try std.fmt.bufPrintZ(buf, ".{x}{s}", .{ @truncate(u64, @intCast(u128, hash) * @intCast(u128, std.time.nanoTimestamp())), extname });
     }
 
     pub var max_fd: FileDescriptorType = 0;
@@ -514,6 +519,15 @@ pub const FileSystem = struct {
             return try std.fs.openDirAbsolute(tmpdir_path, .{ .access_sub_paths = true, .iterate = true });
         }
 
+        pub fn getDefaultTempDir() string {
+            return std.os.getenvZ("BUN_TMPDIR") orelse std.os.getenvZ("TMPDIR") orelse PLATFORM_TMP_DIR;
+        }
+
+        pub fn setTempdir(path: ?string) void {
+            tmpdir_path = path orelse getDefaultTempDir();
+            tmpdir_path_set = true;
+        }
+
         pub fn fetchCacheFile(fs: *RealFS, basename: string) !std.fs.File {
             const file = try fs._fetchCacheFile(basename);
             if (comptime FeatureFlags.store_file_descriptors) {

src/install/extract_tarball.zig

@@ -16,7 +16,8 @@ const Global = @import("../global.zig").Global;
 name: strings.StringOrTinyString,
 resolution: Resolution,
 registry: string,
-cache_dir: string,
+cache_dir: std.fs.Dir,
+temp_dir: std.fs.Dir,
 package_id: PackageID,
 extracted_file_count: usize = 0,
 skip_verify: bool = false,
@@ -144,8 +145,8 @@ threadlocal var abs_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
 threadlocal var abs_buf2: [std.fs.MAX_PATH_BYTES]u8 = undefined;
 
 fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !string {
-    var tmpdir = FileSystem.instance.tmpdir();
-    var tmpname_buf: [128]u8 = undefined;
+    var tmpdir = this.temp_dir;
+    var tmpname_buf: [64]u8 = undefined;
     const name = this.name.slice();
 
     var basename = this.name.slice();
@@ -156,85 +157,84 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !string {
     }
 
     var tmpname = try FileSystem.instance.tmpname(basename, &tmpname_buf, tgz_bytes.len);
+    {
+        var extract_destination = tmpdir.makeOpenPath(std.mem.span(tmpname), .{ .iterate = true }) catch |err| {
+            Output.panic("err: {s} when create temporary directory named {s} (while extracting {s})", .{ @errorName(err), tmpname, name });
+        };
 
-    var cache_dir = tmpdir.makeOpenPath(std.mem.span(tmpname), .{ .iterate = true }) catch |err| {
-        Output.panic("err: {s} when create temporary directory named {s} (while extracting {s})", .{ @errorName(err), tmpname, name });
-    };
-    var temp_destination = std.os.getFdPath(cache_dir.fd, &abs_buf) catch |err| {
-        Output.panic("err: {s} when resolve path for temporary directory named {s} (while extracting {s})", .{ @errorName(err), tmpname, name });
-    };
-    cache_dir.close();
+        defer extract_destination.close();
 
-    if (PackageManager.verbose_install) {
-        Output.prettyErrorln("[{s}] Start extracting {s}<r>", .{ name, tmpname });
-        Output.flush();
+        if (PackageManager.verbose_install) {
+            Output.prettyErrorln("[{s}] Start extracting {s}<r>", .{ name, tmpname });
+            Output.flush();
+        }
+
+        const Archive = @import("../libarchive/libarchive.zig").Archive;
+        const Zlib = @import("../zlib.zig");
+        var zlib_pool = Npm.Registry.BodyPool.get(default_allocator);
+        zlib_pool.data.reset();
+        defer Npm.Registry.BodyPool.release(zlib_pool);
+
+        var zlib_entry = try Zlib.ZlibReaderArrayList.init(tgz_bytes, &zlib_pool.data.list, default_allocator);
+        zlib_entry.readAll() catch |err| {
+            Output.prettyErrorln(
+                "<r><red>Error {s}<r> decompressing {s}",
+                .{
+                    @errorName(err),
+                    name,
+                },
+            );
+            Output.flush();
+            Global.crash();
+        };
+        _ = if (PackageManager.verbose_install)
+            try Archive.extractToDir(
+                zlib_pool.data.list.items,
+                extract_destination,
+                null,
+                void,
+                void{},
+                // for npm packages, the root dir is always "package"
+                1,
+                true,
+                true,
+            )
+        else
+            try Archive.extractToDir(
+                zlib_pool.data.list.items,
+                extract_destination,
+                null,
+                void,
+                void{},
+                // for npm packages, the root dir is always "package"
+                1,
+                true,
+                false,
+            );
+
+        if (PackageManager.verbose_install) {
+            Output.prettyErrorln(
+                "[{s}] Extracted<r>",
+                .{
+                    name,
+                },
+            );
+            Output.flush();
+        }
     }
-
-    const Archive = @import("../libarchive/libarchive.zig").Archive;
-    const Zlib = @import("../zlib.zig");
-    var zlib_pool = Npm.Registry.BodyPool.get(default_allocator);
-    zlib_pool.data.reset();
-    defer Npm.Registry.BodyPool.release(zlib_pool);
-
-    var zlib_entry = try Zlib.ZlibReaderArrayList.init(tgz_bytes, &zlib_pool.data.list, default_allocator);
-    zlib_entry.readAll() catch |err| {
-        Output.prettyErrorln(
-            "<r><red>Error {s}<r> decompressing {s}",
-            .{
-                @errorName(err),
-                name,
-            },
-        );
-        Output.flush();
-        Global.crash();
-    };
-    _ = if (PackageManager.verbose_install)
-        try Archive.extractToDisk(
-            zlib_pool.data.list.items,
-            temp_destination,
-            null,
-            void,
-            void{},
-            // for npm packages, the root dir is always "package"
-            1,
-            true,
-            true,
-        )
-    else
-        try Archive.extractToDisk(
-            zlib_pool.data.list.items,
-            temp_destination,
-            null,
-            void,
-            void{},
-            // for npm packages, the root dir is always "package"
-            1,
-            true,
-            false,
-        );
-
-    if (PackageManager.verbose_install) {
-        Output.prettyErrorln(
-            "[{s}] Extracted<r>",
-            .{
-                name,
-            },
-        );
-        Output.flush();
-    }
-
     var folder_name = PackageManager.cachedNPMPackageFolderNamePrint(&abs_buf2, name, this.resolution.value.npm);
     if (folder_name.len == 0 or (folder_name.len == 1 and folder_name[0] == '/')) @panic("Tried to delete root and stopped it");
-    PackageManager.instance.cache_directory.deleteTree(folder_name) catch {};
+    var cache_dir = this.cache_dir;
+    cache_dir.deleteTree(folder_name) catch {};
 
     // e.g. @next
     // if it's a namespace package, we need to make sure the @name folder exists
     if (basename.len != name.len) {
-        PackageManager.instance.cache_directory.makeDir(std.mem.trim(u8, name[0 .. name.len - basename.len], "/")) catch {};
+        cache_dir.makeDir(std.mem.trim(u8, name[0 .. name.len - basename.len], "/")) catch {};
     }
 
     // Now that we've extracted the archive, we rename.
-    std.os.renameatZ(tmpdir.fd, tmpname, PackageManager.instance.cache_directory.fd, folder_name) catch |err| {
+    std.os.renameatZ(tmpdir.fd, tmpname, cache_dir.fd, folder_name) catch |err| {
         Output.prettyErrorln(
             "<r><red>Error {s}<r> moving {s} to cache dir:\n   From: {s}    To: {s}",
             .{
@@ -250,7 +250,7 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !string {
 
     // We return a resolved absolute absolute file path to the cache dir.
     // To get that directory, we open the directory again.
-    var final_dir = PackageManager.instance.cache_directory.openDirZ(folder_name, .{ .iterate = true }) catch |err| {
+    var final_dir = cache_dir.openDirZ(folder_name, .{ .iterate = true }) catch |err| {
         Output.prettyErrorln(
             "<r><red>Error {s}<r> failed to verify cache dir for {s}",
             .{

src/install/install.zig

@@ -3587,8 +3587,8 @@ pub const CacheLevel = struct {
 // 1. Download all packages, parsing their dependencies and enqueuing all dependnecies for resolution
 // 2.
 pub const PackageManager = struct {
-    cache_directory_path: string = "",
-    cache_directory: std.fs.Dir = undefined,
+    cache_directory_: ?std.fs.Dir = null,
+    temp_dir_: ?std.fs.Dir = null,
     root_dir: *Fs.FileSystem.DirEntry,
     env_loader: *DotEnv.Loader,
     allocator: std.mem.Allocator,
@@ -3669,6 +3669,114 @@ pub const PackageManager = struct {
 
     var cached_package_folder_name_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
 
+    pub inline fn getCacheDirectory(this: *PackageManager) std.fs.Dir {
+        return this.cache_directory_ orelse brk: {
+            this.cache_directory_ = this.ensureCacheDirectory();
+            break :brk this.cache_directory_.?;
+        };
+    }
+
+    pub inline fn getTemporaryDirectory(this: *PackageManager) std.fs.Dir {
+        return this.temp_dir_ orelse brk: {
+            this.temp_dir_ = this.ensureTemporaryDirectory();
+            break :brk this.temp_dir_.?;
+        };
+    }
+
+    noinline fn ensureCacheDirectory(this: *PackageManager) std.fs.Dir {
+        loop: while (true) {
+            if (this.options.enable.cache) {
+                const cache_dir = fetchCacheDirectoryPath(this.env_loader);
+                return std.fs.cwd().makeOpenPath(cache_dir.path, .{ .iterate = true }) catch {
+                    this.options.enable.cache = false;
+                    continue :loop;
+                };
+            }
+
+            return std.fs.cwd().makeOpenPath("node_modules/.cache", .{ .iterate = true }) catch |err| {
+                Output.prettyErrorln("<r><red>error<r>: Bun is unable to write files: {s}", .{@errorName(err)});
+                Output.flush();
+                Global.crash();
+            };
+        }
+        unreachable;
+    }
+
+    // We need a temporary directory that can be rename()
+    // This is important for extracting files.
+    //
+    // However, we want it to be reused! Otherwise a cache is silly.
+    //   Error RenameAcrossMountPoints moving react-is to cache dir:
+    noinline fn ensureTemporaryDirectory(this: *PackageManager) std.fs.Dir {
+        var cache_directory = this.getCacheDirectory();
+        // The chosen tempdir must be on the same filesystem as the cache directory
+        // This makes renameat() work
+        const default_tempdir = Fs.FileSystem.RealFS.getDefaultTempDir();
+        var tried_dot_tmp = false;
+        var tempdir: std.fs.Dir = std.fs.cwd().makeOpenPath(default_tempdir, .{ .iterate = true }) catch brk: {
+            tried_dot_tmp = true;
+            break :brk cache_directory.makeOpenPath(".tmp", .{ .iterate = true }) catch |err| {
+                Output.prettyErrorln("<r><red>error<r>: Bun is unable to access tempdir: {s}", .{@errorName(err)});
+                Output.flush();
+                Global.crash();
+            };
+        };
+        var tmpbuf: ["18446744073709551615".len + 8]u8 = undefined;
+        const tmpname = Fs.FileSystem.instance.tmpname("hm", &tmpbuf, 999) catch unreachable;
+        var timer: std.time.Timer = if (this.options.log_level != .silent) std.time.Timer.start() catch unreachable else undefined;
+        brk: while (true) {
+            _ = tempdir.createFileZ(tmpname, .{ .truncate = true }) catch |err2| {
+                if (!tried_dot_tmp) {
+                    tried_dot_tmp = true;
+
+                    tempdir = cache_directory.makeOpenPath(".tmp", .{ .iterate = true }) catch |err| {
+                        Output.prettyErrorln("<r><red>error<r>: Bun is unable to access tempdir: {s}", .{@errorName(err)});
+                        Output.flush();
+                        Global.crash();
+                    };
+                    continue :brk;
+                }
+                Output.prettyErrorln("<r><red>error<r>: {s} accessing temporary directory. Please set <b>$BUN_TMPDIR<r> or <b>$BUN_INSTALL_DIR<r>", .{
+                    @errorName(err2),
+                });
+                Output.flush();
+                Global.crash();
+            };
+
+            std.os.renameatZ(tempdir.fd, tmpname, cache_directory.fd, tmpname) catch |err| {
+                if (!tried_dot_tmp) {
+                    tried_dot_tmp = true;
+                    tempdir = cache_directory.makeOpenPath(".tmp", .{ .iterate = true }) catch |err2| {
+                        Output.prettyErrorln("<r><red>error<r>: Bun is unable to write files to tempdir: {s}", .{@errorName(err2)});
+                        Output.flush();
+                        Global.crash();
+                    };
+                    continue :brk;
+                }
+
+                Output.prettyErrorln("<r><red>error<r>: {s} accessing temporary directory. Please set <b>$BUN_TMPDIR<r> or <b>$BUN_INSTALL_DIR<r>", .{
+                    @errorName(err),
+                });
+                Output.flush();
+                Global.crash();
+            };
+            cache_directory.deleteFileZ(tmpname) catch {};
+            break;
+        }
+        if (this.options.log_level != .silent) {
+            const elapsed = timer.read();
+            if (elapsed > std.time.ns_per_ms * 10) {
+                var cache_dir_path = std.os.getFdPath(cache_directory.fd, &path_buf) catch "it's";
+                Output.prettyErrorln(
+                    "<r><yellow>warn<r>: Slow filesystem detected. If {s} is a network drive, consider setting $BUN_INSTALL_CACHE_DIR to a local folder.",
+                    .{cache_dir_path},
+                );
+            }
+        }
+
+        return tempdir;
+    }
+
     pub var instance: PackageManager = undefined;
 
     pub fn getNetworkTask(this: *PackageManager) *NetworkTask {
@@ -3717,7 +3825,7 @@ pub const PackageManager = struct {
 
     pub fn isFolderInCache(this: *PackageManager, folder_path: stringZ) bool {
         // TODO: is this slow?
-        var dir = this.cache_directory.openDirZ(folder_path, .{ .iterate = true }) catch return false;
+        var dir = this.getCacheDirectory().openDirZ(folder_path, .{ .iterate = true }) catch return false;
         dir.close();
         return true;
     }
@@ -3837,7 +3945,8 @@ pub const PackageManager = struct {
                     strings.StringOrTinyString.init(this.lockfile.str(package.name)),
 
                 .resolution = package.resolution,
-                .cache_dir = this.cache_directory_path,
+                .cache_dir = this.getCacheDirectory(),
+                .temp_dir = this.getTemporaryDirectory(),
                 .registry = this.registry.url.href,
                 .package_id = package.meta.id,
                 .extracted_file_count = package.meta.file_count,
@@ -4152,7 +4261,7 @@ pub const PackageManager = struct {
                             var network_entry = try this.network_dedupe_map.getOrPutContext(this.allocator, task_id, .{});
                             if (!network_entry.found_existing) {
                                 if (this.options.enable.manifest_cache) {
-                                    if (Npm.PackageManifest.Serializer.load(this.allocator, this.cache_directory, name_str) catch null) |manifest_| {
+                                    if (Npm.PackageManifest.Serializer.load(this.allocator, this.getCacheDirectory(), name_str) catch null) |manifest_| {
                                         const manifest: Npm.PackageManifest = manifest_;
                                         loaded_manifest = manifest;
 
@@ -4185,6 +4294,11 @@ pub const PackageManager = struct {
                                             _ = this.network_dedupe_map.remove(task_id);
                                             continue :retry_from_manifests_ptr;
                                         }
+
+                                        // We want to make sure the temporary directory & cache directory are loaded on the main thread
+                                        // so that we don't run into weird threading issues
+                                        // the call to getCacheDirectory() above handles the cache dir
+                                        _ = this.getTemporaryDirectory();
                                     }
                                 }
 
@@ -4303,36 +4417,34 @@ pub const PackageManager = struct {
         this.network_resolve_batch = .{};
     }
 
+    const CacheDir = struct { path: string, is_node_modules: bool };
     pub fn fetchCacheDirectoryPath(
-        _: std.mem.Allocator,
         env_loader: *DotEnv.Loader,
-        _: *Fs.FileSystem.DirEntry,
-    ) ?string {
+    ) CacheDir {
         if (env_loader.map.get("BUN_INSTALL_CACHE_DIR")) |dir| {
-            return dir;
+            return CacheDir{ .path = dir, .is_node_modules = false };
+        }
+
+        if (env_loader.map.get("HOME")) |dir| {
+            FileSystem.instance.setInsideHomeDir(dir);
+            if (FileSystem.instance.inside_home_dir) {
+                var parts = [_]string{ dir, ".bun/", "install/", "cache/" };
+                return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false };
+            }
         }
 
         if (env_loader.map.get("BUN_INSTALL")) |dir| {
             var parts = [_]string{ dir, "install/", "cache/" };
-            return Fs.FileSystem.instance.abs(&parts);
-        }
-
-        if (env_loader.map.get("HOME")) |dir| {
-            var parts = [_]string{ dir, ".bun/", "install/", "cache/" };
-            return Fs.FileSystem.instance.abs(&parts);
+            return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false };
         }
 
         if (env_loader.map.get("XDG_CACHE_HOME")) |dir| {
             var parts = [_]string{ dir, ".bun/", "install/", "cache/" };
-            return Fs.FileSystem.instance.abs(&parts);
+            return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false };
         }
 
-        if (env_loader.map.get("TMPDIR")) |dir| {
-            var parts = [_]string{ dir, ".bun-cache" };
-            return Fs.FileSystem.instance.abs(&parts);
-        }
-
-        return null;
+        var fallback_parts = [_]string{"node_modules/.bun-cache"};
+        return CacheDir{ .is_node_modules = true, .path = Fs.FileSystem.instance.abs(&fallback_parts) };
     }
 
     fn runTasks(
@@ -4399,8 +4511,7 @@ pub const PackageManager = struct {
                             entry.value_ptr.* = manifest;
                             entry.value_ptr.*.pkg.public_max_age = @truncate(u32, @intCast(u64, @maximum(0, std.time.timestamp()))) + 300;
                             {
-                                var tmpdir = Fs.FileSystem.instance.tmpdir();
-                                Npm.PackageManifest.Serializer.save(entry.value_ptr, tmpdir, PackageManager.instance.cache_directory) catch {};
+                                Npm.PackageManifest.Serializer.save(entry.value_ptr, PackageManager.instance.getTemporaryDirectory(), PackageManager.instance.getCacheDirectory()) catch {};
                             }
 
                             var dependency_list_entry = manager.task_queue.getEntry(task.task_id).?;
@@ -5142,7 +5253,6 @@ pub const PackageManager = struct {
 
         var entries_option = try fs.fs.readDirectory(fs.top_level_dir, null);
         var options = Options{};
-        var cache_directory: std.fs.Dir = undefined;
 
         var env_loader: *DotEnv.Loader = brk: {
             var map = try ctx.allocator.create(DotEnv.Map);
@@ -5160,17 +5270,6 @@ pub const PackageManager = struct {
             PackageManager.verbose_install = true;
         }
 
-        if (PackageManager.fetchCacheDirectoryPath(ctx.allocator, env_loader, &entries_option.entries)) |cache_dir_path| {
-            options.cache_directory = try fs.dirname_store.append(@TypeOf(cache_dir_path), cache_dir_path);
-            cache_directory = std.fs.cwd().makeOpenPath(options.cache_directory, .{ .iterate = true }) catch |err| brk: {
-                options.enable.cache = false;
-                options.enable.manifest_cache = false;
-                options.enable.manifest_cache_control = false;
-                Output.prettyErrorln("Cache is disabled due to error: {s}", .{@errorName(err)});
-                break :brk undefined;
-            };
-        } else {}
-
         if (PackageManager.verbose_install) {
             Output.prettyErrorln("Cache Dir: {s}", .{options.cache_directory});
             Output.flush();
@@ -5190,7 +5289,6 @@ pub const PackageManager = struct {
         manager.* = PackageManager{
             .options = options,
             .network_task_fifo = NetworkQueue.init(),
-            .cache_directory = cache_directory,
             .env_loader = env_loader,
             .allocator = ctx.allocator,
             .log = ctx.log,
@@ -6016,7 +6114,7 @@ pub const PackageManager = struct {
             switch (resolution.tag) {
                 .npm => {
                     var installer = PackageInstall{
-                        .cache_dir = this.manager.cache_directory,
+                        .cache_dir = this.manager.getCacheDirectory(),
                         .progress = this.progress,
                         .cache_dir_subpath = PackageManager.cachedNPMPackageFolderName(name, resolution.value.npm),
                         .destination_dir = this.node_modules_folder,

src/install/npm.zig

@@ -124,9 +124,7 @@ pub const Registry = struct {
             @truncate(u32, @intCast(u64, @maximum(0, std.time.timestamp()))) + 300,
         )) |package| {
             if (PackageManager.instance.options.enable.manifest_cache) {
-                var tmpdir = FileSystem.instance.tmpdir();
-
-                PackageManifest.Serializer.save(&package, tmpdir, PackageManager.instance.cache_directory) catch {};
+                PackageManifest.Serializer.save(&package, PackageManager.instance.getTemporaryDirectory(), PackageManager.instance.getCacheDirectory()) catch {};
             }
 
             return PackageVersionResponse{ .fresh = package };