Compare commits

...

3 Commits

Author SHA1 Message Date
autofix-ci[bot]
47d57fd3e2 [autofix.ci] apply automated fixes 2025-08-01 20:39:25 +00:00
Dylan Conway
a6bf7ccbee chore: remove implementation plan document 2025-08-01 13:36:41 -07:00
Dylan Conway
6961f696e3 feat: implement filesystem-aware cache for package installation
This adds filesystem detection logic to Bun's cache directory so that
the cache is placed on the same mount/filesystem as the install location
(node_modules). This enables efficient hardlink creation during package
installation.

Key changes:
- Add filesystem_utils.zig with utilities for filesystem detection
- Update PackageManagerDirectories to check filesystem boundaries
- Add fetchCacheDirectoryPathWithInstallDir that considers install location
- Add findOptimalCacheDir to walk up directory tree for writable cache location
- Add comprehensive tests for filesystem-aware cache behavior

The implementation will:
1. Use default cache when on same filesystem as node_modules
2. Find optimal cache location on same filesystem when different
3. Walk up from mount point toward project to find writable location
4. Fall back to project-local cache if no suitable location found
5. Respect explicit BUN_INSTALL_CACHE_DIR settings

🤖 Generated with Claude Code

Co-Authored-By: Claude <noreply@anthropic.com>
2025-08-01 13:35:53 -07:00
4 changed files with 549 additions and 16 deletions

View File

@@ -578,6 +578,7 @@ src/install/bin.zig
src/install/dependency.zig
src/install/ExternalSlice.zig
src/install/extract_tarball.zig
src/install/filesystem_utils.zig
src/install/hoisted_install.zig
src/install/install_binding.zig
src/install/install.zig

View File

@@ -23,7 +23,23 @@ pub inline fn getTemporaryDirectory(this: *PackageManager) std.fs.Dir {
noinline fn ensureCacheDirectory(this: *PackageManager) std.fs.Dir {
loop: while (true) {
if (this.options.enable.cache) {
const cache_dir = fetchCacheDirectoryPath(this.env, &this.options);
// Get the install directory (node_modules location)
const install_dir = brk: {
if (this.options.global) {
break :brk this.globalLinkDirPath();
} else {
var node_modules_path_buf: bun.PathBuffer = undefined;
const node_modules = Path.joinAbsStringBuf(
Fs.FileSystem.instance.top_level_dir,
&node_modules_path_buf,
&[_][]const u8{"node_modules"},
.auto,
);
break :brk node_modules;
}
};
const cache_dir = fetchCacheDirectoryPathWithInstallDir(this.env, &this.options, install_dir);
this.cache_directory_path = this.allocator.dupeZ(u8, cache_dir.path) catch bun.outOfMemory();
return std.fs.cwd().makeOpenPath(cache_dir.path, .{}) catch {
@@ -136,35 +152,68 @@ noinline fn ensureTemporaryDirectory(this: *PackageManager) std.fs.Dir {
return tempdir;
}
const CacheDir = struct { path: string, is_node_modules: bool };
const CacheDir = struct { path: string, is_node_modules: bool, is_same_filesystem: bool = true };
pub fn fetchCacheDirectoryPath(env: *DotEnv.Loader, options: ?*const Options) CacheDir {
return fetchCacheDirectoryPathWithInstallDir(env, options, null);
}
pub fn fetchCacheDirectoryPathWithInstallDir(env: *DotEnv.Loader, options: ?*const Options, install_dir: ?[]const u8) CacheDir {
if (env.get("BUN_INSTALL_CACHE_DIR")) |dir| {
return CacheDir{ .path = Fs.FileSystem.instance.abs(&[_]string{dir}), .is_node_modules = false };
const cache_path = Fs.FileSystem.instance.abs(&[_]string{dir});
const is_same_fs = if (install_dir) |install| filesystem_utils.isSameFilesystem(cache_path, install) catch true else true;
return CacheDir{ .path = cache_path, .is_node_modules = false, .is_same_filesystem = is_same_fs };
}
if (options) |opts| {
if (opts.cache_directory.len > 0) {
return CacheDir{ .path = Fs.FileSystem.instance.abs(&[_]string{opts.cache_directory}), .is_node_modules = false };
const cache_path = Fs.FileSystem.instance.abs(&[_]string{opts.cache_directory});
const is_same_fs = if (install_dir) |install| filesystem_utils.isSameFilesystem(cache_path, install) catch true else true;
return CacheDir{ .path = cache_path, .is_node_modules = false, .is_same_filesystem = is_same_fs };
}
}
if (env.get("BUN_INSTALL")) |dir| {
var parts = [_]string{ dir, "install/", "cache/" };
return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false };
const default_cache = brk: {
if (env.get("BUN_INSTALL")) |dir| {
var parts = [_]string{ dir, "install/", "cache/" };
break :brk Fs.FileSystem.instance.abs(&parts);
}
if (env.get("XDG_CACHE_HOME")) |dir| {
var parts = [_]string{ dir, ".bun/", "install/", "cache/" };
break :brk Fs.FileSystem.instance.abs(&parts);
}
if (env.get(bun.DotEnv.home_env)) |dir| {
var parts = [_]string{ dir, ".bun/", "install/", "cache/" };
break :brk Fs.FileSystem.instance.abs(&parts);
}
break :brk null;
};
// If install_dir is provided and we have a default cache, check if they're on the same filesystem
if (install_dir) |install| {
if (default_cache) |cache_path| {
const is_same_fs = filesystem_utils.isSameFilesystem(cache_path, install) catch false;
if (is_same_fs) {
return CacheDir{ .path = cache_path, .is_node_modules = false, .is_same_filesystem = true };
}
// Different filesystem - find optimal cache location
const optimal_cache = findOptimalCacheDir(install) catch null;
if (optimal_cache) |opt_cache| {
return CacheDir{ .path = opt_cache, .is_node_modules = false, .is_same_filesystem = true };
}
}
}
if (env.get("XDG_CACHE_HOME")) |dir| {
var parts = [_]string{ dir, ".bun/", "install/", "cache/" };
return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false };
}
if (env.get(bun.DotEnv.home_env)) |dir| {
var parts = [_]string{ dir, ".bun/", "install/", "cache/" };
return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false };
// Fallback to default cache or node_modules/.bun-cache
if (default_cache) |cache_path| {
return CacheDir{ .path = cache_path, .is_node_modules = false, .is_same_filesystem = false };
}
var fallback_parts = [_]string{"node_modules/.bun-cache"};
return CacheDir{ .is_node_modules = true, .path = Fs.FileSystem.instance.abs(&fallback_parts) };
return CacheDir{ .is_node_modules = true, .path = Fs.FileSystem.instance.abs(&fallback_parts), .is_same_filesystem = true };
}
pub fn cachedGitFolderNamePrint(buf: []u8, resolved: string, patch_hash: ?u64) stringZ {
@@ -739,9 +788,59 @@ const PatchHashFmt = struct {
var using_fallback_temp_dir: bool = false;
fn findOptimalCacheDir(install_dir: []const u8) ![]const u8 {
// Get the mount point of the install directory
const mount_point = try filesystem_utils.getMountPoint(install_dir);
// Get absolute path of install directory
var install_abs_buf: bun.PathBuffer = undefined;
const install_abs = try std.fs.cwd().realpath(install_dir, &install_abs_buf);
// Start from mount point and walk down toward the project
var current_path_buf: bun.PathBuffer = undefined;
@memcpy(current_path_buf[0..mount_point.len], mount_point);
current_path_buf[mount_point.len] = 0;
var current_path = current_path_buf[0..mount_point.len :0];
const install_parent = std.fs.path.dirname(install_abs) orelse install_abs;
while (true) {
var cache_path_buf: bun.PathBuffer = undefined;
const cache_dir_name = ".bun-cache";
const cache_path = Path.joinZBuf(&cache_path_buf, &[_][]const u8{ current_path, cache_dir_name }, .auto);
// Try to create cache directory at this level
if (filesystem_utils.canCreateDir(cache_path)) {
return bun.default_allocator.dupeZ(u8, cache_path) catch bun.outOfMemory();
}
// Stop if we've reached the install directory's parent
if (strings.eql(current_path, install_parent)) {
break;
}
// Move down one level toward the project
const next_component = filesystem_utils.getNextPathComponent(current_path, install_abs) orelse break;
const new_len = current_path.len + 1 + next_component.len;
if (new_len >= bun.MAX_PATH_BYTES) break;
current_path_buf[current_path.len] = std.fs.path.sep;
@memcpy(current_path_buf[current_path.len + 1 .. new_len], next_component);
current_path_buf[new_len] = 0;
current_path = current_path_buf[0..new_len :0];
}
// Last resort: project-local cache
var final_cache_buf: bun.PathBuffer = undefined;
const project_cache = Path.joinZBuf(&final_cache_buf, &[_][]const u8{ install_parent, ".bun-cache" }, .auto);
return bun.default_allocator.dupeZ(u8, project_cache) catch bun.outOfMemory();
}
const string = []const u8;
const stringZ = [:0]const u8;
const filesystem_utils = @import("../filesystem_utils.zig");
const std = @import("std");
const bun = @import("bun");
@@ -754,6 +853,7 @@ const Output = bun.Output;
const Path = bun.path;
const Progress = bun.Progress;
const default_allocator = bun.default_allocator;
const strings = bun.strings;
const Command = bun.cli.Command;
const File = bun.sys.File;

View File

@@ -0,0 +1,123 @@
pub fn isSameFilesystem(path1: []const u8, path2: []const u8) !bool {
if (comptime Environment.isWindows) {
var volume1: bun.PathBuffer = undefined;
var volume2: bun.PathBuffer = undefined;
const vol1 = try getVolumePathName(path1, &volume1);
const vol2 = try getVolumePathName(path2, &volume2);
return strings.eql(vol1, vol2);
} else {
var path1_buf: bun.PathBuffer = undefined;
var path2_buf: bun.PathBuffer = undefined;
const path1_z = bun.path.joinZBuf(&path1_buf, &[_][]const u8{path1}, .auto);
const path2_z = bun.path.joinZBuf(&path2_buf, &[_][]const u8{path2}, .auto);
const stat1 = try bun.sys.stat(path1_z).unwrap();
const stat2 = try bun.sys.stat(path2_z).unwrap();
return stat1.dev == stat2.dev;
}
}
pub fn getMountPoint(path: []const u8) ![]const u8 {
if (comptime Environment.isWindows) {
var volume: bun.PathBuffer = undefined;
return try getVolumePathName(path, &volume);
} else {
return try getMountPointUnix(path);
}
}
fn getVolumePathName(path: []const u8, buf: *bun.PathBuffer) ![]const u8 {
if (comptime !Environment.isWindows) {
@compileError("Windows only");
}
var path_buf: bun.PathBuffer = undefined;
const abs_path = Path.joinAbsStringBuf(Fs.FileSystem.instance.cwd, &path_buf, &[_][]const u8{path}, .windows);
var wide_path: bun.WPathBuffer = undefined;
const wide_len = bun.strings.toWPathNormalized(&wide_path, abs_path);
var volume_wide: bun.WPathBuffer = undefined;
const result = bun.windows.GetVolumePathNameW(wide_path[0..wide_len :0].ptr, &volume_wide, volume_wide.len);
if (result == 0) {
return error.GetVolumePathNameFailed;
}
const volume_len = bun.strings.fromWPath(buf, volume_wide[0..result]);
return buf[0..volume_len];
}
fn getMountPointUnix(path: []const u8) ![]const u8 {
var current_path = try std.fs.realpathAlloc(bun.default_allocator, path);
defer bun.default_allocator.free(current_path);
var path_buf: bun.PathBuffer = undefined;
const current_path_z = bun.path.joinZBuf(&path_buf, &[_][]const u8{current_path}, .auto);
const initial_stat = try bun.sys.stat(current_path_z).unwrap();
var current_dev = initial_stat.dev;
while (true) {
const parent = std.fs.path.dirname(current_path) orelse return current_path;
if (strings.eql(parent, current_path)) {
return current_path;
}
var parent_buf: bun.PathBuffer = undefined;
const parent_z = bun.path.joinZBuf(&parent_buf, &[_][]const u8{parent}, .auto);
const parent_stat = try bun.sys.stat(parent_z).unwrap();
if (parent_stat.dev != current_dev) {
return current_path;
}
const new_path = try bun.default_allocator.dupe(u8, parent);
bun.default_allocator.free(current_path);
current_path = new_path;
current_dev = parent_stat.dev;
}
}
pub fn canCreateDir(path: []const u8) bool {
std.fs.cwd().makePath(path) catch |err| {
switch (err) {
error.PathAlreadyExists => {
std.fs.cwd().access(path, .{ .mode = .write_only }) catch {
return false;
};
return true;
},
else => return false,
}
};
std.fs.cwd().deleteDir(path) catch {};
return true;
}
pub fn getNextPathComponent(from: []const u8, to: []const u8) ?[]const u8 {
if (!strings.startsWith(to, from)) return null;
var remainder = to[from.len..];
if (remainder.len == 0) return null;
if (remainder[0] == std.fs.path.sep) {
remainder = remainder[1..];
}
const sep_index = strings.indexOfChar(remainder, std.fs.path.sep) orelse remainder.len;
if (sep_index == 0) return null;
return remainder[0..sep_index];
}
const std = @import("std");
const bun = @import("bun");
const Environment = bun.Environment;
const Fs = bun.fs;
const Path = bun.path;
const strings = bun.strings;

View File

@@ -0,0 +1,309 @@
import { spawn } from "bun";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test";
import { writeFileSync } from "fs";
import { exists, mkdir } from "fs/promises";
import { bunExe, bunEnv as env, VerdaccioRegistry } from "harness";
import { dirname, join } from "path";
let registry: VerdaccioRegistry;
let projectDir: string;
let packageJson: string;
let testRoot: string;
beforeAll(async () => {
setDefaultTimeout(1000 * 60 * 5);
registry = new VerdaccioRegistry();
await registry.start();
});
afterAll(async () => {
registry.stop();
});
beforeEach(async () => {
// Clean up environment
delete env.BUN_INSTALL_CACHE_DIR;
delete env.BUN_INSTALL;
delete env.XDG_CACHE_HOME;
// Create test directory using registry helper
({ packageDir: projectDir, packageJson } = await registry.createTestDir({ saveTextLockfile: false }));
testRoot = dirname(projectDir);
// Set up environment to isolate cache behavior
env.BUN_TMPDIR = env.TMPDIR = env.TEMP = join(projectDir, ".bun-tmp");
// Set HOME to ensure cache doesn't go to real home directory
env.HOME = env.USERPROFILE = testRoot;
});
afterEach(async () => {
// Clean up is handled by registry.createTestDir
});
describe("filesystem-aware cache", () => {
test("uses default cache when on same filesystem", async () => {
// Set up a default cache location
const defaultCache = join(dirname(projectDir), "default-cache");
env.BUN_INSTALL_CACHE_DIR = defaultCache;
// Create a simple package.json with registry
const pkg = {
name: "test-project",
dependencies: {
"no-deps": "1.0.0", // Use a simple package from the test registry
},
};
await writeFileSync(packageJson, JSON.stringify(pkg));
// Run bun install
await using proc = spawn({
cmd: [bunExe(), "install"],
cwd: projectDir,
env,
stdout: "pipe",
stderr: "pipe",
});
const exitCode = await proc.exited;
expect(exitCode).toBe(0);
// Verify cache was created in the default location
expect(await exists(defaultCache)).toBe(true);
// Verify that packages were installed
expect(await exists(join(projectDir, "node_modules", "no-deps"))).toBe(true);
});
test("creates filesystem-specific cache when on different filesystem", async () => {
// This test simulates different filesystems by checking if the optimal cache
// location is created when the default would be on a different filesystem
// For testing, we'll check that .bun-cache is created in the project directory
// when no other cache location is specified
const pkg = {
name: "test-project",
dependencies: {
"no-deps": "1.0.0",
},
};
await writeFileSync(packageJson, JSON.stringify(pkg));
// Run bun install without specifying cache dir
await using proc = spawn({
cmd: [bunExe(), "install"],
cwd: projectDir,
env,
stdout: "pipe",
stderr: "pipe",
});
const [stdout, stderr] = await Promise.all([proc.stdout.text(), proc.stderr.text()]);
const exitCode = await proc.exited;
if (exitCode !== 0) {
console.error("Install failed:", { stdout, stderr, exitCode });
}
expect(exitCode).toBe(0);
// Check for possible cache locations
const possibleCaches = [
join(projectDir, ".bun-cache"),
join(projectDir, "node_modules", ".bun-cache"),
join(projectDir, "node_modules", ".cache"),
join(dirname(projectDir), ".bun-cache"),
];
let cacheFound = false;
let foundCache = "";
for (const cache of possibleCaches) {
if (await exists(cache)) {
cacheFound = true;
foundCache = cache;
break;
}
}
if (!cacheFound) {
// List directory contents to debug
console.error("No cache found. Project dir contents:");
await Bun.$`ls -la ${projectDir}`.quiet(false);
console.error("Node modules contents:");
await Bun.$`ls -la ${join(projectDir, "node_modules")}`.quiet(false).catch(() => {});
}
expect(cacheFound).toBe(true);
expect(await exists(join(projectDir, "node_modules", "no-deps"))).toBe(true);
});
test("walks up directory tree to find writable cache location", async () => {
// Create a nested project structure
const nestedProject = join(projectDir, "nested", "deep", "project");
await mkdir(nestedProject, { recursive: true });
const pkg = {
name: "nested-project",
dependencies: {
"no-deps": "1.0.0",
},
};
// Write package.json to nested location
await mkdir(dirname(join(nestedProject, "package.json")), { recursive: true });
writeFileSync(join(nestedProject, "package.json"), JSON.stringify(pkg));
// Also need .npmrc for registry
writeFileSync(join(nestedProject, ".npmrc"), `registry=${registry.registryUrl()}`);
// Run bun install in the nested directory
await using proc = spawn({
cmd: [bunExe(), "install"],
cwd: nestedProject,
env,
stdout: "pipe",
stderr: "pipe",
});
const [stdout, stderr] = await Promise.all([proc.stdout.text(), proc.stderr.text()]);
const exitCode = await proc.exited;
if (exitCode !== 0) {
console.error("Install failed:", { stdout, stderr, exitCode });
}
expect(exitCode).toBe(0);
// Check that a cache was created somewhere in the hierarchy
const possibleCaches = [
join(nestedProject, ".bun-cache"),
join(nestedProject, "node_modules", ".bun-cache"),
join(nestedProject, "node_modules", ".cache"),
join(dirname(nestedProject), ".bun-cache"),
join(dirname(dirname(nestedProject)), ".bun-cache"),
join(dirname(dirname(dirname(nestedProject))), ".bun-cache"),
join(projectDir, ".bun-cache"),
join(testRoot, ".bun-cache"),
join(testRoot, ".bun", "install", "cache"), // Default HOME-based cache
];
let cacheFound = false;
let cacheLocation = "";
for (const cache of possibleCaches) {
if (await exists(cache)) {
cacheFound = true;
cacheLocation = cache;
break;
}
}
if (!cacheFound) {
console.error("No cache found. Nested project dir:", nestedProject);
console.error("Possible cache locations checked:", possibleCaches);
// Check what actually exists
console.error("Looking for .bun-cache or .cache directories...");
await Bun.$`find ${testRoot} -name ".bun-cache" -o -name ".cache" 2>/dev/null || true`.quiet(false);
// Also check if there's a default cache being used
if (env.HOME) {
const homeCache = join(env.HOME, ".bun", "install", "cache");
console.error("Checking HOME cache:", homeCache);
if (await exists(homeCache)) {
console.error("Found cache in HOME directory");
await Bun.$`ls -la ${homeCache}`.quiet(false).catch(() => {});
}
}
}
expect(cacheFound).toBe(true);
expect(await exists(join(nestedProject, "node_modules", "no-deps"))).toBe(true);
// Verify the cache contains the package
if (cacheLocation && cacheFound) {
const cacheContents = await Bun.$`ls ${cacheLocation}`.text();
expect(cacheContents.length).toBeGreaterThan(0);
}
});
test("respects BUN_INSTALL_CACHE_DIR even on different filesystem", async () => {
// When BUN_INSTALL_CACHE_DIR is explicitly set, it should always be used
const explicitCache = join(dirname(projectDir), "explicit-cache");
env.BUN_INSTALL_CACHE_DIR = explicitCache;
const pkg = {
name: "explicit-cache-test",
dependencies: {
"no-deps": "1.0.0",
},
};
await writeFileSync(packageJson, JSON.stringify(pkg));
await using proc = spawn({
cmd: [bunExe(), "install"],
cwd: projectDir,
env,
stdout: "pipe",
stderr: "pipe",
});
const exitCode = await proc.exited;
expect(exitCode).toBe(0);
// Verify the explicit cache was used
expect(await exists(explicitCache)).toBe(true);
expect(await exists(join(projectDir, "node_modules", "no-deps"))).toBe(true);
// Verify no other cache was created
expect(await exists(join(projectDir, ".bun-cache"))).toBe(false);
expect(await exists(join(projectDir, "node_modules", ".bun-cache"))).toBe(false);
});
test("falls back to node_modules/.bun-cache when no writable location found", async () => {
// This test verifies the ultimate fallback behavior
const pkg = {
name: "fallback-test",
dependencies: {
"no-deps": "1.0.0",
},
};
await writeFileSync(packageJson, JSON.stringify(pkg));
// Run install without any cache configuration
await using proc = spawn({
cmd: [bunExe(), "install"],
cwd: projectDir,
env,
stdout: "pipe",
stderr: "pipe",
});
const exitCode = await proc.exited;
expect(exitCode).toBe(0);
// Package should be installed
expect(await exists(join(projectDir, "node_modules", "no-deps"))).toBe(true);
// Some cache should exist
const possibleCaches = [
join(projectDir, "node_modules", ".bun-cache"),
join(projectDir, ".bun-cache"),
join(dirname(projectDir), ".bun-cache"),
];
let cacheFound = false;
for (const cache of possibleCaches) {
if (await exists(cache)) {
cacheFound = true;
break;
}
}
expect(cacheFound).toBe(true);
});
});