diff --git a/.github/workflows/bun-linux-build.yml b/.github/workflows/bun-linux-build.yml index 694adcd9fb..ba0a136acf 100644 --- a/.github/workflows/bun-linux-build.yml +++ b/.github/workflows/bun-linux-build.yml @@ -233,16 +233,18 @@ jobs: run: | # If this hangs, it means something is seriously wrong with the build bun --version - - id: install-dependnecies - name: Install dependencies - run: | - sudo apt-get update && sudo apt-get install -y openssl - bun install --verbose - bun install --cwd=test --verbose - bun install --cwd=packages/bun-internal-test --verbose - - bun install --cwd=test/js/third_party/prisma --verbose + # Split these into multiple steps to make it clear which one fails + - name: Install dependencies (apt-get) + run: sudo apt-get update && sudo apt-get install -y openssl + - name: Install dependencies (root) + run: bun install --verbose + - name: Install dependencies (test) + run: bun install --cwd=test --verbose + - name: Install dependencies (runner) + run: bun install --cwd=packages/bun-internal-test --verbose + - name: Install dependencies (prisma) + run: bun install --cwd=test/js/third_party/prisma --verbose # This is disabled because the cores are ~5.5gb each # so it is easy to hit 50gb coredump downloads. Only enable if you need to retrive one diff --git a/.github/workflows/bun-mac-aarch64.yml b/.github/workflows/bun-mac-aarch64.yml index b5bd764a69..463db45841 100644 --- a/.github/workflows/bun-mac-aarch64.yml +++ b/.github/workflows/bun-mac-aarch64.yml @@ -47,8 +47,8 @@ jobs: tag: bun-obj-darwin-aarch64 steps: - uses: actions/checkout@v4 - # - name: Checkout submodules - # run: git submodule update --init --recursive --depth=1 --progress --force + - name: Checkout submodules + run: git submodule update --init --recursive --depth=1 --progress --force - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -416,12 +416,13 @@ jobs: run: | # If this hangs, it means something is seriously wrong with the build bun --version - - id: install - name: Install dependencies - run: | - bun install --verbose - bun install --cwd=test --verbose - bun install --cwd=packages/bun-internal-test --verbose + # Split these into multiple steps to make it clear which one fails + - name: "Install dependencies (root)" + run: bun install --verbose + - name: "Install dependencies (test)" + run: bun install --cwd=test --verbose + - name: "Install dependencies (runner)" + run: bun install --cwd=packages/bun-internal-test --verbose - id: test name: Test (node runner) env: diff --git a/.github/workflows/bun-mac-x64-baseline.yml b/.github/workflows/bun-mac-x64-baseline.yml index 3c5c13e367..83ae54cc09 100644 --- a/.github/workflows/bun-mac-x64-baseline.yml +++ b/.github/workflows/bun-mac-x64-baseline.yml @@ -53,6 +53,8 @@ jobs: # tag: bun-obj-darwin-aarch64 steps: - uses: actions/checkout@v4 + - name: Checkout submodules + run: git submodule update --init --recursive --depth=1 --progress --force - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/bun-mac-x64.yml b/.github/workflows/bun-mac-x64.yml index 686d5f2b6f..ffc3edfe7a 100644 --- a/.github/workflows/bun-mac-x64.yml +++ b/.github/workflows/bun-mac-x64.yml @@ -50,6 +50,8 @@ jobs: tag: bun-obj-darwin-x64 steps: - uses: actions/checkout@v4 + - name: Checkout submodules + run: git submodule update --init --recursive --depth=1 --progress --force - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -400,12 +402,13 @@ jobs: run: | # If this hangs, it means something is seriously wrong with the build bun --version - - id: install - name: Install dependencies - run: | - bun install --verbose - bun install --cwd=test --verbose - bun install --cwd=packages/bun-internal-test --verbose + # Split these into multiple steps to make it clear which one fails + - name: "Install dependencies (root)" + run: bun install --verbose + - name: "Install dependencies (test)" + run: bun install --cwd=test --verbose + - name: "Install dependencies (runner)" + run: bun install --cwd=packages/bun-internal-test --verbose - id: test name: Test (node runner) env: diff --git a/.github/workflows/bun-windows.yml b/.github/workflows/bun-windows.yml index 5c106b3ecf..35c3c1be76 100644 --- a/.github/workflows/bun-windows.yml +++ b/.github/workflows/bun-windows.yml @@ -61,6 +61,8 @@ jobs: steps: - run: git config --global core.autocrlf false && git config --global core.eol lf - uses: actions/checkout@v4 + - name: Checkout submodules + run: git submodule update --init --recursive --depth=1 --progress --force - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -424,11 +426,13 @@ jobs: - uses: secondlife/setup-cygwin@v3 with: packages: bash - - name: Install dependencies - run: | - bun install --verbose - bun install --cwd=test --verbose - bun install --cwd=packages/bun-internal-test --verbose + # Split these into multiple steps to make it clear which one fails + - name: Install dependencies (root) + run: bun install --verbose + - name: Install dependencies (test) + run: bun install --cwd=test --verbose + - name: Install dependencies (runner) + run: bun install --cwd=packages/bun-internal-test --verbose - id: test name: Run tests env: diff --git a/.gitmodules b/.gitmodules index 6617e9af8f..51f5cdc69f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -83,3 +83,10 @@ ignore = dirty depth = 1 shallow = true fetchRecurseSubmodules = false +[submodule "zig"] + path = src/deps/zig + url = https://github.com/oven-sh/zig + branch = bun + depth = 1 + shallow = true + fetchRecurseSubmodules = false diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c96f17775..d5177da40b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -315,6 +315,10 @@ option(USE_STATIC_LIBATOMIC "Statically link libatomic, requires the presence of option(USE_LTO "Enable Link-Time Optimization" ${DEFAULT_LTO}) +if(NOT ZIG_LIB_DIR) + cmake_path(SET ZIG_LIB_DIR NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/src/deps/zig/lib") +endif() + if(USE_VALGRIND) # Disable SIMD set(USE_BASELINE_BUILD ON) @@ -743,11 +747,14 @@ endif() # requires all the JS files to be known, but also Bun will use all cores during bundling anyways. if(NOT NO_CODEGEN) file(GLOB BUN_TS_MODULES ${CONFIGURE_DEPENDS} - "${BUN_SRC}/js/builtins/*.js" - "${BUN_SRC}/js/builtins/*.ts" - "${BUN_SRC}/js/bun/*.js" + "${BUN_SRC}/js/node/*.ts" + "${BUN_SRC}/js/node/*.js" "${BUN_SRC}/js/bun/*.ts" - "${BUN_SRC}/js/internal-for-testing.ts" + "${BUN_SRC}/js/bun/*.js" + "${BUN_SRC}/js/builtins/*.ts" + "${BUN_SRC}/js/builtins/*.js" + "${BUN_SRC}/js/thirdparty/*.js" + "${BUN_SRC}/js/thirdparty/*.ts" "${BUN_SRC}/js/internal/*.js" "${BUN_SRC}/js/internal/*.ts" "${BUN_SRC}/js/node/*.js" @@ -849,8 +856,10 @@ if(NOT BUN_LINK_ONLY AND NOT BUN_CPP_ONLY) OUTPUT "${BUN_ZIG_OBJ}" COMMAND "${ZIG_COMPILER}" "build" "obj" + "--zig-lib-dir" "${ZIG_LIB_DIR}" "-Doutput-file=${BUN_ZIG_OBJ}" "-Dgenerated-code=${BUN_WORKDIR}/codegen" + "-freference-trace=10" "-Dversion=${Bun_VERSION}" "-Dcanary=${CANARY}" "-Doptimize=${ZIG_OPTIMIZE}" diff --git a/Dockerfile b/Dockerfile index ca3a651bf3..538dbd8b61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -429,6 +429,7 @@ RUN mkdir -p build \ -DBUN_ZIG_OBJ="/tmp/bun-zig.o" \ -DCANARY="${CANARY}" \ -DZIG_COMPILER=system \ + -DZIG_LIB_DIR=$BUN_DIR/src/deps/zig/lib \ && ONLY_ZIG=1 ninja "/tmp/bun-zig.o" -v FROM scratch as build_release_obj diff --git a/src/bun.js/bindings/ZigSourceProvider.cpp b/src/bun.js/bindings/ZigSourceProvider.cpp index 06a6fd8a4c..b5a6eae772 100644 --- a/src/bun.js/bindings/ZigSourceProvider.cpp +++ b/src/bun.js/bindings/ZigSourceProvider.cpp @@ -28,22 +28,22 @@ using SourceOrigin = JSC::SourceOrigin; using String = WTF::String; using SourceProviderSourceType = JSC::SourceProviderSourceType; -SourceOrigin toSourceOrigin(const String& specifier, bool isBuiltin) +SourceOrigin toSourceOrigin(const String& sourceURL, bool isBuiltin) { - ASSERT_WITH_MESSAGE(!specifier.startsWith("file://"_s), "specifier should not already be a file URL"); + ASSERT_WITH_MESSAGE(!sourceURL.startsWith("file://"_s), "specifier should not already be a file URL"); if (isBuiltin) { - if (specifier.startsWith("node:"_s)) { - return SourceOrigin(WTF::URL(makeString("builtin://node/", specifier.substring(5)))); - } else if (specifier.startsWith("bun:"_s)) { - return SourceOrigin(WTF::URL(makeString("builtin://bun/", specifier.substring(4)))); + if (sourceURL.startsWith("node:"_s)) { + return SourceOrigin(WTF::URL(makeString("builtin://node/", sourceURL.substring(5)))); + } else if (sourceURL.startsWith("bun:"_s)) { + return SourceOrigin(WTF::URL(makeString("builtin://bun/", sourceURL.substring(4)))); } else { - return SourceOrigin(WTF::URL(makeString("builtin://", specifier))); + return SourceOrigin(WTF::URL(makeString("builtin://", sourceURL))); } } - return SourceOrigin(WTF::URL::fileURLWithFileSystemPath(specifier)); + return SourceOrigin(WTF::URL::fileURLWithFileSystemPath(sourceURL)); } extern "C" int ByteRangeMapping__getSourceID(void* mappings, BunString sourceURL); diff --git a/src/bun.js/node/dir_iterator.zig b/src/bun.js/node/dir_iterator.zig index cef5d72c3b..74a4bf92ea 100644 --- a/src/bun.js/node/dir_iterator.zig +++ b/src/bun.js/node/dir_iterator.zig @@ -391,7 +391,7 @@ pub fn NewIterator(comptime use_windows_ospath: bool) type { }; } -const PathType = enum { u8, u16 }; +pub const PathType = enum { u8, u16 }; pub fn NewWrappedIterator(comptime path_type: PathType) type { const IteratorType = if (path_type == .u16) IteratorW else Iterator; diff --git a/src/bun.js/node/node_fs.zig b/src/bun.js/node/node_fs.zig index 38d8fe75e8..3981e18767 100644 --- a/src/bun.js/node/node_fs.zig +++ b/src/bun.js/node/node_fs.zig @@ -4365,7 +4365,9 @@ pub const NodeFS = struct { ) Maybe(Return.Mkdir) { const VTable = struct { pub fn onCreateDir(c: Ctx, dirpath: bun.OSPathSliceZ) void { - c.onCreateDir(dirpath); + if (Ctx != void) { + c.onCreateDir(dirpath); + } return; } }; @@ -4812,7 +4814,7 @@ pub const NodeFS = struct { const fd = switch (switch (Environment.os) { else => Syscall.openat(atfd, basename, flags, 0), // windows bun.sys.open does not pass iterable=true, - .windows => bun.sys.openDirAtWindowsA(atfd, basename, true, false), + .windows => bun.sys.openDirAtWindowsA(atfd, basename, .{ .no_follow = true, .iterable = true }), }) { .err => |err| { if (comptime !is_root) { @@ -5106,7 +5108,7 @@ pub const NodeFS = struct { const fd = switch (switch (Environment.os) { else => Syscall.open(path, flags, 0), // windows bun.sys.open does not pass iterable=true, - .windows => bun.sys.openDirAtWindowsA(bun.toFD(std.fs.cwd().fd), path, true, false), + .windows => bun.sys.openDirAtWindowsA(bun.toFD(std.fs.cwd().fd), path, .{ .iterable = true }), }) { .err => |err| return .{ .err = err.withPath(args.path.slice()), @@ -5360,7 +5362,7 @@ pub const NodeFS = struct { const dirfd_path = buffer[0..dirfd_path_len]; const parent_path = bun.Dirname.dirname(u16, dirfd_path).?; if (std.mem.startsWith(u16, parent_path, &bun.windows.nt_maxpath_prefix)) @constCast(parent_path)[1] = '?'; - const newdirfd = switch (bun.sys.openDirAtWindows(bun.invalid_fd, parent_path, false, true)) { + const newdirfd = switch (bun.sys.openDirAtWindows(bun.invalid_fd, parent_path, .{ .no_follow = true })) { .result => |fd| fd, .err => |err| { return .{ .err = err.withPath(path) }; diff --git a/src/bun.js/webcore/streams.zig b/src/bun.js/webcore/streams.zig index 30068b8b9d..7703efa216 100644 --- a/src/bun.js/webcore/streams.zig +++ b/src/bun.js/webcore/streams.zig @@ -3707,10 +3707,33 @@ pub const FileReader = struct { } } else if (this.pending.state == .pending) { if (buf.len == 0) { - this.pending.result = .{ .done = {} }; + { + if (this.buffered.items.len == 0) { + if (this.buffered.capacity > 0) { + this.buffered.clearAndFree(bun.default_allocator); + } + + if (this.reader.buffer().items.len != 0) { + this.buffered = this.reader.buffer().moveToUnmanaged(); + } + } + + var buffer = &this.buffered; + defer buffer.clearAndFree(bun.default_allocator); + if (buffer.items.len > 0) { + if (this.pending_view.len >= buffer.items.len) { + @memcpy(this.pending_view[0..buffer.items.len], buffer.items); + this.pending.result = .{ .into_array_and_done = .{ .value = this.pending_value.get() orelse .zero, .len = @truncate(buffer.items.len) } }; + } else { + this.pending.result = .{ .owned_and_done = bun.ByteList.fromList(buffer.*) }; + buffer.* = .{}; + } + } else { + this.pending.result = .{ .done = {} }; + } + } this.pending_value.clear(); this.pending_view = &.{}; - this.reader.buffer().clearAndFree(); this.pending.run(); return false; } diff --git a/src/bun.zig b/src/bun.zig index 71743f4d64..3f37d6d87b 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -458,6 +458,45 @@ pub fn hash(content: []const u8) u64 { return std.hash.Wyhash.hash(0, content); } +/// Get a random-ish value +pub fn fastRandom() u64 { + const pcrng = struct { + const random_seed = struct { + var seed_value: std.atomic.Value(u64) = std.atomic.Value(u64).init(0); + pub fn get() u64 { + // This is slightly racy but its fine because this memoization is done as a performance optimization + // and we only need to do it once per process + var value = seed_value.load(.Monotonic); + while (value == 0) : (value = seed_value.load(.Monotonic)) { + if (comptime Environment.isDebug) outer: { + if (getenvZ("BUN_DEBUG_HASH_RANDOM_SEED")) |env| { + value = std.fmt.parseInt(u64, env, 10) catch break :outer; + seed_value.store(value, .Monotonic); + return value; + } + } + rand(std.mem.asBytes(&value)); + seed_value.store(value, .Monotonic); + } + + return value; + } + }; + + var prng_: ?std.rand.DefaultPrng = null; + + pub fn get() u64 { + if (prng_ == null) { + prng_ = std.rand.DefaultPrng.init(random_seed.get()); + } + + return prng_.?.random().uintAtMost(u64, std.math.maxInt(u64)); + } + }; + + return pcrng.get(); +} + pub fn hashWithSeed(seed: u64, content: []const u8) u64 { return std.hash.Wyhash.hash(seed, content); } @@ -691,7 +730,7 @@ pub fn openFile(path_: []const u8, open_flags: std.fs.File.OpenFlags) !std.fs.Fi pub fn openDir(dir: std.fs.Dir, path_: [:0]const u8) !std.fs.Dir { if (comptime Environment.isWindows) { - const res = try sys.openDirAtWindowsA(toFD(dir.fd), path_, true, false).unwrap(); + const res = try sys.openDirAtWindowsA(toFD(dir.fd), path_, .{ .iterable = true, .can_rename_or_delete = true }).unwrap(); return res.asDir(); } else { const fd = try sys.openat(toFD(dir.fd), path_, std.os.O.DIRECTORY | std.os.O.CLOEXEC | std.os.O.RDONLY, 0).unwrap(); @@ -701,7 +740,7 @@ pub fn openDir(dir: std.fs.Dir, path_: [:0]const u8) !std.fs.Dir { pub fn openDirA(dir: std.fs.Dir, path_: []const u8) !std.fs.Dir { if (comptime Environment.isWindows) { - const res = try sys.openDirAtWindowsA(toFD(dir.fd), path_, true, false).unwrap(); + const res = try sys.openDirAtWindowsA(toFD(dir.fd), path_, .{ .iterable = true, .can_rename_or_delete = true }).unwrap(); return res.asDir(); } else { const fd = try sys.openatA(toFD(dir.fd), path_, std.os.O.DIRECTORY | std.os.O.CLOEXEC | std.os.O.RDONLY, 0).unwrap(); @@ -711,7 +750,7 @@ pub fn openDirA(dir: std.fs.Dir, path_: []const u8) !std.fs.Dir { pub fn openDirAbsolute(path_: []const u8) !std.fs.Dir { if (comptime Environment.isWindows) { - const res = try sys.openDirAtWindowsA(invalid_fd, path_, true, false).unwrap(); + const res = try sys.openDirAtWindowsA(invalid_fd, path_, .{ .iterable = true, .can_rename_or_delete = true }).unwrap(); return res.asDir(); } else { const fd = try sys.openA(path_, std.os.O.DIRECTORY | std.os.O.CLOEXEC | std.os.O.RDONLY, 0).unwrap(); @@ -1253,6 +1292,7 @@ pub fn getFdPathW(fd_: anytype, buf: *WPathBuffer) ![]u16 { if (comptime Environment.isWindows) { const wide_slice = try std.os.windows.GetFinalPathNameByHandle(fd, .{}, buf); + return wide_slice; } @@ -1870,6 +1910,8 @@ pub inline fn toFD(fd: anytype) FileDescriptor { FDImpl.System => FDImpl.fromSystem(fd), FDImpl.UV, i32, comptime_int => FDImpl.fromUV(fd), FileDescriptor => FDImpl.decode(fd), + std.fs.Dir => FDImpl.fromSystem(fd.fd), + sys.File, std.fs.File => FDImpl.fromSystem(fd.handle), // TODO: remove u32 u32 => FDImpl.fromUV(@intCast(fd)), else => @compileError("toFD() does not support type \"" ++ @typeName(T) ++ "\""), @@ -1879,6 +1921,8 @@ pub inline fn toFD(fd: anytype) FileDescriptor { // even though file descriptors are always positive, linux/mac repesents them as signed integers return switch (T) { FileDescriptor => fd, // TODO: remove the toFD call from these places and make this a @compileError + std.fs.File, sys.File => toFD(fd.handle), + std.fs.Dir => @enumFromInt(@as(i32, @intCast(fd.fd))), c_int, i32, u32, comptime_int => @enumFromInt(fd), usize, i64 => @enumFromInt(@as(i32, @intCast(fd))), else => @compileError("bun.toFD() not implemented for: " ++ @typeName(T)), @@ -2347,14 +2391,139 @@ pub inline fn OSPathLiteral(comptime literal: anytype) *const [literal.len:0]OSP const builtin = @import("builtin"); pub const MakePath = struct { + const w = std.os.windows; + + // TODO(@paperdave): upstream making this public into zig std + // there is zero reason this must be copied + // + /// Calls makeOpenDirAccessMaskW iteratively to make an entire path + /// (i.e. creating any parent directories that do not exist). + /// Opens the dir if the path already exists and is a directory. + /// This function is not atomic, and if it returns an error, the file system may + /// have been modified regardless. + fn makeOpenPathAccessMaskW(self: std.fs.Dir, comptime T: type, sub_path: []const T, access_mask: u32, no_follow: bool) !std.fs.Dir { + const Iterator = std.fs.path.ComponentIterator(.windows, T); + var it = try Iterator.init(sub_path); + // If there are no components in the path, then create a dummy component with the full path. + var component = it.last() orelse Iterator.Component{ + .name = &.{}, + .path = sub_path, + }; + + while (true) { + const sub_path_w = if (comptime T == u16) + try w.wToPrefixedFileW(self.fd, + // TODO: report this bug + // they always copy it + // it doesn't need to be [:0]const u16 + @ptrCast(component.path)) + else + try w.sliceToPrefixedFileW(self.fd, component.path); + const is_last = it.peekNext() == null; + _ = is_last; // autofix + var result = makeOpenDirAccessMaskW(self, sub_path_w.span().ptr, access_mask, .{ + .no_follow = no_follow, + .create_disposition = w.FILE_OPEN_IF, + }) catch |err| switch (err) { + error.FileNotFound => |e| { + component = it.previous() orelse return e; + continue; + }, + else => |e| return e, + }; + + component = it.next() orelse return result; + // Don't leak the intermediate file handles + result.close(); + } + } + const MakeOpenDirAccessMaskWOptions = struct { + no_follow: bool, + create_disposition: u32, + }; + + fn makeOpenDirAccessMaskW(self: std.fs.Dir, sub_path_w: [*:0]const u16, access_mask: u32, flags: MakeOpenDirAccessMaskWOptions) !std.fs.Dir { + var result = std.fs.Dir{ + .fd = undefined, + }; + + const path_len_bytes = @as(u16, @intCast(std.mem.sliceTo(sub_path_w, 0).len * 2)); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @constCast(sub_path_w), + }; + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w)) null else self.fd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + const open_reparse_point: w.DWORD = if (flags.no_follow) w.FILE_OPEN_REPARSE_POINT else 0x0; + var status: w.IO_STATUS_BLOCK = undefined; + const rc = w.ntdll.NtCreateFile( + &result.fd, + access_mask, + &attr, + &status, + null, + w.FILE_ATTRIBUTE_NORMAL, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + flags.create_disposition, + w.FILE_DIRECTORY_FILE | w.FILE_SYNCHRONOUS_IO_NONALERT | w.FILE_OPEN_FOR_BACKUP_INTENT | w.FILE_WRITE_THROUGH | open_reparse_point, + null, + 0, + ); + + switch (rc) { + .SUCCESS => return result, + .OBJECT_NAME_INVALID => return error.BadPathName, + .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, + .NOT_A_DIRECTORY => return error.NotDir, + // This can happen if the directory has 'List folder contents' permission set to 'Deny' + // and the directory is trying to be opened for iteration. + .ACCESS_DENIED => return error.AccessDenied, + .INVALID_PARAMETER => return error.BadPathName, + .SHARING_VIOLATION => return error.SharingViolation, + else => return w.unexpectedStatus(rc), + } + } + + pub fn makeOpenPath(self: std.fs.Dir, sub_path: anytype, opts: std.fs.Dir.OpenDirOptions) !std.fs.Dir { + if (comptime Environment.isWindows) { + return makeOpenPathAccessMaskW( + self, + std.meta.Elem(@TypeOf(sub_path)), + sub_path, + w.STANDARD_RIGHTS_READ | + w.FILE_READ_ATTRIBUTES | + w.FILE_READ_EA | + w.SYNCHRONIZE | + w.FILE_TRAVERSE, + false, + ); + } + + return self.makeOpenPath(sub_path, opts); + } + /// copy/paste of `std.fs.Dir.makePath` and related functions and modified to support u16 slices. /// inside `MakePath` scope to make deleting later easier. /// TODO(dylan-conway) delete `MakePath` pub fn makePath(comptime T: type, self: std.fs.Dir, sub_path: []const T) !void { + if (Environment.isWindows) { + var dir = try makeOpenPath(self, sub_path, .{}); + dir.close(); + return; + } + var it = try componentIterator(T, sub_path); var component = it.last() orelse return; while (true) { - (if (T == u16) makeDirW else std.fs.Dir.makeDir)(self, component.path) catch |err| switch (err) { + std.fs.Dir.makeDir(self, component.path) catch |err| switch (err) { error.PathAlreadyExists => { // TODO stat the file and return an error if it's not a directory // this is important because otherwise a dangling symlink @@ -2370,10 +2539,6 @@ pub const MakePath = struct { } } - fn makeDirW(self: std.fs.Dir, sub_path: []const u16) !void { - try std.os.mkdiratW(self.fd, sub_path, 0o755); - } - fn componentIterator(comptime T: type, path_: []const T) !std.fs.path.ComponentIterator(switch (builtin.target.os.tag) { .windows => .windows, .uefi => .uefi, diff --git a/src/cli/bunx_command.zig b/src/cli/bunx_command.zig index 2e40bab6d0..f8efa60d4b 100644 --- a/src/cli/bunx_command.zig +++ b/src/cli/bunx_command.zig @@ -63,43 +63,19 @@ pub const BunxCommand = struct { const nanoseconds_cache_valid = seconds_cache_valid * 1000000000; fn getBinNameFromSubpath(bundler: *bun.Bundler, dir_fd: bun.FileDescriptor, subpath_z: [:0]const u8) ![]const u8 { - const target_package_json_fd = try std.os.openatZ(dir_fd.cast(), subpath_z, std.os.O.RDONLY, 0); - const target_package_json = std.fs.File{ .handle = target_package_json_fd }; - - const is_stale = is_stale: { - if (Environment.isWindows) { - var io_status_block: std.os.windows.IO_STATUS_BLOCK = undefined; - var info: std.os.windows.FILE_BASIC_INFORMATION = undefined; - const rc = std.os.windows.ntdll.NtQueryInformationFile(target_package_json_fd, &io_status_block, &info, @sizeOf(std.os.windows.FILE_BASIC_INFORMATION), .FileBasicInformation); - switch (rc) { - .SUCCESS => { - const time = std.os.windows.fromSysTime(info.LastWriteTime); - const now = std.time.nanoTimestamp(); - break :is_stale (now - time > nanoseconds_cache_valid); - }, - // treat failures to stat as stale - else => break :is_stale true, - } - } else { - var stat: std.os.Stat = undefined; - const rc = std.c.fstat(target_package_json_fd, &stat); - if (rc != 0) { - break :is_stale true; - } - break :is_stale std.time.timestamp() - stat.mtime().tv_sec > seconds_cache_valid; - } - }; - - if (is_stale) { - target_package_json.close(); - // If delete fails, oh well. Hope installation takes care of it. - dir_fd.asDir().deleteTree(subpath_z) catch {}; - return error.NeedToInstall; - } + const target_package_json_fd = try bun.sys.openat(dir_fd, subpath_z, std.os.O.RDONLY, 0).unwrap(); + const target_package_json = bun.sys.File{ .handle = target_package_json_fd }; defer target_package_json.close(); - const package_json_contents = try target_package_json.readToEndAlloc(bundler.allocator, std.math.maxInt(u32)); + const package_json_read = target_package_json.readToEnd(bundler.allocator); + + // TODO: make this better + if (package_json_read.err) |err| { + try (bun.JSC.Maybe(void){ .err = err }).unwrap(); + } + + const package_json_contents = package_json_read.bytes.items; const source = bun.logger.Source.initPathString(bun.span(subpath_z), package_json_contents); bun.JSAst.Expr.Data.Store.create(default_allocator); @@ -134,9 +110,9 @@ pub const BunxCommand = struct { if (expr.asProperty("directories")) |dirs| { if (dirs.expr.asProperty("bin")) |bin_prop| { if (bin_prop.expr.asString(bundler.allocator)) |dir_name| { - const bin_dir = try std.os.openat(dir_fd.cast(), dir_name, std.os.O.RDONLY, 0); - defer std.os.close(bin_dir); - const dir = std.fs.Dir{ .fd = bin_dir }; + const bin_dir = try bun.sys.openatA(dir_fd, dir_name, std.os.O.RDONLY | std.os.O.DIRECTORY, 0).unwrap(); + defer _ = bun.sys.close(bin_dir); + const dir = std.fs.Dir{ .fd = bin_dir.cast() }; var iterator = bun.DirIterator.iterate(dir, .u8); var entry = iterator.next(); while (true) : (entry = iterator.next()) { @@ -159,17 +135,56 @@ pub const BunxCommand = struct { fn getBinNameFromProjectDirectory(bundler: *bun.Bundler, dir_fd: bun.FileDescriptor, package_name: []const u8) ![]const u8 { var subpath: [bun.MAX_PATH_BYTES]u8 = undefined; - const subpath_z = std.fmt.bufPrintZ(&subpath, "node_modules/{s}/package.json", .{package_name}) catch unreachable; + const subpath_z = std.fmt.bufPrintZ(&subpath, bun.pathLiteral("node_modules/{s}/package.json"), .{package_name}) catch unreachable; return try getBinNameFromSubpath(bundler, dir_fd, subpath_z); } - fn getBinNameFromTempDirectory(bundler: *bun.Bundler, tempdir_name: []const u8, package_name: []const u8) ![]const u8 { + fn getBinNameFromTempDirectory(bundler: *bun.Bundler, tempdir_name: []const u8, package_name: []const u8, with_stale_check: bool) ![]const u8 { var subpath: [bun.MAX_PATH_BYTES]u8 = undefined; + if (with_stale_check) { + const subpath_z = std.fmt.bufPrintZ( + &subpath, + bun.pathLiteral("{s}/package.json"), + .{tempdir_name}, + ) catch unreachable; + const target_package_json_fd = bun.sys.openat(bun.toFD(std.fs.cwd().fd), subpath_z, std.os.O.RDONLY, 0).unwrap() catch return error.NeedToInstall; + const target_package_json = bun.sys.File{ .handle = target_package_json_fd }; + + const is_stale = is_stale: { + if (Environment.isWindows) { + var io_status_block: std.os.windows.IO_STATUS_BLOCK = undefined; + var info: std.os.windows.FILE_BASIC_INFORMATION = undefined; + const rc = std.os.windows.ntdll.NtQueryInformationFile(target_package_json_fd.cast(), &io_status_block, &info, @sizeOf(std.os.windows.FILE_BASIC_INFORMATION), .FileBasicInformation); + switch (rc) { + .SUCCESS => { + const time = std.os.windows.fromSysTime(info.LastWriteTime); + const now = std.time.nanoTimestamp(); + break :is_stale (now - time > nanoseconds_cache_valid); + }, + // treat failures to stat as stale + else => break :is_stale true, + } + } else { + const stat = target_package_json.stat().unwrap() catch break :is_stale true; + break :is_stale std.time.timestamp() - stat.mtime().tv_sec > seconds_cache_valid; + } + }; + + if (is_stale) { + _ = target_package_json.close(); + // If delete fails, oh well. Hope installation takes care of it. + std.fs.cwd().deleteTree(tempdir_name) catch {}; + return error.NeedToInstall; + } + _ = target_package_json.close(); + } + const subpath_z = std.fmt.bufPrintZ( &subpath, - "{s}/node_modules/{s}/package.json", + bun.pathLiteral("{s}/node_modules/{s}/package.json"), .{ tempdir_name, package_name }, ) catch unreachable; + return try getBinNameFromSubpath(bundler, bun.toFD(std.fs.cwd().fd), subpath_z); } @@ -182,7 +197,7 @@ pub const BunxCommand = struct { return error.NoBinFound; } - return getBinNameFromTempDirectory(bundler, tempdir_name, package_name) catch |err2| { + return getBinNameFromTempDirectory(bundler, tempdir_name, package_name, true) catch |err2| { if (err2 == error.NoBinFound) { return error.NoBinFound; } @@ -520,7 +535,7 @@ pub const BunxCommand = struct { if (getBinName(&this_bundler, root_dir_fd, bunx_cache_dir, initial_bin_name)) |package_name_for_bin| { // if we check the bin name and its actually the same, we don't need to check $PATH here again if (!strings.eqlLong(package_name_for_bin, initial_bin_name, true)) { - absolute_in_cache_dir = std.fmt.bufPrint(&absolute_in_cache_dir_buf, "{s}/node_modules/.bin/{s}{s}", .{ bunx_cache_dir, package_name_for_bin, bun.exe_suffix }) catch unreachable; + absolute_in_cache_dir = std.fmt.bufPrint(&absolute_in_cache_dir_buf, bun.pathLiteral("{s}/node_modules/.bin/{s}{s}"), .{ bunx_cache_dir, package_name_for_bin, bun.exe_suffix }) catch unreachable; // Only use the system-installed version if there is no version specified if (update_request.version.literal.isEmpty()) { @@ -559,7 +574,6 @@ pub const BunxCommand = struct { } } } - const bunx_install_dir = try std.fs.cwd().makeOpenPath(bunx_cache_dir, .{}); create_package_json: { @@ -674,7 +688,7 @@ pub const BunxCommand = struct { } // 2. The "bin" is possibly not the same as the package name, so we load the package.json to figure out what "bin" to use - if (getBinNameFromTempDirectory(&this_bundler, bunx_cache_dir, result_package_name)) |package_name_for_bin| { + if (getBinNameFromTempDirectory(&this_bundler, bunx_cache_dir, result_package_name, false)) |package_name_for_bin| { if (!strings.eqlLong(package_name_for_bin, initial_bin_name, true)) { absolute_in_cache_dir = std.fmt.bufPrint(&absolute_in_cache_dir_buf, "{s}/node_modules/.bin/{s}{s}", .{ bunx_cache_dir, package_name_for_bin, bun.exe_suffix }) catch unreachable; diff --git a/src/deps/zig b/src/deps/zig new file mode 160000 index 0000000000..7fe33d94ea --- /dev/null +++ b/src/deps/zig @@ -0,0 +1 @@ +Subproject commit 7fe33d94eaeb1af7705e9c5f43a3b243aa895436 diff --git a/src/dir.zig b/src/dir.zig new file mode 100644 index 0000000000..c8580046b1 --- /dev/null +++ b/src/dir.zig @@ -0,0 +1,9 @@ +const bun = @import("root").bun; +const JSC = bun.JSC; +const std = @import("std"); +const builtin = @import("builtin"); +const FileDescriptor = bun.FileDescriptor; + +pub const Dir = struct { + fd: FileDescriptor, +}; diff --git a/src/fs.zig b/src/fs.zig index 26f286d912..25e7be0146 100644 --- a/src/fs.zig +++ b/src/fs.zig @@ -710,11 +710,7 @@ pub const FileSystem = struct { const flags = std.os.O.CREAT | std.os.O.WRONLY | std.os.O.CLOEXEC; - this.fd = brk: { - const fd = try bun.sys.openat(bun.toFD(tmpdir_.fd), name, flags, 0).unwrap(); - errdefer _ = bun.sys.close(fd); - break :brk try bun.toLibUVOwnedFD(fd); - }; + this.fd = try bun.sys.openat(bun.toFD(tmpdir_.fd), name, flags, 0).unwrap(); var buf: [bun.MAX_PATH_BYTES]u8 = undefined; const existing_path = try bun.getFdPath(this.fd, &buf); this.existing_path = try bun.default_allocator.dupe(u8, existing_path); @@ -940,7 +936,7 @@ pub const FileSystem = struct { pub fn openDir(_: *RealFS, unsafe_dir_string: string) !std.fs.Dir { const dirfd = if (Environment.isWindows) - bun.sys.openDirAtWindowsA(bun.invalid_fd, unsafe_dir_string, true, true) + bun.sys.openDirAtWindowsA(bun.invalid_fd, unsafe_dir_string, .{ .iterable = true, .no_follow = false }) else bun.sys.openA( unsafe_dir_string, diff --git a/src/install/extract_tarball.zig b/src/install/extract_tarball.zig index c0ab8c1011..e997b017b9 100644 --- a/src/install/extract_tarball.zig +++ b/src/install/extract_tarball.zig @@ -159,7 +159,7 @@ threadlocal var json_path_buf: bun.PathBuffer = undefined; fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractData { const tmpdir = this.temp_dir; - var tmpname_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + var tmpname_buf: if (Environment.isWindows) bun.WPathBuffer else bun.PathBuffer = undefined; const name = this.name.slice(); const basename = brk: { var tmp = name; @@ -183,22 +183,9 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD }; var resolved: string = ""; - const tmpname = try FileSystem.instance.tmpname(basename[0..@min(basename.len, 32)], &tmpname_buf, tgz_bytes.len); - const extract_fd_on_windows = brk: { - var extract_destination = switch (Environment.os) { - .windows => makeOpenPathAccessMaskW( - tmpdir, - std.mem.span(tmpname), - w.STANDARD_RIGHTS_READ | - w.FILE_READ_ATTRIBUTES | - w.FILE_READ_EA | - w.SYNCHRONIZE | - w.FILE_TRAVERSE | - w.DELETE, - false, - ), - else => tmpdir.makeOpenPath(std.mem.span(tmpname), .{}), - } catch |err| { + const tmpname = try FileSystem.instance.tmpname(basename[0..@min(basename.len, 32)], std.mem.asBytes(&tmpname_buf), bun.fastRandom()); + { + var extract_destination = bun.MakePath.makeOpenPath(tmpdir, bun.span(tmpname), .{}) catch |err| { this.package_manager.log.addErrorFmt( null, logger.Loc.Empty, @@ -209,8 +196,7 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD return error.InstallFailed; }; - errdefer if (Environment.isWindows) extract_destination.close(); - defer if (!Environment.isWindows) extract_destination.close(); + defer extract_destination.close(); if (PackageManager.verbose_install) { Output.prettyErrorln("[{s}] Start extracting {s}", .{ name, tmpname }); @@ -229,8 +215,8 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD null, logger.Loc.Empty, this.package_manager.allocator, - "{s} decompressing \"{s}\"", - .{ @errorName(err), name }, + "{s} decompressing \"{s}\" to \"{}\"", + .{ @errorName(err), name, bun.fmt.fmtPath(u8, std.mem.span(tmpname), .{}) }, ) catch unreachable; return error.InstallFailed; }; @@ -291,11 +277,7 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD Output.prettyErrorln("[{s}] Extracted", .{name}); Output.flush(); } - - if (Environment.isWindows) { - break :brk bun.toFD(extract_destination.fd); - } - }; + } const folder_name = switch (this.resolution.tag) { .npm => this.package_manager.cachedNPMPackageFolderNamePrint(&folder_name_buf, name, this.resolution.value.npm.version), .github => PackageManager.cachedGitHubFolderNamePrint(&folder_name_buf, resolved), @@ -307,29 +289,73 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD // e.g. @next // if it's a namespace package, we need to make sure the @name folder exists - if (basename.len != name.len and !this.resolution.tag.isGit()) { - cache_dir.makePath(std.mem.trim(u8, name[0 .. name.len - basename.len], "/")) catch {}; - } + const create_subdir = basename.len != name.len and !this.resolution.tag.isGit(); // Now that we've extracted the archive, we rename. if (comptime Environment.isWindows) { - defer _ = bun.sys.close(extract_fd_on_windows); + var did_retry = false; + var path2_buf: bun.WPathBuffer = undefined; + const path2 = bun.strings.toWPathNormalized(&path2_buf, folder_name); + var close_target_dir = false; + var target_dir = brk: { + if (create_subdir) { + if (bun.Dirname.dirname(u16, path2)) |folder| { + if (bun.MakePath.makeOpenPath(cache_dir, folder, .{})) |targ| { + close_target_dir = true; + break :brk targ; + } else |_| {} + } + } - var folder_name_wbuf: bun.WPathBuffer = undefined; - const folder_name_w = bun.strings.toWPathNormalized(&folder_name_wbuf, folder_name); + break :brk cache_dir; + }; + defer if (close_target_dir) target_dir.close(); - switch (bun.C.moveOpenedFileAtLoose(extract_fd_on_windows, bun.toFD(cache_dir.fd), folder_name_w, false)) { - .err => |err| { + while (true) { + const dir_to_move = bun.sys.openDirAtWindowsA(bun.toFD(this.temp_dir.fd), bun.span(tmpname), .{ .can_rename_or_delete = true, .create = false, .iterable = false }).unwrap() catch |err| { + // i guess we just this.package_manager.log.addErrorFmt( null, logger.Loc.Empty, this.package_manager.allocator, - "moving \"{s}\" to cache dir failed: {}\n From: {s}\n To: {}", - .{ name, err, tmpname, bun.fmt.utf16(folder_name_w) }, + "moving \"{s}\" to cache dir failed\n{}\n From: {s}\n To: {s}", + .{ name, err, tmpname, folder_name }, ) catch unreachable; return error.InstallFailed; - }, - .result => {}, + }; + + switch (bun.C.moveOpenedFileAt(dir_to_move, bun.toFD(target_dir.fd), path2[if (std.mem.lastIndexOfScalar(u16, path2, '\\')) |i| i + 1 else 0..], true)) { + .err => |err| { + if (!did_retry) { + switch (err.getErrno()) { + .PERM, .BUSY, .EXIST => { + // before we attempt to delete the destination, let's close the source dir. + _ = bun.sys.close(dir_to_move); + + // two copies of bun are trying to extract the same package version to the same folder + cache_dir.deleteTree(bun.span(folder_name)) catch {}; + did_retry = true; + continue; + }, + else => {}, + } + } + _ = bun.sys.close(dir_to_move); + this.package_manager.log.addErrorFmt( + null, + logger.Loc.Empty, + this.package_manager.allocator, + "moving \"{s}\" to cache dir failed\n{}\n From: {s}\n To: {s}", + .{ name, err, tmpname, folder_name }, + ) catch unreachable; + return error.InstallFailed; + }, + .result => { + _ = bun.sys.close(dir_to_move); + }, + } + + break; } } else { // Attempt to gracefully handle duplicate concurrent `bun install` calls @@ -342,6 +368,12 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD // const src = bun.sliceTo(tmpname, 0); + if (create_subdir) { + if (bun.Dirname.dirname(u8, folder_name)) |folder| { + bun.MakePath.makePath(u8, cache_dir, folder) catch {}; + } + } + var did_atomically_replace = false; if (did_atomically_replace and PackageManager.using_fallback_temp_dir) tmpdir.deleteTree(src) catch {}; @@ -393,7 +425,7 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD // We return a resolved absolute absolute file path to the cache dir. // To get that directory, we open the directory again. - var final_dir = cache_dir.openDirZ(folder_name, .{}) catch |err| { + var final_dir = bun.openDir(cache_dir, folder_name) catch |err| { this.package_manager.log.addErrorFmt( null, logger.Loc.Empty, @@ -419,9 +451,46 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD return error.InstallFailed; }; + var json_path: []u8 = ""; + var json_buf: []u8 = ""; + if (switch (this.resolution.tag) { + // TODO remove extracted files not matching any globs under "files" + .github, .local_tarball, .remote_tarball => true, + else => this.package_manager.lockfile.trusted_dependencies != null and + this.package_manager.lockfile.trusted_dependencies.?.contains(@truncate(Semver.String.Builder.stringHash(name))), + }) { + const json_file, json_buf = bun.sys.File.readFileFrom( + bun.toFD(cache_dir.fd), + bun.path.joinZ(&[_]string{ folder_name, "package.json" }, .auto), + bun.default_allocator, + ).unwrap() catch |err| { + this.package_manager.log.addErrorFmt( + null, + logger.Loc.Empty, + this.package_manager.allocator, + "\"package.json\" for \"{s}\" failed to open: {s}", + .{ name, @errorName(err) }, + ) catch unreachable; + return error.InstallFailed; + }; + defer json_file.close(); + json_path = json_file.getPath( + &json_path_buf, + ).unwrap() catch |err| { + this.package_manager.log.addErrorFmt( + null, + logger.Loc.Empty, + this.package_manager.allocator, + "\"package.json\" for \"{s}\" failed to resolve: {s}", + .{ name, @errorName(err) }, + ) catch unreachable; + return error.InstallFailed; + }; + } + // create an index storing each version of a package installed if (strings.indexOfChar(basename, '/') == null) create_index: { - var index_dir = cache_dir.makeOpenPath(name, .{}) catch break :create_index; + var index_dir = bun.MakePath.makeOpenPath(cache_dir, name, .{}) catch break :create_index; defer index_dir.close(); index_dir.symLink( final_path, @@ -435,45 +504,6 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD ) catch break :create_index; } - var json_path: []u8 = ""; - var json_buf: []u8 = ""; - var json_len: usize = 0; - if (switch (this.resolution.tag) { - // TODO remove extracted files not matching any globs under "files" - .github, .local_tarball, .remote_tarball => true, - else => this.package_manager.lockfile.trusted_dependencies != null and - this.package_manager.lockfile.trusted_dependencies.?.contains(@truncate(Semver.String.Builder.stringHash(name))), - }) { - const json_file = final_dir.openFileZ("package.json", .{ .mode = .read_only }) catch |err| { - this.package_manager.log.addErrorFmt( - null, - logger.Loc.Empty, - this.package_manager.allocator, - "\"package.json\" for \"{s}\" failed to open: {s}", - .{ name, @errorName(err) }, - ) catch unreachable; - return error.InstallFailed; - }; - defer json_file.close(); - const json_stat_size = try json_file.getEndPos(); - json_buf = try this.package_manager.allocator.alloc(u8, json_stat_size + 64); - json_len = try json_file.preadAll(json_buf, 0); - - json_path = bun.getFdPath( - json_file.handle, - &json_path_buf, - ) catch |err| { - this.package_manager.log.addErrorFmt( - null, - logger.Loc.Empty, - this.package_manager.allocator, - "\"package.json\" for \"{s}\" failed to resolve: {s}", - .{ name, @errorName(err) }, - ) catch unreachable; - return error.InstallFailed; - }; - } - const ret_json_path = try FileSystem.instance.dirname_store.append(@TypeOf(json_path), json_path); const url = try FileSystem.instance.dirname_store.append(@TypeOf(this.url.slice()), this.url.slice()); @@ -482,96 +512,5 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD .resolved = resolved, .json_path = ret_json_path, .json_buf = json_buf, - .json_len = json_len, }; } - -// TODO(@paperdave): upstream making this public into zig std -// there is zero reason this must be copied -// -/// Calls makeOpenDirAccessMaskW iteratively to make an entire path -/// (i.e. creating any parent directories that do not exist). -/// Opens the dir if the path already exists and is a directory. -/// This function is not atomic, and if it returns an error, the file system may -/// have been modified regardless. -fn makeOpenPathAccessMaskW(self: std.fs.Dir, sub_path: []const u8, access_mask: u32, no_follow: bool) !std.fs.Dir { - var it = try std.fs.path.componentIterator(sub_path); - // If there are no components in the path, then create a dummy component with the full path. - var component = it.last() orelse std.fs.path.NativeUtf8ComponentIterator.Component{ - .name = "", - .path = sub_path, - }; - - while (true) { - const sub_path_w = try w.sliceToPrefixedFileW(self.fd, component.path); - const is_last = it.peekNext() == null; - var result = makeOpenDirAccessMaskW(self, sub_path_w.span().ptr, access_mask, .{ - .no_follow = no_follow, - .create_disposition = if (is_last) w.FILE_OPEN_IF else w.FILE_CREATE, - }) catch |err| switch (err) { - error.FileNotFound => |e| { - component = it.previous() orelse return e; - continue; - }, - else => |e| return e, - }; - - component = it.next() orelse return result; - // Don't leak the intermediate file handles - result.close(); - } -} -const MakeOpenDirAccessMaskWOptions = struct { - no_follow: bool, - create_disposition: u32, -}; - -fn makeOpenDirAccessMaskW(self: std.fs.Dir, sub_path_w: [*:0]const u16, access_mask: u32, flags: MakeOpenDirAccessMaskWOptions) !std.fs.Dir { - var result = std.fs.Dir{ - .fd = undefined, - }; - - const path_len_bytes = @as(u16, @intCast(std.mem.sliceTo(sub_path_w, 0).len * 2)); - var nt_name = w.UNICODE_STRING{ - .Length = path_len_bytes, - .MaximumLength = path_len_bytes, - .Buffer = @constCast(sub_path_w), - }; - var attr = w.OBJECT_ATTRIBUTES{ - .Length = @sizeOf(w.OBJECT_ATTRIBUTES), - .RootDirectory = if (std.fs.path.isAbsoluteWindowsW(sub_path_w)) null else self.fd, - .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. - .ObjectName = &nt_name, - .SecurityDescriptor = null, - .SecurityQualityOfService = null, - }; - const open_reparse_point: w.DWORD = if (flags.no_follow) w.FILE_OPEN_REPARSE_POINT else 0x0; - var io: w.IO_STATUS_BLOCK = undefined; - const rc = w.ntdll.NtCreateFile( - &result.fd, - access_mask, - &attr, - &io, - null, - w.FILE_ATTRIBUTE_NORMAL, - w.FILE_SHARE_READ | w.FILE_SHARE_WRITE, - flags.create_disposition, - w.FILE_DIRECTORY_FILE | w.FILE_SYNCHRONOUS_IO_NONALERT | w.FILE_OPEN_FOR_BACKUP_INTENT | open_reparse_point, - null, - 0, - ); - - switch (rc) { - .SUCCESS => return result, - .OBJECT_NAME_INVALID => return error.BadPathName, - .OBJECT_NAME_NOT_FOUND => return error.FileNotFound, - .OBJECT_PATH_NOT_FOUND => return error.FileNotFound, - .NOT_A_DIRECTORY => return error.NotDir, - // This can happen if the directory has 'List folder contents' permission set to 'Deny' - // and the directory is trying to be opened for iteration. - .ACCESS_DENIED => return error.AccessDenied, - .INVALID_PARAMETER => return error.BadPathName, - .SHARING_VIOLATION => return error.SharingViolation, - else => return w.unexpectedStatus(rc), - } -} diff --git a/src/install/install.zig b/src/install/install.zig index 6d7a37e7b5..cbf14e6455 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -13,7 +13,7 @@ const std = @import("std"); const uws = @import("../deps/uws.zig"); const JSC = bun.JSC; const DirInfo = @import("../resolver/dir_info.zig"); - +const File = bun.sys.File; const JSLexer = bun.js_lexer; const logger = bun.logger; @@ -57,6 +57,7 @@ threadlocal var initialized_store = false; const Futex = @import("../futex.zig"); pub const Lockfile = @import("./lockfile.zig"); +const Walker = @import("../walker_skippable.zig"); // these bytes are skipped // so we just make it repeat bun bun bun bun bun bun bun bun bun @@ -858,7 +859,6 @@ pub const ExtractData = struct { resolved: string = "", json_path: string = "", json_buf: []u8 = "", - json_len: usize = 0, }; pub const PackageInstall = struct { @@ -1017,14 +1017,6 @@ pub const PackageInstall = struct { var total: usize = 0; var read: usize = 0; - bun.copy(u8, this.destination_dir_subpath_buf[this.destination_dir_subpath.len..], std.fs.path.sep_str ++ "package.json"); - this.destination_dir_subpath_buf[this.destination_dir_subpath.len + std.fs.path.sep_str.len + "package.json".len] = 0; - const package_json_path: [:0]u8 = this.destination_dir_subpath_buf[0 .. this.destination_dir_subpath.len + std.fs.path.sep_str.len + "package.json".len :0]; - defer this.destination_dir_subpath_buf[this.destination_dir_subpath.len] = 0; - - var package_json_file = this.destination_dir.openFileZ(package_json_path, .{ .mode = .read_only }) catch return false; - defer package_json_file.close(); - var body_pool = Npm.Registry.BodyPool.get(allocator); var mutable: MutableString = body_pool.data; defer { @@ -1032,34 +1024,49 @@ pub const PackageInstall = struct { Npm.Registry.BodyPool.release(body_pool); } - mutable.reset(); - mutable.list.expandToCapacity(); - - // Heuristic: most package.jsons will be less than 2048 bytes. - read = package_json_file.read(mutable.list.items[total..]) catch return false; - var remain = mutable.list.items[@min(total, read)..]; - if (read > 0 and remain.len < 1024) { - mutable.growBy(4096) catch return false; + // Read the file + // Return false on any error. + // Don't keep it open while we're parsing the JSON. + // The longer the file stays open, the more likely it causes issues for + // other processes on Windows. + const source = brk: { + mutable.reset(); mutable.list.expandToCapacity(); - } + bun.copy(u8, this.destination_dir_subpath_buf[this.destination_dir_subpath.len..], std.fs.path.sep_str ++ "package.json"); + this.destination_dir_subpath_buf[this.destination_dir_subpath.len + std.fs.path.sep_str.len + "package.json".len] = 0; + const package_json_path: [:0]u8 = this.destination_dir_subpath_buf[0 .. this.destination_dir_subpath.len + std.fs.path.sep_str.len + "package.json".len :0]; + defer this.destination_dir_subpath_buf[this.destination_dir_subpath.len] = 0; - while (read > 0) : (read = package_json_file.read(remain) catch return false) { - total += read; + var package_json_file = File.openat(this.destination_dir, package_json_path, std.os.O.RDONLY, 0).unwrap() catch return false; + defer package_json_file.close(); - mutable.list.expandToCapacity(); - remain = mutable.list.items[total..]; - - if (remain.len < 1024) { + // Heuristic: most package.jsons will be less than 2048 bytes. + read = package_json_file.read(mutable.list.items[total..]).unwrap() catch return false; + var remain = mutable.list.items[@min(total, read)..]; + if (read > 0 and remain.len < 1024) { mutable.growBy(4096) catch return false; + mutable.list.expandToCapacity(); } - mutable.list.expandToCapacity(); - remain = mutable.list.items[total..]; - } - // If it's not long enough to have {"name": "foo", "version": "1.2.0"}, there's no way it's valid - if (total < "{\"name\":\"\",\"version\":\"\"}".len + this.package_name.len + this.package_version.len) return false; + while (read > 0) : (read = package_json_file.read(remain).unwrap() catch return false) { + total += read; + + mutable.list.expandToCapacity(); + remain = mutable.list.items[total..]; + + if (remain.len < 1024) { + mutable.growBy(4096) catch return false; + } + mutable.list.expandToCapacity(); + remain = mutable.list.items[total..]; + } + + // If it's not long enough to have {"name": "foo", "version": "1.2.0"}, there's no way it's valid + if (total < "{\"name\":\"\",\"version\":\"\"}".len + this.package_name.len + this.package_version.len) return false; + + break :brk logger.Source.initPathString(bun.span(package_json_path), mutable.list.items[0..total]); + }; - const source = logger.Source.initPathString(bun.span(package_json_path), mutable.list.items[0..total]); var log = logger.Log.init(allocator); defer log.deinit(); @@ -1115,6 +1122,10 @@ pub const PackageInstall = struct { } }, + pub inline fn success() Result { + return .{ .success = {} }; + } + pub fn fail(err: anyerror, step: Step) Result { return .{ .fail = .{ @@ -1124,6 +1135,13 @@ pub const PackageInstall = struct { }; } + pub fn isFail(this: @This()) bool { + return switch (this) { + .success => false, + .fail => true, + }; + } + pub const Tag = enum { success, fail, @@ -1143,8 +1161,6 @@ pub const PackageInstall = struct { Method.hardlink; fn installWithClonefileEachDir(this: *PackageInstall) !Result { - const Walker = @import("../walker_skippable.zig"); - var cached_package_dir = bun.openDir(this.cache_dir, this.cache_dir_subpath) catch |err| return Result{ .fail = .{ .err = err, .step = .opening_cache_dir }, }; @@ -1169,7 +1185,7 @@ pub const PackageInstall = struct { while (try walker.next()) |entry| { switch (entry.kind) { .directory => { - std.os.mkdirat(destination_dir_.fd, entry.path, 0o755) catch {}; + _ = bun.sys.mkdirat(bun.toFD(destination_dir_.fd), entry.path, 0o755); }, .file => { bun.copy(u8, &stackpath, entry.path); @@ -1258,79 +1274,197 @@ pub const PackageInstall = struct { }, }; } - fn installWithCopyfile(this: *PackageInstall) Result { - const Walker = @import("../walker_skippable.zig"); - var cached_package_dir = bun.openDir(this.cache_dir, this.cache_dir_subpath) catch |err| return Result{ + const InstallDirState = struct { + cached_package_dir: std.fs.Dir = undefined, + walker: Walker = undefined, + subdir: std.fs.Dir = if (Environment.isWindows) std.fs.Dir{ .fd = std.os.windows.INVALID_HANDLE_VALUE } else undefined, + buf: bun.windows.WPathBuffer = if (Environment.isWindows) undefined else {}, + buf2: bun.windows.WPathBuffer = if (Environment.isWindows) undefined else {}, + to_copy_buf: if (Environment.isWindows) []u16 else void = if (Environment.isWindows) undefined else {}, + to_copy_buf2: if (Environment.isWindows) []u16 else void = if (Environment.isWindows) undefined else {}, + + pub fn deinit(this: *@This()) void { + if (!Environment.isWindows) { + this.subdir.close(); + } + defer this.walker.deinit(); + defer this.cached_package_dir.close(); + } + }; + + threadlocal var node_fs_for_package_installer: bun.JSC.Node.NodeFS = .{}; + + fn initInstallDir(this: *PackageInstall, state: *InstallDirState) Result { + const destbase = this.destination_dir; + const destpath = this.destination_dir_subpath; + + state.cached_package_dir = bun.openDir(this.cache_dir, this.cache_dir_subpath) catch |err| return Result{ .fail = .{ .err = err, .step = .opening_cache_dir }, }; - defer cached_package_dir.close(); - var walker_ = Walker.walk( - cached_package_dir, + state.walker = Walker.walk( + state.cached_package_dir, this.allocator, &[_]bun.OSPathSlice{}, &[_]bun.OSPathSlice{}, - ) catch |err| return Result{ - .fail = .{ .err = err, .step = .opening_cache_dir }, + ) catch |err| { + state.cached_package_dir.close(); + return Result.fail(err, .opening_cache_dir); }; - defer walker_.deinit(); + + if (!Environment.isWindows) { + state.subdir = destbase.makeOpenPath(bun.span(destpath), .{ + .iterate = true, + .access_sub_paths = true, + }) catch |err| { + state.cached_package_dir.close(); + state.walker.deinit(); + return Result.fail(err, .copying_files); + }; + return Result.success(); + } + + const dest_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(destbase.fd, &state.buf, state.buf.len, 0); + if (dest_path_length == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + state.cached_package_dir.close(); + state.walker.deinit(); + return Result.fail(err, .copying_files); + } + + var i: usize = dest_path_length; + if (state.buf[i] != '\\') { + state.buf[i] = '\\'; + i += 1; + } + + i += bun.strings.toWPathNormalized(state.buf[i..], destpath).len; + state.buf[i] = std.fs.path.sep_windows; + i += 1; + state.buf[i] = 0; + const fullpath = state.buf[0..i :0]; + + _ = node_fs_for_package_installer.mkdirRecursiveOSPathImpl(void, {}, fullpath, 0, false).unwrap() catch |err| { + state.cached_package_dir.close(); + state.walker.deinit(); + return Result.fail(err, .copying_files); + }; + + const cache_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(state.cached_package_dir.fd, &state.buf2, state.buf2.len, 0); + if (cache_path_length == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + state.cached_package_dir.close(); + state.walker.deinit(); + return Result.fail(err, .copying_files); + } + const cache_path = state.buf2[0..cache_path_length]; + var to_copy_buf2: []u16 = undefined; + if (state.buf2[cache_path.len - 1] != '\\') { + state.buf2[cache_path.len] = '\\'; + to_copy_buf2 = state.buf2[cache_path.len + 1 ..]; + } else { + to_copy_buf2 = state.buf2[cache_path.len..]; + } + state.to_copy_buf = state.buf[fullpath.len..]; + state.to_copy_buf2 = to_copy_buf2; + + return Result.success(); + } + + fn installWithCopyfile(this: *PackageInstall) Result { + var state = InstallDirState{}; + const res = this.initInstallDir(&state); + if (res.isFail()) return res; + defer state.deinit(); const FileCopier = struct { pub fn copy( destination_dir_: std.fs.Dir, walker: *Walker, progress_: *Progress, + to_copy_into1: if (Environment.isWindows) []u16 else void, + head1: if (Environment.isWindows) []u16 else void, + to_copy_into2: if (Environment.isWindows) []u16 else void, + head2: if (Environment.isWindows) []u16 else void, ) !u32 { var real_file_count: u32 = 0; - var in_buf: if (Environment.isWindows) bun.OSPathBuffer else void = undefined; - var out_buf: if (Environment.isWindows) bun.OSPathBuffer else void = undefined; var copy_file_state: bun.CopyFileState = .{}; while (try walker.next()) |entry| { - if (entry.kind != .file) continue; - real_file_count += 1; - - const createFile = if (comptime Environment.isWindows) std.fs.Dir.createFileW else std.fs.Dir.createFile; - - var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { - if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { - bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; - } - break :brk createFile(destination_dir_, entry.path, .{}) catch |err| { - progress_.root.end(); - - progress_.refresh(); - - Output.prettyErrorln("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - Global.crash(); - }; - }; - defer outfile.close(); - - const openFile = if (comptime Environment.isWindows) std.fs.Dir.openFileW else std.fs.Dir.openFile; - - var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); - defer in_file.close(); - if (comptime Environment.isWindows) { - const in_path = bun.getFdPathW(in_file.handle, &in_buf) catch @panic("Failed to copyfile"); - in_buf[in_path.len] = 0; - const in = in_buf[0..in_path.len :0]; + switch (entry.kind) { + .directory, .file => {}, + else => continue, + } - const out_path = bun.getFdPathW(outfile.handle, &out_buf) catch @panic("Failed to copyfile"); - out_buf[out_path.len] = 0; - const out = out_buf[0..out_path.len :0]; + if (entry.path.len > to_copy_into1.len or entry.path.len > to_copy_into2.len) { + return error.NameTooLong; + } - bun.copyFile(in, out) catch |err| { - progress_.root.end(); + @memcpy(to_copy_into1[0..entry.path.len], entry.path); + head1[entry.path.len + (head1.len - to_copy_into1.len)] = 0; + const dest: [:0]u16 = head1[0 .. entry.path.len + head1.len - to_copy_into1.len :0]; - progress_.refresh(); + @memcpy(to_copy_into2[0..entry.path.len], entry.path); + head2[entry.path.len + (head1.len - to_copy_into2.len)] = 0; + const src: [:0]u16 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; - Output.prettyError("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); - Global.crash(); - }; + switch (entry.kind) { + .directory => { + if (bun.windows.CreateDirectoryExW(src.ptr, dest.ptr, null) == 0) { + bun.MakePath.makePath(u16, destination_dir_, entry.path) catch {}; + } + }, + .file => { + if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) == 0) { + if (bun.Dirname.dirname(u16, entry.path)) |entry_dirname| { + bun.MakePath.makePath(u16, destination_dir_, entry_dirname) catch {}; + if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) != 0) { + continue; + } + } + + progress_.root.end(); + progress_.refresh(); + + if (bun.windows.Win32Error.get().toSystemErrno()) |err| { + Output.prettyError("{s}: copying file {}", .{ @tagName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + } else { + Output.prettyError("error copying file {}", .{bun.fmt.fmtOSPath(entry.path, .{})}); + } + + Global.crash(); + } + }, + else => unreachable, // handled above + } } else { + if (entry.kind != .file) continue; + real_file_count += 1; + const openFile = std.fs.Dir.openFile; + const createFile = std.fs.Dir.createFile; + + var in_file = try openFile(entry.dir, entry.basename, .{ .mode = .read_only }); + defer in_file.close(); + + var outfile = createFile(destination_dir_, entry.path, .{}) catch brk: { + if (bun.Dirname.dirname(bun.OSPathChar, entry.path)) |entry_dirname| { + bun.MakePath.makePath(bun.OSPathChar, destination_dir_, entry_dirname) catch {}; + } + break :brk createFile(destination_dir_, entry.path, .{}) catch |err| { + progress_.root.end(); + + progress_.refresh(); + + Output.prettyErrorln("{s}: copying file {}", .{ @errorName(err), bun.fmt.fmtOSPath(entry.path, .{}) }); + Global.crash(); + }; + }; + defer outfile.close(); + if (comptime Environment.isPosix) { const stat = in_file.stat() catch continue; _ = C.fchmod(outfile.handle, @intCast(stat.mode)); @@ -1351,13 +1485,15 @@ pub const PackageInstall = struct { } }; - var subdir = this.destination_dir.makeOpenPath(bun.span(this.destination_dir_subpath), .{}) catch |err| return Result{ - .fail = .{ .err = err, .step = .opening_cache_dir }, - }; - - defer subdir.close(); - - this.file_count = FileCopier.copy(subdir, &walker_, this.progress) catch |err| return Result{ + this.file_count = FileCopier.copy( + state.subdir, + &state.walker, + this.progress, + if (Environment.isWindows) state.to_copy_buf else void{}, + if (Environment.isWindows) &state.buf else void{}, + if (Environment.isWindows) state.to_copy_buf2 else void{}, + if (Environment.isWindows) &state.buf2 else void{}, + ) catch |err| return Result{ .fail = .{ .err = err, .step = .copying_files }, }; @@ -1367,63 +1503,10 @@ pub const PackageInstall = struct { } fn installWithHardlink(this: *PackageInstall) !Result { - const Walker = @import("../walker_skippable.zig"); - - var cached_package_dir = bun.openDir(this.cache_dir, this.cache_dir_subpath) catch |err| return Result.fail(err, .opening_cache_dir); - defer cached_package_dir.close(); - var walker_ = Walker.walk( - cached_package_dir, - this.allocator, - &[_]bun.OSPathSlice{}, - &[_]bun.OSPathSlice{}, - ) catch |err| return Result.fail(err, .opening_cache_dir); - defer walker_.deinit(); - - var subdir = this.destination_dir.makeOpenPath(bun.span(this.destination_dir_subpath), .{}) catch |err| return Result.fail(err, .opening_cache_dir); - - defer subdir.close(); - - var buf: bun.windows.WPathBuffer = undefined; - var buf2: bun.windows.WPathBuffer = undefined; - var to_copy_buf: []u16 = undefined; - var to_copy_buf2: []u16 = undefined; - if (comptime Environment.isWindows) { - const dest_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(subdir.fd, &buf, buf.len, 0); - if (dest_path_length == 0) { - const e = bun.windows.Win32Error.get(); - const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else brk: { - // If this code path is reached, it should have a toSystemErrno mapping - Output.warn("Failed to get destination path for package \"{s}\" during installation: {s}", .{ this.package_name, @tagName(e) }); - break :brk error.Unexpected; - }; - return Result.fail(err, .copying_files); - } - const dest_path = buf[0..dest_path_length]; - if (buf[dest_path.len - 1] != '\\') { - buf[dest_path.len] = '\\'; - to_copy_buf = buf[dest_path.len + 1 ..]; - } else { - to_copy_buf = buf[dest_path.len..]; - } - - const cache_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(cached_package_dir.fd, &buf2, buf2.len, 0); - if (cache_path_length == 0) { - const e = bun.windows.Win32Error.get(); - const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else brk: { - // If this code path is reached, it should have a toSystemErrno mapping - Output.warn("Failed to get cache path for package \"{s}\" during installation: {s}", .{ this.package_name, @tagName(e) }); - break :brk error.Unexpected; - }; - return Result.fail(err, .copying_files); - } - const cache_path = buf2[0..cache_path_length]; - if (buf2[cache_path.len - 1] != '\\') { - buf2[cache_path.len] = '\\'; - to_copy_buf2 = buf2[cache_path.len + 1 ..]; - } else { - to_copy_buf2 = buf2[cache_path.len..]; - } - } + var state = InstallDirState{}; + const res = this.initInstallDir(&state); + if (res.isFail()) return res; + defer state.deinit(); const FileCopier = struct { pub fn copy( @@ -1436,38 +1519,12 @@ pub const PackageInstall = struct { ) !u32 { var real_file_count: u32 = 0; while (try walker.next()) |entry| { - switch (entry.kind) { - .directory => { - const mkdirat = if (comptime Environment.isWindows) std.os.mkdiratW else std.os.mkdirat; - mkdirat(destination_dir.fd, entry.path, 0o755) catch {}; - }, - .file => { - if (comptime Environment.isWindows) { - if (entry.path.len > to_copy_into1.len or entry.path.len > to_copy_into2.len) { - return error.NameTooLong; - } - - @memcpy(to_copy_into1[0..entry.path.len], entry.path); - head1[entry.path.len + (head1.len - to_copy_into1.len)] = 0; - const dest: [:0]u16 = head1[0 .. entry.path.len + head1.len - to_copy_into1.len :0]; - - @memcpy(to_copy_into2[0..entry.path.len], entry.path); - head2[entry.path.len + (head1.len - to_copy_into2.len)] = 0; - const src: [:0]u16 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; - - // Windows limits hardlinks to 1023 per file - if (bun.windows.CreateHardLinkW(dest, src, null) == 0) { - if (bun.windows.CopyFileW(src, dest, 0) == 0) { - const e = bun.windows.Win32Error.get(); - if (e.toSystemErrno()) |err| { - return bun.errnoToZigErr(err); - } - // If this code path is reached, it should have a toSystemErrno mapping - Output.warn("Failed to copy file during installation: {s}", .{@tagName(e)}); - return error.FailedToCopyFile; - } - } - } else { + if (comptime Environment.isPosix) { + switch (entry.kind) { + .directory => { + bun.MakePath.makePath(std.meta.Elem(@TypeOf(entry.path)), destination_dir, entry.path) catch {}; + }, + .file => { std.os.linkat(entry.dir.fd, entry.basename, destination_dir.fd, entry.path, 0) catch |err| { if (err != error.PathAlreadyExists) { return err; @@ -1476,11 +1533,65 @@ pub const PackageInstall = struct { std.os.unlinkat(destination_dir.fd, entry.path, 0) catch {}; try std.os.linkat(entry.dir.fd, entry.basename, destination_dir.fd, entry.path, 0); }; - } - real_file_count += 1; - }, - else => {}, + real_file_count += 1; + }, + else => {}, + } + } else { + switch (entry.kind) { + .file => {}, + else => continue, + } + + if (entry.path.len > to_copy_into1.len or entry.path.len > to_copy_into2.len) { + return error.NameTooLong; + } + + @memcpy(to_copy_into1[0..entry.path.len], entry.path); + head1[entry.path.len + (head1.len - to_copy_into1.len)] = 0; + const dest: [:0]u16 = head1[0 .. entry.path.len + head1.len - to_copy_into1.len :0]; + + @memcpy(to_copy_into2[0..entry.path.len], entry.path); + head2[entry.path.len + (head1.len - to_copy_into2.len)] = 0; + const src: [:0]u16 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; + + if (bun.windows.CreateHardLinkW(dest.ptr, src.ptr, null) != 0) { + continue; + } + + dest[dest.len - entry.basename.len - 1] = 0; + const dirpath = dest[0 .. dest.len - entry.basename.len - 1 :0]; + _ = node_fs_for_package_installer.mkdirRecursiveOSPathImpl(void, {}, dirpath, 0, false).unwrap() catch {}; + dest[dest.len - entry.basename.len - 1] = std.fs.path.sep; + if (bun.windows.CreateHardLinkW(dest.ptr, src.ptr, null) != 0) { + continue; + } + + if (PackageManager.verbose_install) { + const once_log = struct { + var once = false; + + pub fn get() bool { + const prev = once; + once = true; + return !prev; + } + }.get(); + + if (once_log) { + Output.warn("CreateHardLinkW failed, falling back to CopyFileW: {} -> {}\n", .{ + bun.fmt.fmtOSPath(src, .{}), + bun.fmt.fmtOSPath(dest, .{}), + }); + } + } + + if (bun.windows.CopyFileW(src.ptr, dest.ptr, 0) != 0) { + continue; + } + + return bun.errnoToZigErr(bun.windows.getLastErrno()); } } @@ -1489,12 +1600,12 @@ pub const PackageInstall = struct { }; this.file_count = FileCopier.copy( - subdir, - &walker_, - if (Environment.isWindows) to_copy_buf else void{}, - if (Environment.isWindows) &buf else void{}, - if (Environment.isWindows) to_copy_buf2 else void{}, - if (Environment.isWindows) &buf2 else void{}, + state.subdir, + &state.walker, + state.to_copy_buf, + if (Environment.isWindows) &state.buf else void{}, + state.to_copy_buf2, + if (Environment.isWindows) &state.buf2 else void{}, ) catch |err| { if (comptime Environment.isWindows) { if (err == error.FailedToCopyFile) { @@ -1512,73 +1623,117 @@ pub const PackageInstall = struct { } fn installWithSymlink(this: *PackageInstall) !Result { - const Walker = @import("../walker_skippable.zig"); + var state = InstallDirState{}; + const res = this.initInstallDir(&state); + if (res.isFail()) return res; + defer state.deinit(); - var cached_package_dir = bun.openDir(this.cache_dir, this.cache_dir_subpath) catch |err| return Result{ - .fail = .{ .err = err, .step = .opening_cache_dir }, - }; - defer cached_package_dir.close(); - var walker_ = Walker.walk( - cached_package_dir, - this.allocator, - &[_]bun.OSPathSlice{}, - &[_]bun.OSPathSlice{ - bun.OSPathLiteral("node_modules"), - bun.OSPathLiteral(".git"), - }, - ) catch |err| return Result{ - .fail = .{ .err = err, .step = .opening_cache_dir }, - }; - defer walker_.deinit(); + var buf2: bun.PathBuffer = undefined; + var to_copy_buf2: []u8 = undefined; + if (Environment.isPosix) { + const cache_dir_path = try bun.getFdPath(state.cached_package_dir.fd, &buf2); + if (cache_dir_path.len > 0 and cache_dir_path[cache_dir_path.len - 1] != std.fs.path.sep) { + buf2[cache_dir_path.len] = std.fs.path.sep; + to_copy_buf2 = buf2[cache_dir_path.len + 1 ..]; + } else { + to_copy_buf2 = buf2[cache_dir_path.len..]; + } + } const FileCopier = struct { pub fn copy( - dest_dir_fd: bun.FileDescriptor, - cache_dir_fd: bun.FileDescriptor, + destination_dir: std.fs.Dir, walker: *Walker, + to_copy_into1: if (Environment.isWindows) []u16 else void, + head1: if (Environment.isWindows) []u16 else void, + to_copy_into2: []if (Environment.isWindows) u16 else u8, + head2: []if (Environment.isWindows) u16 else u8, ) !u32 { var real_file_count: u32 = 0; - var buf: [bun.MAX_PATH_BYTES]u8 = undefined; - const cache_dir_path = try bun.getFdPath(cache_dir_fd, &buf); - - var remain = buf[cache_dir_path.len..]; - var cache_dir_offset = cache_dir_path.len; - if (cache_dir_path.len > 0 and cache_dir_path[cache_dir_path.len - 1] != std.fs.path.sep) { - remain[0] = std.fs.path.sep; - cache_dir_offset += 1; - remain = remain[1..]; - } - var dest_buf: [bun.MAX_PATH_BYTES]u8 = undefined; - const dest_base = try bun.getFdPath(dest_dir_fd, &dest_buf); - var dest_remaining = dest_buf[dest_base.len..]; - var dest_dir_offset = dest_base.len; - if (dest_base.len > 0 and dest_buf[dest_base.len - 1] != std.fs.path.sep) { - dest_remaining[0] = std.fs.path.sep; - dest_remaining = dest_remaining[1..]; - dest_dir_offset += 1; - } - while (try walker.next()) |entry| { - switch (entry.kind) { - // directories are created - .directory => { - std.os.mkdirat(dest_dir_fd.cast(), entry.path, 0o755) catch {}; - }, - // but each file in the directory is a symlink - .file => { - @memcpy(remain[0..entry.path.len], entry.path); - remain[entry.path.len] = 0; - const from_path = buf[0 .. cache_dir_offset + entry.path.len :0]; + if (comptime Environment.isPosix) { + switch (entry.kind) { + .directory => { + bun.MakePath.makePath(std.meta.Elem(@TypeOf(entry.path)), destination_dir, entry.path) catch {}; + }, + .file => { + @memcpy(to_copy_into2[0..entry.path.len], entry.path); + head2[entry.path.len + (head2.len - to_copy_into2.len)] = 0; + const target: [:0]u8 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; - @memcpy(dest_remaining[0..entry.path.len], entry.path); - dest_remaining[entry.path.len] = 0; - const to_path = dest_buf[0 .. dest_dir_offset + entry.path.len :0]; + std.os.symlinkat(target, destination_dir.fd, entry.path) catch |err| { + if (err != error.PathAlreadyExists) { + return err; + } - try std.os.symlinkZ(from_path, to_path); + std.os.unlinkat(destination_dir.fd, entry.path, 0) catch {}; + try std.os.symlinkat(entry.basename, destination_dir.fd, entry.path); + }; - real_file_count += 1; - }, - else => {}, + real_file_count += 1; + }, + else => {}, + } + } else { + switch (entry.kind) { + .directory, .file => {}, + else => continue, + } + + if (entry.path.len > to_copy_into1.len or entry.path.len > to_copy_into2.len) { + return error.NameTooLong; + } + + @memcpy(to_copy_into1[0..entry.path.len], entry.path); + head1[entry.path.len + (head1.len - to_copy_into1.len)] = 0; + const dest: [:0]u16 = head1[0 .. entry.path.len + head1.len - to_copy_into1.len :0]; + + @memcpy(to_copy_into2[0..entry.path.len], entry.path); + head2[entry.path.len + (head1.len - to_copy_into2.len)] = 0; + const src: [:0]u16 = head2[0 .. entry.path.len + head2.len - to_copy_into2.len :0]; + + switch (entry.kind) { + .directory => { + if (bun.windows.CreateDirectoryExW(src.ptr, dest.ptr, null) == 0) { + bun.MakePath.makePath(u16, destination_dir, entry.path) catch {}; + } + }, + .file => { + switch (bun.sys.symlinkW(dest, src, .{})) { + .err => |err| { + if (bun.Dirname.dirname(u16, entry.path)) |entry_dirname| { + bun.MakePath.makePath(u16, destination_dir, entry_dirname) catch {}; + if (bun.sys.symlinkW(dest, src, .{}) == .result) { + continue; + } + } + + if (PackageManager.verbose_install) { + const once_log = struct { + var once = false; + + pub fn get() bool { + const prev = once; + once = true; + return !prev; + } + }.get(); + + if (once_log) { + Output.warn("CreateHardLinkW failed, falling back to CopyFileW: {} -> {}\n", .{ + bun.fmt.fmtOSPath(src, .{}), + bun.fmt.fmtOSPath(dest, .{}), + }); + } + } + + return bun.errnoToZigErr(err.errno); + }, + .result => {}, + } + }, + else => unreachable, // handled above + } } } @@ -1586,22 +1741,22 @@ pub const PackageInstall = struct { } }; - var subdir = this.destination_dir.makeOpenPath(bun.span(this.destination_dir_subpath), .{}) catch |err| return Result{ - .fail = .{ .err = err, .step = .opening_cache_dir }, - }; - - defer subdir.close(); - this.file_count = FileCopier.copy( - bun.toFD(subdir.fd), - bun.toFD(cached_package_dir.fd), - &walker_, - ) catch |err| - return Result{ - .fail = .{ - .err = err, - .step = .copying_files, - }, + state.subdir, + &state.walker, + if (Environment.isWindows) state.to_copy_buf else void{}, + if (Environment.isWindows) &state.buf else void{}, + if (Environment.isWindows) state.to_copy_buf2 else to_copy_buf2, + if (Environment.isWindows) &state.buf2 else &buf2, + ) catch |err| { + if (comptime Environment.isWindows) { + if (err == error.FailedToCopyFile) { + return Result.fail(err, .copying_files); + } + } else if (err == error.NotSameFileSystem or err == error.ENXIO) { + return err; + } + return Result.fail(err, .copying_files); }; return Result{ @@ -1629,7 +1784,6 @@ pub const PackageInstall = struct { // } else { // this.destination_dir.deleteTree(bun.span(this.destination_dir_subpath)) catch {}; // } - this.destination_dir.deleteTree(bun.span(this.destination_dir_subpath)) catch {}; } @@ -1691,25 +1845,8 @@ pub const PackageInstall = struct { if (!skip_delete and !strings.eqlComptime(dest_path, ".")) this.uninstallBeforeInstall(); const subdir = std.fs.path.dirname(dest_path); - var dest_dir = if (subdir) |dir| brk: { - break :brk this.destination_dir.makeOpenPath(dir, .{}) catch |err| return Result{ - .fail = .{ - .err = err, - .step = .linking, - }, - }; - } else this.destination_dir; - defer { - if (subdir != null) dest_dir.close(); - } - var dest_buf: [bun.MAX_PATH_BYTES]u8 = undefined; - const dest_dir_path = dest_dir.realpath(".", &dest_buf) catch |err| return Result{ - .fail = .{ - .err = err, - .step = .linking, - }, - }; + var dest_buf: bun.PathBuffer = undefined; // cache_dir_subpath in here is actually the full path to the symlink pointing to the linked package const symlinked_path = this.cache_dir_subpath; var to_buf: [bun.MAX_PATH_BYTES]u8 = undefined; @@ -1721,23 +1858,52 @@ pub const PackageInstall = struct { }; const dest = std.fs.path.basename(dest_path); + // When we're linking on Windows, we want to avoid keeping the source directory handle open if (comptime Environment.isWindows) { - var dest_buf2: bun.PathBuffer = undefined; - const dest_z = bun.path.joinAbsStringBufZ( - dest_dir_path, - &dest_buf2, - &.{ - dest, - }, - .windows, - ); + var wbuf: bun.WPathBuffer = undefined; + const dest_path_length = bun.windows.kernel32.GetFinalPathNameByHandleW(this.destination_dir.fd, &wbuf, dest_buf.len, 0); + if (dest_path_length == 0) { + const e = bun.windows.Win32Error.get(); + const err = if (e.toSystemErrno()) |sys_err| bun.errnoToZigErr(sys_err) else error.Unexpected; + return Result.fail(err, .linking); + } + + var i: usize = dest_path_length; + if (wbuf[i] != '\\') { + wbuf[i] = '\\'; + i += 1; + } + + if (subdir) |dir| { + i += bun.strings.toWPathNormalized(wbuf[i..], dir).len; + wbuf[i] = std.fs.path.sep_windows; + i += 1; + wbuf[i] = 0; + const fullpath = wbuf[0..i :0]; + + _ = node_fs_for_package_installer.mkdirRecursiveOSPathImpl(void, {}, fullpath, 0, false).unwrap() catch |err| { + return Result.fail(err, .linking); + }; + } + + const res = strings.copyUTF16IntoUTF8(dest_buf[0..], []const u16, wbuf[0..i], true); + var offset: usize = res.written; + if (dest_buf[offset - 1] != std.fs.path.sep_windows) { + dest_buf[offset] = std.fs.path.sep_windows; + offset += 1; + } + @memcpy(dest_buf[offset .. offset + dest.len], dest); + offset += dest.len; + dest_buf[offset] = 0; + + const dest_z = dest_buf[0..offset :0]; to_buf[to_path.len] = 0; const to_path_z = to_buf[0..to_path.len :0]; // https://github.com/npm/cli/blob/162c82e845d410ede643466f9f8af78a312296cc/workspaces/arborist/lib/arborist/reify.js#L738 // https://github.com/npm/cli/commit/0e58e6f6b8f0cd62294642a502c17561aaf46553 - switch (bun.sys.sys_uv.symlinkUV(to_path_z, dest_z, bun.windows.libuv.UV_FS_SYMLINK_JUNCTION)) { + switch (bun.sys.symlinkOrJunctionOnWindows(to_path_z, dest_z)) { .err => |err| { return Result{ .fail = .{ @@ -1749,6 +1915,25 @@ pub const PackageInstall = struct { .result => {}, } } else { + var dest_dir = if (subdir) |dir| brk: { + break :brk bun.MakePath.makeOpenPath(this.destination_dir, dir, .{}) catch |err| return Result{ + .fail = .{ + .err = err, + .step = .linking, + }, + }; + } else this.destination_dir; + defer { + if (subdir != null) dest_dir.close(); + } + + const dest_dir_path = bun.getFdPath(dest_dir.fd, &dest_buf) catch |err| return Result{ + .fail = .{ + .err = err, + .step = .linking, + }, + }; + const target = Path.relative(dest_dir_path, to_path); std.os.symlinkat(target, dest_dir.fd, dest) catch |err| return Result{ .fail = .{ @@ -2524,22 +2709,22 @@ pub const PackageManager = struct { this.temp_dir_name = Fs.FileSystem.RealFS.getDefaultTempDir(); var tried_dot_tmp = false; - var tempdir: std.fs.Dir = std.fs.cwd().makeOpenPath(this.temp_dir_name, .{}) catch brk: { + var tempdir: std.fs.Dir = bun.MakePath.makeOpenPath(std.fs.cwd(), this.temp_dir_name, .{}) catch brk: { tried_dot_tmp = true; - break :brk cache_directory.makeOpenPath(".tmp", .{}) catch |err| { + break :brk bun.MakePath.makeOpenPath(cache_directory, bun.pathLiteral(".tmp"), .{}) catch |err| { Output.prettyErrorln("error: bun is unable to access tempdir: {s}", .{@errorName(err)}); Global.crash(); }; }; var tmpbuf: [bun.MAX_PATH_BYTES]u8 = undefined; - const tmpname = Fs.FileSystem.instance.tmpname("hm", &tmpbuf, 999) catch unreachable; + const tmpname = Fs.FileSystem.instance.tmpname("hm", &tmpbuf, bun.fastRandom()) catch unreachable; var timer: std.time.Timer = if (this.options.log_level != .silent) std.time.Timer.start() catch unreachable else undefined; brk: while (true) { - _ = tempdir.createFileZ(tmpname, .{ .truncate = true }) catch |err2| { + var file = tempdir.createFileZ(tmpname, .{ .truncate = true }) catch |err2| { if (!tried_dot_tmp) { tried_dot_tmp = true; - tempdir = cache_directory.makeOpenPath(".tmp", .{}) catch |err| { + tempdir = bun.MakePath.makeOpenPath(cache_directory, bun.pathLiteral(".tmp"), .{}) catch |err| { Output.prettyErrorln("error: bun is unable to access tempdir: {s}", .{@errorName(err)}); Global.crash(); }; @@ -2555,6 +2740,7 @@ pub const PackageManager = struct { }); Global.crash(); }; + file.close(); std.os.renameatZ(tempdir.fd, tmpname, cache_directory.fd, tmpname) catch |err| { if (!tried_dot_tmp) { @@ -4555,7 +4741,7 @@ pub const PackageManager = struct { .git, .github => { const package_json_source = logger.Source.initPathString( data.json_path, - data.json_buf[0..data.json_len], + data.json_buf, ); var package = Lockfile.Package{}; @@ -4606,7 +4792,7 @@ pub const PackageManager = struct { .local_tarball, .remote_tarball => { const package_json_source = logger.Source.initPathString( data.json_path, - data.json_buf[0..data.json_len], + data.json_buf, ); var package = Lockfile.Package{}; @@ -4654,10 +4840,10 @@ pub const PackageManager = struct { return package; }, - else => if (data.json_len > 0) { + else => if (data.json_buf.len > 0) { const package_json_source = logger.Source.initPathString( data.json_path, - data.json_buf[0..data.json_len], + data.json_buf, ); initializeStore(); const json = json_parser.ParseJSONUTF8( @@ -4690,7 +4876,7 @@ pub const PackageManager = struct { const CacheDir = struct { path: string, is_node_modules: bool }; pub fn fetchCacheDirectoryPath(env: *DotEnv.Loader) CacheDir { if (env.get("BUN_INSTALL_CACHE_DIR")) |dir| { - return CacheDir{ .path = dir, .is_node_modules = false }; + return CacheDir{ .path = Fs.FileSystem.instance.abs(&[_]string{dir}), .is_node_modules = false }; } if (env.get("BUN_INSTALL")) |dir| { @@ -8855,26 +9041,28 @@ pub const PackageManager = struct { var node_modules_is_ok = false; }; if (!Singleton.node_modules_is_ok) { - const stat = bun.sys.fstat(bun.toFD(this.node_modules_folder.fd)).unwrap() catch |err| { - Output.err("EACCES", "Permission denied while installing {s}", .{ - this.names[package_id].slice(buf), - }); - if (Environment.isDebug) { - Output.err(err, "Failed to stat node_modules", .{}); + if (!Environment.isWindows) { + const stat = bun.sys.fstat(bun.toFD(this.node_modules_folder.fd)).unwrap() catch |err| { + Output.err("EACCES", "Permission denied while installing {s}", .{ + this.names[package_id].slice(buf), + }); + if (Environment.isDebug) { + Output.err(err, "Failed to stat node_modules", .{}); + } + Global.exit(1); + }; + + const is_writable = if (stat.uid == bun.C.getuid()) + stat.mode & bun.S.IWUSR > 0 + else if (stat.gid == bun.C.getgid()) + stat.mode & bun.S.IWGRP > 0 + else + stat.mode & bun.S.IWOTH > 0; + + if (!is_writable) { + Output.err("EACCES", "Permission denied while writing packages into node_modules.", .{}); + Global.exit(1); } - Global.exit(1); - }; - - const is_writable = if (Environment.isWindows or stat.uid == bun.C.getuid()) - stat.mode & bun.S.IWUSR > 0 - else if (stat.gid == bun.C.getgid()) - stat.mode & bun.S.IWGRP > 0 - else - stat.mode & bun.S.IWOTH > 0; - - if (!is_writable) { - Output.err("EACCES", "Permission denied while writing packages into node_modules.", .{}); - Global.exit(1); } Singleton.node_modules_is_ok = true; } @@ -9235,8 +9423,16 @@ pub const PackageManager = struct { // no need to download packages you've already installed!! var skip_verify_installed_version_number = false; const cwd = std.fs.cwd(); - const node_modules_folder = bun.openDir(cwd, "node_modules") catch brk: { + const node_modules_folder = brk: { + // Attempt to open the existing node_modules folder + switch (bun.sys.openatOSPath(bun.toFD(cwd), bun.OSPathLiteral("node_modules"), std.os.O.DIRECTORY | std.os.O.RDONLY, 0o755)) { + .result => |fd| break :brk std.fs.Dir{ .fd = fd.cast() }, + .err => {}, + } + skip_verify_installed_version_number = true; + + // Attempt to create a new node_modules folder bun.sys.mkdir("node_modules", 0o755).unwrap() catch |err| { if (err != error.EEXIST) { Output.prettyErrorln("error: {s} creating node_modules folder", .{@errorName(err)}); @@ -9405,7 +9601,7 @@ pub const PackageManager = struct { // We deliberately do not close this folder. // If the package hasn't been downloaded, we will need to install it later // We use this file descriptor to know where to put it. - installer.node_modules_folder = bun.openDir(cwd, node_modules.relative_path) catch brk: { + installer.node_modules_folder = bun.MakePath.makeOpenPath(cwd, node_modules.relative_path, .{ .access_sub_paths = true, .iterate = true }) catch brk: { // Avoid extra mkdir() syscall // // note: this will recursively delete any dangling symlinks diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index 8da6638514..1bec47744a 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -1664,14 +1664,16 @@ pub fn saveToDisk(this: *Lockfile, filename: stringZ) void { } } - // chmod 777 - switch (bun.sys.fchmod(tmpfile.fd, 0o777)) { - .err => |err| { - tmpfile.dir().deleteFileZ(tmpname) catch {}; - Output.prettyErrorln("error: failed to change lockfile permissions: {s}", .{@tagName(err.getErrno())}); - Global.crash(); - }, - .result => {}, + if (comptime Environment.isPosix) { + // chmod 777 on posix + switch (bun.sys.fchmod(tmpfile.fd, 0o777)) { + .err => |err| { + tmpfile.dir().deleteFileZ(tmpname) catch {}; + Output.prettyErrorln("error: failed to change lockfile permissions: {s}", .{@tagName(err.getErrno())}); + Global.crash(); + }, + .result => {}, + } } tmpfile.promoteToCWD(tmpname, filename) catch |err| { @@ -2675,19 +2677,8 @@ pub const Package = extern struct { const json = brk: { const json_src = brk2: { const json_path = bun.path.joinZ([_]string{ folder_name, "package.json" }, .auto); - const json_file_fd = try bun.sys.openat( - bun.toFD(node_modules.fd), - json_path, - std.os.O.RDONLY, - 0, - ).unwrap(); - const json_file = json_file_fd.asFile(); - defer json_file.close(); - const json_stat_size = try json_file.getEndPos(); - const json_buf = try allocator.alloc(u8, json_stat_size + 64); - errdefer allocator.free(json_buf); - const json_len = try json_file.preadAll(json_buf, 0); - break :brk2 logger.Source.initPathString(json_path, json_buf[0..json_len]); + const buf = try bun.sys.File.readFrom(node_modules, json_path, allocator).unwrap(); + break :brk2 logger.Source.initPathString(json_path, buf); }; initializeStore(); @@ -3410,12 +3401,7 @@ pub const Package = extern struct { &[_]string{ path, "package.json" }, .auto, ); - var file = try bun.openFileZ(package_json_path, .{ .mode = .read_only }); - - defer file.close(); - const bytes = try file.readToEndAlloc(allocator, std.math.maxInt(usize)); - defer allocator.free(bytes); - const source = logger.Source.initPathString(path, bytes); + const source = try bun.sys.File.toSource(package_json_path, allocator).unwrap(); var workspace = Package{}; try workspace.parseMain(to_lockfile, allocator, log, source, Features.workspace); @@ -3546,24 +3532,31 @@ pub const Package = extern struct { key_loc: logger.Loc, value_loc: logger.Loc, ) !?Dependency { - var external_version = string_builder.append(String, version); + const external_version = brk: { + if (comptime Environment.isWindows) { + switch (tag orelse Dependency.Version.Tag.infer(version)) { + .workspace, .folder, .symlink, .tarball => { + if (String.canInline(version)) { + var copy = string_builder.append(String, version); + bun.path.dangerouslyConvertPathToPosixInPlace(u8, ©.bytes); + break :brk copy; + } else { + const str_ = string_builder.append(String, version); + const ptr = str_.ptr(); + bun.path.dangerouslyConvertPathToPosixInPlace(u8, lockfile.buffers.string_bytes.items[ptr.off..][0..ptr.len]); + break :brk str_; + } + }, + else => {}, + } + } + + break :brk string_builder.append(String, version); + }; + const buf = lockfile.buffers.string_bytes.items; const sliced = external_version.sliced(buf); - if (comptime Environment.isWindows) { - switch (Dependency.Version.Tag.infer(sliced.slice)) { - .workspace, .folder, .symlink, .tarball => { - if (external_version.isInline()) { - bun.path.pathToPosixInPlace(u8, &external_version.bytes); - } else { - const ptr = external_version.ptr(); - bun.path.pathToPosixInPlace(u8, buf[ptr.off..][0..ptr.len]); - } - }, - else => {}, - } - } - var dependency_version = Dependency.parseWithOptionalTag( allocator, external_alias.value, @@ -3798,6 +3791,13 @@ pub const Package = extern struct { if (comptime Environment.allow_assert) { std.debug.assert(!strings.containsChar(key, std.fs.path.sep_windows)); } + + if (comptime Environment.isDebug) { + if (!bun.sys.exists(key)) { + Output.debugWarn("WorkspaceMap.insert: key {s} does not exist", .{key}); + } + } + const entry = try self.map.getOrPut(key); if (!entry.found_existing) { entry.key_ptr.* = try self.map.allocator.dupe(u8, key); @@ -4132,7 +4132,7 @@ pub const Package = extern struct { // entry_path is contained in filepath_buf so it is safe to constCast and // replace path separators const entry_slice = bun.span(entry_path); - Path.pathToPosixInPlace(u8, @constCast(entry_slice)); + Path.dangerouslyConvertPathToPosixInPlace(u8, @constCast(entry_slice)); break :brk entry_slice; }; @@ -4598,17 +4598,7 @@ pub const Package = extern struct { if (strings.eqlLong(value.name, entry.name, true)) { const note_abs_path = allocator.dupeZ(u8, Path.joinAbsStringZ(cwd, &.{ note_path, "package.json" }, .auto)) catch bun.outOfMemory(); - const note_src = src: { - var workspace_file = std.fs.openFileAbsoluteZ(note_abs_path, .{ .mode = .read_only }) catch { - break :src logger.Source.initEmptyFile(note_abs_path); - }; - defer workspace_file.close(); - - // TODO: when are these bytes supposed to be freed? - const workspace_bytes = try workspace_file.readToEndAlloc(allocator, std.math.maxInt(usize)); - // defer allocator.free(workspace_bytes); - break :src logger.Source.initPathString(note_abs_path, workspace_bytes); - }; + const note_src = bun.sys.File.toSource(note_abs_path, allocator).unwrap() catch logger.Source.initEmptyFile(note_abs_path); notes[i] = .{ .text = "Package name is also declared here", @@ -4622,17 +4612,7 @@ pub const Package = extern struct { const abs_path = Path.joinAbsStringZ(cwd, &.{ path, "package.json" }, .auto); - const src = src: { - var workspace_file = std.fs.openFileAbsoluteZ(abs_path, .{ .mode = .read_only }) catch { - break :src logger.Source.initEmptyFile(abs_path); - }; - defer workspace_file.close(); - - // TODO: when are these bytes supposed to be freed? - const workspace_bytes = try workspace_file.readToEndAlloc(allocator, std.math.maxInt(usize)); - // defer allocator.free(workspace_bytes); - break :src logger.Source.initPathString(abs_path, workspace_bytes); - }; + const src = bun.sys.File.toSource(abs_path, allocator).unwrap() catch logger.Source.initEmptyFile(abs_path); log.addRangeErrorFmtWithNotes( &src, diff --git a/src/install/repository.zig b/src/install/repository.zig index a37acc8cbe..d05aad4f30 100644 --- a/src/install/repository.zig +++ b/src/install/repository.zig @@ -107,25 +107,21 @@ pub const Repository = extern struct { fn exec( allocator: std.mem.Allocator, env: *DotEnv.Loader, - cwd: if (Environment.isWindows) string else std.fs.Dir, argv: []const string, ) !string { var std_map = try env.map.stdEnvMap(allocator); defer std_map.deinit(); const result = if (comptime Environment.isWindows) - try std.ChildProcess.run(.{ + try std.process.Child.run(.{ .allocator = allocator, .argv = argv, - // windows `std.ChildProcess.run` uses `cwd` instead of `cwd_dir` - .cwd = cwd, .env_map = std_map.get(), }) else - try std.ChildProcess.run(.{ + try std.process.Child.run(.{ .allocator = allocator, .argv = argv, - .cwd_dir = cwd, .env_map = std_map.get(), }); @@ -183,16 +179,12 @@ pub const Repository = extern struct { }); return if (cache_dir.openDirZ(folder_name, .{})) |dir| fetch: { - const cwd = if (comptime Environment.isWindows) - Path.joinAbsString(PackageManager.instance.cache_directory_path, &.{folder_name}, .windows) - else - dir; + const path = Path.joinAbsString(PackageManager.instance.cache_directory_path, &.{folder_name}, .auto); _ = exec( allocator, env, - cwd, - &[_]string{ "git", "fetch", "--quiet" }, + &[_]string{ "git", "-C", path, "fetch", "--quiet" }, ) catch |err| { log.addErrorFmt( null, @@ -207,18 +199,15 @@ pub const Repository = extern struct { } else |not_found| clone: { if (not_found != error.FileNotFound) return not_found; - const cwd = if (comptime Environment.isWindows) - PackageManager.instance.cache_directory_path - else - cache_dir; + const target = Path.joinAbsString(PackageManager.instance.cache_directory_path, &.{folder_name}, .auto); - _ = exec(allocator, env, cwd, &[_]string{ + _ = exec(allocator, env, &[_]string{ "git", "clone", "--quiet", "--bare", url, - folder_name, + target, }) catch |err| { log.addErrorFmt( null, @@ -242,21 +231,19 @@ pub const Repository = extern struct { committish: string, task_id: u64, ) !string { - const cwd = if (comptime Environment.isWindows) - Path.joinAbsString(PackageManager.instance.cache_directory_path, &.{try std.fmt.bufPrint(&folder_name_buf, "{any}.git", .{ - bun.fmt.hexIntLower(task_id), - })}, .windows) - else - repo_dir; + const path = Path.joinAbsString(PackageManager.instance.cache_directory_path, &.{try std.fmt.bufPrint(&folder_name_buf, "{any}.git", .{ + bun.fmt.hexIntLower(task_id), + })}, .auto); + + _ = repo_dir; return std.mem.trim(u8, exec( allocator, env, - cwd, if (committish.len > 0) - &[_]string{ "git", "log", "--format=%H", "-1", committish } + &[_]string{ "git", "-C", path, "log", "--format=%H", "-1", committish } else - &[_]string{ "git", "log", "--format=%H", "-1" }, + &[_]string{ "git", "-C", path, "log", "--format=%H", "-1" }, ) catch |err| { log.addErrorFmt( null, @@ -282,21 +269,18 @@ pub const Repository = extern struct { bun.Analytics.Features.git_dependencies += 1; const folder_name = PackageManager.cachedGitFolderNamePrint(&folder_name_buf, resolved); - var package_dir = cache_dir.openDirZ(folder_name, .{}) catch |not_found| brk: { - if (not_found != error.FileNotFound) return not_found; + var package_dir = bun.openDir(cache_dir, folder_name) catch |not_found| brk: { + if (not_found != error.ENOENT) return not_found; - var cwd = if (comptime Environment.isWindows) - PackageManager.instance.cache_directory_path - else - cache_dir; + const target = Path.joinAbsString(PackageManager.instance.cache_directory_path, &.{folder_name}, .auto); - _ = exec(allocator, env, cwd, &[_]string{ + _ = exec(allocator, env, &[_]string{ "git", "clone", "--quiet", "--no-checkout", try bun.getFdPath(repo_dir.fd, &final_path_buf), - folder_name, + target, }) catch |err| { log.addErrorFmt( null, @@ -308,14 +292,9 @@ pub const Repository = extern struct { return err; }; - var dir = try cache_dir.openDirZ(folder_name, .{}); + const folder = Path.joinAbsString(PackageManager.instance.cache_directory_path, &.{folder_name}, .auto); - cwd = if (comptime Environment.isWindows) - Path.joinAbsString(PackageManager.instance.cache_directory_path, &.{folder_name}, .windows) - else - dir; - - _ = exec(allocator, env, cwd, &[_]string{ "git", "checkout", "--quiet", resolved }) catch |err| { + _ = exec(allocator, env, &[_]string{ "git", "-C", folder, "checkout", "--quiet", resolved }) catch |err| { log.addErrorFmt( null, logger.Loc.Empty, @@ -325,6 +304,7 @@ pub const Repository = extern struct { ) catch unreachable; return err; }; + var dir = try bun.openDir(cache_dir, folder_name); dir.deleteTree(".git") catch {}; if (resolved.len > 0) insert_tag: { @@ -339,7 +319,7 @@ pub const Repository = extern struct { }; defer package_dir.close(); - const json_file = package_dir.openFileZ("package.json", .{ .mode = .read_only }) catch |err| { + const json_file, const json_buf = bun.sys.File.readFileFrom(package_dir, "package.json", allocator).unwrap() catch |err| { log.addErrorFmt( null, logger.Loc.Empty, @@ -350,14 +330,10 @@ pub const Repository = extern struct { return error.InstallFailed; }; defer json_file.close(); - const size = try json_file.getEndPos(); - const json_buf = try allocator.alloc(u8, size + 64); - const json_len = try json_file.preadAll(json_buf, 0); - const json_path = bun.getFdPath( - json_file.handle, + const json_path = json_file.getPath( &json_path_buf, - ) catch |err| { + ).unwrap() catch |err| { log.addErrorFmt( null, logger.Loc.Empty, @@ -374,7 +350,6 @@ pub const Repository = extern struct { .resolved = resolved, .json_path = ret_json_path, .json_buf = json_buf, - .json_len = json_len, }; } }; diff --git a/src/install/resolvers/folder_resolver.zig b/src/install/resolvers/folder_resolver.zig index 1026116b03..c7f89c8d12 100644 --- a/src/install/resolvers/folder_resolver.zig +++ b/src/install/resolvers/folder_resolver.zig @@ -167,19 +167,24 @@ pub const FolderResolution = union(Tag) { comptime ResolverType: type, resolver: ResolverType, ) !Lockfile.Package { - var package_json: std.fs.File = try std.fs.cwd().openFileZ(abs, .{ .mode = .read_only }); - defer package_json.close(); - var package = Lockfile.Package{}; var body = Npm.Registry.BodyPool.get(manager.allocator); defer Npm.Registry.BodyPool.release(body); - const len = try package_json.getEndPos(); - body.data.reset(); - body.data.inflate(@max(len, 2048)) catch unreachable; - body.data.list.expandToCapacity(); - const source_buf = try package_json.readAll(body.data.list.items); + const source = brk: { + var file = bun.sys.File.from(try bun.sys.openatA(bun.toFD(std.fs.cwd().fd), abs, std.os.O.RDONLY, 0).unwrap()); + defer file.close(); - const source = logger.Source.initPathString(abs, body.data.list.items[0..source_buf]); + { + body.data.reset(); + var man = body.data.list.toManaged(manager.allocator); + defer body.data.list = man.moveToUnmanaged(); + _ = try file.readToEndWithArrayList(&man).unwrap(); + } + + break :brk logger.Source.initPathString(abs, body.data.list.items); + }; + + var package = Lockfile.Package{}; try package.parse( manager.lockfile, @@ -226,8 +231,8 @@ pub const FolderResolution = union(Tag) { // replace before getting hash. rel may or may not be contained in abs if (comptime bun.Environment.isWindows) { - bun.path.pathToPosixInPlace(u8, @constCast(abs)); - bun.path.pathToPosixInPlace(u8, @constCast(rel)); + bun.path.dangerouslyConvertPathToPosixInPlace(u8, @constCast(abs)); + bun.path.dangerouslyConvertPathToPosixInPlace(u8, @constCast(rel)); } const abs_hash = hash(abs); @@ -275,7 +280,7 @@ pub const FolderResolution = union(Tag) { CacheFolderResolver{ .version = version.value.npm.version.toVersion() }, ), } catch |err| { - if (err == error.FileNotFound) { + if (err == error.FileNotFound or err == error.ENOENT) { entry.value_ptr.* = .{ .err = error.MissingPackageJSON }; } else { entry.value_ptr.* = .{ .err = err }; diff --git a/src/io/PipeReader.zig b/src/io/PipeReader.zig index 45ae7e8289..6c0a26b350 100644 --- a/src/io/PipeReader.zig +++ b/src/io/PipeReader.zig @@ -339,6 +339,7 @@ pub fn WindowsPipeReader( var this = bun.cast(*This, stream.data); const nread_int = nread.int(); + bun.sys.syslog("onStreamRead(0x{d}) = {d}", .{ @intFromPtr(this), nread_int }); // NOTE: pipes/tty need to call stopReading on errors (yeah) diff --git a/src/libarchive/libarchive.zig b/src/libarchive/libarchive.zig index ed06b8d13a..39ccc2730e 100644 --- a/src/libarchive/libarchive.zig +++ b/src/libarchive/libarchive.zig @@ -567,11 +567,7 @@ pub const Archive = struct { mode |= 0o1; if (comptime Environment.isWindows) { - std.os.mkdiratW(dir_fd, pathname, @as(u32, @intCast(mode))) catch |err| { - if (err == error.PathAlreadyExists or err == error.NotDir) break; - try bun.MakePath.makePath(u16, dir, bun.Dirname.dirname(u16, path_slice) orelse return err); - try std.os.mkdiratW(dir_fd, pathname, 0o777); - }; + try bun.MakePath.makePath(u16, dir, pathname); } else { std.os.mkdiratZ(dir_fd, pathname, @as(u32, @intCast(mode))) catch |err| { if (err == error.PathAlreadyExists or err == error.NotDir) break; diff --git a/src/resolver/resolve_path.zig b/src/resolver/resolve_path.zig index 349dcbcd33..82a982a6cf 100644 --- a/src/resolver/resolve_path.zig +++ b/src/resolver/resolve_path.zig @@ -884,6 +884,7 @@ pub const Platform = enum { loose, windows, posix, + nt, pub fn isAbsolute(comptime platform: Platform, path: []const u8) bool { return isAbsoluteT(platform, u8, path); @@ -894,6 +895,7 @@ pub const Platform = enum { return switch (comptime platform) { .auto => (comptime platform.resolve()).isAbsoluteT(T, path), .posix => path.len > 0 and path[0] == '/', + .nt, .windows, .loose, => if (T == u8) @@ -907,7 +909,7 @@ pub const Platform = enum { return comptime switch (platform) { .auto => platform.resolve().separator(), .loose, .posix => std.fs.path.sep_posix, - .windows => std.fs.path.sep_windows, + .nt, .windows => std.fs.path.sep_windows, }; } @@ -915,7 +917,7 @@ pub const Platform = enum { return comptime switch (platform) { .auto => platform.resolve().separatorString(), .loose, .posix => std.fs.path.sep_str_posix, - .windows => std.fs.path.sep_str_windows, + .nt, .windows => std.fs.path.sep_str_windows, }; } @@ -930,7 +932,7 @@ pub const Platform = enum { .loose => { return isSepAny; }, - .windows => { + .nt, .windows => { return isSepAny; }, .posix => { @@ -945,7 +947,7 @@ pub const Platform = enum { .loose => { return isSepAnyT; }, - .windows => { + .nt, .windows => { return isSepAnyT; }, .posix => { @@ -960,7 +962,7 @@ pub const Platform = enum { .loose => { return lastIndexOfSeparatorLoose; }, - .windows => { + .nt, .windows => { return lastIndexOfSeparatorWindows; }, .posix => { @@ -975,7 +977,7 @@ pub const Platform = enum { .loose => { return lastIndexOfSeparatorLooseT; }, - .windows => { + .nt, .windows => { return lastIndexOfSeparatorWindowsT; }, .posix => { @@ -994,7 +996,7 @@ pub const Platform = enum { .loose => { return isSepAnyT(T, char); }, - .windows => { + .nt, .windows => { return isSepAnyT(T, char); }, .posix => { @@ -1006,14 +1008,14 @@ pub const Platform = enum { pub fn trailingSeparator(comptime _platform: Platform) [2]u8 { return comptime switch (_platform) { .auto => _platform.resolve().trailingSeparator(), - .windows => ".\\".*, + .nt, .windows => ".\\".*, .posix, .loose => "./".*, }; } pub fn leadingSeparatorIndex(comptime _platform: Platform, path: anytype) ?usize { switch (comptime _platform.resolve()) { - .windows => { + .nt, .windows => { if (path.len < 1) return null; @@ -1116,7 +1118,7 @@ pub fn normalizeStringBufT( comptime preserve_trailing_slash: bool, ) []T { switch (comptime platform.resolve()) { - .auto => @compileError("unreachable"), + .nt, .auto => @compileError("unreachable"), .windows => { return normalizeStringWindowsT( @@ -1257,6 +1259,14 @@ pub fn joinAbsStringBufZ(cwd: []const u8, buf: []u8, _parts: anytype, comptime _ return _joinAbsStringBuf(true, [:0]const u8, cwd, buf, _parts, _platform); } +pub fn joinAbsStringBufZNT(cwd: []const u8, buf: []u8, _parts: anytype, comptime _platform: Platform) [:0]const u8 { + if ((_platform == .auto or _platform == .loose or _platform == .windows) and bun.Environment.isWindows) { + return _joinAbsStringBuf(true, [:0]const u8, cwd, buf, _parts, .nt); + } + + return _joinAbsStringBuf(true, [:0]const u8, cwd, buf, _parts, _platform); +} + pub fn joinAbsStringBufZTrailingSlash(cwd: []const u8, buf: []u8, _parts: anytype, comptime _platform: Platform) [:0]const u8 { const out = _joinAbsStringBuf(true, [:0]const u8, cwd, buf, _parts, _platform); if (out.len + 2 < buf.len and out.len > 0 and out[out.len - 1] != _platform.separator()) { @@ -1275,6 +1285,16 @@ fn _joinAbsStringBuf(comptime is_sentinel: bool, comptime ReturnType: type, _cwd return _joinAbsStringBufWindows(is_sentinel, ReturnType, _cwd, buf, _parts); } + if (comptime platform.resolve() == .nt) { + const end_path = _joinAbsStringBufWindows(is_sentinel, ReturnType, _cwd, buf[4..], _parts); + buf[0..4].* = "\\\\?\\".*; + if (comptime is_sentinel) { + buf[end_path.len + 4] = 0; + return buf[0 .. end_path.len + 4 :0]; + } + return buf[0 .. end_path.len + 4]; + } + var parts: []const []const u8 = _parts; var temp_buf: [bun.MAX_PATH_BYTES * 2]u8 = undefined; if (parts.len == 0) { @@ -2225,7 +2245,7 @@ pub fn platformToPosixInPlace(comptime T: type, path_buffer: []T) void { } } -pub fn pathToPosixInPlace(comptime T: type, path: []T) void { +pub fn dangerouslyConvertPathToPosixInPlace(comptime T: type, path: []T) void { var idx: usize = 0; while (std.mem.indexOfScalarPos(T, path, idx, std.fs.path.sep_windows)) |index| : (idx = index + 1) { path[index] = '/'; diff --git a/src/resolver/resolver.zig b/src/resolver/resolver.zig index 67f0ed639f..81fd287eaf 100644 --- a/src/resolver/resolver.zig +++ b/src/resolver/resolver.zig @@ -2703,7 +2703,7 @@ pub const Resolver = struct { .{ .no_follow = !follow_symlinks, .iterate = true }, ) else if (comptime Environment.isWindows) open_req: { - const dirfd_result = bun.sys.openDirAtWindowsA(bun.invalid_fd, sentinel, true, !follow_symlinks); + const dirfd_result = bun.sys.openDirAtWindowsA(bun.invalid_fd, sentinel, .{ .iterable = true, .no_follow = !follow_symlinks }); if (dirfd_result.unwrap()) |result| { break :open_req result.asDir(); } else |err| { diff --git a/src/shell/interpreter.zig b/src/shell/interpreter.zig index 83a43b4fce..53920af144 100644 --- a/src/shell/interpreter.zig +++ b/src/shell/interpreter.zig @@ -10850,12 +10850,12 @@ const ShellSyscall = struct { .result => |p| p, .err => |e| return .{ .err = e }, }; - return switch (Syscall.openDirAtWindowsA(dir, p, true, flags & os.O.NOFOLLOW != 0)) { + return switch (Syscall.openDirAtWindowsA(dir, p, .{ .iterable = true, .no_follow = flags & os.O.NOFOLLOW != 0 })) { .result => |fd| bun.sys.toLibUVOwnedFD(fd, .open, .close_on_fail), .err => |e| .{ .err = e.withPath(path) }, }; } - return switch (Syscall.openDirAtWindowsA(dir, path, true, flags & os.O.NOFOLLOW != 0)) { + return switch (Syscall.openDirAtWindowsA(dir, path, .{ .iterable = true, .no_follow = flags & os.O.NOFOLLOW != 0 })) { .result => |fd| bun.sys.toLibUVOwnedFD(fd, .open, .close_on_fail), .err => |e| .{ .err = e.withPath(path) }, }; diff --git a/src/sys.zig b/src/sys.zig index 9e75498c65..db7e9c994c 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -471,6 +471,42 @@ pub fn fstat(fd: bun.FileDescriptor) Maybe(bun.Stat) { return Maybe(bun.Stat){ .result = stat_ }; } +pub fn mkdiratA(dir_fd: bun.FileDescriptor, file_path: []const u8) Maybe(void) { + var buf: bun.WPathBuffer = undefined; + return mkdiratW(dir_fd, bun.strings.toWPathNormalized(&buf, file_path)); +} + +pub fn mkdiratZ(dir_fd: bun.FileDescriptor, file_path: [*:0]const u8, mode: mode_t) Maybe(void) { + return switch (Environment.os) { + .mac => Maybe(void).errnoSysP(system.mkdirat(@intCast(dir_fd.cast()), file_path, mode), .mkdir, file_path) orelse Maybe(void).success, + .linux => Maybe(void).errnoSysP(linux.mkdirat(@intCast(dir_fd.cast()), file_path, mode), .mkdir, file_path) orelse Maybe(void).success, + else => @compileError("mkdir is not implemented on this platform"), + }; +} + +fn mkdiratPosix(dir_fd: bun.FileDescriptor, file_path: []const u8, mode: mode_t) Maybe(void) { + return mkdiratZ( + dir_fd, + &(std.os.toPosixPath(file_path) catch return .{ .err = Error.fromCode(.NAMETOOLONG, .mkdir) }), + mode, + ); +} + +pub const mkdirat = if (Environment.isWindows) + mkdiratW +else + mkdiratPosix; + +pub fn mkdiratW(dir_fd: bun.FileDescriptor, file_path: []const u16, _: i32) Maybe(void) { + const dir_to_make = openDirAtWindowsNtPath(dir_fd, file_path, .{ .iterable = false, .can_rename_or_delete = true, .create = true }); + if (dir_to_make == .err) { + return .{ .err = dir_to_make.err }; + } + + _ = close(dir_to_make.result); + return .{ .result = {} }; +} + pub fn fstatat(fd: bun.FileDescriptor, path: [:0]const u8) Maybe(bun.Stat) { if (Environment.isWindows) @compileError("TODO"); var stat_ = mem.zeroes(bun.Stat); @@ -533,10 +569,18 @@ pub fn mkdirOSPath(file_path: bun.OSPathSliceZ, flags: bun.Mode) Maybe(void) { else => mkdir(file_path, flags), .windows => { assertIsValidWindowsPath(bun.OSPathChar, file_path); - return Maybe(void).errnoSys( - kernel32.CreateDirectoryW(file_path, null), + const rc = kernel32.CreateDirectoryW(file_path, null); + + if (Maybe(void).errnoSys( + rc, .mkdir, - ) orelse Maybe(void).success; + )) |err| { + log("CreateDirectoryW({}) = {s}", .{ bun.fmt.fmtOSPath(file_path, .{}), err.err.name() }); + return err; + } + + log("CreateDirectoryW({}) = 0", .{bun.fmt.fmtOSPath(file_path, .{})}); + return Maybe(void).success; }, }; } @@ -614,16 +658,20 @@ pub fn normalizePathWindows( }; } -pub fn openDirAtWindowsNtPath( +fn openDirAtWindowsNtPath( dirFd: bun.FileDescriptor, path: []const u16, - iterable: bool, - no_follow: bool, + options: WindowsOpenDirOptions, ) Maybe(bun.FileDescriptor) { + const iterable = options.iterable; + const no_follow = options.no_follow; + const can_rename_or_delete = options.can_rename_or_delete; assertIsValidWindowsPath(u16, path); const base_flags = w.STANDARD_RIGHTS_READ | w.FILE_READ_ATTRIBUTES | w.FILE_READ_EA | - w.SYNCHRONIZE | w.FILE_TRAVERSE; - const flags: u32 = if (iterable) base_flags | w.FILE_LIST_DIRECTORY else base_flags; + w.SYNCHRONIZE | w.FILE_TRAVERSE | w.FILE_ADD_FILE | w.FILE_ADD_SUBDIRECTORY; + const iterable_flag: u32 = if (iterable) w.FILE_LIST_DIRECTORY else 0; + const rename_flag: u32 = if (can_rename_or_delete) w.DELETE else 0; + const flags: u32 = iterable_flag | base_flags | rename_flag; const path_len_bytes: u16 = @truncate(path.len * 2); var nt_name = w.UNICODE_STRING{ @@ -647,6 +695,7 @@ pub fn openDirAtWindowsNtPath( const open_reparse_point: w.DWORD = if (no_follow) w.FILE_OPEN_REPARSE_POINT else 0x0; var fd: w.HANDLE = w.INVALID_HANDLE_VALUE; var io: w.IO_STATUS_BLOCK = undefined; + const rc = w.ntdll.NtCreateFile( &fd, flags, @@ -654,8 +703,8 @@ pub fn openDirAtWindowsNtPath( &io, null, 0, - w.FILE_SHARE_READ | w.FILE_SHARE_WRITE, - w.FILE_OPEN, + FILE_SHARE, + if (options.create) w.FILE_OPEN_IF else w.FILE_OPEN, w.FILE_DIRECTORY_FILE | w.FILE_SYNCHRONOUS_IO_NONALERT | w.FILE_OPEN_FOR_BACKUP_INTENT | open_reparse_point, null, 0, @@ -702,12 +751,18 @@ pub fn openDirAtWindowsNtPath( } } -pub fn openDirAtWindowsT( +pub const WindowsOpenDirOptions = packed struct { + iterable: bool = false, + no_follow: bool = false, + can_rename_or_delete: bool = false, + create: bool = false, +}; + +fn openDirAtWindowsT( comptime T: type, dirFd: bun.FileDescriptor, path: []const T, - iterable: bool, - no_follow: bool, + options: WindowsOpenDirOptions, ) Maybe(bun.FileDescriptor) { var wbuf: bun.WPathBuffer = undefined; @@ -716,25 +771,23 @@ pub fn openDirAtWindowsT( .result => |norm| norm, }; - return openDirAtWindowsNtPath(dirFd, norm, iterable, no_follow); + return openDirAtWindowsNtPath(dirFd, norm, options); } pub fn openDirAtWindows( dirFd: bun.FileDescriptor, path: []const u16, - iterable: bool, - no_follow: bool, + options: WindowsOpenDirOptions, ) Maybe(bun.FileDescriptor) { - return openDirAtWindowsT(u16, dirFd, path, iterable, no_follow); + return openDirAtWindowsT(u16, dirFd, path, options); } pub noinline fn openDirAtWindowsA( dirFd: bun.FileDescriptor, path: []const u8, - iterable: bool, - no_follow: bool, + options: WindowsOpenDirOptions, ) Maybe(bun.FileDescriptor) { - return openDirAtWindowsT(u8, dirFd, path, iterable, no_follow); + return openDirAtWindowsT(u8, dirFd, path, options); } /// For this function to open an absolute path, it must start with "\??\". Otherwise @@ -810,7 +863,7 @@ pub fn openFileAtWindowsNtPath( &io, null, attributes, - w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, + FILE_SHARE, disposition, options, null, @@ -932,7 +985,7 @@ pub noinline fn openFileAtWindowsA( pub fn openatWindowsT(comptime T: type, dir: bun.FileDescriptor, path: []const T, flags: bun.Mode) Maybe(bun.FileDescriptor) { if (flags & O.DIRECTORY != 0) { // we interpret O_PATH as meaning that we don't want iteration - return openDirAtWindowsT(T, dir, path, flags & O.PATH == 0, flags & O.NOFOLLOW != 0); + return openDirAtWindowsT(T, dir, path, .{ .iterable = flags & O.PATH == 0, .no_follow = flags & O.NOFOLLOW != 0, .can_rename_or_delete = false }); } const nonblock = flags & O.NONBLOCK != 0; @@ -1652,6 +1705,87 @@ pub fn symlink(from: [:0]const u8, to: [:0]const u8) Maybe(void) { } } +pub const WindowsSymlinkOptions = packed struct { + directory: bool = false, + + var symlink_flags: u32 = w.SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; + pub fn flags(this: WindowsSymlinkOptions) u32 { + if (this.directory) { + symlink_flags |= w.SYMBOLIC_LINK_FLAG_DIRECTORY; + } + + return symlink_flags; + } + + pub fn denied() void { + symlink_flags = 0; + } + + pub var has_failed_to_create_symlink = false; +}; + +pub fn symlinkOrJunctionOnWindows(sym: [:0]const u8, target: [:0]const u8) Maybe(void) { + if (!WindowsSymlinkOptions.has_failed_to_create_symlink) { + var sym16: bun.WPathBuffer = undefined; + var target16: bun.WPathBuffer = undefined; + const sym_path = bun.strings.toNTPath(&sym16, sym); + const target_path = bun.strings.toNTPath(&target16, target); + switch (symlinkW(sym_path, target_path, .{ .directory = true })) { + .result => { + return Maybe(void).success; + }, + .err => {}, + } + } + + return sys_uv.symlinkUV(sym, target, bun.windows.libuv.UV_FS_SYMLINK_JUNCTION); +} + +pub fn symlinkW(sym: [:0]const u16, target: [:0]const u16, options: WindowsSymlinkOptions) Maybe(void) { + while (true) { + const flags = options.flags(); + + if (windows.kernel32.CreateSymbolicLinkW(sym, target, flags) == 0) { + const errno = bun.windows.Win32Error.get(); + log("CreateSymbolicLinkW({}, {}, {any}) = {s}", .{ + bun.fmt.fmtPath(u16, sym, .{}), + bun.fmt.fmtPath(u16, target, .{}), + flags, + @tagName(errno), + }); + switch (errno) { + .INVALID_PARAMETER => { + if ((flags & w.SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE) != 0) { + WindowsSymlinkOptions.denied(); + continue; + } + }, + else => {}, + } + + if (errno.toSystemErrno()) |err| { + WindowsSymlinkOptions.has_failed_to_create_symlink = true; + return .{ + .err = .{ + .errno = @intFromEnum(err), + .syscall = .symlink, + }, + }; + } + } + + log("CreateSymbolicLinkW({}, {}, {any}) = 0", .{ + bun.fmt.fmtPath(u16, sym, .{}), + bun.fmt.fmtPath(u16, target, .{}), + flags, + }); + + return Maybe(void).success; + } + + unreachable; +} + pub fn clonefile(from: [:0]const u8, to: [:0]const u8) Maybe(void) { if (comptime !Environment.isMac) @compileError("macOS only"); @@ -1972,24 +2106,29 @@ pub fn existsAt(fd: bun.FileDescriptor, subpath: []const u8) bool { } if (comptime Environment.isWindows) { - // TODO(dylan-conway): this is not tested - var wbuf: bun.MAX_WPATH = undefined; - const path_to_use = bun.strings.toWPath(&wbuf, subpath); - const nt_name = windows.UNICODE_STRING{ - .Length = path_to_use.len * 2, - .MaximumLength = path_to_use.len * 2, - .Buffer = path_to_use, + var wbuf: bun.WPathBuffer = undefined; + const path = bun.strings.toWPath(&wbuf, subpath); + const path_len_bytes: u16 = @truncate(path.len * 2); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @constCast(path.ptr), }; - const attr = windows.OBJECT_ATTRIBUTES{ - .Length = @sizeOf(windows.OBJECT_ATTRIBUTES), - .RootDirectory = fd, - .Attributes = 0, + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = if (std.fs.path.isAbsoluteWindowsWTF16(path)) + null + else if (fd == bun.invalid_fd) + std.fs.cwd().fd + else + fd.cast(), + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. .ObjectName = &nt_name, .SecurityDescriptor = null, .SecurityQualityOfService = null, }; - const basic_info: windows.FILE_BASIC_INFORMATION = undefined; - return switch (kernel32.NtQueryAttributesFile(&attr, basic_info)) { + var basic_info: w.FILE_BASIC_INFORMATION = undefined; + return switch (kernel32.NtQueryAttributesFile(&attr, &basic_info)) { .SUCCESS => true, else => false, }; @@ -2326,6 +2465,13 @@ pub const File = struct { // "handle" matches std.fs.File handle: bun.FileDescriptor, + pub fn openat(other: anytype, path: anytype, flags: bun.Mode, mode: bun.Mode) Maybe(File) { + return switch (This.openat(bun.toFD(other), path, flags, mode)) { + .result => |fd| .{ .result = .{ .handle = fd } }, + .err => |err| .{ .err = err }, + }; + } + pub fn from(other: anytype) File { const T = @TypeOf(other); @@ -2346,7 +2492,7 @@ pub const File = struct { } if (T == std.fs.Dir) { - return File{ .handle = bun.toFD(other.handle) }; + return File{ .handle = bun.toFD(other.fd) }; } if (comptime Environment.isWindows) { @@ -2441,6 +2587,10 @@ pub const File = struct { return getFileSize(self.handle); } + pub fn stat(self: File) Maybe(bun.Stat) { + return fstat(self.handle); + } + pub const ReadToEndResult = struct { bytes: std.ArrayList(u8) = std.ArrayList(u8).init(default_allocator), err: ?Error = null, @@ -2453,7 +2603,7 @@ pub const File = struct { .result => |s| s, }; - list.ensureUnusedCapacity(size + 16) catch bun.outOfMemory(); + list.ensureTotalCapacityPrecise(size + 16) catch bun.outOfMemory(); var total: i64 = 0; while (true) { @@ -2461,7 +2611,10 @@ pub const File = struct { list.ensureUnusedCapacity(16) catch bun.outOfMemory(); } - switch (bun.sys.pread(this.handle, list.unusedCapacitySlice(), total)) { + switch (if (comptime Environment.isPosix) + bun.sys.pread(this.handle, list.unusedCapacitySlice(), total) + else + bun.sys.read(this.handle, list.unusedCapacitySlice())) { .err => |err| { return .{ .err = err }; }, @@ -2485,6 +2638,60 @@ pub const File = struct { .result => .{ .err = null, .bytes = list }, }; } + + pub fn getPath(this: File, out_buffer: *[MAX_PATH_BYTES]u8) Maybe([]u8) { + return getFdPath(this.handle, out_buffer); + } + + /// 1. Open a file for reading + /// 2. Read the file to a buffer + /// 3. Return the File handle and the buffer + pub fn readFileFrom(dir_fd: anytype, path: [:0]const u8, allocator: std.mem.Allocator) Maybe(struct { File, []u8 }) { + const this = switch (bun.sys.openat(from(dir_fd).handle, path, O.RDONLY, 0)) { + .err => |err| return .{ .err = err }, + .result => |fd| from(fd), + }; + + var result = this.readToEnd(allocator); + + if (result.err) |err| { + this.close(); + result.bytes.deinit(); + return .{ .err = err }; + } + + return .{ .result = .{ this, result.bytes.items } }; + } + + /// 1. Open a file for reading relative to a directory + /// 2. Read the file to a buffer + /// 3. Close the file + /// 4. Return the buffer + pub fn readFrom(dir_fd: anytype, path: [:0]const u8, allocator: std.mem.Allocator) Maybe([]u8) { + const file, const bytes = switch (readFileFrom(dir_fd, path, allocator)) { + .err => |err| return .{ .err = err }, + .result => |result| result, + }; + + file.close(); + return .{ .result = bytes }; + } + + pub fn toSource(path: anytype, allocator: std.mem.Allocator) Maybe(bun.logger.Source) { + if (std.meta.sentinel(@TypeOf(path)) == null) { + return toSource( + &(std.os.toPosixPath(path) catch return .{ + .err = Error.oom, + }), + allocator, + ); + } + + return switch (readFrom(std.fs.cwd(), path, allocator)) { + .err => |err| .{ .err = err }, + .result => |bytes| .{ .result = bun.logger.Source.initPathString(path, bytes) }, + }; + } }; pub inline fn toLibUVOwnedFD( @@ -2512,3 +2719,6 @@ pub inline fn toLibUVOwnedFD( }, }; } + +pub const Dir = @import("./dir.zig"); +const FILE_SHARE = w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE; diff --git a/src/windows.zig b/src/windows.zig index 69c4f027d7..f9bf2f0a33 100644 --- a/src/windows.zig +++ b/src/windows.zig @@ -2987,11 +2987,29 @@ pub extern fn LoadLibraryA( pub extern fn LoadLibraryExW([*:0]const u16, ?HANDLE, DWORD) ?*anyopaque; -pub extern "kernel32" fn CreateHardLinkW( - newFileName: LPCWSTR, - existingFileName: LPCWSTR, - securityAttributes: ?*win32.SECURITY_ATTRIBUTES, -) BOOL; +pub const CreateHardLinkW = struct { + pub fn wrapper(newFileName: LPCWSTR, existingFileName: LPCWSTR, securityAttributes: ?*win32.SECURITY_ATTRIBUTES) BOOL { + const run = struct { + pub extern "kernel32" fn CreateHardLinkW( + newFileName: LPCWSTR, + existingFileName: LPCWSTR, + securityAttributes: ?*win32.SECURITY_ATTRIBUTES, + ) BOOL; + }.CreateHardLinkW; + + const rc = run(newFileName, existingFileName, securityAttributes); + if (comptime Environment.isDebug) + bun.sys.syslog( + "CreateHardLinkW({}, {}) = {d}", + .{ + bun.fmt.fmtOSPath(std.mem.span(newFileName), .{}), + bun.fmt.fmtOSPath(std.mem.span(existingFileName), .{}), + if (rc == 0) @intFromEnum(Win32Error.get()) else 0, + }, + ); + return rc; + } +}.wrapper; pub extern "kernel32" fn CopyFileW( source: LPCWSTR, @@ -3022,7 +3040,12 @@ pub fn translateNTStatusToErrno(err: win32.NTSTATUS) bun.C.E { .OBJECT_NAME_NOT_FOUND => .NOENT, .NOT_A_DIRECTORY => .NOTDIR, .RETRY => .AGAIN, + .DIRECTORY_NOT_EMPTY => .EXIST, .FILE_TOO_LARGE => .@"2BIG", + .SHARING_VIOLATION => if (comptime Environment.isDebug) brk: { + bun.Output.debugWarn("Received SHARING_VIOLATION, indicates file handle should've been opened with FILE_SHARE_DELETE", .{}); + break :brk .BUSY; + } else .BUSY, .OBJECT_NAME_INVALID => if (comptime Environment.isDebug) brk: { bun.Output.debugWarn("Received OBJECT_NAME_INVALID, indicates a file path conversion issue.", .{}); break :brk .INVAL; @@ -3306,8 +3329,10 @@ pub fn winSockErrorToZigError(err: std.os.windows.ws2_32.WinsockError) !void { .WSA_QOS_ESHAPERATEOBJ => error.WSA_QOS_ESHAPERATEOBJ, .WSA_QOS_RESERVED_PETYPE => error.WSA_QOS_RESERVED_PETYPE, _ => |t| { - if (Environment.isDebug) { - bun.Output.debugWarn("Unknown WinSockError: {d}", .{@intFromEnum(t)}); + if (@intFromEnum(t) != 0) { + if (Environment.isDebug) { + bun.Output.debugWarn("Unknown WinSockError: {d}", .{@intFromEnum(t)}); + } } }, }; @@ -3316,3 +3341,14 @@ pub fn winSockErrorToZigError(err: std.os.windows.ws2_32.WinsockError) !void { pub fn WSAGetLastError() !void { return winSockErrorToZigError(std.os.windows.ws2_32.WSAGetLastError()); } + +// BOOL CreateDirectoryExW( +// [in] LPCWSTR lpTemplateDirectory, +// [in] LPCWSTR lpNewDirectory, +// [in, optional] LPSECURITY_ATTRIBUTES lpSecurityAttributes +// ); +pub extern "kernel32" fn CreateDirectoryExW( + lpTemplateDirectory: [*:0]const u16, + lpNewDirectory: [*:0]const u16, + lpSecurityAttributes: ?*win32.SECURITY_ATTRIBUTES, +) callconv(windows.WINAPI) BOOL; diff --git a/src/windows_c.zig b/src/windows_c.zig index 8584659612..7379197854 100644 --- a/src/windows_c.zig +++ b/src/windows_c.zig @@ -1280,7 +1280,7 @@ pub fn renameAtW( old_dir_fd, old_path_w, // access_mask - w.SYNCHRONIZE | w.GENERIC_WRITE | w.DELETE, + w.SYNCHRONIZE | w.GENERIC_WRITE | w.DELETE | w.FILE_WRITE_DATA | w.FILE_TRAVERSE, // create disposition w.FILE_OPEN, // create options @@ -1378,7 +1378,7 @@ pub fn moveOpenedFileAtLoose( if (std.mem.lastIndexOfScalar(u16, new_path, '\\')) |last_slash| { const dirname = new_path[0..last_slash]; - const fd = switch (bun.sys.openDirAtWindows(new_dir_fd, dirname, false, true)) { + const fd = switch (bun.sys.openDirAtWindows(new_dir_fd, dirname, .{ .can_rename_or_delete = true, .iterable = false })) { .err => |e| return .{ .err = e }, .result => |fd| fd, }; diff --git a/test/cli/install/bunx.test.ts b/test/cli/install/bunx.test.ts index 79aae70993..fc5ea2e83b 100644 --- a/test/cli/install/bunx.test.ts +++ b/test/cli/install/bunx.test.ts @@ -35,6 +35,7 @@ beforeEach(async () => { env.TEMP = current_tmpdir; env.BUN_TMPDIR = env.TMPDIR = current_tmpdir; + env.TMPDIR = current_tmpdir; env.BUN_INSTALL_CACHE_DIR = install_cache_dir; await Promise.all(waiting); @@ -83,12 +84,11 @@ it("should choose the tagged versions instead of the PATH versions when a tag is }); }); - const [results, stdouts] = await Promise.all([ - Promise.all(processes.map(p => p.exited)), - Promise.all(processes.map(p => new Response(p.stdout).text())), - ]); + const results = await Promise.all(processes.map(p => p.exited)); expect(results).toEqual(semverVersions.map(() => 0)); - const outputs = stdouts.map(a => a.substring(0, a.indexOf("\n"))); + const outputs = (await Promise.all(processes.map(p => new Response(p.stdout).text()))).map(a => + a.substring(0, a.indexOf("\n")), + ); expect(outputs).toEqual(semverVersions.map(v => "SemVer " + v)); }); diff --git a/test/js/bun/spawn/spawn-stress.test.ts b/test/js/bun/spawn/spawn-stress.test.ts new file mode 100644 index 0000000000..97d0268760 --- /dev/null +++ b/test/js/bun/spawn/spawn-stress.test.ts @@ -0,0 +1,29 @@ +import { spawn } from "bun"; +import { expect, test } from "bun:test"; + +test("spawn stress", async () => { + for (let i = 0; i < 100; i++) { + try { + console.log("--- Begin Iteration " + i, "----"); + const withoutCache = spawn({ + cmd: ["clang", "--version"], + stdout: "pipe", + stderr: "pipe", + stdin: "ignore", + }); + var err = await new Response(withoutCache.stderr).text(); + var out = await new Response(withoutCache.stdout).text(); + console.log("--- End Iteration " + i, "----"); + out = out.trim(); + err = err.trim(); + + expect(out).not.toBe(""); + await Bun.sleep(1); + } catch (e) { + console.log("Failed in Iteration " + i + "\n"); + console.log(out); + console.log(err); + throw e; + } + } +}, 99999999);