From a36355cefb6f98429c60172976b293a0d1f3bdbf Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 15 Jan 2024 17:33:33 -0800 Subject: [PATCH] Some Windows tweaks (#8118) * Some windows tweaks * Make this pub * Update bundle_v2.zig * Fix woopsie * Make this error better * Add assertion for using allocator for HTTP client from another thread. * Do fewer copies in readdir() * Fix crash/failing tests * Update tests * Fix windows build * Update loop.c --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> --- misctools/ntquery.cpp | 192 ++++++++++++++++++++++++++++++ packages/bun-usockets/src/loop.c | 4 +- src/bun.js/bindings/BunString.cpp | 16 +++ src/bun.js/node/dir_iterator.zig | 40 ++++--- src/bun.js/node/node_fs.zig | 52 +++++--- src/bun.zig | 6 +- src/bundler/bundle_v2.zig | 2 +- src/cache.zig | 15 +-- src/fd.zig | 19 ++- src/feature_flags.zig | 2 +- src/fs.zig | 25 ++-- src/http.zig | 6 + src/report.zig | 2 +- src/resolver/resolver.zig | 28 +++-- src/string.zig | 4 + src/sys.zig | 11 +- src/sys_uv.zig | 44 +++---- test/cli/install/bun-run.test.ts | 4 +- 18 files changed, 372 insertions(+), 100 deletions(-) create mode 100644 misctools/ntquery.cpp diff --git a/misctools/ntquery.cpp b/misctools/ntquery.cpp new file mode 100644 index 0000000000..e4345eec4e --- /dev/null +++ b/misctools/ntquery.cpp @@ -0,0 +1,192 @@ + +#include +#include +#include +#include + +typedef struct _FILE_DIRECTORY_INFORMATION { + ULONG NextEntryOffset; + ULONG FileIndex; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER EndOfFile; + LARGE_INTEGER AllocationSize; + ULONG FileAttributes; + ULONG FileNameLength; + WCHAR FileName[1]; +} FILE_DIRECTORY_INFORMATION, *PFILE_DIRECTORY_INFORMATION; + +typedef struct _FILE_BOTH_DIR_INFORMATION { + ULONG NextEntryOffset; + ULONG FileIndex; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER EndOfFile; + LARGE_INTEGER AllocationSize; + ULONG FileAttributes; + ULONG FileNameLength; + ULONG EaSize; + CHAR ShortNameLength; + WCHAR ShortName[12]; + WCHAR FileName[1]; +} FILE_BOTH_DIR_INFORMATION, *PFILE_BOTH_DIR_INFORMATION; + +int main_using_file_directory_information(int argc, char *argv[]) { + if (argc != 2) { + printf("Usage: ntquery \n"); + return 1; + } + + auto *NtQueryDirectoryFile = (NTSTATUS(WINAPI *)( + HANDLE, HANDLE, PVOID, PVOID, PVOID, PVOID, DWORD, FILE_INFORMATION_CLASS, + BOOLEAN, PVOID, BOOLEAN))GetProcAddress(GetModuleHandle("ntdll.dll"), + "NtQueryDirectoryFile"); + + if (NtQueryDirectoryFile == NULL) { + printf("Error getting NtQueryDirectoryFile\n"); + return 1; + } + + // open the current working directory using NT API + HANDLE hFile = CreateFile(argv[1], GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + // use NtQueryDirectoryFile + char buffer[64096]; + + IO_STATUS_BLOCK ioStatusBlock; + NTSTATUS status = NtQueryDirectoryFile( + hFile, NULL, NULL, NULL, &ioStatusBlock, buffer, sizeof(buffer), + FileDirectoryInformation, FALSE, NULL, FALSE); + if (status != STATUS_SUCCESS) { + printf("Error querying directory\n"); + return 1; + } + + PFILE_DIRECTORY_INFORMATION pDirInfo = (PFILE_DIRECTORY_INFORMATION)buffer; + while (TRUE) { + printf("%S\n", pDirInfo->FileName); + if (pDirInfo->NextEntryOffset == 0) { + // if no more entries, continue to next query directory call + status = NtQueryDirectoryFile( + hFile, NULL, NULL, NULL, &ioStatusBlock, buffer, sizeof(buffer), + FileDirectoryInformation, FALSE, NULL, FALSE); + pDirInfo = (PFILE_DIRECTORY_INFORMATION)buffer; + + if (status == STATUS_NO_MORE_FILES) { + break; + } else if (status != STATUS_SUCCESS) { + printf("Error querying directory: %x\n", status); + return 1; + } else { + continue; + } + } + + pDirInfo = (PFILE_DIRECTORY_INFORMATION)((LPBYTE)pDirInfo + + pDirInfo->NextEntryOffset); + } + + CloseHandle(hFile); + return 0; +} + +#define FileBothDirectoryInformation static_cast(3) + +int main_using_file_both_information(int argc, char *argv[]) { + if (argc != 2) { + printf("Usage: ntquery \n"); + return 1; + } + + auto *NtQueryDirectoryFile = (NTSTATUS(WINAPI *)( + HANDLE, HANDLE, PVOID, PVOID, PVOID, PVOID, DWORD, FILE_INFORMATION_CLASS, + BOOLEAN, PVOID, BOOLEAN))GetProcAddress(GetModuleHandle("ntdll.dll"), + "NtQueryDirectoryFile"); + + if (NtQueryDirectoryFile == NULL) { + printf("Error getting NtQueryDirectoryFile\n"); + return 1; + } + + // open the current working directory using NT API + HANDLE hFile = CreateFile(argv[1], GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + // use NtQueryDirectoryFile + char buffer[64096]; + + IO_STATUS_BLOCK ioStatusBlock; + NTSTATUS status = NtQueryDirectoryFile( + hFile, NULL, NULL, NULL, &ioStatusBlock, buffer, sizeof(buffer), + FileBothDirectoryInformation, FALSE, NULL, FALSE); + if (status != STATUS_SUCCESS) { + printf("Error querying directory\n"); + return 1; + } + + PFILE_BOTH_DIR_INFORMATION pDirInfo = (PFILE_BOTH_DIR_INFORMATION)buffer; + while (TRUE) { + printf("%S\n", pDirInfo->FileName); + if (pDirInfo->NextEntryOffset == 0) { + // if no more entries, continue to next query directory call + status = NtQueryDirectoryFile( + hFile, NULL, NULL, NULL, &ioStatusBlock, buffer, sizeof(buffer), + FileBothDirectoryInformation, FALSE, NULL, FALSE); + pDirInfo = (PFILE_BOTH_DIR_INFORMATION)buffer; + + if (status == STATUS_NO_MORE_FILES) { + break; + } else if (status != STATUS_SUCCESS) { + printf("Error querying directory: %x\n", status); + return 1; + } else { + continue; + } + } + + pDirInfo = (PFILE_BOTH_DIR_INFORMATION)((LPBYTE)pDirInfo + + pDirInfo->NextEntryOffset); + } + + CloseHandle(hFile); + return 0; +} + +int main_using_findfirstfile_ex(int argc, char *argv[]) { + if (argc != 2) { + printf("Usage: ntquery \n"); + return 1; + } + + // open the current working directory using NT API + HANDLE hFile = CreateFile(argv[1], GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + char buffer[64096]; + + WIN32_FIND_DATA findData; + HANDLE hFind = FindFirstFileEx(argv[1], FindExInfoBasic, &findData, + FindExSearchNameMatch, NULL, 0); + if (hFind == INVALID_HANDLE_VALUE) { + printf("Error querying directory\n"); + return 1; + } + + do { + char szPath[MAX_PATH]; + + printf("%s\n", findData.cFileName); + } while (FindNextFile(hFind, &findData)); + + FindClose(hFind); + return 0; +} + +int main(int argc, char *argv[]) { + return main_using_findfirstfile_ex(argc, argv); +} \ No newline at end of file diff --git a/packages/bun-usockets/src/loop.c b/packages/bun-usockets/src/loop.c index 976f4ce538..5143653131 100644 --- a/packages/bun-usockets/src/loop.c +++ b/packages/bun-usockets/src/loop.c @@ -342,7 +342,8 @@ void us_internal_dispatch_ready_poll(struct us_poll_t *p, int error, int events) if (length > 0) { s = s->context->on_data(s, loop->data.recv_buf + LIBUS_RECV_BUFFER_PADDING, length); - + // loop->num_ready_polls isn't accessible on Windows. + #ifndef WIN32 // rare case: we're reading a lot of data, there's more to be read, and either: // - the socket has hung up, so we will never get more data from it (only applies to macOS, as macOS will send the event the same tick but Linux will not.) // - the event loop isn't very busy, so we can read multiple times in a row @@ -361,6 +362,7 @@ void us_internal_dispatch_ready_poll(struct us_poll_t *p, int error, int events) } } #undef LOOP_ISNT_VERY_BUSY_THRESHOLD + #endif } else if (!length) { if (us_socket_is_shut_down(0, s)) { /* We got FIN back after sending it */ diff --git a/src/bun.js/bindings/BunString.cpp b/src/bun.js/bindings/BunString.cpp index ba53c2c44f..cf9160dad0 100644 --- a/src/bun.js/bindings/BunString.cpp +++ b/src/bun.js/bindings/BunString.cpp @@ -236,6 +236,22 @@ extern "C" BunString BunString__fromLatin1(const char* bytes, size_t length) return { BunStringTag::WTFStringImpl, { .wtf = &WTF::StringImpl::create(bytes, length).leakRef() } }; } +extern "C" BunString BunString__fromUTF16ToLatin1(const char16_t* bytes, size_t length) +{ + ASSERT(length > 0); + ASSERT_WITH_MESSAGE(simdutf::validate_utf16le(bytes, length), "This function only accepts ascii UTF16 strings"); + size_t outLength = simdutf::latin1_length_from_utf16(length); + LChar* ptr = nullptr; + auto impl = WTF::StringImpl::createUninitialized(outLength, ptr); + if (!ptr) { + return { BunStringTag::Dead }; + } + + size_t latin1_length = simdutf::convert_valid_utf16le_to_latin1(bytes, length, reinterpret_cast(ptr)); + ASSERT_WITH_MESSAGE(latin1_length == outLength, "Failed to convert UTF16 to Latin1"); + return { BunStringTag::WTFStringImpl, { .wtf = &impl.leakRef() } }; +} + extern "C" BunString BunString__fromUTF16(const char16_t* bytes, size_t length) { ASSERT(length > 0); diff --git a/src/bun.js/node/dir_iterator.zig b/src/bun.js/node/dir_iterator.zig index bc6f1e10dd..ff86a317f8 100644 --- a/src/bun.js/node/dir_iterator.zig +++ b/src/bun.js/node/dir_iterator.zig @@ -63,7 +63,7 @@ pub fn NewIterator(comptime use_windows_ospath: bool) type { /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. - const next = switch (builtin.os.tag) { + pub const next = switch (builtin.os.tag) { .macos, .ios => nextDarwin, // .freebsd, .netbsd, .dragonfly, .openbsd => nextBsd, // .solaris => nextSolaris, @@ -177,12 +177,11 @@ pub fn NewIterator(comptime use_windows_ospath: bool) type { }, .windows => struct { dir: Dir, - buf: [8192]u8 align(@alignOf(os.windows.FILE_BOTH_DIR_INFORMATION)), + buf: [8192]u8 align(@alignOf(os.windows.FILE_DIRECTORY_INFORMATION)), index: usize, end_index: usize, first: bool, - name_data: [256]u8, - name_data_w: [128]u16, + name_data: if (use_windows_ospath) [257]u16 else [513]u8, const Self = @This(); @@ -206,17 +205,24 @@ pub fn NewIterator(comptime use_windows_ospath: bool) type { &io, &self.buf, self.buf.len, - .FileBothDirectoryInformation, + .FileDirectoryInformation, w.FALSE, null, if (self.first) @as(w.BOOLEAN, w.TRUE) else @as(w.BOOLEAN, w.FALSE), ); + self.first = false; - if (io.Information == 0) return .{ .result = null }; + if (io.Information == 0) { + bun.sys.syslog("NtQueryDirectoryFile({d}) = 0", .{ + @intFromPtr(self.dir.fd), + }); + return .{ .result = null }; + } self.index = 0; self.end_index = io.Information; // If the handle is not a directory, we'll get STATUS_INVALID_PARAMETER. if (rc == .INVALID_PARAMETER) { + bun.sys.syslog("NtQueryDirectoryFile({d}) = {s}", .{ @intFromPtr(self.dir.fd), @tagName(rc) }); return .{ .err = .{ .errno = @intFromEnum(bun.C.SystemErrno.ENOTDIR), @@ -226,11 +232,14 @@ pub fn NewIterator(comptime use_windows_ospath: bool) type { } if (rc == .NO_MORE_FILES) { + bun.sys.syslog("NtQueryDirectoryFile({d}) = {s}", .{ @intFromPtr(self.dir.fd), @tagName(rc) }); self.end_index = self.index; return .{ .result = null }; } if (rc != .SUCCESS) { + bun.sys.syslog("NtQueryDirectoryFile({d}) = {s}", .{ @intFromPtr(self.dir.fd), @tagName(rc) }); + if ((bun.windows.Win32Error.fromNTStatus(rc).toSystemErrno())) |errno| { return .{ .err = .{ @@ -247,21 +256,20 @@ pub fn NewIterator(comptime use_windows_ospath: bool) type { }, }; } + + bun.sys.syslog("NtQueryDirectoryFile({d}) = {d}", .{ @intFromPtr(self.dir.fd), self.end_index }); } - const dir_info: *w.FILE_BOTH_DIR_INFORMATION = @ptrCast(@alignCast(&self.buf[self.index])); + const dir_info: *w.FILE_DIRECTORY_INFORMATION = @ptrCast(@alignCast(&self.buf[self.index])); if (dir_info.NextEntryOffset != 0) { self.index += dir_info.NextEntryOffset; } else { self.index = self.buf.len; } - const length = dir_info.FileNameLength / 2; - @memcpy(self.name_data_w[0..length], @as([*]u16, @ptrCast(&dir_info.FileName))[0..length]); - self.name_data_w[length] = 0; - const name_utf16le = self.name_data_w[0..length :0]; + const dir_info_name = @as([*]const u16, @ptrCast(&dir_info.FileName))[0 .. dir_info.FileNameLength / 2]; - if (mem.eql(u16, name_utf16le, &[_]u16{'.'}) or mem.eql(u16, name_utf16le, &[_]u16{ '.', '.' })) + if (mem.eql(u16, dir_info_name, &[_]u16{'.'}) or mem.eql(u16, dir_info_name, &[_]u16{ '.', '.' })) continue; const kind = blk: { @@ -272,6 +280,11 @@ pub fn NewIterator(comptime use_windows_ospath: bool) type { }; if (use_windows_ospath) { + const length = dir_info.FileNameLength / 2; + @memcpy(self.name_data[0..length], @as([*]u16, @ptrCast(&dir_info.FileName))[0..length]); + self.name_data[length] = 0; + const name_utf16le = self.name_data[0..length :0]; + return .{ .result = IteratorResultW{ .kind = kind, @@ -281,7 +294,7 @@ pub fn NewIterator(comptime use_windows_ospath: bool) type { } // Trust that Windows gives us valid UTF-16LE - const name_utf8 = strings.fromWPath(self.name_data[0..], name_utf16le); + const name_utf8 = strings.fromWPath(self.name_data[0..], dir_info_name); return .{ .result = IteratorResult{ @@ -405,7 +418,6 @@ pub fn NewWrappedIterator(comptime path_type: PathType) type { .first = true, .buf = undefined, .name_data = undefined, - .name_data_w = undefined, }, .wasi => IteratorType{ .dir = dir, diff --git a/src/bun.js/node/node_fs.zig b/src/bun.js/node/node_fs.zig index 7cb5c79ab4..8101ffb1e2 100644 --- a/src/bun.js/node/node_fs.zig +++ b/src/bun.js/node/node_fs.zig @@ -4629,7 +4629,11 @@ pub const NodeFS = struct { entries: *std.ArrayList(ExpectedType), ) Maybe(void) { const dir = fd.asDir(); - var iterator = DirIterator.iterate(dir, .u8); + const is_u16 = comptime Environment.isWindows and (ExpectedType == bun.String or ExpectedType == Dirent); + var iterator = DirIterator.iterate( + dir, + comptime if (is_u16) .u16 else .u8, + ); var entry = iterator.next(); while (switch (entry) { @@ -4657,21 +4661,37 @@ pub const NodeFS = struct { }, .result => |ent| ent, }) |current| : (entry = iterator.next()) { - const utf8_name = current.name.slice(); - switch (ExpectedType) { - Dirent => { - entries.append(.{ - .name = bun.String.create(utf8_name), - .kind = current.kind, - }) catch bun.outOfMemory(); - }, - Buffer => { - entries.append(Buffer.fromString(utf8_name, bun.default_allocator) catch bun.outOfMemory()) catch bun.outOfMemory(); - }, - bun.String => { - entries.append(bun.String.create(utf8_name)) catch bun.outOfMemory(); - }, - else => @compileError("unreachable"), + if (comptime !is_u16) { + const utf8_name = current.name.slice(); + switch (ExpectedType) { + Dirent => { + entries.append(.{ + .name = bun.String.create(utf8_name), + .kind = current.kind, + }) catch bun.outOfMemory(); + }, + Buffer => { + entries.append(Buffer.fromString(utf8_name, bun.default_allocator) catch bun.outOfMemory()) catch bun.outOfMemory(); + }, + bun.String => { + entries.append(bun.String.create(utf8_name)) catch bun.outOfMemory(); + }, + else => @compileError("unreachable"), + } + } else { + const utf16_name = current.name.slice(); + switch (ExpectedType) { + Dirent => { + entries.append(.{ + .name = bun.String.createUTF16(utf16_name), + .kind = current.kind, + }) catch bun.outOfMemory(); + }, + bun.String => { + entries.append(bun.String.createUTF16(utf16_name)) catch bun.outOfMemory(); + }, + else => @compileError("unreachable"), + } } } diff --git a/src/bun.zig b/src/bun.zig index ef22ae5dfa..05f75c8bcb 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1065,7 +1065,7 @@ pub fn getcwd(buf_: []u8) ![]u8 { const temp_slice = try std.os.getcwd(&temp); // Paths are normalized to use / to make more things reliable, but eventually this will have to change to be the true file sep // It is possible to expose this value to JS land - return path.normalizeBuf(temp_slice, buf_, .loose); + return path.normalizeBuf(temp_slice, buf_, .auto); } pub fn getcwdAlloc(allocator: std.mem.Allocator) ![]u8 { @@ -2430,3 +2430,7 @@ pub const S = if (Environment.isWindows) C.S else std.os.S; pub const trait = @import("./trait.zig"); pub const brotli = @import("./brotli.zig"); + +pub fn iterateDir(dir: std.fs.Dir) DirIterator.Iterator { + return DirIterator.iterate(dir, .u8).iter; +} diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index a4263182c3..2d7d9664a2 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -2586,7 +2586,7 @@ pub const ParseTask = struct { ) catch |err| { const source_ = &Logger.Source.initEmptyFile(log.msgs.allocator.dupe(u8, file_path.text) catch unreachable); switch (err) { - error.FileNotFound => { + error.ENOENT, error.FileNotFound => { log.addErrorFmt( source_, Logger.Loc.Empty, diff --git a/src/cache.zig b/src/cache.zig index 93fe5ed69d..1efaf3156a 100644 --- a/src/cache.zig +++ b/src/cache.zig @@ -170,25 +170,26 @@ pub const Fs = struct { if (_file_handle == null) { if (FeatureFlags.store_file_descriptors and dirname_fd != bun.invalid_fd and dirname_fd.int() > 0) { - file_handle = std.fs.Dir.openFile(dirname_fd.asDir(), std.fs.path.basename(path), .{ .mode = .read_only }) catch |err| brk: { + file_handle = (bun.sys.openatA(dirname_fd, std.fs.path.basename(path), std.os.O.RDONLY, 0).unwrap() catch |err| brk: { switch (err) { - error.FileNotFound => { - const handle = try std.fs.openFileAbsolute(path, .{ .mode = .read_only }); + error.ENOENT => { + const handle = try bun.openFile(path, .{ .mode = .read_only }); Output.prettyErrorln( "Internal error: directory mismatch for directory \"{s}\", fd {d}. You don't need to do anything, but this indicates a bug.", .{ path, dirname_fd }, ); - break :brk handle; + break :brk bun.toFD(handle.handle); }, else => return err, } - }; + }).asFile(); } else { - file_handle = try std.fs.cwd().openFile(path, .{ .mode = .read_only }); + file_handle = try bun.openFile(path, .{ .mode = .read_only }); } } - debug("openat({d}, {s}) = {d}", .{ dirname_fd, path, file_handle.handle }); + if (comptime !Environment.isWindows) // skip on Windows because NTCreateFile will do it. + debug("openat({d}, {s}) = {d}", .{ dirname_fd, path, bun.toFD(file_handle.handle).int() }); const will_close = rfs.needToCloseFiles() and _file_handle == null; defer { diff --git a/src/fd.zig b/src/fd.zig index 4514021c1f..ed8ca31613 100644 --- a/src/fd.zig +++ b/src/fd.zig @@ -217,9 +217,9 @@ pub const FDImpl = packed struct { }; }, .windows => result: { - var req: libuv.fs_t = libuv.fs_t.uninitialized; switch (this.kind) { .uv => { + var req: libuv.fs_t = libuv.fs_t.uninitialized; defer req.deinit(); const rc = libuv.uv_fs_close(libuv.Loop.get(), &req, this.value.as_uv, null); break :result if (rc.errno()) |errno| @@ -230,17 +230,14 @@ pub const FDImpl = packed struct { .system => { std.debug.assert(this.value.as_system != 0); const handle: System = @ptrFromInt(@as(u64, this.value.as_system)); - if (std.os.windows.kernel32.CloseHandle(handle) == 0) { - const errno = switch (std.os.windows.kernel32.GetLastError()) { - .INVALID_HANDLE => @intFromEnum(os.E.BADF), - else => |i| @intFromEnum(i), - }; - break :result bun.sys.Error{ - .errno = errno, + break :result switch (bun.windows.NtClose(handle)) { + .SUCCESS => null, + else => |rc| bun.sys.Error{ + .errno = if (bun.windows.Win32Error.fromNTStatus(rc).toSystemErrno()) |errno| @intFromEnum(errno) else 1, .syscall = .CloseHandle, .fd = this.encode(), - }; - } + }, + }; }, } }, @@ -253,7 +250,7 @@ pub const FDImpl = packed struct { // TODO(@paperdave): Zig Compiler Bug, if you remove `this` from the log. An error is correctly printed, but with the wrong reference trace bun.Output.debugWarn("close({}) = EBADF. This is an indication of a file descriptor UAF", .{this}); } else { - log("close({}) = err {d}", .{ this, err.errno }); + log("close({}) = {}", .{ this, err }); } } else { log("close({})", .{this}); diff --git a/src/feature_flags.zig b/src/feature_flags.zig index 76cf5f70b1..4c3874bc40 100644 --- a/src/feature_flags.zig +++ b/src/feature_flags.zig @@ -7,7 +7,7 @@ pub const print_ast = false; pub const disable_printing_null = false; // This was a ~5% performance improvement -pub const store_file_descriptors = !env.isWindows and !env.isBrowser; +pub const store_file_descriptors = !env.isBrowser; pub const css_in_js_import_behavior = CSSInJSImportBehavior.facade; diff --git a/src/fs.zig b/src/fs.zig index e76c04f388..0b06480707 100644 --- a/src/fs.zig +++ b/src/fs.zig @@ -90,7 +90,11 @@ pub const FileSystem = struct { return; } - max_fd = @enumFromInt(@max(fd, max_fd.int())); + if (comptime @TypeOf(fd) == *anyopaque) { + max_fd = @enumFromInt(@max(@intFromPtr(fd), max_fd.int())); + } else { + max_fd = @enumFromInt(@max(fd, max_fd.int())); + } } pub var instance_loaded: bool = false; pub var instance: FileSystem = undefined; @@ -158,7 +162,8 @@ pub const FileSystem = struct { // // dir.data.remove(name); // } - pub fn addEntry(dir: *DirEntry, prev_map: ?*EntryMap, entry: std.fs.Dir.Entry, allocator: std.mem.Allocator, comptime Iterator: type, iterator: Iterator) !void { + pub fn addEntry(dir: *DirEntry, prev_map: ?*EntryMap, entry: *const bun.DirIterator.IteratorResult, allocator: std.mem.Allocator, comptime Iterator: type, iterator: Iterator) !void { + const name_slice = entry.name.slice(); const _kind: Entry.Kind = switch (entry.kind) { .directory => .dir, // This might be wrong! @@ -171,9 +176,9 @@ pub const FileSystem = struct { if (prev_map) |map| { var stack_fallback = std.heap.stackFallback(512, allocator); const stack = stack_fallback.get(); - const prehashed = bun.StringHashMapContext.PrehashedCaseInsensitive.init(stack, entry.name); + const prehashed = bun.StringHashMapContext.PrehashedCaseInsensitive.init(stack, name_slice); defer prehashed.deinit(stack); - if (map.getAdapted(entry.name, prehashed)) |existing| { + if (map.getAdapted(name_slice, prehashed)) |existing| { existing.mutex.lock(); defer existing.mutex.unlock(); existing.dir = dir.dir; @@ -189,15 +194,15 @@ pub const FileSystem = struct { } } - // entry.name only lives for the duration of the iteration + // name_slice only lives for the duration of the iteration const name = try strings.StringOrTinyString.initAppendIfNeeded( - entry.name, + name_slice, *FileSystem.FilenameStore, &FileSystem.FilenameStore.instance, ); const name_lowercased = try strings.StringOrTinyString.initLowerCaseAppendIfNeeded( - entry.name, + name_slice, *FileSystem.FilenameStore, &FileSystem.FilenameStore.instance, ); @@ -923,7 +928,7 @@ pub const FileSystem = struct { ) !DirEntry { _ = fs; - var iter = handle.iterate(); + var iter = bun.iterateDir(handle); var dir = DirEntry.init(_dir, generation); const allocator = bun.fs_allocator; errdefer dir.deinit(allocator); @@ -933,8 +938,8 @@ pub const FileSystem = struct { dir.fd = bun.toFD(handle.fd); } - while (try iter.next()) |_entry| { - debug("readdir entry {s}", .{_entry.name}); + while (try iter.next().unwrap()) |*_entry| { + debug("readdir entry {s}", .{_entry.name.slice()}); try dir.addEntry(prev_map, _entry, allocator, Iterator, iterator); } diff --git a/src/http.zig b/src/http.zig index 3615c6a054..1958c318ba 100644 --- a/src/http.zig +++ b/src/http.zig @@ -2183,6 +2183,12 @@ pub fn start(this: *HTTPClient, body: HTTPRequestBody, body_out_str: *MutableStr } fn start_(this: *HTTPClient, comptime is_ssl: bool) void { + if (comptime Environment.allow_assert) { + if (this.allocator.vtable == default_allocator.vtable and this.allocator.ptr != default_allocator.ptr) { + @panic("HTTPClient used with threadlocal allocator belonging to another thread. This will cause crashes."); + } + } + // Aborted before connecting if (this.signals.get(.aborted)) { this.fail(error.Aborted); diff --git a/src/report.zig b/src/report.zig index 7ca553ac72..f4cb203090 100644 --- a/src/report.zig +++ b/src/report.zig @@ -571,7 +571,7 @@ pub noinline fn globalError(err: anyerror, trace_: @TypeOf(@errorReturnTrace())) Global.exit(1); } }, - error.FileNotFound => { + error.ENOENT, error.FileNotFound => { Output.prettyError( "\nerror: FileNotFound\nBun could not find a file, and the code that produces this error is missing a better error.\n", .{}, diff --git a/src/resolver/resolver.zig b/src/resolver/resolver.zig index b364914a97..34f268bc45 100644 --- a/src/resolver/resolver.zig +++ b/src/resolver/resolver.zig @@ -566,7 +566,10 @@ pub const Resolver = struct { const pm = PackageManager.initWithRuntime( this.log, this.opts.install, - this.allocator, + + // This cannot be the threadlocal allocator. It goes to the HTTP thread. + bun.default_allocator, + .{}, this.env_loader.?, ) catch @panic("Failed to initialize package manager"); @@ -2056,7 +2059,7 @@ pub const Resolver = struct { var dir_entries_option: *Fs.FileSystem.RealFS.EntriesOption = undefined; var needs_iter = true; var in_place: ?*Fs.FileSystem.DirEntry = null; - var open_dir = std.fs.cwd().openDir(dir_path, .{ .iterate = true }) catch |err| { + const open_dir = std.fs.cwd().openDir(dir_path, .{ .iterate = true }) catch |err| { // TODO: handle this error better r.log.addErrorFmt( null, @@ -2086,11 +2089,11 @@ pub const Resolver = struct { r.generation, ); - var dir_iterator = open_dir.iterate(); - while (dir_iterator.next() catch null) |_value| { + var dir_iterator = bun.iterateDir(open_dir); + while (dir_iterator.next().unwrap() catch null) |_value| { new_entry.addEntry( if (in_place) |existing| &existing.data else null, - _value, + &_value, allocator, void, {}, @@ -2514,13 +2517,19 @@ pub const Resolver = struct { }; } + threadlocal var win32_normalized_dir_info_cache_buf: if (Environment.isWindows) [bun.MAX_PATH_BYTES * 2]u8 else void = undefined; fn dirInfoCachedMaybeLog(r: *ThisResolver, __path: string, comptime enable_logging: bool, comptime follow_symlinks: bool) !?*DirInfo { r.mutex.lock(); defer r.mutex.unlock(); var _path = __path; + if (isDotSlash(_path) or strings.eqlComptime(_path, ".")) _path = r.fs.top_level_dir; + if (comptime Environment.isWindows) { + _path = r.fs.normalizeBuf(&win32_normalized_dir_info_cache_buf, _path); + } + const top_result = try r.dir_cache.getOrPut(_path); if (top_result.status != .unknown) { return r.dir_cache.atIndex(top_result.index); @@ -2744,11 +2753,11 @@ pub const Resolver = struct { r.generation, ); - var dir_iterator = open_dir.iterate(); - while (dir_iterator.next() catch null) |_value| { + var dir_iterator = bun.iterateDir(open_dir); + while (dir_iterator.next().unwrap() catch null) |_value| { new_entry.addEntry( if (in_place) |existing| &existing.data else null, - _value, + &_value, allocator, void, {}, @@ -3999,9 +4008,8 @@ pub const Resolver = struct { var current = tsconfig_json; while (current.extends.len > 0) { const ts_dir_name = Dirname.dirname(current.abs_path); - // not sure why this needs cwd but we'll just pass in the dir of the tsconfig... const abs_path = ResolvePath.joinAbsStringBuf(ts_dir_name, bufs(.tsconfig_path_abs), &[_]string{ ts_dir_name, current.extends }, .auto); - const parent_config_maybe = r.parseTSConfig(abs_path, bun.STDIN_FD) catch |err| { + const parent_config_maybe = r.parseTSConfig(abs_path, bun.invalid_fd) catch |err| { r.log.addDebugFmt(null, logger.Loc.Empty, r.allocator, "{s} loading tsconfig.json extends {}", .{ @errorName(err), bun.fmt.QuotedFormatter{ diff --git a/src/string.zig b/src/string.zig index 969301c8ab..f12f5e25d0 100644 --- a/src/string.zig +++ b/src/string.zig @@ -280,6 +280,7 @@ pub const String = extern struct { extern fn BunString__fromLatin1(bytes: [*]const u8, len: usize) String; extern fn BunString__fromBytes(bytes: [*]const u8, len: usize) String; extern fn BunString__fromUTF16(bytes: [*]const u16, len: usize) String; + extern fn BunString__fromUTF16ToLatin1(bytes: [*]const u16, len: usize) String; extern fn BunString__fromLatin1Unitialized(len: usize) String; extern fn BunString__fromUTF16Unitialized(len: usize) String; @@ -369,6 +370,9 @@ pub const String = extern struct { pub fn createUTF16(bytes: []const u16) String { if (bytes.len == 0) return String.empty; + if (bun.strings.firstNonASCII16CheckMin([]const u16, bytes, false) == null) { + return BunString__fromUTF16ToLatin1(bytes.ptr, bytes.len); + } return BunString__fromUTF16(bytes.ptr, bytes.len); } diff --git a/src/sys.zig b/src/sys.zig index d49ca15687..3da694deb8 100644 --- a/src/sys.zig +++ b/src/sys.zig @@ -426,7 +426,7 @@ pub fn openDirAtWindows( ); if (comptime Environment.allow_assert) { - log("NtCreateFile({d}, {}) = {d} (dir) = {d}", .{ dirFd, bun.fmt.fmtUTF16(path), rc, @intFromPtr(fd) }); + log("NtCreateFile({d}, {s}) = {s} (dir) = {d}", .{ dirFd, bun.fmt.fmtUTF16(path), @tagName(rc), @intFromPtr(fd) }); } switch (windows.Win32Error.fromNTStatus(rc)) { @@ -544,7 +544,7 @@ pub fn openatWindows(dirfd: bun.FileDescriptor, path_: []const u16, flags: bun.M ); if (comptime Environment.allow_assert) { - log("NtCreateFile({d}, {}) = {d} (file) = {d}", .{ dirfd, bun.fmt.fmtUTF16(path), rc, @intFromPtr(result) }); + log("NtCreateFile({d}, {}) = {s} (file) = {d}", .{ dirfd, bun.fmt.fmtUTF16(path), @tagName(rc), @intFromPtr(result) }); } switch (windows.Win32Error.fromNTStatus(rc)) { @@ -1334,6 +1334,7 @@ pub const Error = struct { syscall: Syscall.Tag, path: []const u8 = "", fd: bun.FileDescriptor = bun.invalid_fd, + from_libuv: if (Environment.isWindows) bool else void = if (Environment.isWindows) false else undefined, pub inline fn isRetry(this: *const Error) bool { return this.getErrno() == .AGAIN; @@ -1429,7 +1430,11 @@ pub const Error = struct { const system_errno = brk: { // setRuntimeSafety(false) because we use tagName function, which will be null on invalid enum value. @setRuntimeSafety(false); - break :brk @as(C.SystemErrno, @enumFromInt(@intFromEnum(bun.windows.libuv.translateUVErrorToE(err.errno)))); + if (this.from_libuv) { + break :brk @as(C.SystemErrno, @enumFromInt(@intFromEnum(bun.windows.libuv.translateUVErrorToE(err.errno)))); + } + + break :brk @as(C.SystemErrno, @enumFromInt(this.errno)); }; if (std.enums.tagName(bun.C.SystemErrno, system_errno)) |errname| { err.code = bun.String.static(errname); diff --git a/src/sys_uv.zig b/src/sys_uv.zig index d4bd93f004..c8e9452e0c 100644 --- a/src/sys_uv.zig +++ b/src/sys_uv.zig @@ -43,7 +43,7 @@ pub fn open(file_path: [:0]const u8, flags: bun.Mode, perm: bun.Mode) Maybe(bun. const rc = uv.uv_fs_open(uv.Loop.get(), &req, file_path.ptr, flags, perm, null); log("uv open({s}, {d}, {d}) = {d}", .{ file_path, flags, perm, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .open } } + .{ .err = .{ .errno = errno, .syscall = .open, .from_libuv = true } } else .{ .result = bun.toFD(@as(i32, @intCast(req.result.value))) }; } @@ -55,7 +55,7 @@ pub fn mkdir(file_path: [:0]const u8, flags: bun.Mode) Maybe(void) { log("uv mkdir({s}, {d}) = {d}", .{ file_path, flags, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .mkdir } } + .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } } else .{ .result = {} }; } @@ -67,7 +67,7 @@ pub fn chmod(file_path: [:0]const u8, flags: bun.Mode) Maybe(void) { log("uv chmod({s}, {d}) = {d}", .{ file_path, flags, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .chmod } } + .{ .err = .{ .errno = errno, .syscall = .chmod, .from_libuv = true } } else .{ .result = {} }; } @@ -80,7 +80,7 @@ pub fn fchmod(fd: FileDescriptor, flags: bun.Mode) Maybe(void) { log("uv fchmod({}, {d}) = {d}", .{ uv_fd, flags, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .fchmod } } + .{ .err = .{ .errno = errno, .syscall = .fchmod, .from_libuv = true } } else .{ .result = {} }; } @@ -92,7 +92,7 @@ pub fn chown(file_path: [:0]const u8, uid: uv.uv_uid_t, gid: uv.uv_uid_t) Maybe( log("uv chown({s}, {d}, {d}) = {d}", .{ file_path, uid, gid, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .mkdir } } + .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } } else .{ .result = {} }; } @@ -106,7 +106,7 @@ pub fn fchown(fd: FileDescriptor, uid: uv.uv_uid_t, gid: uv.uv_uid_t) Maybe(void log("uv chown({}, {d}, {d}) = {d}", .{ uv_fd, uid, gid, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .mkdir } } + .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } } else .{ .result = {} }; } @@ -118,7 +118,7 @@ pub fn access(file_path: [:0]const u8, flags: bun.Mode) Maybe(void) { log("uv access({s}, {d}) = {d}", .{ file_path, flags, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .access } } + .{ .err = .{ .errno = errno, .syscall = .access, .from_libuv = true } } else .{ .result = {} }; } @@ -130,7 +130,7 @@ pub fn rmdir(file_path: [:0]const u8) Maybe(void) { log("uv rmdir({s}) = {d}", .{ file_path, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .mkdir } } + .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } } else .{ .result = {} }; } @@ -142,7 +142,7 @@ pub fn unlink(file_path: [:0]const u8) Maybe(void) { log("uv unlink({s}) = {d}", .{ file_path, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .mkdir } } + .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } } else .{ .result = {} }; } @@ -155,14 +155,14 @@ pub fn readlink(file_path: [:0]const u8, buf: []u8) Maybe(usize) { if (rc.errno()) |errno| { log("uv readlink({s}) = {d}, [err]", .{ file_path, rc.value }); - return .{ .err = .{ .errno = errno, .syscall = .mkdir } }; + return .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } }; } else { // Seems like `rc` does not contain the errno? std.debug.assert(rc.value == 0); const slice = bun.span(req.ptrAs([*:0]u8)); if (slice.len > buf.len) { log("uv readlink({s}) = {d}, {s} TRUNCATED", .{ file_path, rc.value, slice }); - return .{ .err = .{ .errno = @intFromEnum(E.NOMEM), .syscall = .mkdir } }; + return .{ .err = .{ .errno = @intFromEnum(E.NOMEM), .syscall = .mkdir, .from_libuv = true } }; } log("uv readlink({s}) = {d}, {s}", .{ file_path, rc.value, slice }); @memcpy(buf[0..slice.len], slice); @@ -177,7 +177,7 @@ pub fn rename(from: [:0]const u8, to: [:0]const u8) Maybe(void) { log("uv rename({s}, {s}) = {d}", .{ from, to, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .mkdir } } + .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } } else .{ .result = {} }; } @@ -190,7 +190,7 @@ pub fn link(from: [:0]const u8, to: [:0]const u8) Maybe(void) { log("uv link({s}, {s}) = {d}", .{ from, to, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .mkdir } } + .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } } else .{ .result = {} }; } @@ -203,7 +203,7 @@ pub fn symlinkUV(from: [:0]const u8, to: [:0]const u8, flags: c_int) Maybe(void) log("uv symlink({s}, {s}) = {d}", .{ from, to, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .mkdir } } + .{ .err = .{ .errno = errno, .syscall = .mkdir, .from_libuv = true } } else .{ .result = {} }; } @@ -216,7 +216,7 @@ pub fn ftruncate(fd: FileDescriptor, size: isize) Maybe(void) { log("uv ftruncate({}, {d}) = {d}", .{ uv_fd, size, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .ftruncate, .fd = fd } } + .{ .err = .{ .errno = errno, .syscall = .ftruncate, .fd = fd, .from_libuv = true } } else .{ .result = {} }; } @@ -229,7 +229,7 @@ pub fn fstat(fd: FileDescriptor) Maybe(bun.Stat) { log("uv fstat({}) = {d}", .{ uv_fd, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .fstat, .fd = fd } } + .{ .err = .{ .errno = errno, .syscall = .fstat, .fd = fd, .from_libuv = true } } else .{ .result = req.statbuf }; } @@ -242,7 +242,7 @@ pub fn fdatasync(fd: FileDescriptor) Maybe(void) { log("uv fdatasync({}) = {d}", .{ uv_fd, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .fstat, .fd = fd } } + .{ .err = .{ .errno = errno, .syscall = .fstat, .fd = fd, .from_libuv = true } } else .{ .result = {} }; } @@ -255,7 +255,7 @@ pub fn fsync(fd: FileDescriptor) Maybe(void) { log("uv fsync({d}) = {d}", .{ uv_fd, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .fstat, .fd = fd } } + .{ .err = .{ .errno = errno, .syscall = .fstat, .fd = fd, .from_libuv = true } } else .{ .result = {} }; } @@ -267,7 +267,7 @@ pub fn stat(path: [:0]const u8) Maybe(bun.Stat) { log("uv stat({s}) = {d}", .{ path, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .stat } } + .{ .err = .{ .errno = errno, .syscall = .stat, .from_libuv = true } } else .{ .result = req.statbuf }; } @@ -279,7 +279,7 @@ pub fn lstat(path: [:0]const u8) Maybe(bun.Stat) { log("uv lstat({s}) = {d}", .{ path, rc.value }); return if (rc.errno()) |errno| - .{ .err = .{ .errno = errno, .syscall = .fstat } } + .{ .err = .{ .errno = errno, .syscall = .fstat, .from_libuv = true } } else .{ .result = req.statbuf }; } @@ -320,7 +320,7 @@ pub fn preadv(fd: FileDescriptor, bufs: []const bun.PlatformIOVec, position: i64 } if (rc.errno()) |errno| { - return .{ .err = .{ .errno = errno, .fd = fd, .syscall = .read } }; + return .{ .err = .{ .errno = errno, .fd = fd, .syscall = .read, .from_libuv = true } }; } else { return .{ .result = @as(usize, @intCast(rc.value)) }; } @@ -354,7 +354,7 @@ pub fn pwritev(fd: FileDescriptor, bufs: []const bun.PlatformIOVec, position: i6 } if (rc.errno()) |errno| { - return .{ .err = .{ .errno = errno, .fd = fd, .syscall = .write } }; + return .{ .err = .{ .errno = errno, .fd = fd, .syscall = .write, .from_libuv = true } }; } else { return .{ .result = @as(usize, @intCast(rc.value)) }; } diff --git a/test/cli/install/bun-run.test.ts b/test/cli/install/bun-run.test.ts index 33b4265c32..2fcf3b0f0c 100644 --- a/test/cli/install/bun-run.test.ts +++ b/test/cli/install/bun-run.test.ts @@ -200,9 +200,9 @@ logLevel = "debug" }); console.log(run_dir); if (withLogLevel) { - expect(stderr.toString().trim()).toContain("FileNotFound loading tsconfig.json extends"); + expect(stderr.toString().trim()).toContain("ENOENT loading tsconfig.json extends"); } else { - expect(stderr.toString().trim()).not.toContain("FileNotFound loading tsconfig.json extends"); + expect(stderr.toString().trim()).not.toContain("ENOENT loading tsconfig.json extends"); } expect(stdout.toString()).toBe("hi\n");