diff --git a/misctools/tgz.zig b/misctools/tgz.zig index 780b02fad5..6a3f9b5722 100644 --- a/misctools/tgz.zig +++ b/misctools/tgz.zig @@ -88,8 +88,9 @@ pub fn main() anyerror!void { null, void, void{}, - 1, - false, - false, + .{ + .depth_to_skip = 1, + .close_handles = false, + }, ); } diff --git a/src/bun.zig b/src/bun.zig index 1a3cccc2f7..44486feb35 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -3260,6 +3260,19 @@ noinline fn assertionFailure() noreturn { Output.panic("Internal assertion failure", .{}); } +noinline fn assertionFailureWithLocation(src: std.builtin.SourceLocation) noreturn { + if (@inComptime()) { + @compileError("assertion failure"); + } + + @setCold(true); + Output.panic("Internal assertion failure {s}:{d}:{d}", .{ + src.file, + src.line, + src.column, + }); +} + pub inline fn debugAssert(cheap_value_only_plz: bool) void { if (comptime !Environment.isDebug) { return; @@ -3281,6 +3294,17 @@ pub fn assert(value: bool) callconv(callconv_inline) void { } } +pub fn assertWithLocation(value: bool, src: std.builtin.SourceLocation) callconv(callconv_inline) void { + if (comptime !Environment.allow_assert) { + return; + } + + if (!value) { + if (comptime Environment.isDebug) unreachable; + assertionFailureWithLocation(src); + } +} + /// This has no effect on the real code but capturing 'a' and 'b' into parameters makes assertion failures much easier inspect in a debugger. pub inline fn assert_eql(a: anytype, b: anytype) void { return assert(a == b); diff --git a/src/cli/create_command.zig b/src/cli/create_command.zig index e3b7312ebf..2cf3b9a0ea 100644 --- a/src/cli/create_command.zig +++ b/src/cli/create_command.zig @@ -433,9 +433,9 @@ pub const CreateCommand = struct { &archive_context, void, {}, - 1, - false, - false, + .{ + .depth_to_skip = 1, + }, ); if (!create_options.skip_package_json) { diff --git a/src/compile_target.zig b/src/compile_target.zig index 2f5d32c160..1ba17bb304 100644 --- a/src/compile_target.zig +++ b/src/compile_target.zig @@ -269,10 +269,10 @@ pub fn downloadToPath(this: *const CompileTarget, env: *bun.DotEnv.Loader, alloc null, void, {}, - // "package/bin" - 2, - true, - false, + .{ + // "package/bin" + .depth_to_skip = 2, + }, ) catch |err| { node.end(); Output.err(err, diff --git a/src/install/extract_tarball.zig b/src/install/extract_tarball.zig index 2635fe9e8d..ba6ad50858 100644 --- a/src/install/extract_tarball.zig +++ b/src/install/extract_tarball.zig @@ -240,10 +240,11 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD null, *DirnameReader, &dirname_reader, - // for GitHub tarballs, the root dir is always -- - 1, - true, - log, + .{ + // for GitHub tarballs, the root dir is always -- + .depth_to_skip = 1, + .log = log, + }, ), } @@ -265,10 +266,13 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD null, void, {}, - // for npm packages, the root dir is always "package" - 1, - true, - log, + .{ + .log = log, + // packages usually have root directory `package/`, and scoped packages usually have root `/` + // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/pacote/lib/fetcher.js#L442 + .depth_to_skip = 1, + .npm = true, + }, ), }, } diff --git a/src/install/install.zig b/src/install/install.zig index f81df0c815..0e8d572782 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -289,47 +289,6 @@ const NetworkTask = struct { header_builder.count("npm-auth-type", "legacy"); } - // The first time this happened! - // - // "peerDependencies": { - // "@ianvs/prettier-plugin-sort-imports": "*", - // "prettier-plugin-twig-melody": "*" - // }, - // "peerDependenciesMeta": { - // "@ianvs/prettier-plugin-sort-imports": { - // "optional": true - // }, - // Example case ^ - // `@ianvs/prettier-plugin-sort-imports` is peer and also optional but was not marked optional because - // the offset would be 0 and the current loop index is also 0. - // const invalidate_manifest_cache_because_optional_peer_dependencies_were_not_marked_as_optional_if_the_optional_peer_dependency_offset_was_equal_to_the_current_index = 1697871350; - // ---- - // The second time this happened! - // - // pre-release sorting when the number of segments between dots were different, was sorted incorrectly - // so we must invalidate the manifest cache once again. - // - // example: - // - // 1.0.0-pre.a.b > 1.0.0-pre.a - // before ordering said the left was smaller than the right - // - // const invalidate_manifest_cache_because_prerelease_segments_were_sorted_incorrectly_sometimes = 1697871350; - // - // ---- - // The third time this happened! - // - // pre-release sorting bug again! If part of the pre-release segment is a number, and the other pre-release part is a string, - // it would order them incorrectly by comparing them as strings. - // - // example: - // - // 1.0.0-alpha.22 < 1.0.0-alpha.1beta - // before: false - // after: true - // - const invalidate_manifest_cache_because_prerelease_segments_were_sorted_incorrectly_sometimes = 1702425477; - pub fn forManifest( this: *NetworkTask, name: string, @@ -407,10 +366,8 @@ const NetworkTask = struct { var last_modified: string = ""; var etag: string = ""; if (loaded_manifest) |manifest| { - if (manifest.pkg.public_max_age > invalidate_manifest_cache_because_prerelease_segments_were_sorted_incorrectly_sometimes) { - last_modified = manifest.pkg.last_modified.slice(manifest.string_buf); - etag = manifest.pkg.etag.slice(manifest.string_buf); - } + last_modified = manifest.pkg.last_modified.slice(manifest.string_buf); + etag = manifest.pkg.etag.slice(manifest.string_buf); } var header_builder = HeaderBuilder{}; @@ -3626,6 +3583,19 @@ pub const PackageManager = struct { } }; + pub const CacheVersion = struct { + pub const current = 1; + pub const Formatter = struct { + version_number: ?usize = null, + + pub fn format(this: *const @This(), comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { + if (this.version_number) |version| { + try writer.print("@@@{d}", .{version}); + } + } + }; + }; + pub fn cachedGitFolderNamePrintAuto(this: *const PackageManager, repository: *const Repository, patch_hash: ?u64) stringZ { if (!repository.resolved.isEmpty()) { return this.cachedGitFolderName(repository, patch_hash); @@ -3635,9 +3605,10 @@ pub const PackageManager = struct { const string_buf = this.lockfile.buffers.string_bytes.items; return std.fmt.bufPrintZ( &cached_package_folder_name_buf, - "@G@{any}{}", + "@G@{any}{}{}", .{ repository.committish.fmt(string_buf), + CacheVersion.Formatter{ .version_number = CacheVersion.current }, PatchHashFmt{ .hash = patch_hash }, }, ) catch unreachable; @@ -3647,7 +3618,11 @@ pub const PackageManager = struct { } pub fn cachedGitHubFolderNamePrint(buf: []u8, resolved: string, patch_hash: ?u64) stringZ { - return std.fmt.bufPrintZ(buf, "@GH@{s}{}", .{ resolved, PatchHashFmt{ .hash = patch_hash } }) catch unreachable; + return std.fmt.bufPrintZ(buf, "@GH@{s}{}{}", .{ + resolved, + CacheVersion.Formatter{ .version_number = CacheVersion.current }, + PatchHashFmt{ .hash = patch_hash }, + }) catch unreachable; } pub fn cachedGitHubFolderName(this: *const PackageManager, repository: *const Repository, patch_hash: ?u64) stringZ { @@ -3657,8 +3632,14 @@ pub const PackageManager = struct { fn cachedGitHubFolderNamePrintGuess(buf: []u8, string_buf: []const u8, repository: *const Repository, patch_hash: ?u64) stringZ { return std.fmt.bufPrintZ( buf, - "@GH@{any}-{any}-{any}{}", - .{ repository.owner.fmt(string_buf), repository.repo.fmt(string_buf), repository.committish.fmt(string_buf), PatchHashFmt{ .hash = patch_hash } }, + "@GH@{any}-{any}-{any}{}{}", + .{ + repository.owner.fmt(string_buf), + repository.repo.fmt(string_buf), + repository.committish.fmt(string_buf), + CacheVersion.Formatter{ .version_number = CacheVersion.current }, + PatchHashFmt{ .hash = patch_hash }, + }, ) catch unreachable; } @@ -3678,21 +3659,31 @@ pub const PackageManager = struct { pub fn cachedNPMPackageFolderNamePrint(this: *const PackageManager, buf: []u8, name: string, version: Semver.Version, patch_hash: ?u64) stringZ { const scope = this.scopeForPackageName(name); - const basename = cachedNPMPackageFolderPrintBasename(buf, name, version, null); - if (scope.name.len == 0 and !this.options.did_override_default_scope) { - if (patch_hash != null) return cachedNPMPackageFolderPrintBasename(buf, name, version, patch_hash); - return basename; + const include_version_number = true; + return cachedNPMPackageFolderPrintBasename(buf, name, version, patch_hash, include_version_number); } + const include_version_number = false; + const basename = cachedNPMPackageFolderPrintBasename(buf, name, version, null, include_version_number); + const spanned = bun.span(basename); const available = buf[spanned.len..]; var end: []u8 = undefined; if (scope.url.hostname.len > 32 or available.len < 64) { const visible_hostname = scope.url.hostname[0..@min(scope.url.hostname.len, 12)]; - end = std.fmt.bufPrint(available, "@@{s}__{any}{}", .{ visible_hostname, bun.fmt.hexIntLower(String.Builder.stringHash(scope.url.href)), PatchHashFmt{ .hash = patch_hash } }) catch unreachable; + end = std.fmt.bufPrint(available, "@@{s}__{any}{}{}", .{ + visible_hostname, + bun.fmt.hexIntLower(String.Builder.stringHash(scope.url.href)), + CacheVersion.Formatter{ .version_number = CacheVersion.current }, + PatchHashFmt{ .hash = patch_hash }, + }) catch unreachable; } else { - end = std.fmt.bufPrint(available, "@@{s}{}", .{ scope.url.hostname, PatchHashFmt{ .hash = patch_hash } }) catch unreachable; + end = std.fmt.bufPrint(available, "@@{s}{}{}", .{ + scope.url.hostname, + CacheVersion.Formatter{ .version_number = CacheVersion.current }, + PatchHashFmt{ .hash = patch_hash }, + }) catch unreachable; } buf[spanned.len + end.len] = 0; @@ -3700,21 +3691,23 @@ pub const PackageManager = struct { return result; } - pub fn cachedNPMPackageFolderBasename(name: string, version: Semver.Version) stringZ { - return cachedNPMPackageFolderPrintBasename(&cached_package_folder_name_buf, name, version); - } - pub fn cachedNPMPackageFolderName(this: *const PackageManager, name: string, version: Semver.Version, patch_hash: ?u64) stringZ { return this.cachedNPMPackageFolderNamePrint(&cached_package_folder_name_buf, name, version, patch_hash); } // TODO: normalize to alphanumeric - pub fn cachedNPMPackageFolderPrintBasename(buf: []u8, name: string, version: Semver.Version, patch_hash: ?u64) stringZ { + pub fn cachedNPMPackageFolderPrintBasename( + buf: []u8, + name: string, + version: Semver.Version, + patch_hash: ?u64, + include_cache_version: bool, + ) stringZ { if (version.tag.hasPre()) { if (version.tag.hasBuild()) { return std.fmt.bufPrintZ( buf, - "{s}@{d}.{d}.{d}-{any}+{any}{}", + "{s}@{d}.{d}.{d}-{any}+{any}{}{}", .{ name, version.major, @@ -3722,19 +3715,21 @@ pub const PackageManager = struct { version.patch, bun.fmt.hexIntLower(version.tag.pre.hash), bun.fmt.hexIntUpper(version.tag.build.hash), + CacheVersion.Formatter{ .version_number = if (include_cache_version) CacheVersion.current else null }, PatchHashFmt{ .hash = patch_hash }, }, ) catch unreachable; } return std.fmt.bufPrintZ( buf, - "{s}@{d}.{d}.{d}-{any}{}", + "{s}@{d}.{d}.{d}-{any}{}{}", .{ name, version.major, version.minor, version.patch, bun.fmt.hexIntLower(version.tag.pre.hash), + CacheVersion.Formatter{ .version_number = if (include_cache_version) CacheVersion.current else null }, PatchHashFmt{ .hash = patch_hash }, }, ) catch unreachable; @@ -3742,28 +3737,34 @@ pub const PackageManager = struct { if (version.tag.hasBuild()) { return std.fmt.bufPrintZ( buf, - "{s}@{d}.{d}.{d}+{any}{}", + "{s}@{d}.{d}.{d}+{any}{}{}", .{ name, version.major, version.minor, version.patch, bun.fmt.hexIntUpper(version.tag.build.hash), + CacheVersion.Formatter{ .version_number = if (include_cache_version) CacheVersion.current else null }, PatchHashFmt{ .hash = patch_hash }, }, ) catch unreachable; } - return std.fmt.bufPrintZ(buf, "{s}@{d}.{d}.{d}{}", .{ + return std.fmt.bufPrintZ(buf, "{s}@{d}.{d}.{d}{}{}", .{ name, version.major, version.minor, version.patch, + CacheVersion.Formatter{ .version_number = if (include_cache_version) CacheVersion.current else null }, PatchHashFmt{ .hash = patch_hash }, }) catch unreachable; } pub fn cachedTarballFolderNamePrint(buf: []u8, url: string, patch_hash: ?u64) stringZ { - return std.fmt.bufPrintZ(buf, "@T@{any}{}", .{ bun.fmt.hexIntLower(String.Builder.stringHash(url)), PatchHashFmt{ .hash = patch_hash } }) catch unreachable; + return std.fmt.bufPrintZ(buf, "@T@{any}{}{}", .{ + bun.fmt.hexIntLower(String.Builder.stringHash(url)), + CacheVersion.Formatter{ .version_number = CacheVersion.current }, + PatchHashFmt{ .hash = patch_hash }, + }) catch unreachable; } pub fn cachedTarballFolderName(this: *const PackageManager, url: String, patch_hash: ?u64) stringZ { @@ -3778,25 +3779,25 @@ pub const PackageManager = struct { this: *PackageManager, buf: *bun.PathBuffer, package_name: []const u8, - npm: Semver.Version, + version: Semver.Version, ) ![]u8 { - var package_name_version_buf: bun.PathBuffer = undefined; + var cache_path_buf: bun.PathBuffer = undefined; + + const cache_path = this.cachedNPMPackageFolderNamePrint(&cache_path_buf, package_name, version, null); + + if (comptime Environment.allow_assert) { + bun.assertWithLocation(cache_path[package_name.len] == '@', @src()); + } + + cache_path_buf[package_name.len] = std.fs.path.sep; - const subpath = std.fmt.bufPrintZ( - &package_name_version_buf, - "{s}" ++ std.fs.path.sep_str ++ "{any}", - .{ - package_name, - npm.fmt(this.lockfile.buffers.string_bytes.items), - }, - ) catch unreachable; return this.getCacheDirectory().readLink( - subpath, + cache_path, buf, ) catch |err| { // if we run into an error, delete the symlink // so that we don't repeatedly try to read it - std.os.unlinkat(this.getCacheDirectory().fd, subpath, 0) catch {}; + std.os.unlinkat(this.getCacheDirectory().fd, cache_path, 0) catch {}; return err; }; } diff --git a/src/libarchive/libarchive.zig b/src/libarchive/libarchive.zig index a1c565c44c..43cbe5354a 100644 --- a/src/libarchive/libarchive.zig +++ b/src/libarchive/libarchive.zig @@ -469,15 +469,20 @@ pub const Archive = struct { } } + pub const ExtractOptions = struct { + depth_to_skip: usize, + close_handles: bool = true, + log: bool = false, + npm: bool = false, + }; + pub fn extractToDir( file_buffer: []const u8, dir: std.fs.Dir, ctx: ?*Archive.Context, comptime ContextType: type, appender: ContextType, - comptime depth_to_skip: usize, - comptime close_handles: bool, - comptime log: bool, + options: ExtractOptions, ) !u32 { var entry: *lib.archive_entry = undefined; @@ -489,10 +494,10 @@ pub const Archive = struct { var count: u32 = 0; const dir_fd = dir.fd; - var w_path_buf: if (Environment.isWindows) bun.WPathBuffer else void = undefined; + var normalized_buf: bun.OSPathBuffer = undefined; loop: while (true) { - const r = @as(Status, @enumFromInt(lib.archive_read_next_header(archive, &entry))); + const r: Status = @enumFromInt(lib.archive_read_next_header(archive, &entry)); switch (r) { Status.eof => break :loop, @@ -507,27 +512,10 @@ pub const Archive = struct { // // Ideally, we find a way to tell libarchive to not convert the strings to wide characters and also to not // replace path separators. We can do both of these with our own normalization and utf8/utf16 string conversion code. - var pathname: bun.OSPathSliceZ = if (comptime Environment.isWindows) brk: { - const normalized = bun.path.normalizeBufT( - u16, - std.mem.span(lib.archive_entry_pathname_w(entry)), - &w_path_buf, - .windows, - ); - - // When writing files on Windows, translate the characters to their - // 0xf000 higher-encoded versions. - // https://github.com/isaacs/node-tar/blob/0510c9ea6d000c40446d56674a7efeec8e72f052/lib/winchars.js - for (normalized) |*c| { - switch (c.*) { - '|', '<', '>', '?', ':' => c.* += 0xf000, - else => {}, - } - } - - w_path_buf[normalized.len] = 0; - break :brk w_path_buf[0..normalized.len :0]; - } else std.mem.sliceTo(lib.archive_entry_pathname(entry), 0); + var pathname: bun.OSPathSliceZ = if (comptime Environment.isWindows) + std.mem.sliceTo(lib.archive_entry_pathname_w(entry), 0) + else + std.mem.sliceTo(lib.archive_entry_pathname(entry), 0); if (comptime ContextType != void and @hasDecl(std.meta.Child(ContextType), "onFirstDirectoryName")) { if (appender.needs_first_dirname) { @@ -543,22 +531,53 @@ pub const Archive = struct { } } - var tokenizer = std.mem.tokenizeScalar(bun.OSPathChar, pathname, std.fs.path.sep); - comptime var depth_i: usize = 0; + const kind = C.kindFromMode(lib.archive_entry_filetype(entry)); - inline while (depth_i < depth_to_skip) : (depth_i += 1) { + if (options.npm) { + // - ignore entries other than files (`true` can only be returned if type is file) + // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/pacote/lib/fetcher.js#L419-L441 + if (kind != .file) continue; + + // TODO: .npmignore, or .gitignore if it doesn't exist + // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/pacote/lib/fetcher.js#L434 + } + + // strip and normalize the path + var tokenizer = std.mem.tokenizeScalar(bun.OSPathChar, pathname, '/'); + for (0..options.depth_to_skip) |_| { if (tokenizer.next() == null) continue :loop; } - const pathname_ = tokenizer.rest(); - pathname = @as([*]const bun.OSPathChar, @ptrFromInt(@intFromPtr(pathname_.ptr)))[0..pathname_.len :0]; - if (pathname.len == 0) continue; + const rest = tokenizer.rest(); + pathname = rest.ptr[0..rest.len :0]; - const kind = C.kindFromMode(lib.archive_entry_filetype(entry)); + const normalized = bun.path.normalizeBufT(bun.OSPathChar, pathname, &normalized_buf, .auto); + normalized_buf[normalized.len] = 0; + const path: [:0]bun.OSPathChar = normalized_buf[0..normalized.len :0]; + if (path.len == 0 or path.len == 1 and path[0] == '.') continue; - const path_slice: bun.OSPathSlice = pathname.ptr[0..pathname.len]; + if (options.npm and Environment.isWindows) { + // When writing files on Windows, translate the characters to their + // 0xf000 higher-encoded versions. + // https://github.com/isaacs/node-tar/blob/0510c9ea6d000c40446d56674a7efeec8e72f052/lib/winchars.js + var remain = path; + if (strings.startsWithWindowsDriveLetterT(bun.OSPathChar, remain)) { + // don't encode `:` from the drive letter + // https://github.com/npm/cli/blob/93883bb6459208a916584cad8c6c72a315cf32af/node_modules/tar/lib/unpack.js#L327 + remain = remain[2..]; + } - if (comptime log) { + for (remain) |*c| { + switch (c.*) { + '|', '<', '>', '?', ':' => c.* += 0xf000, + else => {}, + } + } + } + + const path_slice: bun.OSPathSlice = path.ptr[0..path.len]; + + if (options.log) { Output.prettyln(" {}", .{bun.fmt.fmtOSPath(path_slice, .{})}); } @@ -578,23 +597,26 @@ pub const Archive = struct { mode |= 0o1; if (comptime Environment.isWindows) { - try bun.MakePath.makePath(u16, dir, pathname); + try bun.MakePath.makePath(u16, dir, path); } else { - std.os.mkdiratZ(dir_fd, pathname, @as(u32, @intCast(mode))) catch |err| { - if (err == error.PathAlreadyExists or err == error.NotDir) break; - try bun.makePath(dir, std.fs.path.dirname(path_slice) orelse return err); - try std.os.mkdiratZ(dir_fd, pathname, 0o777); + std.os.mkdiratZ(dir_fd, path, @as(u32, @intCast(mode))) catch |err| { + // It's possible for some tarballs to return a directory twice, with and + // without `./` in the beginning. So if it already exists, continue to the + // next entry. + if (err == error.PathAlreadyExists or err == error.NotDir) continue; + bun.makePath(dir, std.fs.path.dirname(path_slice) orelse return err) catch {}; + std.os.mkdiratZ(dir_fd, path, 0o777) catch {}; }; } }, Kind.sym_link => { const link_target = lib.archive_entry_symlink(entry).?; if (Environment.isPosix) { - std.os.symlinkatZ(link_target, dir_fd, pathname) catch |err| brk: { + std.os.symlinkatZ(link_target, dir_fd, path) catch |err| brk: { switch (err) { error.AccessDenied, error.FileNotFound => { dir.makePath(std.fs.path.dirname(path_slice) orelse return err) catch {}; - break :brk try std.os.symlinkatZ(link_target, dir_fd, pathname); + break :brk try std.os.symlinkatZ(link_target, dir_fd, path); }, else => { return err; @@ -609,12 +631,12 @@ pub const Archive = struct { const file_handle_native = brk: { if (Environment.isWindows) { const flags = std.os.O.WRONLY | std.os.O.CREAT | std.os.O.TRUNC; - switch (bun.sys.openatWindows(bun.toFD(dir_fd), pathname, flags)) { + switch (bun.sys.openatWindows(bun.toFD(dir_fd), path, flags)) { .result => |fd| break :brk fd, .err => |e| switch (e.errno) { @intFromEnum(bun.C.E.PERM), @intFromEnum(bun.C.E.NOENT) => { bun.MakePath.makePath(u16, dir, bun.Dirname.dirname(u16, path_slice) orelse return bun.errnoToZigErr(e.errno)) catch {}; - break :brk try bun.sys.openatWindows(bun.toFD(dir_fd), pathname, flags).unwrap(); + break :brk try bun.sys.openatWindows(bun.toFD(dir_fd), path, flags).unwrap(); }, else => { return bun.errnoToZigErr(e.errno); @@ -622,11 +644,11 @@ pub const Archive = struct { }, } } else { - break :brk (dir.createFileZ(pathname, .{ .truncate = true, .mode = mode }) catch |err| { + break :brk (dir.createFileZ(path, .{ .truncate = true, .mode = mode }) catch |err| { switch (err) { error.AccessDenied, error.FileNotFound => { dir.makePath(std.fs.path.dirname(path_slice) orelse return err) catch {}; - break :brk (try dir.createFileZ(pathname, .{ + break :brk (try dir.createFileZ(path, .{ .truncate = true, .mode = mode, })).handle; @@ -643,7 +665,8 @@ pub const Archive = struct { break :brk try bun.toLibUVOwnedFD(file_handle_native); }; - defer if (comptime close_handles) { + var plucked_file = false; + defer if (options.close_handles and !plucked_file) { // On windows, AV hangs these closes really badly. // 'bun i @mui/icons-material' takes like 20 seconds to extract // mostly spend on waiting for things to close closing @@ -684,6 +707,7 @@ pub const Archive = struct { try plucker_.contents.inflate(@as(usize, @intCast(read))); plucker_.found = read > 0; plucker_.fd = file_handle; + plucked_file = true; continue :loop; } } @@ -706,7 +730,7 @@ pub const Archive = struct { lib.ARCHIVE_EOF => break :loop, lib.ARCHIVE_OK => break :possibly_retry, lib.ARCHIVE_RETRY => { - if (comptime log) { + if (options.log) { Output.err("libarchive error", "extracting {}, retry {d} / {d}", .{ bun.fmt.fmtOSPath(path_slice, .{}), retries_remaining, @@ -715,7 +739,7 @@ pub const Archive = struct { } }, else => { - if (comptime log) { + if (options.log) { const archive_error = std.mem.span(lib.archive_error_string(archive)); Output.err("libarchive error", "extracting {}: {s}", .{ bun.fmt.fmtOSPath(path_slice, .{}), @@ -743,9 +767,7 @@ pub const Archive = struct { ctx: ?*Archive.Context, comptime FilePathAppender: type, appender: FilePathAppender, - comptime depth_to_skip: usize, - comptime close_handles: bool, - comptime log: bool, + comptime options: ExtractOptions, ) !u32 { var dir: std.fs.Dir = brk: { const cwd = std.fs.cwd(); @@ -760,7 +782,7 @@ pub const Archive = struct { } }; - defer if (comptime close_handles) dir.close(); - return try extractToDir(file_buffer, dir, ctx, FilePathAppender, appender, depth_to_skip, close_handles, log); + defer if (comptime options.close_handles) dir.close(); + return try extractToDir(file_buffer, dir, ctx, FilePathAppender, appender, options); } }; diff --git a/src/string_immutable.zig b/src/string_immutable.zig index 3cb523a902..1bb8f90640 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -28,7 +28,11 @@ pub inline fn containsChar(self: string, char: u8) bool { } pub inline fn contains(self: string, str: string) bool { - return indexOf(self, str) != null; + return containsT(u8, self, str); +} + +pub inline fn containsT(comptime T: type, self: []const T, str: []const T) bool { + return indexOfT(T, self, str) != null; } pub inline fn removeLeadingDotSlash(slice: []const u8) []const u8 { @@ -5131,6 +5135,10 @@ pub inline fn charIsAnySlash(char: u8) bool { } pub inline fn startsWithWindowsDriveLetter(s: []const u8) bool { + return startsWithWindowsDriveLetterT(u8, s); +} + +pub inline fn startsWithWindowsDriveLetterT(comptime T: type, s: []const T) bool { return s.len > 2 and s[1] == ':' and switch (s[0]) { 'a'...'z', 'A'...'Z' => true, else => false, diff --git a/test/cli/install/bun-add.test.ts b/test/cli/install/bun-add.test.ts index 2186842669..7c34d12d28 100644 --- a/test/cli/install/bun-add.test.ts +++ b/test/cli/install/bun-add.test.ts @@ -614,7 +614,7 @@ it("should add dependency (GitHub)", async () => { expect(requested).toBe(0); expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "uglify-js"]); expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toHaveBins(["uglifyjs"]); - expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual(["@GH@mishoo-UglifyJS-e219a9a"]); + expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual(["@GH@mishoo-UglifyJS-e219a9a@@@1"]); expect(await readdirSorted(join(package_dir, "node_modules", "uglify-js"))).toEqual([ ".bun-tag", ".gitattributes", @@ -829,14 +829,14 @@ it("should add aliased dependency (GitHub)", async () => { expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "uglify"]); expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toHaveBins(["uglifyjs"]); expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual([ - "@GH@mishoo-UglifyJS-e219a9a", + "@GH@mishoo-UglifyJS-e219a9a@@@1", "uglify", ]); expect(await readdirSorted(join(package_dir, "node_modules", ".cache", "uglify"))).toEqual([ - "mishoo-UglifyJS-e219a9a", + "mishoo-UglifyJS-e219a9a@@@1", ]); - expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a"))).toBe( - join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a"), + expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a@@@1"))).toBe( + join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a@@@1"), ); expect(await readdirSorted(join(package_dir, "node_modules", "uglify"))).toEqual([ ".bun-tag", @@ -1527,7 +1527,7 @@ it("should add dependency without duplication (GitHub)", async () => { expect(await exited1).toBe(0); expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "uglify-js"]); expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toHaveBins(["uglifyjs"]); - expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual(["@GH@mishoo-UglifyJS-e219a9a"]); + expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual(["@GH@mishoo-UglifyJS-e219a9a@@@1"]); expect(await readdirSorted(join(package_dir, "node_modules", "uglify-js"))).toEqual([ ".bun-tag", ".gitattributes", @@ -1587,7 +1587,7 @@ it("should add dependency without duplication (GitHub)", async () => { expect(await exited2).toBe(0); expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "uglify-js"]); expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toHaveBins(["uglifyjs"]); - expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual(["@GH@mishoo-UglifyJS-e219a9a"]); + expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual(["@GH@mishoo-UglifyJS-e219a9a@@@1"]); expect(await readdirSorted(join(package_dir, "node_modules", "uglify-js"))).toEqual([ ".bun-tag", ".gitattributes", diff --git a/test/cli/install/bun-install.test.ts b/test/cli/install/bun-install.test.ts index 6d30eaf6ef..5cfe56af15 100644 --- a/test/cli/install/bun-install.test.ts +++ b/test/cli/install/bun-install.test.ts @@ -3134,14 +3134,14 @@ it("should handle GitHub URL in dependencies (user/repo#commit-id)", async () => expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "uglify"]); expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toHaveBins(["uglifyjs"]); expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual([ - "@GH@mishoo-UglifyJS-e219a9a", + "@GH@mishoo-UglifyJS-e219a9a@@@1", "uglify", ]); expect(await readdirSorted(join(package_dir, "node_modules", ".cache", "uglify"))).toEqual([ - "mishoo-UglifyJS-e219a9a", + "mishoo-UglifyJS-e219a9a@@@1", ]); - expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a"))).toBe( - join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a"), + expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a@@@1"))).toBe( + join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a@@@1"), ); expect(await readdirSorted(join(package_dir, "node_modules", "uglify"))).toEqual([ ".bun-tag", @@ -3199,14 +3199,14 @@ it("should handle GitHub URL in dependencies (user/repo#tag)", async () => { expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "uglify"]); expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toHaveBins(["uglifyjs"]); expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual([ - "@GH@mishoo-UglifyJS-e219a9a", + "@GH@mishoo-UglifyJS-e219a9a@@@1", "uglify", ]); expect(await readdirSorted(join(package_dir, "node_modules", ".cache", "uglify"))).toEqual([ - "mishoo-UglifyJS-e219a9a", + "mishoo-UglifyJS-e219a9a@@@1", ]); - expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a"))).toBe( - join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a"), + expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a@@@1"))).toBe( + join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a@@@1"), ); expect(await readdirSorted(join(package_dir, "node_modules", "uglify"))).toEqual([ ".bun-tag", @@ -3420,14 +3420,14 @@ it("should handle GitHub URL in dependencies (github:user/repo#tag)", async () = expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toHaveBins(["uglifyjs"]); expect(join(package_dir, "node_modules", ".bin", "uglifyjs")).toBeValidBin(join("..", "uglify", "bin", "uglifyjs")); expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual([ - "@GH@mishoo-UglifyJS-e219a9a", + "@GH@mishoo-UglifyJS-e219a9a@@@1", "uglify", ]); expect(await readdirSorted(join(package_dir, "node_modules", ".cache", "uglify"))).toEqual([ - "mishoo-UglifyJS-e219a9a", + "mishoo-UglifyJS-e219a9a@@@1", ]); - expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a"))).toBe( - join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a"), + expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a@@@1"))).toBe( + join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a@@@1"), ); expect(await readdirSorted(join(package_dir, "node_modules", "uglify"))).toEqual([ ".bun-tag", @@ -3537,14 +3537,14 @@ it("should handle GitHub URL in dependencies (git://github.com/user/repo.git#com expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toHaveBins(["uglifyjs"]); expect(join(package_dir, "node_modules", ".bin", "uglifyjs")).toBeValidBin(join("..", "uglify", "bin", "uglifyjs")); expect(await readdirSorted(join(package_dir, "node_modules", ".cache"))).toEqual([ - "@GH@mishoo-UglifyJS-e219a9a", + "@GH@mishoo-UglifyJS-e219a9a@@@1", "uglify", ]); expect(await readdirSorted(join(package_dir, "node_modules", ".cache", "uglify"))).toEqual([ - "mishoo-UglifyJS-e219a9a", + "mishoo-UglifyJS-e219a9a@@@1", ]); - expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a"))).toBe( - join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a"), + expect(await readlink(join(package_dir, "node_modules", ".cache", "uglify", "mishoo-UglifyJS-e219a9a@@@1"))).toBe( + join(package_dir, "node_modules", ".cache", "@GH@mishoo-UglifyJS-e219a9a@@@1"), ); expect(await readdirSorted(join(package_dir, "node_modules", "uglify"))).toEqual([ ".bun-tag", diff --git a/test/cli/install/bun-run.test.ts b/test/cli/install/bun-run.test.ts index 89dc568c6f..203f8f9676 100644 --- a/test/cli/install/bun-run.test.ts +++ b/test/cli/install/bun-run.test.ts @@ -279,8 +279,8 @@ console.log(minify("print(6 * 7)").code); expect(err1).toBe(""); expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); expect(await readdirSorted(join(run_dir, ".cache"))).toContain("uglify-js"); - expect(await readdirSorted(join(run_dir, ".cache", "uglify-js"))).toEqual(["3.17.4"]); - expect(await exists(join(run_dir, ".cache", "uglify-js", "3.17.4", "package.json"))).toBeTrue(); + expect(await readdirSorted(join(run_dir, ".cache", "uglify-js"))).toEqual(["3.17.4@@@1"]); + expect(await exists(join(run_dir, ".cache", "uglify-js", "3.17.4@@@1", "package.json"))).toBeTrue(); const out1 = await new Response(stdout1).text(); expect(out1.split(/\r?\n/)).toEqual(["print(42);", ""]); expect(await exited1).toBe(0); @@ -304,7 +304,7 @@ console.log(minify("print(6 * 7)").code); expect(err2).toBe(""); expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); expect(await readdirSorted(join(run_dir, ".cache"))).toContain("uglify-js"); - expect(await readdirSorted(join(run_dir, ".cache", "uglify-js"))).toEqual(["3.17.4"]); + expect(await readdirSorted(join(run_dir, ".cache", "uglify-js"))).toEqual(["3.17.4@@@1"]); const out2 = await new Response(stdout2).text(); expect(out2.split(/\r?\n/)).toEqual(["print(42);", ""]); expect(await exited2).toBe(0); @@ -343,9 +343,9 @@ for (const entry of await decompress(Buffer.from(buffer))) { expect(err1).toBe(""); expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); expect(await readdirSorted(join(run_dir, ".cache"))).toContain("decompress"); - expect(await readdirSorted(join(run_dir, ".cache", "decompress"))).toEqual(["4.2.1"]); - expect(await exists(join(run_dir, ".cache", "decompress", "4.2.1", "package.json"))).toBeTrue(); - expect(await file(join(run_dir, ".cache", "decompress", "4.2.1", "index.js")).text()).toContain( + expect(await readdirSorted(join(run_dir, ".cache", "decompress"))).toEqual(["4.2.1@@@1"]); + expect(await exists(join(run_dir, ".cache", "decompress", "4.2.1@@@1", "package.json"))).toBeTrue(); + expect(await file(join(run_dir, ".cache", "decompress", "4.2.1@@@1", "index.js")).text()).toContain( "\nmodule.exports = ", ); const out1 = await new Response(stdout1).text(); @@ -376,9 +376,9 @@ for (const entry of await decompress(Buffer.from(buffer))) { if (err2) throw new Error(err2); expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); expect(await readdirSorted(join(run_dir, ".cache"))).toContain("decompress"); - expect(await readdirSorted(join(run_dir, ".cache", "decompress"))).toEqual(["4.2.1"]); - expect(await exists(join(run_dir, ".cache", "decompress", "4.2.1", "package.json"))).toBeTrue(); - expect(await file(join(run_dir, ".cache", "decompress", "4.2.1", "index.js")).text()).toContain( + expect(await readdirSorted(join(run_dir, ".cache", "decompress"))).toEqual(["4.2.1@@@1"]); + expect(await exists(join(run_dir, ".cache", "decompress", "4.2.1@@@1", "package.json"))).toBeTrue(); + expect(await file(join(run_dir, ".cache", "decompress", "4.2.1@@@1", "index.js")).text()).toContain( "\nmodule.exports = ", ); const out2 = await new Response(stdout2).text(); diff --git a/test/cli/install/registry/bun-install-registry.test.ts b/test/cli/install/registry/bun-install-registry.test.ts index ff984e1d90..792b681f9a 100644 --- a/test/cli/install/registry/bun-install-registry.test.ts +++ b/test/cli/install/registry/bun-install-registry.test.ts @@ -8603,3 +8603,77 @@ describe("yarn tests", () => { expect(await exited).toBe(0); }); }); + +test("tarball `./` prefix, duplicate directory with file, and empty directory", async () => { + await write( + join(packageDir, "package.json"), + JSON.stringify({ + name: "foo", + dependencies: { + "tarball-without-package-prefix": "1.0.0", + }, + }), + ); + + // Entries in this tarball: + // + // ./ + // ./package1000.js + // ./package2/ + // ./package3/ + // ./package4/ + // ./package.json + // ./package/ + // ./package1000/ + // ./package/index.js + // ./package4/package5/ + // ./package4/package.json + // ./package3/package6/ + // ./package3/package6/index.js + // ./package2/index.js + // package3/ + // package3/package6/ + // package3/package6/index.js + // + // The directory `package3` is added twice, but because one doesn't start + // with `./`, it is stripped from the path and a copy of `package6` is placed + // at the root of the output directory. Also `package1000` is not included in + // the output because it is an empty directory. + + await runBunInstall(env, packageDir); + const prefix = join(packageDir, "node_modules", "tarball-without-package-prefix"); + const results = await Promise.all([ + file(join(prefix, "package.json")).json(), + file(join(prefix, "package1000.js")).text(), + file(join(prefix, "package", "index.js")).text(), + file(join(prefix, "package2", "index.js")).text(), + file(join(prefix, "package3", "package6", "index.js")).text(), + file(join(prefix, "package4", "package.json")).json(), + exists(join(prefix, "package4", "package5")), + exists(join(prefix, "package1000")), + file(join(prefix, "package6", "index.js")).text(), + ]); + expect(results).toEqual([ + { + name: "tarball-without-package-prefix", + version: "1.0.0", + }, + "hi", + "ooops", + "ooooops", + "oooooops", + { + "name": "tarball-without-package-prefix", + "version": "2.0.0", + }, + false, + false, + "oooooops", + ]); + expect(await file(join(packageDir, "node_modules", "tarball-without-package-prefix", "package.json")).json()).toEqual( + { + name: "tarball-without-package-prefix", + version: "1.0.0", + }, + ); +}); diff --git a/test/cli/install/registry/packages/tarball-without-package-prefix/package.json b/test/cli/install/registry/packages/tarball-without-package-prefix/package.json new file mode 100644 index 0000000000..a86a8910e8 --- /dev/null +++ b/test/cli/install/registry/packages/tarball-without-package-prefix/package.json @@ -0,0 +1,41 @@ +{ + "name": "tarball-without-package-prefix", + "versions": { + "1.0.0": { + "name": "tarball-without-package-prefix", + "version": "1.0.0", + "_id": "tarball-without-package-prefix@1.0.0", + "_integrity": "sha512-y0COXVWAOSOmlEUSs3lO2FrF6ezZYwDukqADrMMaJap5/wO6gvMQ8XI/GoT0ht187owxlrvUjQEwdfTpSqORRg==", + "_resolved": "/Users/dylan/code/tarball-without-package-prefix-1.0.0.tgz", + "_from": "file:tarball-without-package-prefix-1.0.0.tgz", + "_nodeVersion": "22.2.0", + "_npmVersion": "10.7.0", + "dist": { + "integrity": "sha512-y0COXVWAOSOmlEUSs3lO2FrF6ezZYwDukqADrMMaJap5/wO6gvMQ8XI/GoT0ht187owxlrvUjQEwdfTpSqORRg==", + "shasum": "f5dd68309b11f1b61802d8be9db02e7e97d98907", + "tarball": "http://localhost:4873/tarball-without-package-prefix/-/tarball-without-package-prefix-1.0.0.tgz" + }, + "contributors": [] + } + }, + "time": { + "modified": "2024-06-14T07:12:59.803Z", + "created": "2024-06-14T07:12:59.803Z", + "1.0.0": "2024-06-14T07:12:59.803Z" + }, + "users": {}, + "dist-tags": { + "latest": "1.0.0" + }, + "_uplinks": {}, + "_distfiles": {}, + "_attachments": { + "tarball-without-package-prefix-1.0.0.tgz": { + "shasum": "f5dd68309b11f1b61802d8be9db02e7e97d98907", + "version": "1.0.0" + } + }, + "_rev": "", + "_id": "tarball-without-package-prefix", + "readme": "ERROR: No README data found!" +} \ No newline at end of file diff --git a/test/cli/install/registry/packages/tarball-without-package-prefix/tarball-without-package-prefix-1.0.0.tgz b/test/cli/install/registry/packages/tarball-without-package-prefix/tarball-without-package-prefix-1.0.0.tgz new file mode 100644 index 0000000000..35f6bbaa9f Binary files /dev/null and b/test/cli/install/registry/packages/tarball-without-package-prefix/tarball-without-package-prefix-1.0.0.tgz differ