mirrors/bun.sh
1
0
Fork 0
mirror of https://github.com/oven-sh/bun synced 2026-02-09 18:38:55 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
cd320835d19b6741db655fdbad03fe25669aa8e7
bun.sh/src/install/migration.zig
dave caruso cd320835d1 chore: just a little bit of cleanup (#9319) 
* just a little bit of cleanup

* oops
2024-03-09 07:28:54 -08:00

1050 lines
51 KiB
Zig
Raw Blame History

const std = @import("std");
const Allocator = std.mem.Allocator;
const bun = @import("root").bun;
const string = bun.string;
const Output = bun.Output;
const Global = bun.Global;
const Environment = bun.Environment;
const strings = bun.strings;
const MutableString = bun.MutableString;
const stringZ = bun.stringZ;
const logger = bun.logger;
const Install = @import("./install.zig");
const Resolution = @import("./resolution.zig").Resolution;
const Dependency = @import("./dependency.zig");
const VersionedURL = @import("./versioned_url.zig");
const Npm = @import("./npm.zig");
const Integrity = @import("./integrity.zig").Integrity;
const Bin = @import("./bin.zig").Bin;
const Semver = @import("./semver.zig");
const String = Semver.String;
const ExternalString = Semver.ExternalString;
const stringHash = String.Builder.stringHash;
const Lockfile = @import("./lockfile.zig");
const LoadFromDiskResult = Lockfile.LoadFromDiskResult;
const JSAst = bun.JSAst;
const Expr = JSAst.Expr;
const B = JSAst.B;
const E = JSAst.E;
const G = JSAst.G;
const S = JSAst.S;
const debug = Output.scoped(.migrate, false);
pub fn detectAndLoadOtherLockfile(this: *Lockfile, allocator: Allocator, log: *logger.Log, bun_lockfile_path: stringZ) LoadFromDiskResult {
const dirname = bun_lockfile_path[0 .. strings.lastIndexOfChar(bun_lockfile_path, '/') orelse 0];
// check for package-lock.json, yarn.lock, etc...
// if it exists, do an in-memory migration
var buf: [bun.MAX_PATH_BYTES]u8 = undefined;
@memcpy(buf[0..dirname.len], dirname);
const cwd = std.fs.cwd();
npm: {
const npm_lockfile_name = "package-lock.json";
@memcpy(buf[dirname.len .. dirname.len + npm_lockfile_name.len], npm_lockfile_name);
buf[dirname.len + npm_lockfile_name.len] = 0;
const lockfile_path = buf[0 .. dirname.len + npm_lockfile_name.len :0];
var timer = std.time.Timer.start() catch unreachable;
const file = cwd.openFileZ(lockfile_path, .{ .mode = .read_only }) catch break :npm;
defer file.close();
const data = file.readToEndAlloc(allocator, std.math.maxInt(usize)) catch |err| {
return LoadFromDiskResult{ .err = .{ .step = .migrating, .value = err } };
};
const lockfile = migrateNPMLockfile(this, allocator, log, data, lockfile_path) catch |err| {
if (err == error.NPMLockfileVersionMismatch) {
Output.prettyErrorln(
\\<red><b>error<r><d>:<r> Please upgrade package-lock.json to lockfileVersion 2 or 3
\\
\\Run 'npm i --lockfile-version 3 --frozen-lockfile' to upgrade your lockfile without changing dependencies.
, .{});
Global.exit(1);
}
if (Environment.allow_assert) {
const maybe_trace = @errorReturnTrace();
Output.prettyErrorln("Error: {s}", .{@errorName(err)});
log.printForLogLevel(Output.errorWriter()) catch {};
if (maybe_trace) |trace| {
std.debug.dumpStackTrace(trace.*);
}
Output.prettyErrorln("Invalid NPM package-lock.json\nIn a release build, this would ignore and do a fresh install.\nAborting", .{});
Global.exit(1);
}
return LoadFromDiskResult{ .err = .{ .step = .migrating, .value = err } };
};
if (lockfile == .ok) {
Output.printElapsed(@as(f64, @floatFromInt(timer.read())) / std.time.ns_per_ms);
Output.prettyError(" ", .{});
Output.prettyErrorln("<d>migrated lockfile from <r><green>package-lock.json<r>", .{});
Output.flush();
}
return lockfile;
}
return LoadFromDiskResult{ .not_found = {} };
}
const ResolvedURLsMap = std.StringHashMapUnmanaged(string);
const IdMap = std.StringHashMapUnmanaged(IdMapValue);
const IdMapValue = struct {
/// index into the old package-lock.json package entries.
old_json_index: u32,
/// this is the new package id for the bun lockfile
///
/// - if this new_package_id is set to `package_id_is_link`, it means it's a link
/// and to get the actual package id, you need to lookup `.resolved` in the hashmap.
/// - if it is `package_id_is_bundled`, it means it's a bundled dependency that was not
/// marked by npm, which can happen to some transitive dependencies.
new_package_id: u32,
};
const package_id_is_link = std.math.maxInt(u32);
const package_id_is_bundled = std.math.maxInt(u32) - 1;
const unset_package_id = Install.invalid_package_id - 1;
const dependency_keys = .{
.dependencies,
.devDependencies,
.peerDependencies,
.optionalDependencies,
};
pub fn migrateNPMLockfile(this: *Lockfile, allocator: Allocator, log: *logger.Log, data: string, path: string) !LoadFromDiskResult {
debug("begin lockfile migration", .{});
try this.initEmpty(allocator);
Install.initializeStore();
const json_src = logger.Source.initPathString(path, data);
const json = bun.JSON.ParseJSONUTF8(&json_src, log, allocator) catch return error.InvalidNPMLockfile;
if (json.data != .e_object) {
return error.InvalidNPMLockfile;
}
if (json.get("lockfileVersion")) |version| {
if (!(version.data == .e_number and
version.data.e_number.value >= 2 and
version.data.e_number.value <= 3))
{
return error.NPMLockfileVersionMismatch;
}
} else {
return error.InvalidNPMLockfile;
}
// Count pass
var builder_ = this.stringBuilder();
var builder = &builder_;
const name = (if (json.get("name")) |expr| expr.asString(allocator) else null) orelse "";
builder.count(name);
var root_package: *E.Object = undefined;
var packages_properties = brk: {
const obj = json.get("packages") orelse return error.InvalidNPMLockfile;
if (obj.data != .e_object) return error.InvalidNPMLockfile;
if (obj.data.e_object.properties.len == 0) return error.InvalidNPMLockfile;
const prop1 = obj.data.e_object.properties.at(0);
if (prop1.key) |k| {
if (k.data != .e_string) return error.InvalidNPMLockfile;
// first key must be the "", self reference
if (k.data.e_string.data.len != 0) return error.InvalidNPMLockfile;
if (prop1.value.?.data != .e_object) return error.InvalidNPMLockfile;
root_package = prop1.value.?.data.e_object;
} else return error.InvalidNPMLockfile;
break :brk obj.data.e_object.properties;
};
var num_deps: u32 = 0;
const workspace_map: ?Lockfile.Package.WorkspaceMap = workspace_map: {
if (root_package.get("workspaces")) |wksp| {
var workspaces = Lockfile.Package.WorkspaceMap.init(allocator);
const json_array = switch (wksp.data) {
.e_array => |arr| arr,
.e_object => |obj| if (obj.get("packages")) |packages| switch (packages.data) {
.e_array => |arr| arr,
else => return error.InvalidNPMLockfile,
} else return error.InvalidNPMLockfile,
else => return error.InvalidNPMLockfile,
};
const workspace_packages_count = try Lockfile.Package.processWorkspaceNamesArray(
&workspaces,
allocator,
log,
json_array,
&json_src,
wksp.loc,
builder,
);
debug("found {d} workspace packages", .{workspace_packages_count});
num_deps += workspace_packages_count;
break :workspace_map workspaces;
}
break :workspace_map null;
};
// constructed "resolved" urls
var resolved_urls = ResolvedURLsMap{};
defer {
var itr = resolved_urls.iterator();
while (itr.next()) |entry| {
allocator.free(entry.value_ptr.*);
}
resolved_urls.deinit(allocator);
}
// Counting Phase
// This "IdMap" is used to make object key lookups faster for the `packages` object
// it also lets us resolve linked and bundled packages.
var id_map = IdMap{};
try id_map.ensureTotalCapacity(allocator, packages_properties.len);
var num_extern_strings: u32 = 0;
var package_idx: u32 = 0;
for (packages_properties.slice(), 0..) |entry, i| {
const pkg_path = entry.key.?.asString(allocator).?;
if (entry.value.?.data != .e_object)
return error.InvalidNPMLockfile;
const pkg = entry.value.?.data.e_object;
if (pkg.get("link") != null) {
id_map.putAssumeCapacity(
pkg_path,
IdMapValue{
.old_json_index = @truncate(i),
.new_package_id = package_id_is_link,
},
);
continue;
}
if (pkg.get("inBundle")) |x| if (x.data == .e_boolean and x.data.e_boolean.value) {
id_map.putAssumeCapacity(
pkg_path,
IdMapValue{
.old_json_index = @truncate(i),
.new_package_id = package_id_is_bundled,
},
);
continue;
};
if (pkg.get("extraneous")) |x| if (x.data == .e_boolean and x.data.e_boolean.value) {
continue;
};
id_map.putAssumeCapacity(
pkg_path,
IdMapValue{
.old_json_index = @truncate(i),
.new_package_id = package_idx,
},
);
package_idx += 1;
inline for (dependency_keys) |dep_key| {
if (pkg.get(@tagName(dep_key))) |deps| {
if (deps.data != .e_object) {
return error.InvalidNPMLockfile;
}
num_deps +|= @as(u32, deps.data.e_object.properties.len);
for (deps.data.e_object.properties.slice()) |dep| {
const dep_name = dep.key.?.asString(allocator).?;
const version_string = dep.value.?.asString(allocator) orelse return error.InvalidNPMLockfile;
builder.count(dep_name);
builder.count(version_string);
// If it's a folder or workspace, pessimistically assume we will need a maximum path
switch (Dependency.Version.Tag.infer(version_string)) {
.folder, .workspace => builder.cap += bun.MAX_PATH_BYTES,
else => {},
}
}
}
}
if (pkg.get("bin")) |bin| {
if (bin.data != .e_object) return error.InvalidNPMLockfile;
switch (bin.data.e_object.properties.len) {
0 => return error.InvalidNPMLockfile,
1 => {
const first_bin = bin.data.e_object.properties.at(0);
const key = first_bin.key.?.asString(allocator).?;
const workspace_entry = if (workspace_map) |map| map.map.get(pkg_path) else null;
const is_workspace = workspace_entry != null;
const pkg_name = if (is_workspace)
workspace_entry.?.name
else if (entry.value.?.get("name")) |set_name|
(set_name.asString(this.allocator) orelse return error.InvalidNPMLockfile)
else
packageNameFromPath(pkg_path);
if (!strings.eql(key, pkg_name)) {
builder.count(key);
}
builder.count(first_bin.value.?.asString(allocator) orelse return error.InvalidNPMLockfile);
},
else => {
for (bin.data.e_object.properties.slice()) |bin_entry| {
builder.count(bin_entry.key.?.asString(allocator).?);
builder.count(bin_entry.value.?.asString(allocator) orelse return error.InvalidNPMLockfile);
}
num_extern_strings += @truncate(bin.data.e_object.properties.len * 2);
},
}
}
if (pkg.get("resolved")) |resolved_expr| {
const resolved = resolved_expr.asString(allocator) orelse return error.InvalidNPMLockfile;
if (strings.hasPrefixComptime(resolved, "file:")) {
builder.count(resolved[5..]);
} else if (strings.hasPrefixComptime(resolved, "git+")) {
builder.count(resolved[4..]);
} else {
builder.count(resolved);
// this is over-counting but whatever. it would be too hard to determine if the case here
// is an `npm`/`dist_tag` version (the only times this is actually used)
if (pkg.get("version")) |v| if (v.asString(allocator)) |s| {
builder.count(s);
};
}
} else {
const version_prop = pkg.get("version");
const pkg_name = packageNameFromPath(pkg_path);
if (version_prop != null and pkg_name.len > 0) {
// construct registry url
const registry = Install.PackageManager.instance.scopeForPackageName(pkg_name);
var count: usize = 0;
count += registry.url.href.len + pkg_name.len + "/-/".len;
if (pkg_name[0] == '@') {
// scoped
const slash_index = strings.indexOfChar(pkg_name, '/') orelse return error.InvalidNPMLockfile;
if (slash_index >= pkg_name.len - 1) return error.InvalidNPMLockfile;
count += pkg_name[slash_index + 1 ..].len;
} else {
count += pkg_name.len;
}
const version_str = version_prop.?.asString(allocator) orelse return error.InvalidNPMLockfile;
count += "-.tgz".len + version_str.len;
const resolved_url = allocator.alloc(u8, count) catch unreachable;
var remain = resolved_url;
@memcpy(remain[0..registry.url.href.len], registry.url.href);
remain = remain[registry.url.href.len..];
@memcpy(remain[0..pkg_name.len], pkg_name);
remain = remain[pkg_name.len..];
remain[0.."/-/".len].* = "/-/".*;
remain = remain["/-/".len..];
if (pkg_name[0] == '@') {
const slash_index = strings.indexOfChar(pkg_name, '/') orelse unreachable;
@memcpy(remain[0..pkg_name[slash_index + 1 ..].len], pkg_name[slash_index + 1 ..]);
remain = remain[pkg_name[slash_index + 1 ..].len..];
} else {
@memcpy(remain[0..pkg_name.len], pkg_name);
remain = remain[pkg_name.len..];
}
remain[0] = '-';
remain = remain[1..];
@memcpy(remain[0..version_str.len], version_str);
remain = remain[version_str.len..];
remain[0..".tgz".len].* = ".tgz".*;
builder.count(resolved_url);
try resolved_urls.put(allocator, pkg_path, resolved_url);
} else {
builder.count(pkg_path);
}
}
}
if (num_deps == std.math.maxInt(u32)) return error.InvalidNPMLockfile; // lol
debug("counted {d} dependencies", .{num_deps});
debug("counted {d} extern strings", .{num_extern_strings});
debug("counted {d} packages", .{package_idx});
try this.buffers.dependencies.ensureTotalCapacity(allocator, num_deps);
try this.buffers.resolutions.ensureTotalCapacity(allocator, num_deps);
try this.buffers.extern_strings.ensureTotalCapacity(allocator, num_extern_strings);
try this.packages.ensureTotalCapacity(allocator, package_idx);
// The package index is overallocated, but we know the upper bound
try this.package_index.ensureTotalCapacity(package_idx);
try builder.allocate();
if (workspace_map) |wksp| {
try this.workspace_paths.ensureTotalCapacity(allocator, wksp.map.unmanaged.entries.len);
try this.workspace_versions.ensureTotalCapacity(allocator, wksp.map.unmanaged.entries.len);
var path_buf: if (Environment.isWindows) bun.PathBuffer else void = undefined;
for (wksp.map.keys(), wksp.map.values()) |k, v| {
const name_hash = stringHash(v.name);
this.workspace_paths.putAssumeCapacity(
name_hash,
builder.append(
String,
if (comptime Environment.isWindows)
bun.path.normalizeBuf(k, &path_buf, .windows)
else
k,
),
);
if (v.version) |version_string| {
const sliced_version = Semver.SlicedString.init(version_string, version_string);
const result = Semver.Version.parse(sliced_version);
if (result.valid and result.wildcard == .none) {
this.workspace_versions.putAssumeCapacity(name_hash, result.version.min());
}
}
}
}
// Package Building Phase
// This initializes every package and sets the resolution to uninitialized
for (packages_properties.slice()) |entry| {
// this pass is allowed to make more assumptions because we already checked things during
// the counting pass
const pkg = entry.value.?.data.e_object;
if (pkg.get("link") != null or if (pkg.get("inBundle") orelse pkg.get("extraneous")) |x| x.data == .e_boolean and x.data.e_boolean.value else false) continue;
const pkg_path = entry.key.?.asString(allocator).?;
const workspace_entry = if (workspace_map) |map| map.map.get(pkg_path) else null;
const is_workspace = workspace_entry != null;
const pkg_name = if (is_workspace)
workspace_entry.?.name
else if (pkg.get("name")) |set_name|
(set_name.asString(this.allocator) orelse unreachable)
else
packageNameFromPath(pkg_path);
const name_hash = stringHash(pkg_name);
const package_id: Install.PackageID = @intCast(this.packages.len);
if (Environment.allow_assert) {
// If this is false, then it means we wrote wrong resolved ids
// During counting phase we assign all the packages an id.
std.debug.assert(package_id == id_map.get(pkg_path).?.new_package_id);
}
// Instead of calling this.appendPackage, manually append
// the other function has some checks that will fail since we have not set resolution+dependencies yet.
this.packages.appendAssumeCapacity(Lockfile.Package{
.name = builder.appendWithHash(String, pkg_name, name_hash),
.name_hash = name_hash,
// For non workspace packages these are set to .uninitialized, then in the third phase
// they are resolved. This is because the resolution uses the dependant's version
// specifier as a "hint" to resolve the dependency.
.resolution = if (is_workspace) Resolution.init(.{
// This string is counted by `processWorkspaceNamesArray`
.workspace = builder.append(String, pkg_path),
}) else Resolution{},
// we fill this data in later
.dependencies = undefined,
.resolutions = undefined,
.meta = .{
.id = package_id,
.origin = if (package_id == 0) .local else .npm,
.arch = if (pkg.get("cpu")) |cpu_array| arch: {
if (cpu_array.data != .e_array) return error.InvalidNPMLockfile;
var arch: Npm.Architecture = .none;
for (cpu_array.data.e_array.items.slice()) |item| {
if (item.data != .e_string) return error.InvalidNPMLockfile;
arch = arch.apply(item.data.e_string.data);
}
break :arch arch;
} else .all,
.os = if (pkg.get("os")) |cpu_array| arch: {
if (cpu_array.data != .e_array) return error.InvalidNPMLockfile;
var os: Npm.OperatingSystem = .none;
for (cpu_array.data.e_array.items.slice()) |item| {
if (item.data != .e_string) return error.InvalidNPMLockfile;
os = os.apply(item.data.e_string.data);
}
break :arch os;
} else .all,
.man_dir = String{},
.has_install_script = if (pkg.get("hasInstallScript")) |has_install_script_expr| brk: {
if (has_install_script_expr.data != .e_boolean) return error.InvalidNPMLockfile;
break :brk if (has_install_script_expr.data.e_boolean.value)
.true
else
.false;
} else .false,
.integrity = if (pkg.get("integrity")) |integrity|
try Integrity.parse(
integrity.asString(this.allocator) orelse
return error.InvalidNPMLockfile,
)
else
Integrity{},
},
.bin = if (pkg.get("bin")) |bin| bin: {
// we already check these conditions during counting
std.debug.assert(bin.data == .e_object);
std.debug.assert(bin.data.e_object.properties.len > 0);
// in npm lockfile, the bin is always an object, even if it is only a single one
// we need to detect if it's a single entry and lower it to a file.
if (bin.data.e_object.properties.len == 1) {
const prop = bin.data.e_object.properties.at(0);
const key = prop.key.?.asString(this.allocator) orelse return error.InvalidNPMLockfile;
const script_value = prop.value.?.asString(this.allocator) orelse return error.InvalidNPMLockfile;
if (strings.eql(key, pkg_name)) {
break :bin .{
.tag = .file,
.value = Bin.Value.init(.{
.file = builder.append(String, script_value),
}),
};
}
break :bin .{
.tag = .named_file,
.value = Bin.Value.init(.{
.named_file = .{
builder.append(String, key),
builder.append(String, script_value),
},
}),
};
}
const view: Install.ExternalStringList = .{
.off = @truncate(this.buffers.extern_strings.items.len),
.len = @intCast(bin.data.e_object.properties.len * 2),
};
for (bin.data.e_object.properties.slice()) |bin_entry| {
const key = bin_entry.key.?.asString(this.allocator) orelse return error.InvalidNPMLockfile;
const script_value = bin_entry.value.?.asString(this.allocator) orelse return error.InvalidNPMLockfile;
this.buffers.extern_strings.appendAssumeCapacity(builder.append(ExternalString, key));
this.buffers.extern_strings.appendAssumeCapacity(builder.append(ExternalString, script_value));
}
if (Environment.allow_assert) {
std.debug.assert(this.buffers.extern_strings.items.len == view.off + view.len);
std.debug.assert(this.buffers.extern_strings.items.len <= this.buffers.extern_strings.capacity);
}
break :bin .{
.tag = .map,
.value = Bin.Value.init(.{
.map = view,
}),
};
} else Bin.init(),
.scripts = .{},
});
if (is_workspace) {
std.debug.assert(package_id != 0); // root package should not be in it's own workspace
// we defer doing getOrPutID for non-workspace packages because it depends on the resolution being set.
try this.getOrPutID(package_id, name_hash);
}
}
if (Environment.allow_assert) {
std.debug.assert(this.packages.len == package_idx);
}
// ignoring length check because we pre-allocated it. the length may shrink later
// so it's faster if we ignore the underlying length buffer and just assign it at the very end.
var dependencies_buf = this.buffers.dependencies.items.ptr[0..num_deps];
var resolutions_buf = this.buffers.resolutions.items.ptr[0..num_deps];
// pre-initialize the dependencies and resolutions to `unset_package_id`
if (Environment.allow_assert) {
@memset(dependencies_buf, Dependency{});
@memset(resolutions_buf, unset_package_id);
}
var resolutions = this.packages.items(.resolution);
var metas = this.packages.items(.meta);
var dependencies_list = this.packages.items(.dependencies);
var resolution_list = this.packages.items(.resolutions);
if (Environment.allow_assert) {
for (resolutions) |r| {
std.debug.assert(r.tag == .uninitialized or r.tag == .workspace);
}
}
// Root resolution isn't hit through dependency tracing.
resolutions[0] = Resolution.init(.{ .root = {} });
metas[0].origin = .local;
try this.getOrPutID(0, this.packages.items(.name_hash)[0]);
// made it longer than max path just in case something stupid happens
var name_checking_buf: [bun.MAX_PATH_BYTES * 2]u8 = undefined;
// Dependency Linking Phase
package_idx = 0;
var is_first = true;
for (packages_properties.slice()) |entry| {
// this pass is allowed to make more assumptions because we already checked things during
// the counting pass
const pkg = entry.value.?.data.e_object;
if (pkg.get("link") != null or if (pkg.get("inBundle") orelse pkg.get("extraneous")) |x| x.data == .e_boolean and x.data.e_boolean.value else false) continue;
const pkg_path = entry.key.?.asString(allocator).?;
const dependencies_start = dependencies_buf.ptr;
const resolutions_start = resolutions_buf.ptr;
// this is in a defer because there are two places we end this loop iteration at.
defer {
if (dependencies_start == dependencies_buf.ptr) {
dependencies_list[package_idx] = .{ .len = 0 };
resolution_list[package_idx] = .{ .len = 0 };
} else {
// Calculate the offset + length by pointer arithmetic
const len: u32 = @truncate((@intFromPtr(resolutions_buf.ptr) - @intFromPtr(resolutions_start)) / @sizeOf(Install.PackageID));
if (Environment.allow_assert) {
std.debug.assert(len > 0);
std.debug.assert(len == ((@intFromPtr(dependencies_buf.ptr) - @intFromPtr(dependencies_start)) / @sizeOf(Dependency)));
}
dependencies_list[package_idx] = .{
.off = @truncate((@intFromPtr(dependencies_start) - @intFromPtr(this.buffers.dependencies.items.ptr)) / @sizeOf(Dependency)),
.len = len,
};
resolution_list[package_idx] = .{
.off = @truncate((@intFromPtr(resolutions_start) - @intFromPtr(this.buffers.resolutions.items.ptr)) / @sizeOf(Install.PackageID)),
.len = len,
};
}
package_idx += 1;
}
// a feature no one has heard about: https://docs.npmjs.com/cli/v10/configuring-npm/package-json#bundledependencies
const bundled_dependencies = if (pkg.get("bundleDependencies") orelse pkg.get("bundledDependencies")) |expr| deps: {
if (expr.data == .e_boolean) {
if (expr.data.e_boolean.value) continue;
break :deps null;
}
if (expr.data != .e_array) return error.InvalidNPMLockfile;
const arr: *E.Array = expr.data.e_array;
var map = std.StringArrayHashMapUnmanaged(void){};
try map.ensureTotalCapacity(allocator, arr.items.len);
for (arr.items.slice()) |item| {
map.putAssumeCapacity(item.asString(allocator) orelse return error.InvalidNPMLockfile, {});
}
break :deps map;
} else null;
if (is_first) {
is_first = false;
if (workspace_map) |wksp| {
for (wksp.keys(), wksp.values()) |key, value| {
const entry1 = id_map.get(key) orelse return error.InvalidNPMLockfile;
const name_hash = stringHash(value.name);
const wksp_name = builder.append(String, value.name);
const wksp_path = builder.append(String, key);
dependencies_buf[0] = Dependency{
.name = wksp_name,
.name_hash = name_hash,
.version = .{
.tag = .workspace,
.literal = wksp_path,
.value = .{
.workspace = wksp_path,
},
},
.behavior = .{
.workspace = true,
},
};
resolutions_buf[0] = entry1.new_package_id;
dependencies_buf = dependencies_buf[1..];
resolutions_buf = resolutions_buf[1..];
}
}
}
inline for (dependency_keys) |dep_key| {
if (pkg.get(@tagName(dep_key))) |deps| {
// fetch the peerDependenciesMeta if it exists
// this is only done for peerDependencies, obviously
const peer_dep_meta = if (dep_key == .peerDependencies)
if (pkg.get("peerDependenciesMeta")) |expr| peer_dep_meta: {
if (expr.data != .e_object) return error.InvalidNPMLockfile;
break :peer_dep_meta expr.data.e_object;
} else null
else
void{};
if (deps.data != .e_object) return error.InvalidNPMLockfile;
const properties = deps.data.e_object.properties;
dep_loop: for (properties.slice()) |prop| {
const name_bytes = prop.key.?.asString(this.allocator).?;
if (bundled_dependencies != null and bundled_dependencies.?.getIndex(name_bytes) != null) continue :dep_loop;
const version_bytes = prop.value.?.asString(this.allocator) orelse return error.InvalidNPMLockfile;
const name_hash = stringHash(name_bytes);
const dep_name = builder.appendWithHash(String, name_bytes, name_hash);
const dep_version = builder.append(String, version_bytes);
const sliced = dep_version.sliced(this.buffers.string_bytes.items);
debug("parsing {s}, {s}\n", .{ name_bytes, version_bytes });
const version = Dependency.parse(
this.allocator,
dep_name,
name_hash,
sliced.slice,
&sliced,
log,
) orelse {
return error.InvalidNPMLockfile;
};
debug("-> {s}, {}\n", .{ @tagName(version.tag), version.value });
if (Environment.allow_assert) {
std.debug.assert(version.tag != .uninitialized);
}
const str_node_modules = if (pkg_path.len == 0) "node_modules/" else "/node_modules/";
const suffix_len = str_node_modules.len + name_bytes.len;
var buf_len: u32 = @as(u32, @intCast(pkg_path.len + suffix_len));
if (buf_len > name_checking_buf.len) {
return error.PathTooLong;
}
bun.copy(u8, name_checking_buf[0..pkg_path.len], pkg_path);
bun.copy(u8, name_checking_buf[pkg_path.len .. pkg_path.len + str_node_modules.len], str_node_modules);
bun.copy(u8, name_checking_buf[pkg_path.len + str_node_modules.len .. pkg_path.len + suffix_len], name_bytes);
while (true) {
debug("checking {s}", .{name_checking_buf[0..buf_len]});
if (id_map.get(name_checking_buf[0..buf_len])) |found_| {
var found = found_;
if (found.new_package_id == package_id_is_link) {
// it is a workspace package, resolve from the "link": true entry to the real entry.
const ref_pkg = packages_properties.at(found.old_json_index).value.?.data.e_object;
// the `else` here is technically possible to hit
const resolved_v = ref_pkg.get("resolved") orelse return error.LockfileWorkspaceMissingResolved;
const resolved = resolved_v.asString(this.allocator) orelse return error.InvalidNPMLockfile;
found = (id_map.get(resolved) orelse return error.InvalidNPMLockfile);
} else if (found.new_package_id == package_id_is_bundled) {
debug("skipping bundled dependency {s}", .{name_bytes});
continue :dep_loop;
}
const id = found.new_package_id;
const is_workspace = resolutions[id].tag == .workspace;
dependencies_buf[0] = Dependency{
.name = dep_name,
.name_hash = name_hash,
.version = version,
.behavior = .{
.normal = dep_key == .dependencies,
.optional = dep_key == .optionalDependencies,
.dev = dep_key == .devDependencies,
.peer = dep_key == .peerDependencies,
.workspace = is_workspace,
},
};
resolutions_buf[0] = id;
dependencies_buf = dependencies_buf[1..];
resolutions_buf = resolutions_buf[1..];
// If the package resolution is not set, resolve the target package
// using the information we have from the dependency declaration.
if (resolutions[id].tag == .uninitialized) {
debug("resolving '{s}'", .{name_bytes});
const res = resolved: {
const dep_pkg = packages_properties.at(found.old_json_index).value.?.data.e_object;
const dep_resolved: string = dep_resolved: {
if (dep_pkg.get("resolved")) |resolved| {
break :dep_resolved resolved.asString(this.allocator) orelse return error.InvalidNPMLockfile;
}
if (version.tag == .npm) {
if (resolved_urls.get(name_checking_buf[0..buf_len])) |resolved_url| {
break :dep_resolved resolved_url;
}
}
break :resolved Resolution.init(.{
.folder = builder.append(
String,
packages_properties.at(found.old_json_index).key.?.asString(allocator).?,
),
});
};
break :resolved switch (version.tag) {
.uninitialized => std.debug.panic("Version string {s} resolved to `.uninitialized`", .{version_bytes}),
.npm, .dist_tag => res: {
// It is theoretically possible to hit this in a case where the resolved dependency is NOT
// an npm dependency, but that case is so convoluted that it is not worth handling.
//
// Deleting 'package-lock.json' would completely break the installation of the project.
//
// We assume that the given URL is to *some* npm registry, or the resolution is to a workspace package.
// If it is a workspace package, then this branch will not be hit as the resolution was already set earlier.
const dep_actual_version = (dep_pkg.get("version") orelse return error.InvalidNPMLockfile)
.asString(this.allocator) orelse return error.InvalidNPMLockfile;
const dep_actual_version_str = builder.append(String, dep_actual_version);
const dep_actual_version_sliced = dep_actual_version_str.sliced(this.buffers.string_bytes.items);
break :res Resolution.init(.{
.npm = .{
.url = builder.append(String, dep_resolved),
.version = Semver.Version.parse(dep_actual_version_sliced).version.min(),
},
});
},
.tarball => if (strings.hasPrefixComptime(dep_resolved, "file:"))
Resolution.init(.{ .local_tarball = builder.append(String, dep_resolved[5..]) })
else
Resolution.init(.{ .remote_tarball = builder.append(String, dep_resolved) }),
.folder => Resolution.init(.{ .folder = builder.append(String, dep_resolved) }),
// not sure if this is possible to hit
.symlink => Resolution.init(.{ .folder = builder.append(String, dep_resolved) }),
.workspace => workspace: {
var input = builder.append(String, dep_resolved).sliced(this.buffers.string_bytes.items);
if (strings.hasPrefixComptime(input.slice, "workspace:")) {
input = input.sub(input.slice["workspace:".len..]);
}
break :workspace Resolution.init(.{
.workspace = input.value(),
});
},
.git => res: {
const str = (if (strings.hasPrefixComptime(dep_resolved, "git+"))
builder.append(String, dep_resolved[4..])
else
builder.append(String, dep_resolved))
.sliced(this.buffers.string_bytes.items);
const hash_index = strings.lastIndexOfChar(str.slice, '#') orelse return error.InvalidNPMLockfile;
const commit = str.sub(str.slice[hash_index + 1 ..]).value();
break :res Resolution.init(.{
.git = .{
.owner = version.value.git.owner,
.repo = str.sub(str.slice[0..hash_index]).value(),
.committish = commit,
.resolved = commit,
.package_name = dep_name,
},
});
},
.github => res: {
const str = (if (strings.hasPrefixComptime(dep_resolved, "git+"))
builder.append(String, dep_resolved[4..])
else
builder.append(String, dep_resolved))
.sliced(this.buffers.string_bytes.items);
const hash_index = strings.lastIndexOfChar(str.slice, '#') orelse return error.InvalidNPMLockfile;
const commit = str.sub(str.slice[hash_index + 1 ..]).value();
break :res Resolution.init(.{
.git = .{
.owner = version.value.github.owner,
.repo = str.sub(str.slice[0..hash_index]).value(),
.committish = commit,
.resolved = commit,
.package_name = dep_name,
},
});
},
};
};
debug("-> {}", .{res.fmtForDebug(this.buffers.string_bytes.items)});
resolutions[id] = res;
metas[id].origin = switch (res.tag) {
// This works?
.root => .local,
else => .npm,
};
try this.getOrPutID(id, this.packages.items(.name_hash)[id]);
}
continue :dep_loop;
}
// step down each `node_modules/` of the source
if (strings.lastIndexOf(name_checking_buf[0..buf_len -| ("node_modules/".len + name_bytes.len)], "node_modules/")) |idx| {
debug("found 'node_modules/' at {d}", .{idx});
buf_len = @intCast(idx + "node_modules/".len + name_bytes.len);
bun.copy(u8, name_checking_buf[idx + "node_modules/".len .. idx + "node_modules/".len + name_bytes.len], name_bytes);
} else if (!strings.hasPrefixComptime(name_checking_buf[0..buf_len], "node_modules/")) {
// this is hit if you are at something like `packages/etc`, from `packages/etc/node_modules/xyz`
// we need to hit the root `node_modules/{name}`
buf_len = @intCast("node_modules/".len + name_bytes.len);
bun.copy(u8, name_checking_buf[0..buf_len], "node_modules/");
bun.copy(u8, name_checking_buf[buf_len - name_bytes.len .. buf_len], name_bytes);
} else {
// optional peer dependencies can be ... optional
if (dep_key == .peerDependencies) {
if (peer_dep_meta) |o| if (o.get(name_bytes)) |meta| {
if (meta.data != .e_object) return error.InvalidNPMLockfile;
if (meta.data.e_object.get("optional")) |optional| {
if (optional.data != .e_boolean) return error.InvalidNPMLockfile;
if (optional.data.e_boolean.value) {
dependencies_buf[0] = Dependency{
.name = dep_name,
.name_hash = name_hash,
.version = version,
.behavior = .{
.normal = dep_key == .dependencies,
.optional = true,
.dev = dep_key == .devDependencies,
.peer = dep_key == .peerDependencies,
.workspace = false,
},
};
resolutions_buf[0] = Install.invalid_package_id;
dependencies_buf = dependencies_buf[1..];
resolutions_buf = resolutions_buf[1..];
continue :dep_loop;
}
}
};
}
// it is technically possible to get a package-lock.json without a dependency.
// it's very unlikely, but possible. when NPM sees this, it essentially doesnt install the package, and treats it like it doesn't exist.
// in test/cli/install/migrate-fixture, you can observe this for `iconv-lite`
debug("could not find package '{s}' in '{s}'", .{ name_bytes, pkg_path });
continue :dep_loop;
}
}
}
}
}
}
this.buffers.resolutions.items.len = (@intFromPtr(resolutions_buf.ptr) - @intFromPtr(this.buffers.resolutions.items.ptr)) / @sizeOf(Install.PackageID);
this.buffers.dependencies.items.len = this.buffers.resolutions.items.len;
// In allow_assert, we prefill this buffer with uninitialized values that we can detect later
// It is our fault if we hit an error here, making it safe to disable in release.
if (Environment.allow_assert) {
std.debug.assert(this.buffers.dependencies.items.len == (@intFromPtr(dependencies_buf.ptr) - @intFromPtr(this.buffers.dependencies.items.ptr)) / @sizeOf(Dependency));
std.debug.assert(this.buffers.dependencies.items.len <= num_deps);
var crash = false;
for (this.buffers.dependencies.items, 0..) |r, i| {
// 'if behavior is uninitialized'
if (r.behavior.eq(.{})) {
debug("dependency index '{d}' was not set", .{i});
crash = true;
}
}
for (this.buffers.resolutions.items, 0..) |r, i| {
if (r == unset_package_id) {
debug("resolution index '{d}' was not set", .{i});
crash = true;
}
}
if (crash) {
std.debug.panic("Assertion failure, see above", .{});
}
}
// A package not having a resolution, however, is not our fault.
// This can be triggered by a bad lockfile with extra packages. NPM should trim packages out automatically.
var is_missing_resolutions = false;
for (resolutions, 0..) |r, i| {
if (r.tag == .uninitialized) {
Output.warn("Could not resolve package '{s}' in lockfile during migration", .{this.packages.items(.name)[i].slice(this.buffers.string_bytes.items)});
is_missing_resolutions = true;
} else if (Environment.allow_assert) {
// Assertion from appendPackage. If we do this too early it will always fail as we dont have the resolution written
// but after we write all the data, there is no excuse for this to fail.
//
// If this is hit, it means getOrPutID was not called on this package id. Look for where 'resolution[i]' is set
std.debug.assert(this.getPackageID(this.packages.items(.name_hash)[i], null, &r) != null);
}
}
if (is_missing_resolutions) {
return error.NotAllPackagesGotResolved;
}
// if (Environment.isDebug) {
// const dump_file = try std.fs.cwd().createFileZ("before-clean.json", .{});
// defer dump_file.close();
// try std.json.stringify(this, .{ .whitespace = .indent_2 }, dump_file.writer());
// }
// This is definitely a memory leak, but it's fine because there is no install api, so this can only be leaked once per process.
// This operation is neccecary because callers of `loadFromDisk` assume the data is written into the passed `this`.
// You'll find that not cleaning the lockfile will cause `bun install` to not actually install anything since it doesnt have any hoisted trees.
this.* = (try this.cleanWithLogger(&[_]Install.PackageManager.UpdateRequest{}, log, false)).*;
// if (Environment.isDebug) {
// const dump_file = try std.fs.cwd().createFileZ("after-clean.json", .{});
// defer dump_file.close();
// try std.json.stringify(this, .{ .whitespace = .indent_2 }, dump_file.writer());
// }
if (Environment.allow_assert) {
try this.verifyData();
}
this.meta_hash = try this.generateMetaHash(false, this.packages.len);
return LoadFromDiskResult{
.ok = .{
.lockfile = this,
.was_migrated = true,
.serializer_result = .{},
},
};
}
fn packageNameFromPath(pkg_path: []const u8) []const u8 {
if (pkg_path.len == 0) return "";
const pkg_name_start: usize = if (strings.lastIndexOf(pkg_path, "/node_modules/")) |last_index|
last_index + "/node_modules/".len
else if (strings.hasPrefixComptime(pkg_path, "node_modules/"))
"node_modules/".len
else
strings.lastIndexOf(pkg_path, "/") orelse 0;
return pkg_path[pkg_name_start..];
}
Reference in New Issue View Git Blame Copy Permalink