From 4b8ca51b8701f91d4d78fb701cc605899b3dc167 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Thu, 31 Oct 2024 12:28:07 -0700 Subject: [PATCH 01/17] Clean up some code in node validators (#14897) --- src/bun.js/bindings/NodeValidator.cpp | 53 ++++++++++++++++++++------- src/bun.js/bindings/NodeValidator.h | 1 + 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/bun.js/bindings/NodeValidator.cpp b/src/bun.js/bindings/NodeValidator.cpp index 18a897532f..c259609ef6 100644 --- a/src/bun.js/bindings/NodeValidator.cpp +++ b/src/bun.js/bindings/NodeValidator.cpp @@ -12,6 +12,7 @@ #include #include +#include "JSAbortSignal.h" #include "JSBufferEncodingType.h" #include "BunProcess.h" #include "ErrorCode.h" @@ -94,6 +95,15 @@ JSC_DEFINE_HOST_FUNCTION(jsFunction_validateString, (JSC::JSGlobalObject * globa auto name = callFrame->argument(1); return V::validateString(scope, globalObject, value, name); } + +JSC::EncodedJSValue V::validateString(JSC::ThrowScope& scope, JSC::JSGlobalObject* globalObject, JSValue value, ASCIILiteral name) +{ + if (!value.isString()) { + return Bun::ERR::INVALID_ARG_TYPE(scope, globalObject, name, "string"_s, value); + } + return JSValue::encode(jsUndefined()); +} + JSC::EncodedJSValue V::validateString(JSC::ThrowScope& scope, JSC::JSGlobalObject* globalObject, JSValue value, JSValue name) { if (!value.isString()) { @@ -138,12 +148,12 @@ JSC_DEFINE_HOST_FUNCTION(jsFunction_checkRangesOrGetDefault, (JSC::JSGlobalObjec auto name = callFrame->argument(1); auto lower = callFrame->argument(2); auto upper = callFrame->argument(3); - auto def = callFrame->argument(4); auto finite = Bun::V::validateFiniteNumber(scope, globalObject, number, name); RETURN_IF_EXCEPTION(scope, {}); auto finite_real = JSValue::decode(finite).asBoolean(); if (!finite_real) { + auto def = callFrame->argument(4); return JSValue::encode(def); } @@ -265,10 +275,13 @@ JSC_DEFINE_HOST_FUNCTION(jsFunction_validateAbortSignal, (JSC::JSGlobalObject * auto name = callFrame->argument(1); if (!signal.isUndefined()) { - if (signal.isNull()) return Bun::ERR::INVALID_ARG_TYPE(scope, globalObject, name, "AbortSignal"_s, signal); - if (!signal.isObject()) return Bun::ERR::INVALID_ARG_TYPE(scope, globalObject, name, "AbortSignal"_s, signal); + auto* object = signal.getObject(); + if (!object) return Bun::ERR::INVALID_ARG_TYPE(scope, globalObject, name, "AbortSignal"_s, signal); + if (object->inherits()) { + return JSValue::encode(jsUndefined()); + } - auto propin = signal.getObject()->hasProperty(globalObject, Identifier::fromString(vm, "aborted"_s)); + auto propin = object->hasProperty(globalObject, Identifier::fromString(vm, "aborted"_s)); RETURN_IF_EXCEPTION(scope, {}); if (!propin) return Bun::ERR::INVALID_ARG_TYPE(scope, globalObject, name, "AbortSignal"_s, signal); } @@ -378,19 +391,32 @@ JSC_DEFINE_HOST_FUNCTION(jsFunction_validateEncoding, (JSC::JSGlobalObject * glo JSC::VM& vm = globalObject->vm(); auto scope = DECLARE_THROW_SCOPE(vm); - auto data = callFrame->argument(0); auto encoding = callFrame->argument(1); auto normalized = WebCore::parseEnumeration(*globalObject, encoding); - // no check to throw ERR_UNKNOWN_ENCODING ? it's not in node but feels like it would be apt here + if (normalized == BufferEncodingType::hex) { + auto data = callFrame->argument(0); - auto length = data.get(globalObject, Identifier::fromString(vm, "length"_s)); - RETURN_IF_EXCEPTION(scope, {}); - auto length_num = length.toNumber(globalObject); - RETURN_IF_EXCEPTION(scope, {}); - if (normalized == BufferEncodingType::hex && std::fmod(length_num, 2.0) != 0) { - return Bun::ERR::INVALID_ARG_VALUE(scope, globalObject, "encoding"_s, encoding, makeString("is invalid for data of length "_s, length_num)); + size_t length = 0; + if (data.isString()) { + length = data.toString(globalObject)->length(); + } else if (auto* view = JSC::jsDynamicCast(data)) { + length = view->length(); + } else if (auto* buffer = JSC::jsDynamicCast(data)) { + if (auto* impl = buffer->impl()) { + length = impl->byteLength(); + } + } else if (auto* object = data.getObject()) { + JSValue lengthValue = object->getIfPropertyExists(globalObject, vm.propertyNames->length); + RETURN_IF_EXCEPTION(scope, {}); + length = lengthValue.toLength(globalObject); + RETURN_IF_EXCEPTION(scope, {}); + } + if (length % 2 != 0) { + return Bun::ERR::INVALID_ARG_VALUE(scope, globalObject, "encoding"_s, encoding, makeString("is invalid for data of length "_s, length)); + } } + return JSValue::encode(jsUndefined()); } @@ -400,9 +426,9 @@ JSC_DEFINE_HOST_FUNCTION(jsFunction_validatePlainFunction, (JSC::JSGlobalObject auto scope = DECLARE_THROW_SCOPE(vm); auto value = callFrame->argument(0); - auto name = callFrame->argument(1); if (!value.isCallable()) { + auto name = callFrame->argument(1); return Bun::ERR::INVALID_ARG_TYPE(scope, globalObject, name, "function"_s, value); } return JSValue::encode(jsUndefined()); @@ -437,5 +463,4 @@ JSC_DEFINE_HOST_FUNCTION(jsFunction_validateBuffer, (JSC::JSGlobalObject * globa } return JSValue::encode(jsUndefined()); } - } diff --git a/src/bun.js/bindings/NodeValidator.h b/src/bun.js/bindings/NodeValidator.h index 1d5adaed95..b691c7f5e0 100644 --- a/src/bun.js/bindings/NodeValidator.h +++ b/src/bun.js/bindings/NodeValidator.h @@ -30,6 +30,7 @@ JSC::EncodedJSValue validateInteger(JSC::ThrowScope& scope, JSC::JSGlobalObject* JSC::EncodedJSValue validateNumber(JSC::ThrowScope& scope, JSC::JSGlobalObject* globalObject, JSC::JSValue value, JSC::JSValue name, JSC::JSValue min, JSC::JSValue max); JSC::EncodedJSValue validateFiniteNumber(JSC::ThrowScope& scope, JSC::JSGlobalObject* globalObject, JSC::JSValue number, JSC::JSValue name); JSC::EncodedJSValue validateString(JSC::ThrowScope& scope, JSC::JSGlobalObject* globalObject, JSValue value, JSValue name); +JSC::EncodedJSValue validateString(JSC::ThrowScope& scope, JSC::JSGlobalObject* globalObject, JSValue value, ASCIILiteral name); } From 353d44f1ae58cf74f40171b06b624c1579feb66f Mon Sep 17 00:00:00 2001 From: Ashcon Partovi Date: Thu, 31 Oct 2024 12:50:09 -0700 Subject: [PATCH 02/17] ci: If only tests change, use artifacts from last successful build (#14927) --- .buildkite/ci.mjs | 94 +++++++++++++++++++++++++++++++++++------ scripts/runner.node.mjs | 9 +++- 2 files changed, 89 insertions(+), 14 deletions(-) diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index 919b81fc34..bfe09735cc 100644 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -60,7 +60,7 @@ function isPullRequest() { async function getChangedFiles() { const repository = getRepository(); const head = getCommit(); - const base = isMainBranch() ? `${head}^1` : getMainBranch(); + const base = `${head}^1`; try { const response = await fetch(`https://api.github.com/repos/${repository}/compare/${base}...${head}`); @@ -73,6 +73,46 @@ async function getChangedFiles() { } } +function getBuildUrl() { + return getEnv("BUILDKITE_BUILD_URL"); +} + +async function getBuildIdWithArtifacts() { + let depth = 0; + let url = getBuildUrl(); + + while (url) { + const response = await fetch(`${url}.json`, { + headers: { + "Accept": "application/json", + }, + }); + + if (!response.ok) { + return; + } + + const { id, state, prev_branch_build: lastBuild, steps } = await response.json(); + if (depth++) { + if (state === "failed" || state === "passed") { + const buildSteps = steps.filter(({ label }) => label.endsWith("build-bun")); + if (buildSteps.length) { + if (buildSteps.every(({ outcome }) => outcome === "passed")) { + return id; + } + return; + } + } + } + + if (!lastBuild) { + return; + } + + url = url.replace(/\/builds\/[0-9]+/, `/builds/${lastBuild["number"]}`); + } +} + function isDocumentation(filename) { return /^(\.vscode|\.github|bench|docs|examples)|\.(md)$/.test(filename); } @@ -86,6 +126,10 @@ function toYaml(obj, indent = 0) { let result = ""; for (const [key, value] of Object.entries(obj)) { + if (value === undefined) { + continue; + } + if (value === null) { result += `${spaces}${key}: null\n`; continue; @@ -125,7 +169,7 @@ function toYaml(obj, indent = 0) { return result; } -function getPipeline() { +function getPipeline(buildId) { /** * Helpers */ @@ -299,16 +343,27 @@ function getPipeline() { parallelism = 10; } + let depends; + let env; + if (buildId) { + env = { + BUILDKITE_ARTIFACT_BUILD_ID: buildId, + }; + } else { + depends = [`${getKey(platform)}-build-bun`]; + } + return { key: `${getKey(platform)}-${distro}-${release.replace(/\./g, "")}-test-bun`, label: `${name} - test-bun`, - depends_on: [`${getKey(platform)}-build-bun`], + depends_on: depends, agents, retry: getRetry(), cancel_on_build_failing: isMergeQueue(), soft_fail: isMainBranch(), parallelism, command, + env, }; }; @@ -350,18 +405,25 @@ function getPipeline() { ...buildPlatforms.map(platform => { const { os, arch, baseline } = platform; - return { - key: getKey(platform), - group: getLabel(platform), - steps: [ + let steps = [ + ...testPlatforms + .filter(platform => platform.os === os && platform.arch === arch && baseline === platform.baseline) + .map(platform => getTestBunStep(platform)), + ]; + + if (!buildId) { + steps.unshift( getBuildVendorStep(platform), getBuildCppStep(platform), getBuildZigStep(platform), getBuildBunStep(platform), - ...testPlatforms - .filter(platform => platform.os === os && platform.arch === arch && baseline === platform.baseline) - .map(platform => getTestBunStep(platform)), - ], + ); + } + + return { + key: getKey(platform), + group: getLabel(platform), + steps, }; }), ], @@ -377,6 +439,7 @@ async function main() { console.log(" - Is Merge Queue:", isMergeQueue()); console.log(" - Is Pull Request:", isPullRequest()); + let buildId; const changedFiles = await getChangedFiles(); if (changedFiles) { console.log( @@ -389,11 +452,16 @@ async function main() { } if (changedFiles.every(filename => isTest(filename) || isDocumentation(filename))) { - // TODO: console.log("Since changed files contain tests, skipping build..."); + buildId = await getBuildIdWithArtifacts(); + if (buildId) { + console.log("Since changed files are only tests, using build artifacts from previous build...", buildId); + } else { + console.log("Changed files are only tests, but could not find previous build artifacts..."); + } } } - const pipeline = getPipeline(); + const pipeline = getPipeline(buildId); const content = toYaml(pipeline); const contentPath = join(process.cwd(), ".buildkite", "ci.yml"); writeFileSync(contentPath, content); diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs index 2d44f3f51c..2f9b069907 100755 --- a/scripts/runner.node.mjs +++ b/scripts/runner.node.mjs @@ -1010,9 +1010,16 @@ async function getExecPathFromBuildKite(target) { const releasePath = join(cwd, "release"); mkdirSync(releasePath, { recursive: true }); + + const args = ["artifact", "download", "**", releasePath, "--step", target]; + const buildId = process.env["BUILDKITE_ARTIFACT_BUILD_ID"]; + if (buildId) { + args.push("--build", buildId); + } + await spawnSafe({ command: "buildkite-agent", - args: ["artifact", "download", "**", releasePath, "--step", target], + args, }); let zipPath; From 7035a1107ebef293e4827ceaa7258e0281697646 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Thu, 31 Oct 2024 14:26:19 -0700 Subject: [PATCH 03/17] Fixes #14918 (#14921) --- src/bun.js/webcore/blob.zig | 6 ++++-- test/js/web/html/FormData.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/bun.js/webcore/blob.zig b/src/bun.js/webcore/blob.zig index de458d86c9..f5848a6e07 100644 --- a/src/bun.js/webcore/blob.zig +++ b/src/bun.js/webcore/blob.zig @@ -1542,7 +1542,7 @@ pub const Blob = struct { } } - this.reported_estimated_size = size + (this.content_type.len * @intFromBool(this.content_type_allocated)); + this.reported_estimated_size = size + (this.content_type.len * @intFromBool(this.content_type_allocated)) + this.name.byteSlice().len; } pub fn estimatedSize(this: *Blob) usize { @@ -3784,7 +3784,7 @@ pub const Blob = struct { return true; } if (value.isString()) { - this.name.deref(); + const old_name = this.name; this.name = bun.String.tryFromJS(value, globalThis) orelse { // Handle allocation failure. @@ -3793,6 +3793,7 @@ pub const Blob = struct { }; // We don't need to increment the reference count since tryFromJS already did it. Blob.nameSetCached(jsThis, globalThis, value); + old_name.deref(); return true; } return false; @@ -4181,6 +4182,7 @@ pub const Blob = struct { } else if (duped.content_type_allocated and duped.allocator != null and include_content_type) { duped.content_type = bun.default_allocator.dupe(u8, this.content_type) catch bun.outOfMemory(); } + duped.name = duped.name.dupeRef(); duped.allocator = null; return duped; diff --git a/test/js/web/html/FormData.test.ts b/test/js/web/html/FormData.test.ts index 6adaee48b0..83f35cb1b4 100644 --- a/test/js/web/html/FormData.test.ts +++ b/test/js/web/html/FormData.test.ts @@ -620,4 +620,29 @@ describe("FormData", () => { expect(fileSlice.size).toBe(result.size); }); }); + + // The minimum repro for this was to not call the .name and .type getter on the Blob + // But the crux of the issue is that we called dupe() on the Blob, without also incrementing the reference count of the name string. + // https://github.com/oven-sh/bun/issues/14918 + it("should increment reference count of the name string on Blob", async () => { + const buffer = new File([Buffer.from(Buffer.alloc(48 * 1024, "abcdefh").toString("base64"), "base64")], "ok.jpg"); + function test() { + let file = new File([buffer], "ok.jpg"); + file.name; + file.type; + + let formData = new FormData(); + formData.append("foo", file); + formData.get("foo"); + formData.get("foo")!.name; + formData.get("foo")!.type; + return formData; + } + for (let i = 0; i < 100000; i++) { + test(); + if (i % 5000 === 0) { + Bun.gc(); + } + } + }); }); From b1e9e3b31ba06d11389fb1878056975619b2c662 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Thu, 31 Oct 2024 17:54:07 -0700 Subject: [PATCH 04/17] Add `bytesWritten` property to Bun.Socket, fix encoding issue in node:net (#14516) Co-authored-by: Ciro Spaciari --- packages/bun-types/bun.d.ts | 5 + src/baby_list.zig | 2 - src/bun.js/api/bun/socket.zig | 466 ++++++++++++------ src/bun.js/api/sockets.classes.ts | 13 + src/bun.js/bindings/ErrorCode.ts | 3 + src/bun.js/bindings/bindings.zig | 6 +- src/bun.js/node/types.zig | 49 +- src/bun.js/webcore/encoding.zig | 46 +- src/deps/uws.zig | 77 +-- src/js/node/net.ts | 122 +++-- src/string.zig | 2 +- .../parallel/net-bytes-written-large.test.js | 73 +++ 12 files changed, 588 insertions(+), 276 deletions(-) create mode 100644 test/js/node/test/parallel/net-bytes-written-large.test.js diff --git a/packages/bun-types/bun.d.ts b/packages/bun-types/bun.d.ts index c3e4a4fe4b..b20dd1df66 100644 --- a/packages/bun-types/bun.d.ts +++ b/packages/bun-types/bun.d.ts @@ -4362,6 +4362,11 @@ declare module "bun" { * @param [size=16384] The maximum TLS fragment size. The maximum value is `16384`. */ setMaxSendFragment(size: number): boolean; + + /** + * The number of bytes written to the socket. + */ + readonly bytesWritten: number; } interface SocketListener extends Disposable { diff --git a/src/baby_list.zig b/src/baby_list.zig index f613dad125..adf41b1620 100644 --- a/src/baby_list.zig +++ b/src/baby_list.zig @@ -13,7 +13,6 @@ pub fn BabyList(comptime Type: type) type { cap: u32 = 0, pub const Elem = Type; - pub fn parse(input: *bun.css.Parser) bun.css.Result(ListType) { return switch (input.parseCommaSeparated(Type, bun.css.generic.parseFor(Type))) { .result => |v| return .{ .result = ListType{ @@ -36,7 +35,6 @@ pub fn BabyList(comptime Type: type) type { } return true; } - pub fn set(this: *@This(), slice_: []Type) void { this.ptr = slice_.ptr; this.len = @as(u32, @truncate(slice_.len)); diff --git a/src/bun.js/api/bun/socket.zig b/src/bun.js/api/bun/socket.zig index c29e7b98a5..7cc8057156 100644 --- a/src/bun.js/api/bun/socket.zig +++ b/src/bun.js/api/bun/socket.zig @@ -1315,16 +1315,15 @@ fn NewSocket(comptime ssl: bool) type { flags: Flags = .{}, ref_count: u32 = 1, - wrapped: WrappedType = .none, handlers: *Handlers, this_value: JSC.JSValue = .zero, poll_ref: Async.KeepAlive = Async.KeepAlive.init(), - last_4: [4]u8 = .{ 0, 0, 0, 0 }, connection: ?Listener.UnixOrHost = null, protos: ?[]const u8, server_name: ?[]const u8 = null, - bytesWritten: u64 = 0, + buffered_data_for_node_net: bun.ByteList = .{}, + bytes_written: u64 = 0, // TODO: switch to something that uses `visitAggregate` and have the // `Listener` keep a list of all the sockets JSValue in there @@ -1516,6 +1515,7 @@ fn NewSocket(comptime ssl: bool) type { // Ensure the socket is still alive for any defer's we have this.ref(); defer this.deref(); + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); const needs_deref = !this.socket.isDetached(); this.socket = Socket.detached; @@ -1597,6 +1597,8 @@ fn NewSocket(comptime ssl: bool) type { pub fn closeAndDetach(this: *This, code: uws.CloseCode) void { const socket = this.socket; + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); + this.socket.detach(); this.detachNativeCallback(); @@ -2021,14 +2023,9 @@ fn NewSocket(comptime ssl: bool) type { return JSValue.jsNumber(@as(i32, -1)); } - const args = callframe.arguments(4); + var args = callframe.argumentsUndef(5); - if (args.len == 0) { - globalObject.throw("Expected 1 - 4 arguments, got 0", .{}); - return .zero; - } - - return switch (this.writeOrEnd(globalObject, args.ptr[0..args.len], false)) { + return switch (this.writeOrEnd(globalObject, args.mut(), false, false)) { .fail => .zero, .success => |result| JSValue.jsNumber(result.wrote), }; @@ -2074,171 +2071,296 @@ fn NewSocket(comptime ssl: bool) type { return -1; } // we don't cork yet but we might later - if (comptime ssl) { // TLS wrapped but in TCP mode if (this.wrapped == .tcp) { const res = this.socket.rawWrite(buffer, is_end); - if (res > 0) { - this.bytesWritten += @intCast(res); - } + const uwrote: usize = @intCast(@max(res, 0)); + this.bytes_written += uwrote; log("write({d}, {any}) = {d}", .{ buffer.len, is_end, res }); return res; } } const res = this.socket.write(buffer, is_end); - if (res > 0) { - this.bytesWritten += @intCast(res); - } + const uwrote: usize = @intCast(@max(res, 0)); + this.bytes_written += uwrote; log("write({d}, {any}) = {d}", .{ buffer.len, is_end, res }); return res; } - fn writeOrEnd(this: *This, globalObject: *JSC.JSGlobalObject, args: []const JSC.JSValue, is_end: bool) WriteResult { - if (args.len == 0) return .{ .success = .{} }; - if (args.ptr[0].asArrayBuffer(globalObject)) |array_buffer| { - var slice = array_buffer.slice(); - - if (args.len > 1) { - if (!args.ptr[1].isAnyInt()) { - globalObject.throw("Expected offset integer, got {any}", .{args.ptr[1].getZigString(globalObject)}); - return .{ .fail = {} }; - } - - const offset = @min(args.ptr[1].toUInt64NoTruncate(), slice.len); - slice = slice[offset..]; - - if (args.len > 2) { - if (!args.ptr[2].isAnyInt()) { - globalObject.throw("Expected length integer, got {any}", .{args.ptr[2].getZigString(globalObject)}); - return .{ .fail = {} }; - } - - const length = @min(args.ptr[2].toUInt64NoTruncate(), slice.len); - slice = slice[0..length]; - } - } - - // sending empty can be used to ensure that we'll cycle through internal openssl's state - if (comptime ssl == false) { - if (slice.len == 0) return .{ .success = .{} }; - } - - return .{ - .success = .{ - .wrote = this.writeMaybeCorked(slice, is_end), - .total = slice.len, - }, - }; - } else if (args.ptr[0].jsType() == .DOMWrapper) { - const blob: JSC.WebCore.AnyBlob = getter: { - if (args.ptr[0].as(JSC.WebCore.Blob)) |blob| { - break :getter JSC.WebCore.AnyBlob{ .Blob = blob.* }; - } else if (args.ptr[0].as(JSC.WebCore.Response)) |response| { - response.body.value.toBlobIfPossible(); - - if (response.body.value.tryUseAsAnyBlob()) |blob| { - break :getter blob; - } - - globalObject.throw("Only Blob/buffered bodies are supported for now", .{}); - return .{ .fail = {} }; - } else if (args.ptr[0].as(JSC.WebCore.Request)) |request| { - request.body.value.toBlobIfPossible(); - if (request.body.value.tryUseAsAnyBlob()) |blob| { - break :getter blob; - } - - globalObject.throw("Only Blob/buffered bodies are supported for now", .{}); - return .{ .fail = {} }; - } - - globalObject.throw("Expected Blob, Request or Response", .{}); - return .{ .fail = {} }; - }; - - if (!blob.needsToReadFile()) { - var slice = blob.slice(); - - if (args.len > 1) { - if (!args.ptr[1].isAnyInt()) { - globalObject.throw("Expected offset integer, got {any}", .{args.ptr[1].getZigString(globalObject)}); - return .{ .fail = {} }; - } - - const offset = @min(args.ptr[1].toUInt64NoTruncate(), slice.len); - slice = slice[offset..]; - - if (args.len > 2) { - if (!args.ptr[2].isAnyInt()) { - globalObject.throw("Expected length integer, got {any}", .{args.ptr[2].getZigString(globalObject)}); - return .{ .fail = {} }; - } - - const length = @min(args.ptr[2].toUInt64NoTruncate(), slice.len); - slice = slice[0..length]; - } - } - - // sending empty can be used to ensure that we'll cycle through internal openssl's state - if (comptime ssl == false) { - if (slice.len == 0) return .{ .success = .{} }; - } - - return .{ - .success = .{ - .wrote = this.writeMaybeCorked(slice, is_end), - .total = slice.len, - }, - }; - } - - globalObject.throw("sendfile() not implemented yet", .{}); - return .{ .fail = {} }; - } else if (bun.String.tryFromJS(args.ptr[0], globalObject)) |bun_str| { - defer bun_str.deref(); - var zig_str = bun_str.toUTF8WithoutRef(bun.default_allocator); - defer zig_str.deinit(); - - var slice = zig_str.slice(); - - if (args.len > 1) { - if (!args.ptr[1].isAnyInt()) { - globalObject.throw("Expected offset integer, got {any}", .{args.ptr[1].getZigString(globalObject)}); - return .{ .fail = {} }; - } - - const offset = @min(args.ptr[1].toUInt64NoTruncate(), slice.len); - slice = slice[offset..]; - - if (args.len > 2) { - if (!args.ptr[2].isAnyInt()) { - globalObject.throw("Expected length integer, got {any}", .{args.ptr[2].getZigString(globalObject)}); - return .{ .fail = {} }; - } - - const length = @min(args.ptr[2].toUInt64NoTruncate(), slice.len); - slice = slice[0..length]; - } - } - - // sending empty can be used to ensure that we'll cycle through internal openssl's state - if (comptime ssl == false) { - if (slice.len == 0) return .{ .success = .{} }; - } - - return .{ - .success = .{ - .wrote = this.writeMaybeCorked(slice, is_end), - .total = slice.len, - }, - }; - } else { - if (!globalObject.hasException()) - globalObject.throw("Expected ArrayBufferView, a string, or a Blob", .{}); - return .{ .fail = {} }; + pub fn writeBuffered( + this: *This, + globalObject: *JSC.JSGlobalObject, + callframe: *JSC.CallFrame, + ) JSValue { + if (this.socket.isDetached()) { + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); + return JSValue.jsBoolean(false); } + + const args = callframe.argumentsUndef(2); + + return switch (this.writeOrEndBuffered(globalObject, args.ptr[0], args.ptr[1], false)) { + .fail => .zero, + .success => |result| if (@max(result.wrote, 0) == result.total) .true else .false, + }; + } + + pub fn endBuffered( + this: *This, + globalObject: *JSC.JSGlobalObject, + callframe: *JSC.CallFrame, + ) JSValue { + if (this.socket.isDetached()) { + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); + return JSValue.jsBoolean(false); + } + + const args = callframe.argumentsUndef(2); + + return switch (this.writeOrEndBuffered(globalObject, args.ptr[0], args.ptr[1], false)) { + .fail => .zero, + .success => |result| brk: { + if (result.wrote == result.total) { + this.socket.flush(); + // markInactive does .detached = true + this.markInactive(); + } + + break :brk JSValue.jsBoolean(@as(usize, @max(result.wrote, 0)) == result.total); + }, + }; + } + + fn writeOrEndBuffered(this: *This, globalObject: *JSC.JSGlobalObject, data_value: JSC.JSValue, encoding_value: JSC.JSValue, comptime is_end: bool) WriteResult { + if (this.buffered_data_for_node_net.len == 0) { + var values = [4]JSC.JSValue{ data_value, .undefined, .undefined, encoding_value }; + return this.writeOrEnd(globalObject, &values, true, is_end); + } + + var stack_fallback = std.heap.stackFallback(16 * 1024, bun.default_allocator); + const buffer: JSC.Node.StringOrBuffer = if (data_value.isUndefined()) + JSC.Node.StringOrBuffer.empty + else + JSC.Node.StringOrBuffer.fromJSWithEncodingValueMaybeAsync(globalObject, stack_fallback.get(), data_value, encoding_value, false) orelse { + if (!globalObject.hasException()) { + _ = globalObject.throwInvalidArgumentTypeValue("data", "string, buffer, or blob", data_value); + } + return .fail; + }; + defer buffer.deinit(); + if (this.socket.isShutdown() or this.socket.isClosed()) { + return .{ + .success = .{ + .wrote = -1, + .total = buffer.slice().len + this.buffered_data_for_node_net.len, + }, + }; + } + + const total_to_write: usize = buffer.slice().len + @as(usize, this.buffered_data_for_node_net.len); + if (total_to_write == 0) { + return .{ .success = .{} }; + } + + const wrote: i32 = brk: { + if (comptime !ssl and Environment.isPosix) { + // fast-ish path: use writev() to avoid cloning to another buffer. + if (this.socket.socket == .connected and buffer.slice().len > 0) { + const rc = this.socket.socket.connected.write2(this.buffered_data_for_node_net.slice(), buffer.slice()); + const written: usize = @intCast(@max(rc, 0)); + const leftover = total_to_write -| written; + if (leftover == 0) { + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); + this.buffered_data_for_node_net = .{}; + break :brk rc; + } + + const remaining_in_buffered_data = this.buffered_data_for_node_net.slice()[@min(written, this.buffered_data_for_node_net.len)..]; + const remaining_in_input_data = buffer.slice()[@min(this.buffered_data_for_node_net.len -| written, buffer.slice().len)..]; + + if (written > 0) { + if (remaining_in_buffered_data.len > 0) { + var input_buffer = this.buffered_data_for_node_net.slice(); + bun.C.memmove(input_buffer.ptr, input_buffer.ptr[written..], remaining_in_buffered_data.len); + this.buffered_data_for_node_net.len = @truncate(remaining_in_buffered_data.len); + } + } + + if (remaining_in_input_data.len > 0) { + this.buffered_data_for_node_net.append(bun.default_allocator, remaining_in_input_data) catch bun.outOfMemory(); + } + + break :brk rc; + } + } + + // slower-path: clone the data, do one write. + this.buffered_data_for_node_net.append(bun.default_allocator, buffer.slice()) catch bun.outOfMemory(); + const rc = this.writeMaybeCorked(this.buffered_data_for_node_net.slice(), is_end); + if (rc > 0) { + const wrote: usize = @intCast(@max(rc, 0)); + // did we write everything? + // we can free this temporary buffer. + if (wrote == this.buffered_data_for_node_net.len) { + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); + this.buffered_data_for_node_net = .{}; + } else { + // Otherwise, let's move the temporary buffer back. + const len = @as(usize, @intCast(this.buffered_data_for_node_net.len)) - wrote; + bun.debugAssert(len <= this.buffered_data_for_node_net.len); + bun.debugAssert(len <= this.buffered_data_for_node_net.cap); + bun.C.memmove(this.buffered_data_for_node_net.ptr, this.buffered_data_for_node_net.ptr[wrote..], len); + this.buffered_data_for_node_net.len = @truncate(len); + } + } + + break :brk rc; + }; + + return .{ + .success = .{ + .wrote = wrote, + .total = total_to_write, + }, + }; + } + + fn writeOrEnd(this: *This, globalObject: *JSC.JSGlobalObject, args: []JSC.JSValue, buffer_unwritten_data: bool, comptime is_end: bool) WriteResult { + if (args[0].isUndefined()) return .{ .success = .{} }; + + bun.debugAssert(this.buffered_data_for_node_net.len == 0); + var encoding_value: JSC.JSValue = args[3]; + if (args[2].isString()) { + encoding_value = args[2]; + args[2] = .undefined; + } else if (args[1].isString()) { + encoding_value = args[1]; + args[1] = .undefined; + } + + const offset_value = args[1]; + const length_value = args[2]; + + if (encoding_value != .undefined and (offset_value != .undefined or length_value != .undefined)) { + globalObject.throwTODO("Support encoding with offset and length altogether. Only either encoding or offset, length is supported, but not both combinations yet."); + return .fail; + } + + var stack_fallback = std.heap.stackFallback(16 * 1024, bun.default_allocator); + const buffer: JSC.Node.BlobOrStringOrBuffer = if (args[0].isUndefined()) + JSC.Node.BlobOrStringOrBuffer{ .string_or_buffer = JSC.Node.StringOrBuffer.empty } + else + JSC.Node.BlobOrStringOrBuffer.fromJSWithEncodingValueMaybeAsyncAllowRequestResponse(globalObject, stack_fallback.get(), args[0], encoding_value, false, true) orelse { + if (!globalObject.hasException()) { + _ = globalObject.throwInvalidArgumentTypeValue("data", "string, buffer, or blob", args[0]); + } + return .fail; + }; + + defer buffer.deinit(); + if (buffer == .blob and buffer.blob.needsToReadFile()) { + globalObject.throw("File blob not supported yet in this function.", .{}); + return .fail; + } + + const label = if (comptime is_end) "end" else "write"; + + const byte_offset: usize = brk: { + if (offset_value.isUndefined()) break :brk 0; + if (!offset_value.isAnyInt()) { + globalObject.throwInvalidArgumentType(comptime "Socket." ++ label, "byteOffset", "integer"); + return .fail; + } + const i = offset_value.toInt64(); + if (i < 0) { + globalObject.throwRangeError(i, .{ + .field_name = "byteOffset", + .min = 0, + .max = JSC.MAX_SAFE_INTEGER, + }); + return .fail; + } + break :brk @intCast(i); + }; + + const byte_length: usize = brk: { + if (length_value.isUndefined()) break :brk buffer.slice().len; + if (!length_value.isAnyInt()) { + globalObject.throwInvalidArgumentType(comptime "Socket." ++ label, "byteLength", "integer"); + return .fail; + } + + const l = length_value.toInt64(); + + if (l < 0) { + globalObject.throwRangeError(l, .{ + .field_name = "byteLength", + .min = 0, + .max = JSC.MAX_SAFE_INTEGER, + }); + return .fail; + } + break :brk @intCast(l); + }; + + var bytes = buffer.slice(); + + if (byte_offset > bytes.len) { + globalObject.throwRangeError(@as(i64, @intCast(byte_offset)), .{ + .field_name = "byteOffset", + .min = 0, + .max = @intCast(bytes.len), + }); + return .fail; + } + + bytes = bytes[byte_offset..]; + + if (byte_length > bytes.len) { + globalObject.throwRangeError(@as(i64, @intCast(byte_length)), .{ + .field_name = "byteLength", + .min = 0, + .max = @intCast(bytes.len), + }); + return .fail; + } + + bytes = bytes[0..byte_length]; + + if (bytes.len == 0) { + return .{ .success = .{} }; + } + + if (globalObject.hasException()) { + return .fail; + } + + if (this.socket.isShutdown() or this.socket.isClosed()) { + return .{ + .success = .{ + .wrote = -1, + .total = bytes.len, + }, + }; + } + + const wrote = this.writeMaybeCorked(bytes, is_end); + const uwrote: usize = @intCast(@max(wrote, 0)); + if (buffer_unwritten_data) { + const remaining = bytes[uwrote..]; + if (remaining.len > 0) { + this.buffered_data_for_node_net.append(bun.default_allocator, remaining) catch bun.outOfMemory(); + } + } + + return .{ + .success = .{ + .wrote = wrote, + .total = bytes.len, + }, + }; } pub fn flush( @@ -2247,6 +2369,21 @@ fn NewSocket(comptime ssl: bool) type { _: *JSC.CallFrame, ) JSValue { JSC.markBinding(@src()); + if (this.buffered_data_for_node_net.len > 0) { + const written: usize = @intCast(@max(this.socket.write(this.buffered_data_for_node_net.slice(), false), 0)); + + if (written > 0) { + if (this.buffered_data_for_node_net.len > written) { + const remaining = this.buffered_data_for_node_net.slice()[written..]; + bun.C.memmove(this.buffered_data_for_node_net.ptr, remaining.ptr, remaining.len); + this.buffered_data_for_node_net.len = @truncate(remaining.len); + } else { + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); + this.buffered_data_for_node_net = .{}; + } + } + } + this.socket.flush(); return JSValue.jsUndefined(); @@ -2269,6 +2406,7 @@ fn NewSocket(comptime ssl: bool) type { ) JSValue { JSC.markBinding(@src()); const args = callframe.arguments(1); + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); if (args.len > 0 and args.ptr[0].toBoolean()) { this.socket.shutdownRead(); } else { @@ -2285,7 +2423,7 @@ fn NewSocket(comptime ssl: bool) type { ) JSValue { JSC.markBinding(@src()); - const args = callframe.arguments(4); + var args = callframe.argumentsUndef(5); log("end({d} args)", .{args.len}); @@ -2293,7 +2431,7 @@ fn NewSocket(comptime ssl: bool) type { return JSValue.jsNumber(@as(i32, -1)); } - return switch (this.writeOrEnd(globalObject, args.ptr[0..args.len], true)) { + return switch (this.writeOrEnd(globalObject, args.mut(), false, true)) { .fail => .zero, .success => |result| brk: { if (result.wrote == result.total) { @@ -2323,6 +2461,8 @@ fn NewSocket(comptime ssl: bool) type { this.markInactive(); this.detachNativeCallback(); + this.buffered_data_for_node_net.deinitWithAllocator(bun.default_allocator); + this.poll_ref.unref(JSC.VirtualMachine.get()); // need to deinit event without being attached if (this.flags.owned_protos) { @@ -2564,7 +2704,7 @@ fn NewSocket(comptime ssl: bool) type { this: *This, _: *JSC.JSGlobalObject, ) JSValue { - return JSC.JSValue.jsNumber(this.bytesWritten); + return JSC.JSValue.jsNumber(this.bytes_written + this.buffered_data_for_node_net.len); } pub fn getALPNProtocol( this: *This, diff --git a/src/bun.js/api/sockets.classes.ts b/src/bun.js/api/sockets.classes.ts index 3b306cf810..ee3e60e1df 100644 --- a/src/bun.js/api/sockets.classes.ts +++ b/src/bun.js/api/sockets.classes.ts @@ -166,6 +166,9 @@ function generate(ssl) { fn: "reload", length: 1, }, + bytesWritten: { + getter: "getBytesWritten", + }, setServername: { fn: "setServername", @@ -175,6 +178,16 @@ function generate(ssl) { fn: "getServername", length: 0, }, + "writeBuffered": { + fn: "writeBuffered", + length: 2, + privateSymbol: "write", + }, + "endBuffered": { + fn: "endBuffered", + length: 2, + privateSymbol: "end", + }, }, finalize: true, construct: true, diff --git a/src/bun.js/bindings/ErrorCode.ts b/src/bun.js/bindings/ErrorCode.ts index a2b2bd96ec..692ad34a3e 100644 --- a/src/bun.js/bindings/ErrorCode.ts +++ b/src/bun.js/bindings/ErrorCode.ts @@ -52,6 +52,9 @@ export default [ ["ERR_STREAM_WRAP", Error, "Error"], ["ERR_BORINGSSL", Error, "Error"], + //NET + ["ERR_SOCKET_CLOSED_BEFORE_CONNECTION", Error, "Error"], + //HTTP2 ["ERR_INVALID_HTTP_TOKEN", TypeError, "TypeError"], ["ERR_HTTP2_PSEUDOHEADER_NOT_ALLOWED", TypeError, "TypeError"], diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index d3ad3bfb0b..27ff465b7c 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -208,9 +208,13 @@ pub const ZigString = extern struct { } pub fn encode(this: ZigString, encoding: JSC.Node.Encoding) []u8 { + return this.encodeWithAllocator(bun.default_allocator, encoding); + } + + pub fn encodeWithAllocator(this: ZigString, allocator: std.mem.Allocator, encoding: JSC.Node.Encoding) []u8 { return switch (this.as()) { inline else => |repr| switch (encoding) { - inline else => |enc| JSC.WebCore.Encoder.constructFrom(std.meta.Child(@TypeOf(repr)), repr, enc), + inline else => |enc| JSC.WebCore.Encoder.constructFrom(std.meta.Child(@TypeOf(repr)), repr, allocator, enc), }, }; } diff --git a/src/bun.js/node/types.zig b/src/bun.js/node/types.zig index 9b7b7d9ecc..8ff71ffc01 100644 --- a/src/bun.js/node/types.zig +++ b/src/bun.js/node/types.zig @@ -375,12 +375,51 @@ pub const BlobOrStringOrBuffer = union(enum) { } pub fn fromJSWithEncodingValueMaybeAsync(global: *JSC.JSGlobalObject, allocator: std.mem.Allocator, value: JSC.JSValue, encoding_value: JSC.JSValue, is_async: bool) ?BlobOrStringOrBuffer { - if (value.as(JSC.WebCore.Blob)) |blob| { - if (blob.store) |store| { - store.ref(); - } - return .{ .blob = blob.* }; + return fromJSWithEncodingValueMaybeAsyncAllowRequestResponse(global, allocator, value, encoding_value, is_async, false); + } + + pub fn fromJSWithEncodingValueMaybeAsyncAllowRequestResponse(global: *JSC.JSGlobalObject, allocator: std.mem.Allocator, value: JSC.JSValue, encoding_value: JSC.JSValue, is_async: bool, allow_request_response: bool) ?BlobOrStringOrBuffer { + switch (value.jsType()) { + .Blob => { + if (value.as(JSC.WebCore.Blob)) |blob| { + if (blob.store) |store| { + store.ref(); + } + return .{ .blob = blob.* }; + } + }, + .DOMWrapper => { + if (allow_request_response) { + if (value.as(JSC.WebCore.Request)) |request| { + request.body.value.toBlobIfPossible(); + + if (request.body.value.tryUseAsAnyBlob()) |any_blob_| { + var any_blob = any_blob_; + defer any_blob.detach(); + return .{ .blob = any_blob.toBlob(global) }; + } + + global.throwInvalidArguments("Only buffered Request/Response bodies are supported for now.", .{}); + return null; + } + + if (value.as(JSC.WebCore.Response)) |response| { + response.body.value.toBlobIfPossible(); + + if (response.body.value.tryUseAsAnyBlob()) |any_blob_| { + var any_blob = any_blob_; + defer any_blob.detach(); + return .{ .blob = any_blob.toBlob(global) }; + } + + global.throwInvalidArguments("Only buffered Request/Response bodies are supported for now.", .{}); + return null; + } + } + }, + else => {}, } + return .{ .string_or_buffer = StringOrBuffer.fromJSWithEncodingValueMaybeAsync(global, allocator, value, encoding_value, is_async) orelse return null }; } }; diff --git a/src/bun.js/webcore/encoding.zig b/src/bun.js/webcore/encoding.zig index cd8389eefd..608750efbe 100644 --- a/src/bun.js/webcore/encoding.zig +++ b/src/bun.js/webcore/encoding.zig @@ -1039,27 +1039,27 @@ pub const Encoder = struct { } export fn Bun__encoding__constructFromLatin1(globalObject: *JSGlobalObject, input: [*]const u8, len: usize, encoding: u8) JSValue { const slice = switch (@as(JSC.Node.Encoding, @enumFromInt(encoding))) { - .hex => constructFromU8(input, len, .hex), - .ascii => constructFromU8(input, len, .ascii), - .base64url => constructFromU8(input, len, .base64url), - .utf16le => constructFromU8(input, len, .utf16le), - .ucs2 => constructFromU8(input, len, .utf16le), - .utf8 => constructFromU8(input, len, .utf8), - .base64 => constructFromU8(input, len, .base64), + .hex => constructFromU8(input, len, bun.default_allocator, .hex), + .ascii => constructFromU8(input, len, bun.default_allocator, .ascii), + .base64url => constructFromU8(input, len, bun.default_allocator, .base64url), + .utf16le => constructFromU8(input, len, bun.default_allocator, .utf16le), + .ucs2 => constructFromU8(input, len, bun.default_allocator, .utf16le), + .utf8 => constructFromU8(input, len, bun.default_allocator, .utf8), + .base64 => constructFromU8(input, len, bun.default_allocator, .base64), else => unreachable, }; return JSC.JSValue.createBuffer(globalObject, slice, globalObject.bunVM().allocator); } export fn Bun__encoding__constructFromUTF16(globalObject: *JSGlobalObject, input: [*]const u16, len: usize, encoding: u8) JSValue { const slice = switch (@as(JSC.Node.Encoding, @enumFromInt(encoding))) { - .base64 => constructFromU16(input, len, .base64), - .hex => constructFromU16(input, len, .hex), - .base64url => constructFromU16(input, len, .base64url), - .utf16le => constructFromU16(input, len, .utf16le), - .ucs2 => constructFromU16(input, len, .utf16le), - .utf8 => constructFromU16(input, len, .utf8), - .ascii => constructFromU16(input, len, .ascii), - .latin1 => constructFromU16(input, len, .latin1), + .base64 => constructFromU16(input, len, bun.default_allocator, .base64), + .hex => constructFromU16(input, len, bun.default_allocator, .hex), + .base64url => constructFromU16(input, len, bun.default_allocator, .base64url), + .utf16le => constructFromU16(input, len, bun.default_allocator, .utf16le), + .ucs2 => constructFromU16(input, len, bun.default_allocator, .utf16le), + .utf8 => constructFromU16(input, len, bun.default_allocator, .utf8), + .ascii => constructFromU16(input, len, bun.default_allocator, .ascii), + .latin1 => constructFromU16(input, len, bun.default_allocator, .latin1), else => unreachable, }; return JSC.JSValue.createBuffer(globalObject, slice, globalObject.bunVM().allocator); @@ -1440,19 +1440,17 @@ pub const Encoder = struct { } } - pub fn constructFrom(comptime T: type, input: []const T, comptime encoding: JSC.Node.Encoding) []u8 { + pub fn constructFrom(comptime T: type, input: []const T, allocator: std.mem.Allocator, comptime encoding: JSC.Node.Encoding) []u8 { return switch (comptime T) { - u16 => constructFromU16(input.ptr, input.len, encoding), - u8 => constructFromU8(input.ptr, input.len, encoding), + u16 => constructFromU16(input.ptr, input.len, allocator, encoding), + u8 => constructFromU8(input.ptr, input.len, allocator, encoding), else => @compileError("Unsupported type for constructFrom: " ++ @typeName(T)), }; } - pub fn constructFromU8(input: [*]const u8, len: usize, comptime encoding: JSC.Node.Encoding) []u8 { + pub fn constructFromU8(input: [*]const u8, len: usize, allocator: std.mem.Allocator, comptime encoding: JSC.Node.Encoding) []u8 { if (len == 0) return &[_]u8{}; - const allocator = bun.default_allocator; - switch (comptime encoding) { .buffer => { var to = allocator.alloc(u8, len) catch return &[_]u8{}; @@ -1500,11 +1498,9 @@ pub const Encoder = struct { } } - pub fn constructFromU16(input: [*]const u16, len: usize, comptime encoding: JSC.Node.Encoding) []u8 { + pub fn constructFromU16(input: [*]const u16, len: usize, allocator: std.mem.Allocator, comptime encoding: JSC.Node.Encoding) []u8 { if (len == 0) return &[_]u8{}; - const allocator = bun.default_allocator; - switch (comptime encoding) { .utf8 => { return strings.toUTF8AllocWithType(allocator, []const u16, input[0..len]) catch return &[_]u8{}; @@ -1532,7 +1528,7 @@ pub const Encoder = struct { // shouldn't really happen though const transcoded = strings.toUTF8Alloc(allocator, input[0..len]) catch return &[_]u8{}; defer allocator.free(transcoded); - return constructFromU8(transcoded.ptr, transcoded.len, encoding); + return constructFromU8(transcoded.ptr, transcoded.len, allocator, encoding); }, } } diff --git a/src/deps/uws.zig b/src/deps/uws.zig index e6d75a3244..04591f1bf8 100644 --- a/src/deps/uws.zig +++ b/src/deps/uws.zig @@ -9,7 +9,14 @@ pub const u_int32_t = c_uint; pub const u_int64_t = c_ulonglong; pub const LIBUS_LISTEN_DEFAULT: i32 = 0; pub const LIBUS_LISTEN_EXCLUSIVE_PORT: i32 = 1; -pub const Socket = opaque {}; +pub const Socket = opaque { + pub fn write2(this: *Socket, first: []const u8, second: []const u8) i32 { + const rc = us_socket_write2(0, this, first.ptr, first.len, second.ptr, second.len); + debug("us_socket_write2({d}, {d}) = {d}", .{ first.len, second.len, rc }); + return rc; + } + extern "C" fn us_socket_write2(ssl: i32, *Socket, header: ?[*]const u8, len: usize, payload: ?[*]const u8, usize) i32; +}; pub const ConnectingSocket = opaque {}; const debug = bun.Output.scoped(.uws, false); const uws = @This(); @@ -1084,7 +1091,7 @@ pub const WindowsNamedPipe = if (Environment.isWindows) struct { } else void; pub const InternalSocket = union(enum) { - done: *Socket, + connected: *Socket, connecting: *ConnectingSocket, detached: void, upgradedDuplex: *UpgradedDuplex, @@ -1101,7 +1108,7 @@ pub const InternalSocket = union(enum) { pub fn close(this: InternalSocket, comptime is_ssl: bool, code: CloseCode) void { switch (this) { .detached => {}, - .done => |socket| { + .connected => |socket| { debug("us_socket_close({d})", .{@intFromPtr(socket)}); _ = us_socket_close( comptime @intFromBool(is_ssl), @@ -1128,7 +1135,7 @@ pub const InternalSocket = union(enum) { pub fn isClosed(this: InternalSocket, comptime is_ssl: bool) bool { return switch (this) { - .done => |socket| us_socket_is_closed(@intFromBool(is_ssl), socket) > 0, + .connected => |socket| us_socket_is_closed(@intFromBool(is_ssl), socket) > 0, .connecting => |socket| us_connecting_socket_is_closed(@intFromBool(is_ssl), socket) > 0, .detached => true, .upgradedDuplex => |socket| socket.isClosed(), @@ -1138,7 +1145,7 @@ pub const InternalSocket = union(enum) { pub fn get(this: @This()) ?*Socket { return switch (this) { - .done => this.done, + .connected => this.connected, .connecting => null, .detached => null, .upgradedDuplex => null, @@ -1148,25 +1155,25 @@ pub const InternalSocket = union(enum) { pub fn eq(this: @This(), other: @This()) bool { return switch (this) { - .done => switch (other) { - .done => this.done == other.done, + .connected => switch (other) { + .connected => this.connected == other.connected, .upgradedDuplex, .connecting, .detached, .pipe => false, }, .connecting => switch (other) { - .upgradedDuplex, .done, .detached, .pipe => false, + .upgradedDuplex, .connected, .detached, .pipe => false, .connecting => this.connecting == other.connecting, }, .detached => switch (other) { .detached => true, - .upgradedDuplex, .done, .connecting, .pipe => false, + .upgradedDuplex, .connected, .connecting, .pipe => false, }, .upgradedDuplex => switch (other) { .upgradedDuplex => this.upgradedDuplex == other.upgradedDuplex, - .done, .connecting, .detached, .pipe => false, + .connected, .connecting, .detached, .pipe => false, }, .pipe => switch (other) { .pipe => if (Environment.isWindows) other.pipe == other.pipe else false, - .done, .connecting, .detached, .upgradedDuplex => false, + .connected, .connecting, .detached, .upgradedDuplex => false, }, }; } @@ -1189,7 +1196,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { } pub fn verifyError(this: ThisSocket) us_bun_verify_error_t { switch (this.socket) { - .done => |socket| return uws.us_socket_verify_error(comptime ssl_int, socket), + .connected => |socket| return uws.us_socket_verify_error(comptime ssl_int, socket), .upgradedDuplex => |socket| return socket.sslError(), .pipe => |pipe| if (Environment.isWindows) return pipe.sslError() else return std.mem.zeroes(us_bun_verify_error_t), .connecting, .detached => return std.mem.zeroes(us_bun_verify_error_t), @@ -1198,7 +1205,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn isEstablished(this: ThisSocket) bool { switch (this.socket) { - .done => |socket| return us_socket_is_established(comptime ssl_int, socket) > 0, + .connected => |socket| return us_socket_is_established(comptime ssl_int, socket) > 0, .upgradedDuplex => |socket| return socket.isEstablished(), .pipe => |pipe| if (Environment.isWindows) return pipe.isEstablished() else return false, .connecting, .detached => return false, @@ -1209,7 +1216,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { switch (this.socket) { .upgradedDuplex => |socket| socket.setTimeout(seconds), .pipe => |pipe| if (Environment.isWindows) pipe.setTimeout(seconds), - .done => |socket| us_socket_timeout(comptime ssl_int, socket, seconds), + .connected => |socket| us_socket_timeout(comptime ssl_int, socket, seconds), .connecting => |socket| us_connecting_socket_timeout(comptime ssl_int, socket, seconds), .detached => {}, } @@ -1217,7 +1224,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn setTimeout(this: ThisSocket, seconds: c_uint) void { switch (this.socket) { - .done => |socket| { + .connected => |socket| { if (seconds > 240) { us_socket_timeout(comptime ssl_int, socket, 0); us_socket_long_timeout(comptime ssl_int, socket, seconds / 60); @@ -1243,7 +1250,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn setTimeoutMinutes(this: ThisSocket, minutes: c_uint) void { switch (this.socket) { - .done => |socket| { + .connected => |socket| { us_socket_timeout(comptime ssl_int, socket, 0); us_socket_long_timeout(comptime ssl_int, socket, minutes); }, @@ -1403,7 +1410,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn getNativeHandle(this: ThisSocket) ?*NativeSocketHandleType(is_ssl) { return @ptrCast(switch (this.socket) { - .done => |socket| us_socket_get_native_handle(comptime ssl_int, socket), + .connected => |socket| us_socket_get_native_handle(comptime ssl_int, socket), .connecting => |socket| us_connecting_socket_get_native_handle(comptime ssl_int, socket), .detached => null, .upgradedDuplex => |socket| if (is_ssl) @as(*anyopaque, @ptrCast(socket.ssl() orelse return null)) else null, @@ -1438,7 +1445,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { std.meta.alignment(ContextType); const ptr = switch (this.socket) { - .done => |sock| us_socket_ext(comptime ssl_int, sock), + .connected => |sock| us_socket_ext(comptime ssl_int, sock), .connecting => |sock| us_connecting_socket_ext(comptime ssl_int, sock), .detached => return null, .upgradedDuplex => return null, @@ -1451,7 +1458,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { /// This can be null if the socket was closed. pub fn context(this: ThisSocket) ?*SocketContext { switch (this.socket) { - .done => |socket| return us_socket_context(comptime ssl_int, socket), + .connected => |socket| return us_socket_context(comptime ssl_int, socket), .connecting => |socket| return us_connecting_socket_context(comptime ssl_int, socket), .detached => return null, .upgradedDuplex => return null, @@ -1467,7 +1474,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { .pipe => |pipe| { return if (Environment.isWindows) pipe.flush() else return; }, - .done => |socket| { + .connected => |socket| { return us_socket_flush( comptime ssl_int, socket, @@ -1485,7 +1492,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { .pipe => |pipe| { return if (Environment.isWindows) pipe.encodeAndWrite(data, msg_more) else 0; }, - .done => |socket| { + .connected => |socket| { const result = us_socket_write( comptime ssl_int, socket, @@ -1507,7 +1514,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn rawWrite(this: ThisSocket, data: []const u8, msg_more: bool) i32 { switch (this.socket) { - .done => |socket| { + .connected => |socket| { return us_socket_raw_write( comptime ssl_int, socket, @@ -1529,7 +1536,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn shutdown(this: ThisSocket) void { // debug("us_socket_shutdown({d})", .{@intFromPtr(this.socket)}); switch (this.socket) { - .done => |socket| { + .connected => |socket| { return us_socket_shutdown( comptime ssl_int, socket, @@ -1553,7 +1560,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn shutdownRead(this: ThisSocket) void { switch (this.socket) { - .done => |socket| { + .connected => |socket| { // debug("us_socket_shutdown_read({d})", .{@intFromPtr(socket)}); return us_socket_shutdown_read( comptime ssl_int, @@ -1579,7 +1586,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn isShutdown(this: ThisSocket) bool { switch (this.socket) { - .done => |socket| { + .connected => |socket| { return us_socket_is_shut_down( comptime ssl_int, socket, @@ -1611,7 +1618,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { pub fn getError(this: ThisSocket) i32 { switch (this.socket) { - .done => |socket| { + .connected => |socket| { return us_socket_get_error( comptime ssl_int, socket, @@ -1642,7 +1649,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { } pub fn localPort(this: ThisSocket) i32 { switch (this.socket) { - .done => |socket| { + .connected => |socket| { return us_socket_local_port( comptime ssl_int, socket, @@ -1653,7 +1660,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { } pub fn remoteAddress(this: ThisSocket, buf: [*]u8, length: *i32) void { switch (this.socket) { - .done => |socket| { + .connected => |socket| { return us_socket_remote_address( comptime ssl_int, socket, @@ -1676,7 +1683,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { /// This function returns a slice of the buffer on success, or null on failure. pub fn localAddressBinary(this: ThisSocket, buf: []u8) ?[]const u8 { switch (this.socket) { - .done => |socket| { + .connected => |socket| { var length: i32 = @intCast(buf.len); us_socket_local_address( comptime ssl_int, @@ -1753,7 +1760,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { const socket = us_socket_context_connect(comptime ssl_int, socket_ctx, host_, port, 0, @sizeOf(Context), &did_dns_resolve) orelse return null; const socket_ = if (did_dns_resolve == 1) ThisSocket{ - .socket = .{ .done = @ptrCast(socket) }, + .socket = .{ .connected = @ptrCast(socket) }, } else ThisSocket{ @@ -1801,7 +1808,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { this: *This, comptime socket_field_name: ?[]const u8, ) ?ThisSocket { - const socket_ = ThisSocket{ .socket = .{ .done = us_socket_from_fd(ctx, @sizeOf(*anyopaque), bun.socketcast(handle)) orelse return null } }; + const socket_ = ThisSocket{ .socket = .{ .connected = us_socket_from_fd(ctx, @sizeOf(*anyopaque), bun.socketcast(handle)) orelse return null } }; if (socket_.ext(*anyopaque)) |holder| { holder.* = this; @@ -1840,7 +1847,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { const socket = us_socket_context_connect_unix(comptime ssl_int, socket_ctx, path_, path_.len, 0, 8) orelse return error.FailedToOpenSocket; - const socket_ = ThisSocket{ .socket = .{ .done = socket } }; + const socket_ = ThisSocket{ .socket = .{ .connected = socket } }; if (socket_.ext(*anyopaque)) |holder| { holder.* = ctx; } @@ -1878,7 +1885,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { ) orelse return error.FailedToOpenSocket; const socket = if (did_dns_resolve == 1) ThisSocket{ - .socket = .{ .done = @ptrCast(socket_ptr) }, + .socket = .{ .connected = @ptrCast(socket_ptr) }, } else ThisSocket{ @@ -2173,7 +2180,7 @@ pub fn NewSocketHandler(comptime is_ssl: bool) type { } pub fn from(socket: *Socket) ThisSocket { - return ThisSocket{ .socket = .{ .done = socket } }; + return ThisSocket{ .socket = .{ .connected = socket } }; } pub fn fromConnecting(connecting: *ConnectingSocket) ThisSocket { @@ -4294,7 +4301,7 @@ pub const AnySocket = union(enum) { pub fn getNativeHandle(this: AnySocket) ?*anyopaque { return switch (this.socket()) { - .done => |sock| us_socket_get_native_handle( + .connected => |sock| us_socket_get_native_handle( @intFromBool(this.isSSL()), sock, ).?, diff --git a/src/js/node/net.ts b/src/js/node/net.ts index 868de1d27b..63003ae684 100644 --- a/src/js/node/net.ts +++ b/src/js/node/net.ts @@ -71,21 +71,30 @@ const bunSocketServerConnections = Symbol.for("::bunnetserverconnections::"); const bunSocketServerOptions = Symbol.for("::bunnetserveroptions::"); const bunSocketInternal = Symbol.for("::bunnetsocketinternal::"); +const bunFinalCallback = Symbol("::bunFinalCallback::"); const kServerSocket = Symbol("kServerSocket"); +const kBytesWritten = Symbol("kBytesWritten"); const bunTLSConnectOptions = Symbol.for("::buntlsconnectoptions::"); const kRealListen = Symbol("kRealListen"); function endNT(socket, callback, err) { - socket.end(); + socket.$end(); callback(err); } function closeNT(callback, err) { callback(err); } -function detachAfterFinish() { - this[bunSocketInternal] = null; +function detachSocket(self) { + if (!self) self = this; + self[bunSocketInternal] = null; + const finalCallback = self[bunFinalCallback]; + if (finalCallback) { + self[bunFinalCallback] = null; + finalCallback(); + return; + } } var SocketClass; @@ -136,7 +145,6 @@ const Socket = (function (InternalSocket) { open(socket) { const self = socket.data; if (!self) return; - socket.timeout(Math.ceil(self.timeout / 1000)); if (self.#unrefOnConnected) socket.unref(); @@ -150,8 +158,8 @@ const Socket = (function (InternalSocket) { self.setSession(session); } } - if (!self.#upgraded) { + self[kBytesWritten] = socket.bytesWritten; // this is not actually emitted on nodejs when socket used on the connection // this is already emmited on non-TLS socket and on TLS socket is emmited secureConnect after handshake self.emit("connect", self); @@ -216,13 +224,7 @@ const Socket = (function (InternalSocket) { if (!self || self.#closed) return; self.#closed = true; //socket cannot be used after close - self[bunSocketInternal] = null; - const finalCallback = self.#final_callback; - if (finalCallback) { - self.#final_callback = null; - finalCallback(); - return; - } + detachSocket(self); if (!self.#ended) { const queue = self.#readQueue; if (queue.isEmpty()) { @@ -237,16 +239,16 @@ const Socket = (function (InternalSocket) { if (!self) return; const callback = self.#writeCallback; if (callback) { - const chunk = self.#writeChunk; - const written = socket.write(chunk); + const writeChunk = self._pendingData; - if (written < chunk.length) { - self.#writeChunk = chunk.slice(written); - } else { - self.#writeCallback = null; - self.#writeChunk = null; + if (socket.$write(writeChunk || "", "utf8")) { + self._pendingData = self.#writeCallback = null; callback(null); + } else { + self._pendingData = null; } + + self[kBytesWritten] = socket.bytesWritten; } } @@ -355,9 +357,10 @@ const Socket = (function (InternalSocket) { }; bytesRead = 0; + [kBytesWritten] = undefined; #closed = false; #ended = false; - #final_callback = null; + [bunFinalCallback] = null; connecting = false; localAddress = "127.0.0.1"; #readQueue = $createFIFO(); @@ -366,7 +369,8 @@ const Socket = (function (InternalSocket) { [bunTLSConnectOptions] = null; timeout = 0; #writeCallback; - #writeChunk; + _pendingData; + _pendingEncoding; // for compatibility #pendingRead; isServer = false; @@ -423,9 +427,6 @@ const Socket = (function (InternalSocket) { this.once("connect", () => this.emit("ready")); } - get bytesWritten() { - return this[bunSocketInternal]?.bytesWritten || 0; - } address() { return { address: this.localAddress, @@ -438,6 +439,34 @@ const Socket = (function (InternalSocket) { return this.writableLength; } + get _bytesDispatched() { + return this[kBytesWritten] || 0; + } + + get bytesWritten() { + let bytes = this[kBytesWritten] || 0; + const data = this._pendingData; + const writableBuffer = this.writableBuffer; + if (!writableBuffer) return undefined; + + for (const el of writableBuffer) { + bytes += el.chunk instanceof Buffer ? el.chunk.length : Buffer.byteLength(el.chunk, el.encoding); + } + + if ($isArray(data)) { + // Was a writev, iterate over chunks to get total length + for (let i = 0; i < data.length; i++) { + const chunk = data[i]; + + if (data.allBuffers || chunk instanceof Buffer) bytes += chunk.length; + else bytes += Buffer.byteLength(chunk.chunk, chunk.encoding); + } + } else if (data) { + bytes += data.byteLength; + } + return bytes; + } + #attach(port, socket) { this.remotePort = port; socket.data = this; @@ -446,6 +475,7 @@ const Socket = (function (InternalSocket) { this[bunSocketInternal] = socket; this.connecting = false; if (!this.#upgraded) { + this[kBytesWritten] = socket.bytesWritten; // this is not actually emitted on nodejs when socket used on the connection // this is already emmited on non-TLS socket and on TLS socket is emmited secureConnect after handshake this.emit("connect", this); @@ -566,6 +596,13 @@ const Socket = (function (InternalSocket) { // start using existing connection try { + // reset the underlying writable object when establishing a new connection + // this is a function on `Duplex`, originally defined on `Writable` + // https://github.com/nodejs/node/blob/c5cfdd48497fe9bd8dbd55fd1fca84b321f48ec1/lib/net.js#L311 + // https://github.com/nodejs/node/blob/c5cfdd48497fe9bd8dbd55fd1fca84b321f48ec1/lib/net.js#L1126 + this._undestroy(); + this.#readQueue = $createFIFO(); + if (connection) { const socket = connection[bunSocketInternal]; if (!upgradeDuplex && socket) { @@ -682,11 +719,6 @@ const Socket = (function (InternalSocket) { } catch (error) { process.nextTick(emitErrorAndCloseNextTick, this, error); } - // reset the underlying writable object when establishing a new connection - // this is a function on `Duplex`, originally defined on `Writable` - // https://github.com/nodejs/node/blob/c5cfdd48497fe9bd8dbd55fd1fca84b321f48ec1/lib/net.js#L311 - // https://github.com/nodejs/node/blob/c5cfdd48497fe9bd8dbd55fd1fca84b321f48ec1/lib/net.js#L1126 - this._undestroy(); return this; } @@ -703,10 +735,10 @@ const Socket = (function (InternalSocket) { if (this.writableFinished) { // closed we can detach the socket - this[bunSocketInternal] = null; + detachSocket(self); } else { // lets wait for the finish event before detaching the socket - this.once("finish", detachAfterFinish); + this.once("finish", detachSocket); } process.nextTick(closeNT, callback, err); } @@ -718,7 +750,7 @@ const Socket = (function (InternalSocket) { if (this.allowHalfOpen) { // wait socket close event - this.#final_callback = callback; + this[bunFinalCallback] = callback; } else { // emit FIN not allowing half open process.nextTick(endNT, socket, callback); @@ -817,24 +849,26 @@ const Socket = (function (InternalSocket) { } _write(chunk, encoding, callback) { - if (typeof chunk == "string" && encoding !== "ascii") chunk = Buffer.from(chunk, encoding); - var written = this[bunSocketInternal]?.write(chunk); + // If we are still connecting, then buffer this for later. + // The Writable logic will buffer up any more writes while + // waiting for this one to be done. + const socket = this[bunSocketInternal]; + if (!socket) { + // detached but connected? wait for the socket to be attached + this.#writeCallback = callback; + this._pendingEncoding = "buffer"; + this._pendingData = Buffer.from(chunk, encoding); + return; + } - if (written == chunk.length) { + const success = socket?.$write(chunk, encoding); + this[kBytesWritten] = socket.bytesWritten; + if (success) { callback(); } else if (this.#writeCallback) { callback(new Error("overlapping _write()")); } else { - if (written > 0) { - if (typeof chunk == "string") { - chunk = chunk.slice(written); - } else { - chunk = chunk.subarray(written); - } - } - this.#writeCallback = callback; - this.#writeChunk = chunk; } } }, diff --git a/src/string.zig b/src/string.zig index 742dfd7eea..930ee12dd5 100644 --- a/src/string.zig +++ b/src/string.zig @@ -882,7 +882,7 @@ pub const String = extern struct { } pub fn encode(self: String, enc: JSC.Node.Encoding) []u8 { - return self.toZigString().encode(enc); + return self.toZigString().encodeWithAllocator(bun.default_allocator, enc); } pub inline fn utf8(self: String) []const u8 { diff --git a/test/js/node/test/parallel/net-bytes-written-large.test.js b/test/js/node/test/parallel/net-bytes-written-large.test.js new file mode 100644 index 0000000000..715af6ecc7 --- /dev/null +++ b/test/js/node/test/parallel/net-bytes-written-large.test.js @@ -0,0 +1,73 @@ +//#FILE: test-net-bytes-written-large.js +//#SHA1: 9005801147f80a8058f1b2126d772e52abd1f237 +//----------------- +"use strict"; +const net = require("net"); + +const N = 10000000; + +describe("Net bytes written large", () => { + test("Write a Buffer", done => { + const server = net + .createServer(socket => { + socket.end(Buffer.alloc(N), () => { + expect(socket.bytesWritten).toBe(N); + }); + expect(socket.bytesWritten).toBe(N); + }) + .listen(0, () => { + const client = net.connect(server.address().port); + client.resume(); + client.on("close", () => { + expect(client.bytesRead).toBe(N); + server.close(); + done(); + }); + }); + }); + + test("Write a string", done => { + const server = net + .createServer(socket => { + socket.end("a".repeat(N), () => { + expect(socket.bytesWritten).toBe(N); + }); + expect(socket.bytesWritten).toBe(N); + }) + .listen(0, () => { + const client = net.connect(server.address().port); + client.resume(); + client.on("close", () => { + expect(client.bytesRead).toBe(N); + server.close(); + done(); + }); + }); + }); + + test("writev() with mixed data", done => { + const server = net + .createServer(socket => { + socket.cork(); + socket.write("a".repeat(N)); + expect(socket.bytesWritten).toBe(N); + socket.write(Buffer.alloc(N)); + expect(socket.bytesWritten).toBe(2 * N); + socket.end("", () => { + expect(socket.bytesWritten).toBe(2 * N); + }); + socket.uncork(); + }) + .listen(0, () => { + const client = net.connect(server.address().port); + client.resume(); + client.on("close", () => { + expect(client.bytesRead).toBe(2 * N); + server.close(); + done(); + }); + }); + }); +}); + +//<#END_FILE: test-net-bytes-written-large.js From 69332087900096334686e9f440fceb3c6fe32d56 Mon Sep 17 00:00:00 2001 From: Dylan Conway <35280289+dylan-conway@users.noreply.github.com> Date: Thu, 31 Oct 2024 18:21:04 -0700 Subject: [PATCH 05/17] fix(install): only globally link requested packages (#12506) Co-authored-by: Jarred Sumner --- src/install/bin.zig | 15 +-- src/install/install.zig | 111 ++++++++++-------- src/install/lockfile.zig | 7 +- .../registry/bun-install-registry.test.ts | 59 ++++++++++ 4 files changed, 134 insertions(+), 58 deletions(-) diff --git a/src/install/bin.zig b/src/install/bin.zig index e610529bd6..1667455d26 100644 --- a/src/install/bin.zig +++ b/src/install/bin.zig @@ -195,7 +195,7 @@ pub const Bin = extern struct { if (this.done) return null; if (this.dir_iterator == null) { var target = this.bin.value.dir.slice(this.string_buffer); - if (strings.hasPrefix(target, "./")) { + if (strings.hasPrefixComptime(target, "./") or strings.hasPrefixComptime(target, ".\\")) { target = target[2..]; } var parts = [_][]const u8{ this.package_name.slice(this.string_buffer), target }; @@ -229,7 +229,7 @@ pub const Bin = extern struct { this.i += 1; this.done = true; const base = std.fs.path.basename(this.package_name.slice(this.string_buffer)); - if (strings.hasPrefix(base, "./")) + if (strings.hasPrefixComptime(base, "./") or strings.hasPrefixComptime(base, ".\\")) return strings.copy(&this.buf, base[2..]); return strings.copy(&this.buf, base); @@ -239,7 +239,7 @@ pub const Bin = extern struct { this.i += 1; this.done = true; const base = std.fs.path.basename(this.bin.value.named_file[0].slice(this.string_buffer)); - if (strings.hasPrefix(base, "./")) + if (strings.hasPrefixComptime(base, "./") or strings.hasPrefixComptime(base, ".\\")) return strings.copy(&this.buf, base[2..]); return strings.copy(&this.buf, base); }, @@ -259,7 +259,7 @@ pub const Bin = extern struct { this.string_buffer, ), ); - if (strings.hasPrefix(base, "./")) + if (strings.hasPrefixComptime(base, "./") or strings.hasPrefixComptime(base, ".\\")) return strings.copy(&this.buf, base[2..]); return strings.copy(&this.buf, base); }, @@ -305,7 +305,6 @@ pub const Bin = extern struct { /// Used for generating relative paths package_name: strings.StringOrTinyString, - global_bin_dir: std.fs.Dir, global_bin_path: stringZ = "", string_buf: []const u8, @@ -345,8 +344,8 @@ pub const Bin = extern struct { } fn linkBinOrCreateShim(this: *Linker, abs_target: [:0]const u8, abs_dest: [:0]const u8, global: bool) void { - bun.assertWithLocation(std.fs.path.isAbsoluteZ(abs_target), @src()); - bun.assertWithLocation(std.fs.path.isAbsoluteZ(abs_dest), @src()); + bun.assertWithLocation(std.fs.path.isAbsolute(abs_target), @src()); + bun.assertWithLocation(std.fs.path.isAbsolute(abs_dest), @src()); bun.assertWithLocation(abs_target[abs_target.len - 1] != std.fs.path.sep, @src()); bun.assertWithLocation(abs_dest[abs_dest.len - 1] != std.fs.path.sep, @src()); @@ -590,6 +589,8 @@ pub const Bin = extern struct { return remain; } + // target: what the symlink points to + // destination: where the symlink exists on disk pub fn link(this: *Linker, global: bool) void { const package_dir = this.buildTargetPackageDir(); var abs_dest_buf_remain = this.buildDestinationDir(global); diff --git a/src/install/install.zig b/src/install/install.zig index 2c357cef94..407b6779ab 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -7939,7 +7939,7 @@ pub const PackageManager = struct { if (query.expr.asProperty(name)) |value| { if (value.expr.data == .e_string) { - if (!request.resolved_name.isEmpty() and strings.eqlLong(list, dependency_list, true)) { + if (request.package_id != invalid_package_id and strings.eqlLong(list, dependency_list, true)) { replacing += 1; } else { if (manager.subcommand == .update and options.before_install) add_packages_to_update: { @@ -8057,9 +8057,9 @@ pub const PackageManager = struct { var k: usize = 0; while (k < new_dependencies.len) : (k += 1) { if (new_dependencies[k].key) |key| { - if (!request.is_aliased and !request.resolved_name.isEmpty() and key.data.e_string.eql( + if (!request.is_aliased and request.package_id != invalid_package_id and key.data.e_string.eql( string, - request.resolved_name.slice(request.version_buf), + manager.lockfile.packages.items(.name)[request.package_id].slice(request.version_buf), )) { // This actually is a duplicate which we did not // pick up before dependency resolution. @@ -8079,7 +8079,7 @@ pub const PackageManager = struct { else request.version.literal.slice(request.version_buf), )) { - if (request.resolved_name.isEmpty()) { + if (request.package_id == invalid_package_id) { // This actually is a duplicate like "react" // appearing in both "dependencies" and "optionalDependencies". // For this case, we'll just swap remove it @@ -8097,14 +8097,12 @@ pub const PackageManager = struct { } if (new_dependencies[k].key == null) { - new_dependencies[k].key = JSAst.Expr.allocate(allocator, JSAst.E.String, .{ - .data = try allocator.dupe(u8, if (request.is_aliased) - request.name - else if (request.resolved_name.isEmpty()) - request.version.literal.slice(request.version_buf) - else - request.resolved_name.slice(request.version_buf)), - }, logger.Loc.Empty); + new_dependencies[k].key = JSAst.Expr.allocate( + allocator, + JSAst.E.String, + .{ .data = try allocator.dupe(u8, request.getResolvedName(manager.lockfile)) }, + logger.Loc.Empty, + ); new_dependencies[k].value = JSAst.Expr.allocate(allocator, JSAst.E.String, .{ // we set it later @@ -8218,16 +8216,35 @@ pub const PackageManager = struct { } } + const resolutions = if (!options.before_install) manager.lockfile.packages.items(.resolution) else &.{}; for (updates) |*request| { if (request.e_string) |e_string| { - e_string.data = switch (request.resolution.tag) { - .npm => brk: { + if (request.package_id >= resolutions.len or resolutions[request.package_id].tag == .uninitialized) { + e_string.data = uninitialized: { + if (manager.subcommand == .update and manager.options.do.update_to_latest) { + break :uninitialized try allocator.dupe(u8, "latest"); + } + + if (manager.subcommand != .update or !options.before_install or e_string.isBlank() or request.version.tag == .npm) { + break :uninitialized switch (request.version.tag) { + .uninitialized => try allocator.dupe(u8, "latest"), + else => try allocator.dupe(u8, request.version.literal.slice(request.version_buf)), + }; + } else { + break :uninitialized e_string.data; + } + }; + + continue; + } + e_string.data = switch (resolutions[request.package_id].tag) { + .npm => npm: { if (manager.subcommand == .update and (request.version.tag == .dist_tag or request.version.tag == .npm)) { if (manager.updating_packages.fetchSwapRemove(request.name)) |entry| { var alias_at_index: ?usize = null; const new_version = new_version: { - const version_fmt = request.resolution.value.npm.version.fmt(manager.lockfile.buffers.string_bytes.items); + const version_fmt = resolutions[request.package_id].value.npm.version.fmt(manager.lockfile.buffers.string_bytes.items); if (options.exact_versions) { break :new_version try std.fmt.allocPrint(allocator, "{}", .{version_fmt}); } @@ -8254,14 +8271,14 @@ pub const PackageManager = struct { const dep_literal = entry.value.original_version_literal; if (strings.lastIndexOfChar(dep_literal, '@')) |at_index| { - break :brk try std.fmt.allocPrint(allocator, "{s}@{s}", .{ + break :npm try std.fmt.allocPrint(allocator, "{s}@{s}", .{ dep_literal[0..at_index], new_version, }); } } - break :brk new_version; + break :npm new_version; } } if (request.version.tag == .dist_tag or @@ -8272,7 +8289,7 @@ pub const PackageManager = struct { allocator, if (comptime exact_versions) "{}" else "^{}", .{ - request.resolution.value.npm.version.fmt(request.version_buf), + resolutions[request.package_id].value.npm.version.fmt(request.version_buf), }, ), }; @@ -8280,31 +8297,17 @@ pub const PackageManager = struct { if (request.version.tag == .npm and request.version.value.npm.is_alias) { const dep_literal = request.version.literal.slice(request.version_buf); if (strings.indexOfChar(dep_literal, '@')) |at_index| { - break :brk try std.fmt.allocPrint(allocator, "{s}@{s}", .{ + break :npm try std.fmt.allocPrint(allocator, "{s}@{s}", .{ dep_literal[0..at_index], new_version, }); } } - break :brk new_version; + break :npm new_version; } - break :brk try allocator.dupe(u8, request.version.literal.slice(request.version_buf)); - }, - .uninitialized => brk: { - if (manager.subcommand == .update and manager.options.do.update_to_latest) { - break :brk try allocator.dupe(u8, "latest"); - } - - if (manager.subcommand != .update or !options.before_install or e_string.isBlank() or request.version.tag == .npm) { - break :brk switch (request.version.tag) { - .uninitialized => try allocator.dupe(u8, "latest"), - else => try allocator.dupe(u8, request.version.literal.slice(request.version_buf)), - }; - } else { - break :brk e_string.data; - } + break :npm try allocator.dupe(u8, request.version.literal.slice(request.version_buf)); }, .workspace => try allocator.dupe(u8, "workspace:*"), @@ -9107,7 +9110,6 @@ pub const PackageManager = struct { Global.crash(); }, .global_bin_path = manager.options.bin_path, - .global_bin_dir = manager.options.global_bin_dir, // .destination_dir_subpath = destination_dir_subpath, .package_name = strings.StringOrTinyString.init(name), @@ -9254,7 +9256,6 @@ pub const PackageManager = struct { Global.crash(); }, .global_bin_path = manager.options.bin_path, - .global_bin_dir = manager.options.global_bin_dir, .package_name = strings.StringOrTinyString.init(name), .string_buf = lockfile.buffers.string_bytes.items, .extern_string_buf = lockfile.buffers.extern_strings.items, @@ -9961,8 +9962,7 @@ pub const PackageManager = struct { name_hash: PackageNameHash = 0, version: Dependency.Version = .{}, version_buf: []const u8 = "", - resolution: Resolution = .{}, - resolved_name: String = .{}, + package_id: PackageID = invalid_package_id, is_aliased: bool = false, failed: bool = false, // This must be cloned to handle when the AST store resets @@ -9982,13 +9982,13 @@ pub const PackageManager = struct { /// /// `this` needs to be a pointer! If `this` is a copy and the name returned from /// resolved_name is inlined, you will return a pointer to stack memory. - pub fn getResolvedName(this: *UpdateRequest) string { + pub fn getResolvedName(this: *const UpdateRequest, lockfile: *const Lockfile) string { return if (this.is_aliased) this.name - else if (this.resolved_name.isEmpty()) + else if (this.package_id == invalid_package_id) this.version.literal.slice(this.version_buf) else - this.resolved_name.slice(this.version_buf); + lockfile.packages.items(.name)[this.package_id].slice(this.version_buf); } pub fn fromJS(globalThis: *JSC.JSGlobalObject, input: JSC.JSValue) JSC.JSValue { @@ -12038,6 +12038,8 @@ pub const PackageManager = struct { /// Number of installed dependencies. Could be successful or failure. install_count: usize = 0, + pub const Id = Lockfile.Tree.Id; + pub fn deinit(this: *TreeContext, allocator: std.mem.Allocator) void { this.pending_installs.deinit(allocator); this.binaries.deinit(); @@ -12135,7 +12137,7 @@ pub const PackageManager = struct { var link_target_buf: bun.PathBuffer = undefined; var link_dest_buf: bun.PathBuffer = undefined; var link_rel_buf: bun.PathBuffer = undefined; - this.linkTreeBins(tree, destination_dir, &link_target_buf, &link_dest_buf, &link_rel_buf, log_level); + this.linkTreeBins(tree, tree_id, destination_dir, &link_target_buf, &link_dest_buf, &link_rel_buf, log_level); } } @@ -12149,6 +12151,7 @@ pub const PackageManager = struct { pub fn linkTreeBins( this: *PackageInstaller, tree: *TreeContext, + tree_id: TreeContext.Id, destination_dir: std.fs.Dir, link_target_buf: []u8, link_dest_buf: []u8, @@ -12169,7 +12172,6 @@ pub const PackageManager = struct { var bin_linker: Bin.Linker = .{ .bin = bin, .global_bin_path = this.options.bin_path, - .global_bin_dir = this.options.global_bin_dir, .package_name = strings.StringOrTinyString.init(alias), .string_buf = string_buf, .extern_string_buf = lockfile.buffers.extern_strings.items, @@ -12181,7 +12183,24 @@ pub const PackageManager = struct { .rel_buf = link_rel_buf, }; - bin_linker.link(this.manager.options.global); + // globally linked packages shouls always belong to the root + // tree (0). + const global = if (!this.manager.options.global) + false + else if (tree_id != 0) + false + else global: { + for (this.manager.update_requests) |request| { + if (request.package_id == package_id) { + break :global true; + } + } + + break :global false; + }; + + bin_linker.link(global); + if (bin_linker.err) |err| { if (log_level != .silent) { this.manager.log.addErrorFmtNoLoc( @@ -12232,7 +12251,7 @@ pub const PackageManager = struct { }; defer destination_dir.close(); - this.linkTreeBins(tree, destination_dir, &link_target_buf, &link_dest_buf, &link_rel_buf, log_level); + this.linkTreeBins(tree, @intCast(tree_id), destination_dir, &link_target_buf, &link_dest_buf, &link_rel_buf, log_level); } } } diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index 3b25947b29..c16356f72d 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -1033,8 +1033,6 @@ pub fn cleanWithLogger( if (updates.len > 0) { const string_buf = new.buffers.string_bytes.items; const slice = new.packages.slice(); - const names = slice.items(.name); - const resolutions = slice.items(.resolution); // updates might be applied to the root package.json or one // of the workspace package.json files. @@ -1046,14 +1044,13 @@ pub fn cleanWithLogger( const resolved_ids: []const PackageID = res_list.get(new.buffers.resolutions.items); request_updated: for (updates) |*update| { - if (update.resolution.tag == .uninitialized) { + if (update.package_id == invalid_package_id) { for (resolved_ids, workspace_deps) |package_id, dep| { if (update.matches(dep, string_buf)) { if (package_id > new.packages.len) continue; update.version_buf = string_buf; update.version = dep.version; - update.resolution = resolutions[package_id]; - update.resolved_name = names[package_id]; + update.package_id = package_id; continue :request_updated; } diff --git a/test/cli/install/registry/bun-install-registry.test.ts b/test/cli/install/registry/bun-install-registry.test.ts index 038cd2e0a2..4e5f931221 100644 --- a/test/cli/install/registry/bun-install-registry.test.ts +++ b/test/cli/install/registry/bun-install-registry.test.ts @@ -2958,6 +2958,65 @@ describe("binaries", () => { expect(await file(join(packageDir, "bin-1.0.0.txt")).text()).toEqual("success!"); expect(await file(join(packageDir, "bin-1.0.1.txt")).text()).toEqual("success!"); }); + + test("will only link global binaries for requested packages", async () => { + await Promise.all([ + write( + join(packageDir, "bunfig.toml"), + ` + [install] + cache = false + registry = "http://localhost:${port}/" + globalBinDir = "${join(packageDir, "global-bin-dir").replace(/\\/g, "\\\\")}" + `, + ), + , + ]); + + let { stdout, stderr, exited } = spawn({ + cmd: [bunExe(), "i", "-g", `--config=${join(packageDir, "bunfig.toml")}`, "uses-what-bin"], + cwd: packageDir, + stdout: "pipe", + stderr: "pipe", + env: { ...env, BUN_INSTALL: join(packageDir, "global-install-dir") }, + }); + + let err = await Bun.readableStreamToText(stderr); + expect(err).not.toContain("error:"); + let out = await Bun.readableStreamToText(stdout); + expect(out).toContain("uses-what-bin@1.5.0"); + expect(await exited).toBe(0); + + expect(await exists(join(packageDir, "global-bin-dir", "what-bin"))).toBeFalse(); + + ({ stdout, stderr, exited } = spawn({ + cmd: [bunExe(), "i", "-g", `--config=${join(packageDir, "bunfig.toml")}`, "what-bin"], + cwd: packageDir, + stdout: "pipe", + stderr: "pipe", + env: { ...env, BUN_INSTALL: join(packageDir, "global-install-dir") }, + })); + + err = await Bun.readableStreamToText(stderr); + expect(err).not.toContain("error:"); + out = await Bun.readableStreamToText(stdout); + + expect(out).toContain("what-bin@1.5.0"); + expect(await exited).toBe(0); + + // now `what-bin` should be installed in the global bin directory + if (isWindows) { + expect( + await Promise.all([ + exists(join(packageDir, "global-bin-dir", "what-bin.exe")), + exists(join(packageDir, "global-bin-dir", "what-bin.bunx")), + ]), + ).toEqual([true, true]); + } else { + expect(await exists(join(packageDir, "global-bin-dir", "what-bin"))).toBeTrue(); + } + }); + for (const global of [false, true]) { test(`bin types${global ? " (global)" : ""}`, async () => { if (global) { From 62881ee36b166f49033e717cc19a2a65ca4acb1b Mon Sep 17 00:00:00 2001 From: Dylan Conway <35280289+dylan-conway@users.noreply.github.com> Date: Thu, 31 Oct 2024 18:44:24 -0700 Subject: [PATCH 06/17] Redact secrets in `bunfig.toml` and `npmrc` logs (#14919) --- src/api/schema.zig | 5 +- src/bake/DevServer.zig | 8 +- src/bun.js/BuildMessage.zig | 2 +- src/bun.js/api/BunObject.zig | 8 +- src/bun.js/bindings/bindings.zig | 5 + src/bun.js/javascript.zig | 21 +- src/bun.zig | 2 +- src/bun_js.zig | 4 +- src/bundler.zig | 8 +- src/bundler/bundle_v2.zig | 10 +- src/bunfig.zig | 56 ++- src/cli.zig | 20 +- src/cli/build_command.zig | 6 +- src/cli/create_command.zig | 30 +- src/cli/install_command.zig | 2 +- src/cli/outdated_command.zig | 7 +- src/cli/pack_command.zig | 13 +- src/cli/pm_trusted_command.zig | 4 +- src/cli/publish_command.zig | 12 +- src/cli/run_command.zig | 24 +- src/cli/test_command.zig | 6 +- src/cli/upgrade_command.zig | 12 +- src/fmt.zig | 262 +++++++++++-- src/ini.zig | 268 +++++++------- src/install/install.zig | 64 ++-- src/install/lockfile.zig | 12 +- src/install/migration.zig | 2 +- src/install/npm.zig | 9 +- src/install/patch_install.zig | 33 +- src/js_parser.zig | 4 +- src/logger.zig | 346 ++++++++---------- src/main_wasm.zig | 2 +- src/router.zig | 4 +- src/string.zig | 2 +- src/string_immutable.zig | 136 ++++++- src/toml/toml_lexer.zig | 43 ++- src/toml/toml_parser.zig | 8 +- test/cli/install/redacted-config-logs.test.ts | 102 ++++++ .../registry/bun-install-registry.test.ts | 2 +- 39 files changed, 952 insertions(+), 612 deletions(-) create mode 100644 test/cli/install/redacted-config-logs.test.ts diff --git a/src/api/schema.zig b/src/api/schema.zig index bec43fbde7..fa85186280 100644 --- a/src/api/schema.zig +++ b/src/api/schema.zig @@ -1,6 +1,7 @@ const std = @import("std"); const bun = @import("root").bun; const js_ast = bun.JSAst; +const OOM = bun.OOM; pub const Reader = struct { const Self = @This(); @@ -2825,11 +2826,11 @@ pub const Api = struct { } } - pub fn parseRegistryURLString(this: *Parser, str: *js_ast.E.String) !Api.NpmRegistry { + pub fn parseRegistryURLString(this: *Parser, str: *js_ast.E.String) OOM!Api.NpmRegistry { return try this.parseRegistryURLStringImpl(str.data); } - pub fn parseRegistryURLStringImpl(this: *Parser, str: []const u8) !Api.NpmRegistry { + pub fn parseRegistryURLStringImpl(this: *Parser, str: []const u8) OOM!Api.NpmRegistry { const url = bun.URL.parse(str); var registry = std.mem.zeroes(Api.NpmRegistry); diff --git a/src/bake/DevServer.zig b/src/bake/DevServer.zig index 471b28fb7c..c07599f638 100644 --- a/src/bake/DevServer.zig +++ b/src/bake/DevServer.zig @@ -777,14 +777,14 @@ fn bundle(dev: *DevServer, files: []const BakeEntryPoint) BundleError!void { const bundle_result = bv2.runFromBakeDevServer(files) catch |err| { bun.handleErrorReturnTrace(err, @errorReturnTrace()); - bv2.bundler.log.printForLogLevel(Output.errorWriter()) catch {}; + bv2.bundler.log.print(Output.errorWriter()) catch {}; Output.warn("BundleV2.runFromBakeDevServer returned error.{s}", .{@errorName(err)}); return; }; - bv2.bundler.log.printForLogLevel(Output.errorWriter()) catch {}; + bv2.bundler.log.print(Output.errorWriter()) catch {}; try dev.finalizeBundle(bv2, bundle_result); @@ -1256,7 +1256,7 @@ pub fn handleParseTaskFailure( bun.path.relative(dev.cwd, abs_path), }); Output.flush(); - log.printForLogLevel(Output.errorWriter()) catch {}; + log.print(Output.errorWriter()) catch {}; return switch (graph) { .server => dev.server_graph.insertFailure(abs_path, log, false), @@ -2967,7 +2967,7 @@ pub const SerializedFailure = struct { inline else => |k| @intFromEnum(@field(ErrorKind, "bundler_log_" ++ @tagName(k))), }); try writeLogData(msg.data, w); - const notes = msg.notes orelse &.{}; + const notes = msg.notes; try w.writeInt(u32, @intCast(notes.len), .little); for (notes) |note| { try writeLogData(note, w); diff --git a/src/bun.js/BuildMessage.zig b/src/bun.js/BuildMessage.zig index 652a11dc62..e3952df4c9 100644 --- a/src/bun.js/BuildMessage.zig +++ b/src/bun.js/BuildMessage.zig @@ -28,7 +28,7 @@ pub const BuildMessage = struct { } pub fn getNotes(this: *BuildMessage, globalThis: *JSC.JSGlobalObject) JSC.JSValue { - const notes: []const logger.Data = this.msg.notes orelse &[_]logger.Data{}; + const notes = this.msg.notes; const array = JSC.JSValue.createEmptyArray(globalThis, notes.len); for (notes, 0..) |note, i| { const cloned = note.clone(bun.default_allocator) catch { diff --git a/src/bun.js/api/BunObject.zig b/src/bun.js/api/BunObject.zig index 05358336cd..bef890cb65 100644 --- a/src/bun.js/api/BunObject.zig +++ b/src/bun.js/api/BunObject.zig @@ -3909,7 +3909,7 @@ const TOMLObject = struct { var input_slice = arguments[0].toSlice(globalThis, bun.default_allocator); defer input_slice.deinit(); var source = logger.Source.initPathString("input.toml", input_slice.slice()); - const parse_result = TOMLParser.parse(&source, &log, allocator) catch { + const parse_result = TOMLParser.parse(&source, &log, allocator, false) catch { globalThis.throwValue(log.toJS(globalThis, default_allocator, "Failed to parse toml")); return .zero; }; @@ -5017,8 +5017,10 @@ const InternalTestingAPIs = struct { var buffer = MutableString.initEmpty(bun.default_allocator); defer buffer.deinit(); var writer = buffer.bufferedWriter(); - var formatter = bun.fmt.fmtJavaScript(code.slice(), true); - formatter.limited = false; + const formatter = bun.fmt.fmtJavaScript(code.slice(), .{ + .enable_colors = true, + .check_for_unhighlighted_write = false, + }); std.fmt.format(writer.writer(), "{}", .{formatter}) catch |err| { globalThis.throwError(err, "Error formatting code"); return .zero; diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index 27ff465b7c..f12ae5bb41 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -2986,6 +2986,11 @@ pub const JSGlobalObject = opaque { JSGlobalObject__throwOutOfMemoryError(this); } + pub fn throwOutOfMemoryValue(this: *JSGlobalObject) JSValue { + JSGlobalObject__throwOutOfMemoryError(this); + return .zero; + } + pub fn throwTODO(this: *JSGlobalObject, msg: []const u8) void { const err = this.createErrorInstance("{s}", .{msg}); err.put(this, ZigString.static("name"), bun.String.static("TODOError").toJS(this)); diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 20adca6a23..3b0ae31713 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -178,13 +178,14 @@ pub const SavedSourceMap = struct { try fail.toData(path).writeFormat( Output.errorWriter(), logger.Kind.warn, + false, true, ); } else { try fail.toData(path).writeFormat( Output.errorWriter(), logger.Kind.warn, - + false, false, ); } @@ -1513,11 +1514,7 @@ pub const VirtualMachine = struct { this.global.handleRejectedPromises(); if (this.log.msgs.items.len > 0) { - if (Output.enable_ansi_colors) { - this.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true) catch {}; - } else { - this.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false) catch {}; - } + this.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("\n", .{}); Output.flush(); } @@ -3538,7 +3535,7 @@ pub const VirtualMachine = struct { "{d} | {}" ++ fmt, allow_ansi_color, ), - .{ display_line, bun.fmt.fmtJavaScript(clamped, allow_ansi_color) }, + .{ display_line, bun.fmt.fmtJavaScript(clamped, .{ .enable_colors = allow_ansi_color }) }, ); } else { try writer.print( @@ -3546,7 +3543,7 @@ pub const VirtualMachine = struct { "{d} | {}\n", allow_ansi_color, ), - .{ display_line, bun.fmt.fmtJavaScript(clamped, allow_ansi_color) }, + .{ display_line, bun.fmt.fmtJavaScript(clamped, .{ .enable_colors = allow_ansi_color }) }, ); } } @@ -3583,7 +3580,7 @@ pub const VirtualMachine = struct { "- | {}" ++ fmt, allow_ansi_color, ), - .{bun.fmt.fmtJavaScript(text, allow_ansi_color)}, + .{bun.fmt.fmtJavaScript(text, .{ .enable_colors = allow_ansi_color })}, ); } else { try writer.print( @@ -3591,7 +3588,7 @@ pub const VirtualMachine = struct { "- | {}\n", allow_ansi_color, ), - .{bun.fmt.fmtJavaScript(text, allow_ansi_color)}, + .{bun.fmt.fmtJavaScript(text, .{ .enable_colors = allow_ansi_color })}, ); } @@ -3616,7 +3613,7 @@ pub const VirtualMachine = struct { "{d} | {}" ++ fmt, allow_ansi_color, ), - .{ display_line, bun.fmt.fmtJavaScript(clamped, allow_ansi_color) }, + .{ display_line, bun.fmt.fmtJavaScript(clamped, .{ .enable_colors = allow_ansi_color }) }, ); } else { try writer.print( @@ -3624,7 +3621,7 @@ pub const VirtualMachine = struct { "{d} | {}\n", allow_ansi_color, ), - .{ display_line, bun.fmt.fmtJavaScript(clamped, allow_ansi_color) }, + .{ display_line, bun.fmt.fmtJavaScript(clamped, .{ .enable_colors = allow_ansi_color }) }, ); if (clamped.len < max_line_length_with_divot or top.position.column.zeroBased() > max_line_length_with_divot) { diff --git a/src/bun.zig b/src/bun.zig index f7d9cda396..81803574a8 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -2357,7 +2357,7 @@ pub const win32 = struct { return original_mode; } - const watcherChildEnv: [:0]const u16 = strings.toUTF16LiteralZ("_BUN_WATCHER_CHILD"); + const watcherChildEnv: [:0]const u16 = strings.toUTF16Literal("_BUN_WATCHER_CHILD"); // magic exit code to indicate to the watcher manager that the child process should be re-spawned // this was randomly generated - we need to avoid using a common exit code that might be used by the script itself const watcher_reload_exit: w.DWORD = 3224497970; diff --git a/src/bun_js.zig b/src/bun_js.zig index 822bb43883..fb97c269cb 100644 --- a/src/bun_js.zig +++ b/src/bun_js.zig @@ -495,9 +495,7 @@ noinline fn dumpBuildError(vm: *JSC.VirtualMachine) void { const writer = buffered_writer.writer(); - switch (Output.enable_ansi_colors_stderr) { - inline else => |enable_colors| vm.log.printForLogLevelWithEnableAnsiColors(writer, enable_colors) catch {}, - } + vm.log.print(writer) catch {}; } pub noinline fn failWithBuildError(vm: *JSC.VirtualMachine) noreturn { diff --git a/src/bundler.zig b/src/bundler.zig index c6cd1eeb26..33b89a803b 100644 --- a/src/bundler.zig +++ b/src/bundler.zig @@ -15,7 +15,7 @@ const lex = bun.js_lexer; const logger = bun.logger; const options = @import("options.zig"); const js_parser = bun.js_parser; -const json_parser = bun.JSON; +const JSON = bun.JSON; const js_printer = bun.js_printer; const js_ast = bun.JSAst; const linker = @import("linker.zig"); @@ -1450,11 +1450,11 @@ pub const Bundler = struct { // We allow importing tsconfig.*.json or jsconfig.*.json with comments // These files implicitly become JSONC files, which aligns with the behavior of text editors. if (source.path.isJSONCFile()) - json_parser.parseTSConfig(&source, bundler.log, allocator, false) catch return null + JSON.parseTSConfig(&source, bundler.log, allocator, false) catch return null else - json_parser.parse(&source, bundler.log, allocator, false) catch return null + JSON.parse(&source, bundler.log, allocator, false) catch return null else if (kind == .toml) - TOML.parse(&source, bundler.log, allocator) catch return null + TOML.parse(&source, bundler.log, allocator, false) catch return null else @compileError("unreachable"); diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index 9239ed4d8c..b7cdfd295c 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -1810,8 +1810,8 @@ pub const BundleV2 = struct { }, .err => |err| { log.msgs.append(err) catch unreachable; - log.errors += @as(usize, @intFromBool(err.kind == .err)); - log.warnings += @as(usize, @intFromBool(err.kind == .warn)); + log.errors += @as(u32, @intFromBool(err.kind == .err)); + log.warnings += @as(u32, @intFromBool(err.kind == .warn)); // An error occurred, prevent spinning the event loop forever _ = @atomicRmw(usize, &this.graph.parse_pending, .Sub, 1, .monotonic); @@ -1965,8 +1965,8 @@ pub const BundleV2 = struct { }, .err => |err| { log.msgs.append(err) catch unreachable; - log.errors += @as(usize, @intFromBool(err.kind == .err)); - log.warnings += @as(usize, @intFromBool(err.kind == .warn)); + log.errors += @as(u32, @intFromBool(err.kind == .err)); + log.warnings += @as(u32, @intFromBool(err.kind == .warn)); }, .pending, .consumed => unreachable, } @@ -3439,7 +3439,7 @@ pub const ParseTask = struct { .toml => { const trace = tracer(@src(), "ParseTOML"); defer trace.end(); - const root = try TOML.parse(&source, log, allocator); + const root = try TOML.parse(&source, log, allocator, false); return JSAst.init((try js_parser.newLazyExportAST(allocator, bundler.options.define, opts, log, root, &source, "")).?); }, .text => { diff --git a/src/bunfig.zig b/src/bunfig.zig index 0ebfb9cb5d..5be26533cb 100644 --- a/src/bunfig.zig +++ b/src/bunfig.zig @@ -52,12 +52,20 @@ pub const Bunfig = struct { ctx: Command.Context, fn addError(this: *Parser, loc: logger.Loc, comptime text: string) !void { - this.log.addError(this.source, loc, text) catch unreachable; + this.log.addErrorOpts(text, .{ + .source = this.source, + .loc = loc, + .redact_sensitive_information = true, + }) catch unreachable; return error.@"Invalid Bunfig"; } fn addErrorFormat(this: *Parser, loc: logger.Loc, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { - this.log.addErrorFmt(this.source, loc, allocator, text, args) catch unreachable; + this.log.addErrorFmtOpts(allocator, text, args, .{ + .source = this.source, + .loc = loc, + .redact_sensitive_information = true, + }) catch unreachable; return error.@"Invalid Bunfig"; } @@ -791,9 +799,18 @@ pub const Bunfig = struct { switch (expr.data) { .e_string, .e_utf8_string => {}, else => { - this.log.addErrorFmt(this.source, expr.loc, this.allocator, "expected string but received {}", .{ - @as(js_ast.Expr.Tag, expr.data), - }) catch unreachable; + this.log.addErrorFmtOpts( + this.allocator, + "expected string but received {}", + .{ + @as(js_ast.Expr.Tag, expr.data), + }, + .{ + .source = this.source, + .loc = expr.loc, + .redact_sensitive_information = true, + }, + ) catch unreachable; return error.@"Invalid Bunfig"; }, } @@ -801,10 +818,19 @@ pub const Bunfig = struct { pub fn expect(this: *Parser, expr: js_ast.Expr, token: js_ast.Expr.Tag) !void { if (@as(js_ast.Expr.Tag, expr.data) != token) { - this.log.addErrorFmt(this.source, expr.loc, this.allocator, "expected {} but received {}", .{ - token, - @as(js_ast.Expr.Tag, expr.data), - }) catch unreachable; + this.log.addErrorFmtOpts( + this.allocator, + "expected {} but received {}", + .{ + token, + @as(js_ast.Expr.Tag, expr.data), + }, + .{ + .source = this.source, + .loc = expr.loc, + .redact_sensitive_information = true, + }, + ) catch unreachable; return error.@"Invalid Bunfig"; } } @@ -813,14 +839,20 @@ pub const Bunfig = struct { pub fn parse(allocator: std.mem.Allocator, source: logger.Source, ctx: Command.Context, comptime cmd: Command.Tag) !void { const log_count = ctx.log.errors + ctx.log.warnings; - const expr = if (strings.eqlComptime(source.path.name.ext[1..], "toml")) TOML.parse(&source, ctx.log, allocator) catch |err| { + const expr = if (strings.eqlComptime(source.path.name.ext[1..], "toml")) TOML.parse(&source, ctx.log, allocator, true) catch |err| { if (ctx.log.errors + ctx.log.warnings == log_count) { - ctx.log.addErrorFmt(&source, logger.Loc.Empty, allocator, "Failed to parse", .{}) catch unreachable; + try ctx.log.addErrorOpts("Failed to parse", .{ + .source = &source, + .redact_sensitive_information = true, + }); } return err; } else JSONParser.parseTSConfig(&source, ctx.log, allocator, true) catch |err| { if (ctx.log.errors + ctx.log.warnings == log_count) { - ctx.log.addErrorFmt(&source, logger.Loc.Empty, allocator, "Failed to parse", .{}) catch unreachable; + try ctx.log.addErrorOpts("Failed to parse", .{ + .source = &source, + .redact_sensitive_information = true, + }); } return err; }; diff --git a/src/cli.zig b/src/cli.zig index 001f904d97..d6a863a97e 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -61,7 +61,7 @@ pub const Cli = struct { // var panicker = MainPanicHandler.init(log); // MainPanicHandler.Singleton = &panicker; Command.start(allocator, log) catch |err| { - log.printForLogLevel(Output.errorWriter()) catch {}; + log.print(Output.errorWriter()) catch {}; bun.crash_handler.handleRootError(err, @errorReturnTrace()); }; @@ -362,11 +362,7 @@ pub const Arguments = struct { if (getHomeConfigPath(&config_buf)) |path| { loadConfigPath(allocator, true, path, ctx, comptime cmd) catch |err| { if (ctx.log.hasAny()) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + ctx.log.print(Output.errorWriter()) catch {}; } if (ctx.log.hasAny()) Output.printError("\n", .{}); Output.err(err, "failed to load bunfig", .{}); @@ -421,11 +417,7 @@ pub const Arguments = struct { loadConfigPath(allocator, auto_loaded, config_path, ctx, comptime cmd) catch |err| { if (ctx.log.hasAny()) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + ctx.log.print(Output.errorWriter()) catch {}; } if (ctx.log.hasAny()) Output.printError("\n", .{}); Output.err(err, "failed to load bunfig", .{}); @@ -2277,11 +2269,7 @@ pub const Command = struct { ) catch |err| { bun.handleErrorReturnTrace(err, @errorReturnTrace()); - if (Output.enable_ansi_colors) { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true) catch {}; - } else { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false) catch {}; - } + ctx.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("error: Failed to run {s} due to error {s}", .{ std.fs.path.basename(file_path), diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index 63f80063d3..a8ea3f0d67 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -284,7 +284,7 @@ pub const BuildCommand = struct { ); if (log.hasErrors()) { - try log.printForLogLevel(Output.errorWriter()); + try log.print(Output.errorWriter()); if (result.errors.len > 0 or result.output_files.len == 0) { Output.flush(); @@ -306,7 +306,7 @@ pub const BuildCommand = struct { &input_code_length, ) catch |err| { if (log.msgs.items.len > 0) { - try log.printForLogLevel(Output.errorWriter()); + try log.print(Output.errorWriter()); } else { try Output.errorWriter().print("error: {s}", .{@errorName(err)}); } @@ -474,7 +474,7 @@ pub const BuildCommand = struct { } } - try log.printForLogLevel(Output.errorWriter()); + try log.print(Output.errorWriter()); exitOrWatch(0, ctx.debug.hot_reload == .watch); } } diff --git a/src/cli/create_command.zig b/src/cli/create_command.zig index 2d6577a4be..0c81ee3730 100644 --- a/src/cli/create_command.zig +++ b/src/cli/create_command.zig @@ -714,11 +714,7 @@ pub const CreateCommand = struct { const properties_list = std.ArrayList(js_ast.G.Property).fromOwnedSlice(default_allocator, package_json_expr.data.e_object.properties.slice()); if (ctx.log.errors > 0) { - if (Output.enable_ansi_colors) { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); - } + try ctx.log.print(Output.errorWriter()); package_json_file = null; break :process_package_json; @@ -2080,11 +2076,7 @@ pub const Example = struct { refresher.refresh(); if (ctx.log.errors > 0) { - if (Output.enable_ansi_colors) { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); - } + try ctx.log.print(Output.errorWriter()); Global.exit(1); } else { Output.prettyErrorln("Error parsing package: {s}", .{@errorName(err)}); @@ -2096,11 +2088,7 @@ pub const Example = struct { progress.end(); refresher.refresh(); - if (Output.enable_ansi_colors) { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); - } + try ctx.log.print(Output.errorWriter()); Global.exit(1); } @@ -2216,11 +2204,7 @@ pub const Example = struct { var source = logger.Source.initPathString("examples.json", mutable.list.items); const examples_object = JSON.parseUTF8(&source, ctx.log, ctx.allocator) catch |err| { if (ctx.log.errors > 0) { - if (Output.enable_ansi_colors) { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); - } + try ctx.log.print(Output.errorWriter()); Global.exit(1); } else { Output.prettyErrorln("Error parsing examples: {s}", .{@errorName(err)}); @@ -2229,11 +2213,7 @@ pub const Example = struct { }; if (ctx.log.errors > 0) { - if (Output.enable_ansi_colors) { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); - } + try ctx.log.print(Output.errorWriter()); Global.exit(1); } diff --git a/src/cli/install_command.zig b/src/cli/install_command.zig index 6183d43d86..ca69c32e46 100644 --- a/src/cli/install_command.zig +++ b/src/cli/install_command.zig @@ -9,7 +9,7 @@ pub const InstallCommand = struct { error.InvalidPackageJSON, => { const log = &bun.CLI.Cli.log_; - log.printForLogLevel(bun.Output.errorWriter()) catch {}; + log.print(bun.Output.errorWriter()) catch {}; bun.Global.exit(1); }, else => |e| return e, diff --git a/src/cli/outdated_command.zig b/src/cli/outdated_command.zig index 4c8745823e..7e51e94fef 100644 --- a/src/cli/outdated_command.zig +++ b/src/cli/outdated_command.zig @@ -77,12 +77,7 @@ pub const OutdatedCommand = struct { } if (ctx.log.hasErrors()) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| try manager.log.printForLogLevelWithEnableAnsiColors( - Output.errorWriter(), - enable_ansi_colors, - ), - } + try manager.log.print(Output.errorWriter()); } } diff --git a/src/cli/pack_command.zig b/src/cli/pack_command.zig index 8ef1f9b0b9..7645912fef 100644 --- a/src/cli/pack_command.zig +++ b/src/cli/pack_command.zig @@ -136,12 +136,7 @@ pub const PackCommand = struct { } if (manager.log.hasErrors()) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| try manager.log.printForLogLevelWithEnableAnsiColors( - Output.errorWriter(), - enable_ansi_colors, - ), - } + try manager.log.print(Output.errorWriter()); } Global.crash(); @@ -1090,11 +1085,7 @@ pub const PackCommand = struct { }, .parse_err => |err| { Output.err(err, "failed to parse package.json: {s}", .{abs_package_json_path}); - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + manager.log.print(Output.errorWriter()) catch {}; Global.crash(); }, .entry => |entry| entry, diff --git a/src/cli/pm_trusted_command.zig b/src/cli/pm_trusted_command.zig index ba8a918523..b4a56684c9 100644 --- a/src/cli/pm_trusted_command.zig +++ b/src/cli/pm_trusted_command.zig @@ -378,9 +378,7 @@ pub const TrustCommand = struct { const package_json_source = logger.Source.initPathString(PackageManager.package_json_cwd, package_json_contents); var package_json = bun.JSON.parseUTF8(&package_json_source, ctx.log, ctx.allocator) catch |err| { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}, - } + ctx.log.print(Output.errorWriter()) catch {}; Output.errGeneric("failed to parse package.json: {s}", .{@errorName(err)}); Global.crash(); diff --git a/src/cli/publish_command.zig b/src/cli/publish_command.zig index 03ec775d58..69c3e6d12f 100644 --- a/src/cli/publish_command.zig +++ b/src/cli/publish_command.zig @@ -314,11 +314,7 @@ pub const PublishCommand = struct { } if (manager.log.hasErrors()) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + manager.log.print(Output.errorWriter()) catch {}; } Global.crash(); @@ -367,11 +363,7 @@ pub const PublishCommand = struct { Output.errGeneric("failed to find package.json in tarball '{s}'", .{cli.positionals[1]}); }, error.InvalidPackageJSON => { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + manager.log.print(Output.errorWriter()) catch {}; Output.errGeneric("failed to parse tarball package.json", .{}); }, error.PrivatePackage => { diff --git a/src/cli/run_command.zig b/src/cli/run_command.zig index 0b0d083209..d24ef3600a 100644 --- a/src/cli/run_command.zig +++ b/src/cli/run_command.zig @@ -832,20 +832,12 @@ pub const RunCommand = struct { const root_dir_info = this_bundler.resolver.readDirInfo(this_bundler.fs.top_level_dir) catch |err| { if (!log_errors) return error.CouldntReadCurrentDirectory; - if (Output.enable_ansi_colors) { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true) catch {}; - } else { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false) catch {}; - } + ctx.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("error: {s} loading directory {}", .{ @errorName(err), bun.fmt.QuotedFormatter{ .text = this_bundler.fs.top_level_dir } }); Output.flush(); return err; } orelse { - if (Output.enable_ansi_colors) { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true) catch {}; - } else { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false) catch {}; - } + ctx.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("error loading current directory", .{}); Output.flush(); return error.CouldntReadCurrentDirectory; @@ -1296,7 +1288,7 @@ pub const RunCommand = struct { Run.boot(ctx, ".") catch |err| { bun.handleErrorReturnTrace(err, @errorReturnTrace()); - ctx.log.printForLogLevel(Output.errorWriter()) catch {}; + ctx.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("error: Failed to run {s} due to error {s}", .{ script_name_to_search, @@ -1391,7 +1383,7 @@ pub const RunCommand = struct { Run.boot(ctx, out_path) catch |err| { bun.handleErrorReturnTrace(err, @errorReturnTrace()); - ctx.log.printForLogLevel(Output.errorWriter()) catch {}; + ctx.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("error: Failed to run {s} due to error {s}", .{ std.fs.path.basename(file_path), @@ -1488,7 +1480,7 @@ pub const RunCommand = struct { Run.boot(ctx, ctx.allocator.dupe(u8, script_name_to_search) catch unreachable) catch |err| { bun.handleErrorReturnTrace(err, @errorReturnTrace()); - ctx.log.printForLogLevel(Output.errorWriter()) catch {}; + ctx.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("error: Failed to run {s} due to error {s}", .{ std.fs.path.basename(script_name_to_search), @@ -1514,7 +1506,7 @@ pub const RunCommand = struct { const entry_path = entry_point_buf[0 .. cwd.len + trigger.len]; Run.boot(ctx, ctx.allocator.dupe(u8, entry_path) catch return false) catch |err| { - ctx.log.printForLogLevel(Output.errorWriter()) catch {}; + ctx.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("error: Failed to run {s} due to error {s}", .{ std.fs.path.basename(script_name_to_search), @@ -1631,7 +1623,7 @@ pub const RunCommand = struct { }; Run.boot(ctx, normalized_filename) catch |err| { - ctx.log.printForLogLevel(Output.errorWriter()) catch {}; + ctx.log.print(Output.errorWriter()) catch {}; Output.err(err, "Failed to run script \"{s}\"", .{std.fs.path.basename(normalized_filename)}); Global.exit(1); @@ -1704,7 +1696,7 @@ pub const BunXFastPath = struct { wpath, ) catch return; Run.boot(ctx, utf8) catch |err| { - ctx.log.printForLogLevel(Output.errorWriter()) catch {}; + ctx.log.print(Output.errorWriter()) catch {}; Output.err(err, "Failed to run bin \"{s}\"", .{std.fs.path.basename(utf8)}); Global.exit(1); }; diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig index b0f1000b5b..451f1e2a52 100644 --- a/src/cli/test_command.zig +++ b/src/cli/test_command.zig @@ -1161,11 +1161,7 @@ pub const TestCommand = struct { js_ast.Stmt.Data.Store.reset(); if (vm.log.errors > 0) { - if (Output.enable_ansi_colors) { - vm.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true) catch {}; - } else { - vm.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false) catch {}; - } + vm.log.print(Output.errorWriter()) catch {}; vm.log.msgs.clearRetainingCapacity(); vm.log.errors = 0; } diff --git a/src/cli/upgrade_command.zig b/src/cli/upgrade_command.zig index 8cdb5665c6..8b337f15f9 100644 --- a/src/cli/upgrade_command.zig +++ b/src/cli/upgrade_command.zig @@ -272,11 +272,7 @@ pub const UpgradeCommand = struct { refresher.?.refresh(); if (log.errors > 0) { - if (Output.enable_ansi_colors) { - try log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); - } + try log.print(Output.errorWriter()); Global.exit(1); } else { @@ -293,11 +289,7 @@ pub const UpgradeCommand = struct { progress.?.end(); refresher.?.refresh(); - if (Output.enable_ansi_colors) { - try log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); - } + try log.print(Output.errorWriter()); Global.exit(1); } diff --git a/src/fmt.zig b/src/fmt.zig index b91616e1de..309a278c8a 100644 --- a/src/fmt.zig +++ b/src/fmt.zig @@ -140,6 +140,32 @@ pub fn redactedNpmUrl(str: string) RedactedNpmUrlFormatter { }; } +pub const RedactedSourceFormatter = struct { + text: string, + + pub fn format(this: @This(), comptime _: string, _: std.fmt.FormatOptions, writer: anytype) !void { + var i: usize = 0; + while (i < this.text.len) { + if (strings.startsWithSecret(this.text[i..])) |secret| { + const offset, const len = secret; + try writer.writeAll(this.text[i..][0..offset]); + try writer.writeByteNTimes('*', len); + i += offset + len; + continue; + } + + try writer.writeByte(this.text[i]); + i += 1; + } + } +}; + +pub fn redactedSource(str: string) RedactedSourceFormatter { + return .{ + .text = str, + }; +} + // https://github.com/npm/cli/blob/63d6a732c3c0e9c19fd4d147eaa5cc27c29b168d/node_modules/npm-package-arg/lib/npa.js#L163 pub const DependencyUrlFormatter = struct { url: string, @@ -659,17 +685,29 @@ pub const QuotedFormatter = struct { } }; -pub fn fmtJavaScript(text: []const u8, enable_ansi_colors: bool) QuickAndDirtyJavaScriptSyntaxHighlighter { +pub fn fmtJavaScript(text: []const u8, opts: QuickAndDirtyJavaScriptSyntaxHighlighter.Options) QuickAndDirtyJavaScriptSyntaxHighlighter { return QuickAndDirtyJavaScriptSyntaxHighlighter{ .text = text, - .enable_colors = enable_ansi_colors, + .opts = opts, }; } pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { text: []const u8, - enable_colors: bool = false, - limited: bool = true, + opts: Options, + + pub const Options = struct { + enable_colors: bool, + check_for_unhighlighted_write: bool = true, + + redact_sensitive_information: bool = false, + + pub const default: Options = .{ + .enable_colors = Output.enable_ansi_colors, + .check_for_no_highlighting = true, + .redact_sensitive_information = false, + }; + }; const ColorCode = enum { magenta, @@ -824,18 +862,33 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { pub const Keywords = bun.ComptimeEnumMap(Keyword); + pub const RedactedKeyword = enum { + _auth, + _authToken, + token, + _password, + email, + }; + + pub const RedactedKeywords = bun.ComptimeEnumMap(RedactedKeyword); + pub fn format(this: @This(), comptime unused_fmt: []const u8, _: fmt.FormatOptions, writer: anytype) !void { comptime bun.assert(unused_fmt.len == 0); var text = this.text; - if (this.limited) { - if (!this.enable_colors or text.len > 2048 or text.len == 0 or !strings.isAllASCII(text)) { - try writer.writeAll(text); + if (this.opts.check_for_unhighlighted_write) { + if (!this.opts.enable_colors or text.len > 2048 or text.len == 0 or !strings.isAllASCII(text)) { + if (this.opts.redact_sensitive_information) { + try writer.print("{}", .{redactedSource(text)}); + } else { + try writer.writeAll(text); + } return; } } var prev_keyword: ?Keyword = null; + var should_redact_value = false; outer: while (text.len > 0) { if (js_lexer.isIdentifierStart(text[0])) { @@ -846,11 +899,13 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { } if (Keywords.get(text[0..i])) |keyword| { + should_redact_value = false; if (keyword != .as) prev_keyword = keyword; const code = keyword.colorCode(); try writer.print(Output.prettyFmt("{s}{s}", true), .{ code.color(), text[0..i] }); } else { + should_redact_value = this.opts.redact_sensitive_information and RedactedKeywords.has(text[0..i]); write: { if (prev_keyword) |prev| { switch (prev) { @@ -885,11 +940,45 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { } text = text[i..]; } else { + if (this.opts.redact_sensitive_information and should_redact_value) { + while (text.len > 0 and std.ascii.isWhitespace(text[0])) { + try writer.writeByte(text[0]); + text = text[1..]; + } + + if (text.len > 0 and (text[0] == '=' or text[0] == ':')) { + try writer.writeByte(text[0]); + text = text[1..]; + while (text.len > 0 and std.ascii.isWhitespace(text[0])) { + try writer.writeByte(text[0]); + text = text[1..]; + } + + if (text.len == 0) return; + } + } + switch (text[0]) { - '0'...'9' => { + '0'...'9' => |num| { + if (this.opts.redact_sensitive_information) { + if (should_redact_value) { + should_redact_value = false; + const end = strings.indexOfChar(text, '\n') orelse text.len; + text = text[end..]; + try writer.writeAll(Output.prettyFmt("***", true)); + continue; + } + + if (strings.startsWithUUID(text)) { + text = text[36..]; + try writer.writeAll(Output.prettyFmt("***", true)); + continue; + } + } + prev_keyword = null; var i: usize = 1; - if (text.len > 1 and text[0] == '0' and text[1] == 'x') { + if (text.len > 1 and num == '0' and text[1] == 'x') { i += 1; while (i < text.len and switch (text[i]) { '0'...'9', 'a'...'f', 'A'...'F' => true, @@ -914,7 +1003,8 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { var i: usize = 1; while (i < text.len and text[i] != char) { - if (char == '`') { + // if we're redacting, no need to syntax highlight contents + if (!should_redact_value and char == '`') { if (text[i] == '$' and i + 1 < text.len and text[i + 1] == '{') { const curly_start = i; i += 2; @@ -928,10 +1018,11 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { try writer.print(Output.prettyFmt("{s}", true), .{text[0..curly_start]}); try writer.writeAll("${"); + var opts = this.opts; + opts.check_for_unhighlighted_write = false; const curly_remain = QuickAndDirtyJavaScriptSyntaxHighlighter{ .text = text[curly_start + 2 .. i], - .enable_colors = this.enable_colors, - .limited = false, + .opts = opts, }; if (curly_remain.text.len > 0) { @@ -963,11 +1054,81 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { // Include the trailing quote, if any i += @intFromBool(i < text.len); + if (should_redact_value) { + should_redact_value = false; + if (i > 2 and text[i - 1] == char) { + const len = text[0..i].len - 2; + try writer.print(Output.prettyFmt("{c}", true), .{char}); + try writer.writeByteNTimes('*', len); + try writer.print(Output.prettyFmt("{c}", true), .{char}); + } else { + try writer.writeByteNTimes('*', text[0..i].len); + } + text = text[i..]; + continue; + } else if (this.opts.redact_sensitive_information) { + try_redact: { + var inner = text[1..i]; + if (inner.len > 0 and inner[inner.len - 1] == char) { + inner = inner[0 .. inner.len - 1]; + } + + if (inner.len == 0) { + break :try_redact; + } + + if (inner.len == 36 and strings.isUUID(inner)) { + try writer.print(Output.prettyFmt("{c}", true), .{char}); + try writer.writeByteNTimes('*', 36); + try writer.print(Output.prettyFmt("{c}", true), .{char}); + text = text[i..]; + continue; + } + + const npm_secret_len = strings.startsWithNpmSecret(inner); + if (npm_secret_len != 0) { + try writer.print(Output.prettyFmt("{c}", true), .{char}); + try writer.writeByteNTimes('*', npm_secret_len); + try writer.print(Output.prettyFmt("{c}", true), .{char}); + text = text[i..]; + continue; + } + + if (strings.findUrlPassword(inner)) |url_pass| { + const offset, const len = url_pass; + try writer.print(Output.prettyFmt("{c}{s}", true), .{ + char, + inner[0..offset], + }); + try writer.writeByteNTimes('*', len); + try writer.print(Output.prettyFmt("{s}{c}", true), .{ + inner[offset + len ..], + char, + }); + text = text[i..]; + continue; + } + } + + try writer.print(Output.prettyFmt("{s}", true), .{text[0..i]}); + text = text[i..]; + continue; + } + try writer.print(Output.prettyFmt("{s}", true), .{text[0..i]}); text = text[i..]; }, '/' => { prev_keyword = null; + + if (should_redact_value) { + should_redact_value = false; + const len = strings.indexOfChar(text, '\n') orelse text.len; + try writer.writeByteNTimes('*', len); + text = text[len..]; + continue; + } + var i: usize = 1; // the start of a line comment @@ -985,7 +1146,11 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { i += 1; } - try writer.print(Output.prettyFmt("{s}", true), .{remain_to_print}); + if (this.opts.redact_sensitive_information) { + try writer.print(Output.prettyFmt("{}", true), .{redactedSource(remain_to_print)}); + } else { + try writer.print(Output.prettyFmt("{s}", true), .{remain_to_print}); + } text = text[i..]; continue; } @@ -1005,7 +1170,11 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { break :as_multiline_comment; } - try writer.print(Output.prettyFmt("{s}", true), .{text[0..i]}); + if (this.opts.redact_sensitive_information) { + try writer.print(Output.prettyFmt("{}", true), .{redactedSource(text[0..i])}); + } else { + try writer.print(Output.prettyFmt("{s}", true), .{text[0..i]}); + } text = text[i..]; continue; } @@ -1014,27 +1183,58 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { try writer.writeAll(text[0..i]); text = text[i..]; }, - '}', '{' => { + '}', '{' => |brace| { // support potentially highlighting "from" in an import statement if ((prev_keyword orelse Keyword.@"continue") != .import) { prev_keyword = null; } - try writer.writeAll(text[0..1]); + if (should_redact_value) { + should_redact_value = false; + const len = strings.indexOfChar(text, '\n') orelse text.len; + try writer.writeByteNTimes('*', len); + text = text[len..]; + continue; + } + + try writer.writeByte(brace); text = text[1..]; }, - '[', ']' => { + '[', ']' => |bracket| { prev_keyword = null; - try writer.writeAll(text[0..1]); + if (should_redact_value) { + should_redact_value = false; + const len = strings.indexOfChar(text, '\n') orelse text.len; + try writer.writeByteNTimes('*', len); + text = text[len..]; + continue; + } + try writer.writeByte(bracket); text = text[1..]; }, ';' => { prev_keyword = null; + if (should_redact_value) { + should_redact_value = false; + const len = strings.indexOfChar(text, '\n') orelse text.len; + try writer.writeByteNTimes('*', len); + text = text[len..]; + continue; + } try writer.print(Output.prettyFmt(";", true), .{}); text = text[1..]; }, '.' => { prev_keyword = null; + + if (should_redact_value) { + should_redact_value = false; + const len = strings.indexOfChar(text, '\n') orelse text.len; + try writer.writeByteNTimes('*', len); + text = text[len..]; + continue; + } + var i: usize = 1; if (text.len > 1 and (js_lexer.isIdentifierStart(text[1]) or text[1] == '#')) { i = 2; @@ -1051,11 +1251,18 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { i = 1; } - try writer.writeAll(text[0..1]); + try writer.writeByte(text[0]); text = text[1..]; }, '<' => { + if (should_redact_value) { + should_redact_value = false; + const len = strings.indexOfChar(text, '\n') orelse text.len; + try writer.writeByteNTimes('*', len); + text = text[len..]; + continue; + } var i: usize = 1; // JSX @@ -1095,8 +1302,15 @@ pub const QuickAndDirtyJavaScriptSyntaxHighlighter = struct { text = text[i..]; }, - else => { - try writer.writeAll(text[0..1]); + else => |c| { + if (should_redact_value) { + should_redact_value = false; + const len = strings.indexOfChar(text, '\n') orelse text.len; + try writer.writeByteNTimes('*', len); + text = text[len..]; + continue; + } + try writer.writeByte(c); text = text[1..]; }, } @@ -1517,8 +1731,10 @@ pub const fmt_js_test_bindings = struct { const formatter_id: Formatter = @enumFromInt(args.ptr[1].toInt32()); switch (formatter_id) { .fmtJavaScript => { - var formatter = bun.fmt.fmtJavaScript(code.slice(), true); - formatter.limited = false; + const formatter = bun.fmt.fmtJavaScript(code.slice(), .{ + .enable_colors = true, + .check_for_unhighlighted_write = false, + }); std.fmt.format(writer.writer(), "{}", .{formatter}) catch |err| { globalThis.throwError(err, "Error formatting"); return .zero; diff --git a/src/ini.zig b/src/ini.zig index 0a01c0263e..4db24c0b19 100644 --- a/src/ini.zig +++ b/src/ini.zig @@ -9,6 +9,7 @@ const Rope = js_ast.E.Object.Rope; const Output = bun.Output; const Global = bun.Global; const Registry = bun.install.Npm.Registry; +const OOM = bun.OOM; pub const Parser = struct { opts: Options = .{}, @@ -39,10 +40,6 @@ pub const Parser = struct { this.arena.deinit(); } - pub fn parse(this: *Parser, arena_allocator: Allocator) !void { - try this.parseImpl(arena_allocator); - } - inline fn shouldSkipLine(line: []const u8) bool { if (line.len == 0 or // comments @@ -60,7 +57,7 @@ pub const Parser = struct { return true; } - fn parseImpl(this: *Parser, arena_allocator: Allocator) !void { + fn parse(this: *Parser, arena_allocator: Allocator) OOM!void { var iter = std.mem.splitScalar(u8, this.src, '\n'); var head: *E.Object = this.out.data.e_object; @@ -91,7 +88,7 @@ pub const Parser = struct { const section: *Rope = try this.prepareStr(arena_allocator, ropealloc, line[1..close_bracket_idx], @as(i32, @intCast(@intFromPtr(line.ptr) - @intFromPtr(this.src.ptr))) + 1, .section); defer rope_stack.fixed_buffer_allocator.reset(); const parent_object = this.out.data.e_object.getOrPutObject(section, arena_allocator) catch |e| switch (e) { - error.OutOfMemory => bun.outOfMemory(), + error.OutOfMemory => |oom| return oom, error.Clobber => { // We're in here if key exists but it is not an object // @@ -161,7 +158,7 @@ pub const Parser = struct { else key_raw; - if (bun.strings.eql(key, "__proto__")) continue; + if (bun.strings.eqlComptime(key, "__proto__")) continue; const value_raw: Expr = brk: { if (maybe_eq_sign_idx) |eq_sign_idx| { @@ -193,11 +190,11 @@ pub const Parser = struct { if (head.get(key)) |val| { if (val.data != .e_array) { var arr = E.Array{}; - arr.push(arena_allocator, val) catch bun.outOfMemory(); - head.put(arena_allocator, key, Expr.init(E.Array, arr, Loc.Empty)) catch bun.outOfMemory(); + try arr.push(arena_allocator, val); + try head.put(arena_allocator, key, Expr.init(E.Array, arr, Loc.Empty)); } } else { - head.put(arena_allocator, key, Expr.init(E.Array, E.Array{}, Loc.Empty)) catch bun.outOfMemory(); + try head.put(arena_allocator, key, Expr.init(E.Array, E.Array{}, Loc.Empty)); } } @@ -207,12 +204,12 @@ pub const Parser = struct { if (head.get(key)) |val| { if (val.data == .e_array) { was_already_array = true; - val.data.e_array.push(arena_allocator, value) catch bun.outOfMemory(); - head.put(arena_allocator, key, val) catch bun.outOfMemory(); + try val.data.e_array.push(arena_allocator, value); + try head.put(arena_allocator, key, val); } } if (!was_already_array) { - head.put(arena_allocator, key, value) catch bun.outOfMemory(); + try head.put(arena_allocator, key, value); } } } @@ -224,7 +221,7 @@ pub const Parser = struct { val_: []const u8, offset_: i32, comptime usage: enum { section, key, value }, - ) !switch (usage) { + ) OOM!switch (usage) { .value => Expr, .section => *Rope, .key => []const u8, @@ -268,10 +265,7 @@ pub const Parser = struct { return "[Object object]"; }, else => { - const str = std.fmt.allocPrint(arena_allocator, "{}", .{ToStringFormatter{ .d = json_val.data }}) catch |e| { - this.logger.addErrorFmt(&this.source, Loc{ .start = offset }, arena_allocator, "failed to stringify value: {s}", .{@errorName(e)}) catch bun.outOfMemory(); - return error.ParserError; - }; + const str = try std.fmt.allocPrint(arena_allocator, "{}", .{ToStringFormatter{ .d = json_val.data }}); if (comptime usage == .section) return singleStrRope(ropealloc, str); return str; }, @@ -330,7 +324,7 @@ pub const Parser = struct { not_env_substitution: { if (comptime usage != .value) break :not_env_substitution; - if (this.parseEnvSubstitution(val, i, i, &unesc)) |new_i| { + if (try this.parseEnvSubstitution(val, i, i, &unesc)) |new_i| { // set to true so we heap alloc did_any_escape = true; i = new_i; @@ -348,7 +342,7 @@ pub const Parser = struct { }, '.' => { if (comptime usage == .section) { - this.commitRopePart(arena_allocator, ropealloc, &unesc, &rope); + try this.commitRopePart(arena_allocator, ropealloc, &unesc, &rope); } else { try unesc.append('.'); } @@ -380,7 +374,7 @@ pub const Parser = struct { switch (usage) { .section => { - this.commitRopePart(arena_allocator, ropealloc, &unesc, &rope); + try this.commitRopePart(arena_allocator, ropealloc, &unesc, &rope); return rope.?; }, .value => { @@ -412,7 +406,7 @@ pub const Parser = struct { /// - `i` must be an index into `val` that points to a '$' char /// /// npm/ini uses a regex pattern that will select the inner most ${...} - fn parseEnvSubstitution(this: *Parser, val: []const u8, start: usize, i: usize, unesc: *std.ArrayList(u8)) ?usize { + fn parseEnvSubstitution(this: *Parser, val: []const u8, start: usize, i: usize, unesc: *std.ArrayList(u8)) OOM!?usize { bun.debugAssert(val[i] == '$'); var esc = false; if (i + "{}".len < val.len and val[i + 1] == '{') { @@ -435,21 +429,21 @@ pub const Parser = struct { if (start != i) { const missed = val[start..i]; - unesc.appendSlice(missed) catch bun.outOfMemory(); + try unesc.appendSlice(missed); } const env_var = val[i + 2 .. j]; // https://github.com/npm/cli/blob/534ad7789e5c61f579f44d782bdd18ea3ff1ee20/workspaces/config/lib/env-replace.js#L6 const expanded = this.env.get(env_var) orelse return null; - unesc.appendSlice(expanded) catch bun.outOfMemory(); + try unesc.appendSlice(expanded); return j; } return null; } - fn singleStrRope(ropealloc: Allocator, str: []const u8) *Rope { - const rope = ropealloc.create(Rope) catch bun.outOfMemory(); + fn singleStrRope(ropealloc: Allocator, str: []const u8) OOM!*Rope { + const rope = try ropealloc.create(Rope); rope.* = .{ .head = Expr.init(E.String, E.String.init(str), Loc.Empty), }; @@ -460,15 +454,15 @@ pub const Parser = struct { return std.mem.indexOfScalar(u8, key, '.'); } - fn commitRopePart(this: *Parser, arena_allocator: Allocator, ropealloc: Allocator, unesc: *std.ArrayList(u8), existing_rope: *?*Rope) void { + fn commitRopePart(this: *Parser, arena_allocator: Allocator, ropealloc: Allocator, unesc: *std.ArrayList(u8), existing_rope: *?*Rope) OOM!void { _ = this; // autofix - const slice = arena_allocator.dupe(u8, unesc.items[0..]) catch bun.outOfMemory(); + const slice = try arena_allocator.dupe(u8, unesc.items[0..]); const expr = Expr.init(E.String, E.String{ .data = slice }, Loc.Empty); if (existing_rope.*) |_r| { const r: *Rope = _r; - _ = r.append(expr, ropealloc) catch bun.outOfMemory(); + _ = try r.append(expr, ropealloc); } else { - existing_rope.* = ropealloc.create(Rope) catch bun.outOfMemory(); + existing_rope.* = try ropealloc.create(Rope); existing_rope.*.?.* = Rope{ .head = expr, }; @@ -476,15 +470,15 @@ pub const Parser = struct { unesc.clearRetainingCapacity(); } - fn strToRope(ropealloc: Allocator, key: []const u8) *Rope { + fn strToRope(ropealloc: Allocator, key: []const u8) OOM!*Rope { var dot_idx = nextDot(key) orelse { - const rope = ropealloc.create(Rope) catch bun.outOfMemory(); + const rope = try ropealloc.create(Rope); rope.* = .{ .head = Expr.init(E.String, E.String.init(key), Loc.Empty), }; return rope; }; - var rope = ropealloc.create(Rope) catch bun.outOfMemory(); + var rope = try ropealloc.create(Rope); const head = rope; rope.* = .{ .head = Expr.init(E.String, E.String.init(key[0..dot_idx]), Loc.Empty), @@ -494,11 +488,11 @@ pub const Parser = struct { while (dot_idx + 1 < key.len) { const next_dot_idx = dot_idx + 1 + (nextDot(key[dot_idx + 1 ..]) orelse { const rest = key[dot_idx + 1 ..]; - rope = rope.append(Expr.init(E.String, E.String.init(rest), Loc.Empty), ropealloc) catch bun.outOfMemory(); + rope = try rope.append(Expr.init(E.String, E.String.init(rest), Loc.Empty), ropealloc); break; }); const part = key[dot_idx + 1 .. next_dot_idx]; - rope = rope.append(Expr.init(E.String, E.String.init(part), Loc.Empty), ropealloc) catch bun.outOfMemory(); + rope = try rope.append(Expr.init(E.String, E.String.init(part), Loc.Empty), ropealloc); dot_idx = next_dot_idx; } @@ -542,36 +536,36 @@ pub const IniTestingAPIs = struct { }).init(globalThis, envjs); defer object_iter.deinit(); - envmap.ensureTotalCapacity(object_iter.len) catch bun.outOfMemory(); + envmap.ensureTotalCapacity(object_iter.len) catch return globalThis.throwOutOfMemoryValue(); while (object_iter.next()) |key| { - const keyslice = key.toOwnedSlice(allocator) catch bun.outOfMemory(); + const keyslice = key.toOwnedSlice(allocator) catch return globalThis.throwOutOfMemoryValue(); var value = object_iter.value; if (value == .undefined) continue; const value_str = value.getZigString(globalThis); - const slice = value_str.toOwnedSlice(allocator) catch bun.outOfMemory(); + const slice = value_str.toOwnedSlice(allocator) catch return globalThis.throwOutOfMemoryValue(); envmap.put(keyslice, .{ .value = slice, .conditional = false, - }) catch bun.outOfMemory(); + }) catch return globalThis.throwOutOfMemoryValue(); } - const map = allocator.create(bun.DotEnv.Map) catch bun.outOfMemory(); + const map = allocator.create(bun.DotEnv.Map) catch return globalThis.throwOutOfMemoryValue(); map.* = .{ .map = envmap, }; const env = bun.DotEnv.Loader.init(map, allocator); - const envstable = allocator.create(bun.DotEnv.Loader) catch bun.outOfMemory(); + const envstable = allocator.create(bun.DotEnv.Loader) catch return globalThis.throwOutOfMemoryValue(); envstable.* = env; break :brk envstable; }; - const install = allocator.create(bun.Schema.Api.BunInstall) catch bun.outOfMemory(); + const install = allocator.create(bun.Schema.Api.BunInstall) catch return globalThis.throwOutOfMemoryValue(); install.* = std.mem.zeroes(bun.Schema.Api.BunInstall); - loadNpmrc(allocator, install, env, false, ".npmrc", &log, &source) catch { + loadNpmrc(allocator, install, env, ".npmrc", &log, &source) catch { return log.toJS(globalThis, allocator, "error"); }; @@ -636,11 +630,10 @@ pub const IniTestingAPIs = struct { var parser = Parser.init(bun.default_allocator, "", utf8str.slice(), globalThis.bunVM().bundler.env); defer parser.deinit(); - parser.parse(parser.arena.allocator()) catch |e| { - if (parser.logger.errors > 0) { - parser.logger.printForLogLevel(bun.Output.writer()) catch bun.outOfMemory(); - } else globalThis.throwError(e, "failed to parse"); - return .undefined; + parser.parse(parser.arena.allocator()) catch |err| { + switch (err) { + error.OutOfMemory => return globalThis.throwOutOfMemoryValue(), + } }; return parser.out.toJS(bun.default_allocator, globalThis, .{ .decode_escape_sequences = true }) catch |e| { @@ -738,25 +731,27 @@ pub const ConfigIterator = struct { allocator: Allocator, log: *bun.logger.Log, source: *const bun.logger.Source, - ) ?[]const u8 { + ) OOM!?[]const u8 { if (this.optname.isBase64Encoded()) { if (this.value.len == 0) return ""; const len = bun.base64.decodeLen(this.value); - var slice = allocator.alloc(u8, len) catch bun.outOfMemory(); + var slice = try allocator.alloc(u8, len); const result = bun.base64.decode(slice[0..], this.value); if (result.status != .success) { - log.addErrorFmt( - source, - this.loc, + try log.addErrorFmtOpts( allocator, "{s} is not valid base64", .{@tagName(this.optname)}, - ) catch bun.outOfMemory(); + .{ + .source = source, + .loc = this.loc, + }, + ); return null; } - return allocator.dupe(u8, slice[0..result.count]) catch bun.outOfMemory(); + return try allocator.dupe(u8, slice[0..result.count]); } - return allocator.dupe(u8, this.value) catch bun.outOfMemory(); + return try allocator.dupe(u8, this.value); } pub fn format(this: *const @This(), comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { @@ -764,7 +759,7 @@ pub const ConfigIterator = struct { } }; - pub fn next(this: *ConfigIterator) error{ParserError}!?Option(Item) { + pub fn next(this: *ConfigIterator) ?Option(Item) { if (this.prop_idx >= this.config.properties.len) return null; defer this.prop_idx += 1; @@ -830,7 +825,7 @@ pub const ScopeIterator = struct { const Item = struct { scope: []const u8, registry: bun.Schema.Api.NpmRegistry }; - pub fn next(this: *ScopeIterator) error{ParserError}!?Option(Item) { + pub fn next(this: *ScopeIterator) OOM!?Option(Item) { if (this.prop_idx >= this.config.properties.len) return null; defer this.prop_idx += 1; @@ -851,10 +846,7 @@ pub const ScopeIterator = struct { .source = this.source, .allocator = this.allocator, }; - break :brk parser.parseRegistryURLStringImpl(str) catch |e| { - if (e == error.OutOfMemory) bun.outOfMemory(); - return error.ParserError; - }; + break :brk try parser.parseRegistryURLStringImpl(str); } } return .none; @@ -887,44 +879,32 @@ pub fn loadNpmrcFromFile( }; defer allocator.free(source.contents); - loadNpmrc(allocator, install, env, auto_loaded, npmrc_path, &log, &source) catch { - if (log.errors == 1) - Output.warn("Encountered an error while reading .npmrc:\n", .{}) - else - Output.warn("Encountered errors while reading .npmrc:\n", .{}); + loadNpmrc(allocator, install, env, npmrc_path, &log, &source) catch |err| { + switch (err) { + error.OutOfMemory => bun.outOfMemory(), + } }; - log.printForLogLevel(Output.errorWriter()) catch bun.outOfMemory(); + if (log.hasErrors()) { + if (log.errors == 1) + Output.warn("Encountered an error while reading .npmrc:\n\n", .{}) + else + Output.warn("Encountered errors while reading .npmrc:\n\n", .{}); + Output.flush(); + } + log.print(Output.errorWriter()) catch {}; } pub fn loadNpmrc( allocator: std.mem.Allocator, install: *bun.Schema.Api.BunInstall, env: *bun.DotEnv.Loader, - auto_loaded: bool, npmrc_path: [:0]const u8, log: *bun.logger.Log, source: *const bun.logger.Source, -) !void { +) OOM!void { var parser = bun.ini.Parser.init(allocator, npmrc_path, source.contents, env); defer parser.deinit(); - parser.parse(parser.arena.allocator()) catch |e| { - if (e == error.ParserError) { - parser.logger.printForLogLevel(Output.errorWriter()) catch unreachable; - return e; - } - if (auto_loaded) { - Output.warn("{}\nwhile reading .npmrc \"{s}\"", .{ - e, - npmrc_path, - }); - return; - } - Output.prettyErrorln("{}\nwhile reading .npmrc \"{s}\"", .{ - e, - npmrc_path, - }); - Global.exit(1); - }; + try parser.parse(parser.arena.allocator()); // Need to be very, very careful here with strings. // They are allocated in the Parser's arena, which of course gets @@ -939,16 +919,13 @@ pub fn loadNpmrc( .log = log, .source = source, }; - install.default_registry = p.parseRegistryURLStringImpl(allocator.dupe(u8, str) catch bun.outOfMemory()) catch |e| { - if (e == error.OutOfMemory) bun.outOfMemory(); - return error.ParserError; - }; + install.default_registry = try p.parseRegistryURLStringImpl(try allocator.dupe(u8, str)); } } if (out.asProperty("cache")) |query| { if (query.expr.asUtf8StringLiteral()) |str| { - install.cache_directory = allocator.dupe(u8, str) catch bun.outOfMemory(); + install.cache_directory = try allocator.dupe(u8, str); } else if (query.expr.asBool()) |b| { install.disable_cache = !b; } @@ -1002,13 +979,7 @@ pub fn loadNpmrc( const scope_count = brk: { var count: usize = 0; - while (iter.next() catch { - const prop_idx = iter.prop_idx -| 1; - const prop = iter.config.properties.at(prop_idx); - const loc = prop.key.?.loc; - log.addErrorFmt(source, loc, parser.arena.allocator(), "Found an invalid registry option:", .{}) catch bun.outOfMemory(); - return error.ParserError; - }) |o| { + while (try iter.next()) |o| { if (o == .some) { count += 1; } @@ -1017,19 +988,19 @@ pub fn loadNpmrc( }; defer install.scoped = registry_map; - registry_map.scopes.ensureUnusedCapacity(allocator, scope_count) catch bun.outOfMemory(); + try registry_map.scopes.ensureUnusedCapacity(allocator, scope_count); iter.prop_idx = 0; iter.count = false; - while (iter.next() catch unreachable) |val| { + while (try iter.next()) |val| { if (val.get()) |result| { const registry = result.registry.dupe(allocator); - registry_map.scopes.put( + try registry_map.scopes.put( allocator, - allocator.dupe(u8, result.scope) catch bun.outOfMemory(), + try allocator.dupe(u8, result.scope), registry, - ) catch bun.outOfMemory(); + ); } } } @@ -1076,11 +1047,11 @@ pub fn loadNpmrc( // The line that sets the username would apply to both @myorg and @another var url_map = url_map: { var url_map = bun.StringArrayHashMap(bun.URL).init(parser.arena.allocator()); - url_map.ensureTotalCapacity(registry_map.scopes.keys().len) catch bun.outOfMemory(); + try url_map.ensureTotalCapacity(registry_map.scopes.keys().len); for (registry_map.scopes.keys(), registry_map.scopes.values()) |*k, *v| { const url = bun.URL.parse(v.url); - url_map.put(k.*, url) catch bun.outOfMemory(); + try url_map.put(k.*, url); } break :url_map url_map; @@ -1095,13 +1066,7 @@ pub fn loadNpmrc( .allocator = allocator, }; - while (iter.next() catch { - const prop_idx = iter.prop_idx -| 1; - const prop = iter.config.properties.at(prop_idx); - const loc = prop.key.?.loc; - log.addErrorFmt(source, loc, parser.arena.allocator(), "Found an invalid registry option:", .{}) catch bun.outOfMemory(); - return error.ParserError; - }) |val| { + while (iter.next()) |val| { if (val.get()) |conf_item_| { // `conf_item` will look like: // @@ -1113,7 +1078,7 @@ pub fn loadNpmrc( const conf_item: bun.ini.ConfigIterator.Item = conf_item_; switch (conf_item.optname) { .email, .certfile, .keyfile => { - log.addWarningFmt( + try log.addWarningFmt( source, iter.config.properties.at(iter.prop_idx - 1).key.?.loc, allocator, @@ -1122,7 +1087,7 @@ pub fn loadNpmrc( conf_item, @tagName(conf_item.optname), }, - ) catch bun.outOfMemory(); + ); continue; }, else => {}, @@ -1143,16 +1108,16 @@ pub fn loadNpmrc( switch (conf_item.optname) { ._authToken => { - if (conf_item.dupeValueDecoded(allocator, log, source)) |x| v.token = x; + if (try conf_item.dupeValueDecoded(allocator, log, source)) |x| v.token = x; }, .username => { - if (conf_item.dupeValueDecoded(allocator, log, source)) |x| v.username = x; + if (try conf_item.dupeValueDecoded(allocator, log, source)) |x| v.username = x; }, ._password => { - if (conf_item.dupeValueDecoded(allocator, log, source)) |x| v.password = x; + if (try conf_item.dupeValueDecoded(allocator, log, source)) |x| v.password = x; }, ._auth => { - _ = @"handle _auth"(allocator, v, &conf_item, log, source); + try @"handle _auth"(allocator, v, &conf_item, log, source); }, .email, .certfile, .keyfile => unreachable, } @@ -1170,16 +1135,16 @@ pub fn loadNpmrc( } switch (conf_item.optname) { ._authToken => { - if (conf_item.dupeValueDecoded(allocator, log, source)) |x| v.token = x; + if (try conf_item.dupeValueDecoded(allocator, log, source)) |x| v.token = x; }, .username => { - if (conf_item.dupeValueDecoded(allocator, log, source)) |x| v.username = x; + if (try conf_item.dupeValueDecoded(allocator, log, source)) |x| v.username = x; }, ._password => { - if (conf_item.dupeValueDecoded(allocator, log, source)) |x| v.password = x; + if (try conf_item.dupeValueDecoded(allocator, log, source)) |x| v.password = x; }, ._auth => { - _ = @"handle _auth"(allocator, v, &conf_item, log, source); + try @"handle _auth"(allocator, v, &conf_item, log, source); }, .email, .certfile, .keyfile => unreachable, } @@ -1190,11 +1155,6 @@ pub fn loadNpmrc( } } } - - const had_errors = log.hasErrors(); - if (had_errors) { - return error.ParserError; - } } fn @"handle _auth"( @@ -1203,35 +1163,57 @@ fn @"handle _auth"( conf_item: *const ConfigIterator.Item, log: *bun.logger.Log, source: *const bun.logger.Source, -) void { +) OOM!void { if (conf_item.value.len == 0) { - log.addErrorFmt( - source, - conf_item.loc, - allocator, - "invalid _auth value, expected it to be \"\\:\\\" encoded in base64, but got an empty string", - .{}, - ) catch bun.outOfMemory(); + try log.addErrorOpts( + "invalid _auth value, expected base64 encoded \":\", received an empty string", + .{ + .source = source, + .loc = conf_item.loc, + .redact_sensitive_information = true, + }, + ); return; } const decode_len = bun.base64.decodeLen(conf_item.value); - const decoded = allocator.alloc(u8, decode_len) catch bun.outOfMemory(); + const decoded = try allocator.alloc(u8, decode_len); const result = bun.base64.decode(decoded[0..], conf_item.value); if (!result.isSuccessful()) { defer allocator.free(decoded); - log.addErrorFmt(source, conf_item.loc, allocator, "invalid base64", .{}) catch bun.outOfMemory(); + try log.addErrorOpts( + "invalid _auth value, expected valid base64", + .{ + .source = source, + .loc = conf_item.loc, + .redact_sensitive_information = true, + }, + ); return; } const @"username:password" = decoded[0..result.count]; const colon_idx = std.mem.indexOfScalar(u8, @"username:password", ':') orelse { - defer allocator.free(decoded); - log.addErrorFmt(source, conf_item.loc, allocator, "invalid _auth value, expected it to be \"\\:\\\" encoded in base 64, but got:\n\n{s}", .{decoded}) catch bun.outOfMemory(); + defer allocator.free(@"username:password"); + try log.addErrorOpts( + "invalid _auth value, expected base64 encoded \":\"", + .{ + .source = source, + .loc = conf_item.loc, + .redact_sensitive_information = true, + }, + ); return; }; const username = @"username:password"[0..colon_idx]; if (colon_idx + 1 >= @"username:password".len) { - defer allocator.free(decoded); - log.addErrorFmt(source, conf_item.loc, allocator, "invalid _auth value, expected it to be \"\\:\\\" encoded in base64, but got:\n\n{s}", .{decoded}) catch bun.outOfMemory(); + defer allocator.free(@"username:password"); + try log.addErrorOpts( + "invalid _auth value, expected base64 encoded \":\"", + .{ + .source = source, + .loc = conf_item.loc, + .redact_sensitive_information = true, + }, + ); return; } const password = @"username:password"[colon_idx + 1 ..]; diff --git a/src/install/install.zig b/src/install/install.zig index 407b6779ab..7c457914f4 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -696,7 +696,7 @@ pub const Task = struct { if (pt.callback.apply.logger.errors > 0) { defer pt.callback.apply.logger.deinit(); // this.log.addErrorFmt(null, logger.Loc.Empty, bun.default_allocator, "failed to apply patch: {}", .{e}) catch unreachable; - pt.callback.apply.logger.printForLogLevel(Output.writer()) catch {}; + pt.callback.apply.logger.print(Output.writer()) catch {}; } } } @@ -2754,7 +2754,7 @@ pub const PackageManager = struct { pub fn crash(this: *PackageManager) noreturn { if (this.options.log_level != .silent) { - this.log.printForLogLevel(Output.errorWriter()) catch {}; + this.log.print(Output.errorWriter()) catch {}; } Global.crash(); } @@ -6580,11 +6580,7 @@ pub const PackageManager = struct { _ = manager.decrementPendingTasks(); if (task.log.msgs.items.len > 0) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - try task.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors); - }, - } + try task.log.print(Output.errorWriter()); } switch (task.tag) { @@ -8967,7 +8963,7 @@ pub const PackageManager = struct { error.InvalidPackageJSON, => { const log = &bun.CLI.Cli.log_; - log.printForLogLevel(bun.Output.errorWriter()) catch {}; + log.print(bun.Output.errorWriter()) catch {}; bun.Global.exit(1); return; }, @@ -10404,11 +10400,7 @@ pub const PackageManager = struct { ) !void { if (manager.log.errors > 0) { if (comptime log_level != .silent) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + manager.log.print(Output.errorWriter()) catch {}; } Global.crash(); } @@ -10423,11 +10415,7 @@ pub const PackageManager = struct { }, )) { .parse_err => |err| { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + manager.log.print(Output.errorWriter()) catch {}; Output.errGeneric("failed to parse package.json \"{s}\": {s}", .{ manager.original_package_json_path, @errorName(err), @@ -10627,11 +10615,7 @@ pub const PackageManager = struct { }, )) { .parse_err => |err| { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + manager.log.print(Output.errorWriter()) catch {}; Output.errGeneric("failed to parse package.json \"{s}\": {s}", .{ root_package_json_path, @errorName(err), @@ -11059,11 +11043,7 @@ pub const PackageManager = struct { initializeStore(); const json = JSON.parsePackageJSONUTF8AlwaysDecode(&package_json_source, manager.log, manager.allocator) catch |err| { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + manager.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("{s} parsing package.json in \"{s}\"", .{ @errorName(err), package_json_source.path.prettyDir() }); Global.crash(); }; @@ -11476,11 +11456,7 @@ pub const PackageManager = struct { initializeStore(); const json = JSON.parsePackageJSONUTF8AlwaysDecode(&package_json_source, manager.log, manager.allocator) catch |err| { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + manager.log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("{s} parsing package.json in \"{s}\"", .{ @errorName(err), package_json_source.path.prettyDir() }); Global.crash(); }; @@ -11948,15 +11924,20 @@ pub const PackageManager = struct { pub fn install(ctx: Command.Context) !void { const cli = try CommandLineArguments.parse(ctx.allocator, .install); + + const subcommand: Subcommand = if (cli.positionals.len > 1) .add else .install; + + // TODO(dylan-conway): print `bun install ` or `bun add ` before logs from `init`. + // and cleanup install/add subcommand usage var manager, _ = try init(ctx, cli, .install); // switch to `bun add ` - if (manager.options.positionals.len > 1) { + if (subcommand == .add) { + manager.subcommand = .add; if (manager.options.shouldPrintCommandName()) { Output.prettyln("bun add v" ++ Global.package_json_version_with_sha ++ "\n", .{}); Output.flush(); } - manager.subcommand = .add; return try switch (manager.options.log_level) { inline else => |log_level| manager.updatePackageJSONAndInstallWithManager(ctx, log_level), }; @@ -12203,10 +12184,11 @@ pub const PackageManager = struct { if (bin_linker.err) |err| { if (log_level != .silent) { - this.manager.log.addErrorFmtNoLoc( + this.manager.log.addErrorFmtOpts( this.manager.allocator, "Failed to link {s}: {s}", .{ alias, @errorName(err) }, + .{}, ) catch bun.outOfMemory(); } @@ -14001,11 +13983,7 @@ pub const PackageManager = struct { } if (ctx.log.errors > 0) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - try manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors); - }, - } + try manager.log.print(Output.errorWriter()); } Output.flush(); } @@ -14434,7 +14412,7 @@ pub const PackageManager = struct { } const had_errors_before_cleaning_lockfile = manager.log.hasErrors(); - try manager.log.printForLogLevel(Output.errorWriter()); + try manager.log.print(Output.errorWriter()); manager.log.reset(); // This operation doesn't perform any I/O, so it should be relatively cheap. @@ -14551,7 +14529,7 @@ pub const PackageManager = struct { } if (comptime log_level != .silent) { - try manager.log.printForLogLevel(Output.errorWriter()); + try manager.log.print(Output.errorWriter()); } if (had_errors_before_cleaning_lockfile or manager.log.hasErrors()) Global.crash(); diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index c16356f72d..5648e2d5f1 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -1254,11 +1254,7 @@ pub const Printer = struct { }), } if (log.errors > 0) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - try log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors); - }, - } + try log.print(Output.errorWriter()); } Global.crash(); }, @@ -3943,11 +3939,7 @@ pub const Package = extern struct { ) !void { initializeStore(); const json = JSON.parsePackageJSONUTF8AlwaysDecode(&source, log, allocator) catch |err| { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; - }, - } + log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("{s} parsing package.json in \"{s}\"", .{ @errorName(err), source.path.prettyDir() }); Global.crash(); }; diff --git a/src/install/migration.zig b/src/install/migration.zig index 6c44a9e59a..e0dd624826 100644 --- a/src/install/migration.zig +++ b/src/install/migration.zig @@ -77,7 +77,7 @@ pub fn detectAndLoadOtherLockfile( bun.handleErrorReturnTrace(err, @errorReturnTrace()); Output.prettyErrorln("Error: {s}", .{@errorName(err)}); - log.printForLogLevel(Output.errorWriter()) catch {}; + log.print(Output.errorWriter()) catch {}; Output.prettyErrorln("Invalid NPM package-lock.json\nIn a release build, this would ignore and do a fresh install.\nAborting", .{}); Global.exit(1); } diff --git a/src/install/npm.zig b/src/install/npm.zig index bc4fc4a637..69f5bf6f02 100644 --- a/src/install/npm.zig +++ b/src/install/npm.zig @@ -1204,7 +1204,14 @@ pub const PackageManifest = struct { const file_name = try manifestFileName(&file_path_buf, file_id, scope); const cache_file = File.openat(cache_dir, file_name, bun.O.RDONLY, 0).unwrap() catch return null; defer cache_file.close(); - return loadByFile(allocator, scope, cache_file); + + delete: { + return loadByFile(allocator, scope, cache_file) catch break :delete orelse break :delete; + } + + // delete the outdated/invalid manifest + try bun.sys.unlinkat(bun.toFD(cache_dir), file_name).unwrap(); + return null; } pub fn loadByFile(allocator: std.mem.Allocator, scope: *const Registry.Scope, manifest_file: File) !?PackageManifest { diff --git a/src/install/patch_install.zig b/src/install/patch_install.zig index 7b60e28319..3005a0c548 100644 --- a/src/install/patch_install.zig +++ b/src/install/patch_install.zig @@ -158,7 +158,7 @@ pub const PatchTask = struct { if (this.callback.apply.logger.errors > 0) { defer this.callback.apply.logger.deinit(); Output.errGeneric("failed to apply patchfile ({s})", .{this.callback.apply.patchfilepath}); - this.callback.apply.logger.printForLogLevel(Output.errorWriter()) catch {}; + this.callback.apply.logger.print(Output.errorWriter()) catch {}; } } @@ -183,7 +183,7 @@ pub const PatchTask = struct { } if (calc_hash.logger.errors > 0) { Output.prettyErrorln("\n\n", .{}); - calc_hash.logger.printForLogLevel(Output.errorWriter()) catch {}; + calc_hash.logger.print(Output.errorWriter()) catch {}; } Output.flush(); Global.crash(); @@ -281,20 +281,22 @@ pub const PatchTask = struct { )) { .result => |txt| txt, .err => |e| { - try log.addErrorFmtNoLoc( + try log.addErrorFmtOpts( this.manager.allocator, "failed to read patchfile: {}", .{e.toSystemError()}, + .{}, ); return; }, }; defer this.manager.allocator.free(patchfile_txt); var patchfile = bun.patch.parsePatchFile(patchfile_txt) catch |e| { - try log.addErrorFmtNoLoc( + try log.addErrorFmtOpts( this.manager.allocator, "failed to parse patchfile: {s}", .{@errorName(e)}, + .{}, ); return; }; @@ -333,27 +335,30 @@ pub const PatchTask = struct { switch (pkg_install.installImpl(true, system_tmpdir, .copyfile, this.callback.apply.resolution.tag)) { .success => {}, .fail => |reason| { - return try log.addErrorFmtNoLoc( + return try log.addErrorFmtOpts( this.manager.allocator, "{s} while executing step: {s}", .{ @errorName(reason.err), reason.step.name() }, + .{}, ); }, } - var patch_pkg_dir = system_tmpdir.openDir(tempdir_name, .{}) catch |e| return try log.addErrorFmtNoLoc( + var patch_pkg_dir = system_tmpdir.openDir(tempdir_name, .{}) catch |e| return try log.addErrorFmtOpts( this.manager.allocator, "failed trying to open temporary dir to apply patch to package: {s}", .{@errorName(e)}, + .{}, ); defer patch_pkg_dir.close(); // 4. apply patch if (patchfile.apply(this.manager.allocator, bun.toFD(patch_pkg_dir.fd))) |e| { - return try log.addErrorFmtNoLoc( + return try log.addErrorFmtOpts( this.manager.allocator, "failed applying patch file: {}", .{e}, + .{}, ); } @@ -371,7 +376,12 @@ pub const PatchTask = struct { )) { .result => |fd| fd, .err => |e| { - return try log.addErrorFmtNoLoc(this.manager.allocator, "failed adding bun tag: {}", .{e.withPath(buntagbuf[0 .. bun_tag_prefix.len + hashlen :0])}); + return try log.addErrorFmtOpts( + this.manager.allocator, + "failed adding bun tag: {}", + .{e.withPath(buntagbuf[0 .. bun_tag_prefix.len + hashlen :0])}, + .{}, + ); }, }; _ = bun.sys.close(buntagfd); @@ -391,7 +401,12 @@ pub const PatchTask = struct { bun.toFD(this.callback.apply.cache_dir.fd), this.callback.apply.cache_dir_subpath, .{ .move_fallback = true }, - ).asErr()) |e| return try log.addErrorFmtNoLoc(this.manager.allocator, "renaming changes to cache dir: {}", .{e.withPath(this.callback.apply.cache_dir_subpath)}); + ).asErr()) |e| return try log.addErrorFmtOpts( + this.manager.allocator, + "renaming changes to cache dir: {}", + .{e.withPath(this.callback.apply.cache_dir_subpath)}, + .{}, + ); } pub fn calcHash(this: *PatchTask) ?u64 { diff --git a/src/js_parser.zig b/src/js_parser.zig index 4f62865fcb..68153ae884 100644 --- a/src/js_parser.zig +++ b/src/js_parser.zig @@ -3113,7 +3113,7 @@ pub const Parser = struct { const stmts = p.parseStmtsUpTo(js_lexer.T.t_end_of_file, &opts) catch |err| { if (comptime Environment.isWasm) { Output.print("JSParser.parse: caught error {s} at location: {d}\n", .{ @errorName(err), p.lexer.loc().start }); - p.log.printForLogLevel(Output.writer()) catch {}; + p.log.print(Output.writer()) catch {}; } return err; }; @@ -14700,7 +14700,7 @@ fn NewParser_( } p.log.level = .verbose; - p.log.printForLogLevel(panic_stream.writer()) catch unreachable; + p.log.print(panic_stream.writer()) catch unreachable; Output.panic(fmt ++ "\n{s}", args ++ .{panic_buffer[0..panic_stream.pos]}); } diff --git a/src/logger.zig b/src/logger.zig index 8aaf5f1880..c1b81145fe 100644 --- a/src/logger.zig +++ b/src/logger.zig @@ -18,9 +18,10 @@ const unicode = std.unicode; const Ref = @import("./ast/base.zig").Ref; const expect = std.testing.expect; const assert = bun.assert; -const ArrayList = std.ArrayList; const StringBuilder = @import("./string_builder.zig"); const Index = @import("./ast/base.zig").Index; +const OOM = bun.OOM; +const JSError = bun.JSError; pub const Kind = enum(u8) { err = 0, @@ -210,7 +211,7 @@ pub const Data = struct { allocator.free(d.text); } - pub fn cloneLineText(this: Data, should: bool, allocator: std.mem.Allocator) !Data { + pub fn cloneLineText(this: Data, should: bool, allocator: std.mem.Allocator) OOM!Data { if (!should or this.location == null or this.location.?.line_text == null) return this; @@ -223,7 +224,7 @@ pub const Data = struct { }; } - pub fn clone(this: Data, allocator: std.mem.Allocator) !Data { + pub fn clone(this: Data, allocator: std.mem.Allocator) OOM!Data { return Data{ .text = if (this.text.len > 0) try allocator.dupe(u8, this.text) else "", .location = if (this.location != null) try this.location.?.clone(allocator) else null, @@ -253,6 +254,7 @@ pub const Data = struct { this: *const Data, to: anytype, kind: Kind, + redact_sensitive_information: bool, comptime enable_ansi_colors: bool, ) !void { if (this.text.len == 0) return; @@ -293,7 +295,10 @@ pub const Data = struct { line_offset_for_second_line += std.fmt.count("{d} | ", .{location.line}); } - try to.print("{}\n", .{bun.fmt.fmtJavaScript(line_text, enable_ansi_colors)}); + try to.print("{}\n", .{bun.fmt.fmtJavaScript(line_text, .{ + .enable_colors = enable_ansi_colors, + .redact_sensitive_information = redact_sensitive_information, + })}); try to.writeByteNTimes(' ', line_offset_for_second_line); if ((comptime enable_ansi_colors) and message_color.len > 0) { @@ -379,11 +384,11 @@ pub const BabyString = packed struct { pub const Msg = struct { kind: Kind = Kind.err, data: Data, - metadata: Metadata = .{ .build = 0 }, - // TODO: make this non-optional, empty slice for no notes - notes: ?[]Data = null, + metadata: Metadata = .build, + notes: []Data = &.{}, + redact_sensitive_information: bool = false, - pub fn fromJS(allocator: std.mem.Allocator, globalObject: *bun.JSC.JSGlobalObject, file: string, err: bun.JSC.JSValue) !Msg { + pub fn fromJS(allocator: std.mem.Allocator, globalObject: *bun.JSC.JSGlobalObject, file: string, err: bun.JSC.JSValue) OOM!Msg { var zig_exception_holder: bun.JSC.ZigException.Holder = bun.JSC.ZigException.Holder.init(); if (err.toError()) |value| { value.toZigException(globalObject, zig_exception_holder.zigException()); @@ -410,22 +415,17 @@ pub const Msg = struct { pub fn count(this: *const Msg, builder: *StringBuilder) void { this.data.count(builder); - if (this.notes) |notes| { - for (notes) |note| { - note.count(builder); - } + for (this.notes) |note| { + note.count(builder); } } - pub fn clone(this: *const Msg, allocator: std.mem.Allocator) !Msg { + pub fn clone(this: *const Msg, allocator: std.mem.Allocator) OOM!Msg { return Msg{ .kind = this.kind, .data = try this.data.clone(allocator), .metadata = this.metadata, - .notes = if (this.notes != null and this.notes.?.len > 0) - try bun.clone(this.notes.?, allocator) - else - null, + .notes = try bun.clone(this.notes, allocator), }; } @@ -434,22 +434,18 @@ pub const Msg = struct { .kind = this.kind, .data = this.data.cloneWithBuilder(builder), .metadata = this.metadata, - .notes = if (this.notes != null and this.notes.?.len > 0) brk: { - for (this.notes.?, 0..) |note, i| { + .notes = if (this.notes.len > 0) brk: { + for (this.notes, 0..) |note, i| { notes[i] = note.cloneWithBuilder(builder); } - break :brk notes[0..this.notes.?.len]; - } else null, + break :brk notes[0..this.notes.len]; + } else &.{}, }; } - pub const Metadata = union(Tag) { - build: u0, + pub const Metadata = union(enum) { + build, resolve: Resolve, - pub const Tag = enum(u8) { - build = 1, - resolve = 2, - }; pub const Resolve = struct { specifier: BabyString, @@ -458,34 +454,29 @@ pub const Msg = struct { }; }; - pub fn toAPI(this: *const Msg, allocator: std.mem.Allocator) !Api.Message { - const notes_len = if (this.notes != null) this.notes.?.len else 0; - var _notes = try allocator.alloc( + pub fn toAPI(this: *const Msg, allocator: std.mem.Allocator) OOM!Api.Message { + var notes = try allocator.alloc( Api.MessageData, - notes_len, + this.notes.len, ); const msg = Api.Message{ .level = this.kind.toAPI(), .data = this.data.toAPI(), - .notes = _notes, + .notes = notes, .on = Api.MessageMeta{ .resolve = if (this.metadata == .resolve) this.metadata.resolve.specifier.slice(this.data.text) else "", .build = this.metadata == .build, }, }; - if (this.notes) |notes| { - if (notes.len > 0) { - for (notes, 0..) |note, i| { - _notes[i] = note.toAPI(); - } - } + for (this.notes, 0..) |note, i| { + notes[i] = note.toAPI(); } return msg; } - pub fn toAPIFromList(comptime ListType: type, list: ListType, allocator: std.mem.Allocator) ![]Api.Message { + pub fn toAPIFromList(comptime ListType: type, list: ListType, allocator: std.mem.Allocator) OOM![]Api.Message { var out_list = try allocator.alloc(Api.Message, list.items.len); for (list.items, 0..) |item, i| { out_list[i] = try item.toAPI(allocator); @@ -496,15 +487,13 @@ pub const Msg = struct { pub fn deinit(msg: *Msg, allocator: std.mem.Allocator) void { msg.data.deinit(allocator); - if (msg.notes) |notes| { - for (notes) |*note| { - note.deinit(allocator); - } - - allocator.free(notes); + for (msg.notes) |*note| { + note.deinit(allocator); } - msg.notes = null; + allocator.free(msg.notes); + + msg.notes = &.{}; } pub fn writeFormat( @@ -512,18 +501,16 @@ pub const Msg = struct { to: anytype, comptime enable_ansi_colors: bool, ) !void { - try msg.data.writeFormat(to, msg.kind, enable_ansi_colors); + try msg.data.writeFormat(to, msg.kind, msg.redact_sensitive_information, enable_ansi_colors); - if (msg.notes) |notes| { - if (notes.len > 0) { - try to.writeAll("\n"); - } + if (msg.notes.len > 0) { + try to.writeAll("\n"); + } - for (notes) |note| { - try to.writeAll("\n"); + for (msg.notes) |note| { + try to.writeAll("\n"); - try note.writeFormat(to, .note, enable_ansi_colors); - } + try note.writeFormat(to, .note, msg.redact_sensitive_information, enable_ansi_colors); } } @@ -599,12 +586,9 @@ pub const Range = struct { }; pub const Log = struct { - debug: bool = false, - // TODO: make u32 - warnings: usize = 0, - // TODO: make u32 - errors: usize = 0, - msgs: ArrayList(Msg), + warnings: u32 = 0, + errors: u32 = 0, + msgs: std.ArrayList(Msg), level: Level = if (Environment.isDebug) Level.info else Level.warn, clone_line_text: bool = false, @@ -668,7 +652,7 @@ pub const Log = struct { .{ "error", Level.err }, }); - pub fn fromJS(globalThis: *JSC.JSGlobalObject, value: JSC.JSValue) !?Level { + pub fn fromJS(globalThis: *JSC.JSGlobalObject, value: JSC.JSValue) JSError!?Level { if (value == .zero or value == .undefined) { return null; } @@ -684,28 +668,28 @@ pub const Log = struct { pub fn init(allocator: std.mem.Allocator) Log { return Log{ - .msgs = ArrayList(Msg).init(allocator), + .msgs = std.ArrayList(Msg).init(allocator), .level = default_log_level, }; } pub fn initComptime(allocator: std.mem.Allocator) Log { return Log{ - .msgs = ArrayList(Msg).init(allocator), + .msgs = std.ArrayList(Msg).init(allocator), }; } - pub fn addDebugFmt(log: *Log, source: ?*const Source, l: Loc, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { + pub fn addDebugFmt(log: *Log, source: ?*const Source, l: Loc, allocator: std.mem.Allocator, comptime text: string, args: anytype) OOM!void { if (!Kind.shouldPrint(.debug, log.level)) return; @setCold(true); try log.addMsg(.{ .kind = .debug, - .data = try rangeData(source, Range{ .loc = l }, allocPrint(allocator, text, args) catch unreachable).cloneLineText(log.clone_line_text, log.msgs.allocator), + .data = try rangeData(source, Range{ .loc = l }, try allocPrint(allocator, text, args)).cloneLineText(log.clone_line_text, log.msgs.allocator), }); } - pub fn addVerbose(log: *Log, source: ?*const Source, loc: Loc, text: string) !void { + pub fn addVerbose(log: *Log, source: ?*const Source, loc: Loc, text: string) OOM!void { if (!Kind.shouldPrint(.verbose, log.level)) return; @setCold(true); @@ -757,15 +741,13 @@ pub const Log = struct { return arr; } - pub fn cloneTo(self: *Log, other: *Log) !void { + pub fn cloneTo(self: *Log, other: *Log) OOM!void { var notes_count: usize = 0; for (self.msgs.items) |msg_| { const msg: Msg = msg_; - if (msg.notes) |notes| { - for (notes) |note| { - notes_count += @as(usize, @intCast(@intFromBool(note.text.len > 0))); - } + for (msg.notes) |note| { + notes_count += @as(usize, @intCast(@intFromBool(note.text.len > 0))); } } @@ -773,14 +755,12 @@ pub const Log = struct { var notes = try other.msgs.allocator.alloc(Data, notes_count); var note_i: usize = 0; for (self.msgs.items) |*msg| { - if (msg.notes) |current_notes| { - const start_note_i: usize = note_i; - for (current_notes) |note| { - notes[note_i] = note; - note_i += 1; - } - msg.notes = notes[start_note_i..note_i]; + const start_note_i: usize = note_i; + for (msg.notes) |note| { + notes[note_i] = note; + note_i += 1; } + msg.notes = notes[start_note_i..note_i]; } } @@ -789,12 +769,12 @@ pub const Log = struct { other.errors += self.errors; } - pub fn appendTo(self: *Log, other: *Log) !void { + pub fn appendTo(self: *Log, other: *Log) OOM!void { try self.cloneTo(other); self.msgs.clearAndFree(); } - pub fn cloneToWithRecycled(self: *Log, other: *Log, recycled: bool) !void { + pub fn cloneToWithRecycled(self: *Log, other: *Log, recycled: bool) OOM!void { try other.msgs.appendSlice(self.msgs.items); other.warnings += self.warnings; other.errors += self.errors; @@ -806,9 +786,7 @@ pub const Log = struct { for (self.msgs.items) |msg| { msg.count(&string_builder); - if (msg.notes) |notes| { - notes_count += notes.len; - } + notes_count += msg.notes.len; } } @@ -819,18 +797,18 @@ pub const Log = struct { { for (self.msgs.items, (other.msgs.items.len - self.msgs.items.len)..) |msg, j| { other.msgs.items[j] = msg.cloneWithBuilder(notes_buf[note_i..], &string_builder); - note_i += (msg.notes orelse &[_]Data{}).len; + note_i += msg.notes.len; } } } } - pub fn appendToWithRecycled(self: *Log, other: *Log, recycled: bool) !void { + pub fn appendToWithRecycled(self: *Log, other: *Log, recycled: bool) OOM!void { try self.cloneToWithRecycled(other, recycled); self.msgs.clearAndFree(); } - pub fn appendToMaybeRecycled(self: *Log, other: *Log, source: *const Source) !void { + pub fn appendToMaybeRecycled(self: *Log, other: *Log, source: *const Source) OOM!void { return self.appendToWithRecycled(other, source.contents_is_recycled); } @@ -838,7 +816,7 @@ pub const Log = struct { self.msgs.clearAndFree(); } - pub fn addVerboseWithNotes(log: *Log, source: ?*const Source, loc: Loc, text: string, notes: []Data) !void { + pub fn addVerboseWithNotes(log: *Log, source: ?*const Source, loc: Loc, text: string, notes: []Data) OOM!void { @setCold(true); if (!Kind.shouldPrint(.verbose, log.level)) return; @@ -849,13 +827,13 @@ pub const Log = struct { }); } - inline fn allocPrint(allocator: std.mem.Allocator, comptime fmt: string, args: anytype) !string { + inline fn allocPrint(allocator: std.mem.Allocator, comptime fmt: string, args: anytype) OOM!string { return try switch (Output.enable_ansi_colors) { inline else => |enable_ansi_colors| std.fmt.allocPrint(allocator, Output.prettyFmt(fmt, enable_ansi_colors), args), }; } - inline fn _addResolveErrorWithLevel( + inline fn addResolveErrorWithLevel( log: *Log, source: ?*const Source, r: Range, @@ -864,13 +842,13 @@ pub const Log = struct { args: anytype, import_kind: ImportKind, comptime dupe_text: bool, - comptime is_error: bool, + comptime kind: enum { err, warn }, err: anyerror, - ) !void { + ) OOM!void { const text = try allocPrint(allocator, fmt, args); // TODO: fix this. this is stupid, it should be returned in allocPrint. const specifier = BabyString.in(text, args.@"0"); - if (comptime is_error) { + if (comptime kind == .err) { log.errors += 1; } else { log.warnings += 1; @@ -884,7 +862,7 @@ pub const Log = struct { ); if (_data.location != null) { if (_data.location.?.line_text) |line| { - _data.location.?.line_text = allocator.dupe(u8, line) catch unreachable; + _data.location.?.line_text = try allocator.dupe(u8, line); } } break :brk _data; @@ -895,7 +873,8 @@ pub const Log = struct { ); const msg = Msg{ - .kind = if (comptime is_error) Kind.err else Kind.warn, + // .kind = if (comptime error_type == .err) Kind.err else Kind.warn, + .kind = @field(Kind, @tagName(kind)), .data = data, .metadata = .{ .resolve = Msg.Metadata.Resolve{ .specifier = specifier, @@ -907,34 +886,6 @@ pub const Log = struct { try log.addMsg(msg); } - inline fn _addResolveError( - log: *Log, - source: ?*const Source, - r: Range, - allocator: std.mem.Allocator, - comptime fmt: string, - args: anytype, - import_kind: ImportKind, - comptime dupe_text: bool, - err: anyerror, - ) !void { - return _addResolveErrorWithLevel(log, source, r, allocator, fmt, args, import_kind, dupe_text, true, err); - } - - inline fn _addResolveWarn( - log: *Log, - source: ?*const Source, - r: Range, - allocator: std.mem.Allocator, - comptime fmt: string, - args: anytype, - import_kind: ImportKind, - comptime dupe_text: bool, - err: anyerror, - ) !void { - return _addResolveErrorWithLevel(log, source, r, allocator, fmt, args, import_kind, dupe_text, false, err); - } - pub fn addResolveError( log: *Log, source: ?*const Source, @@ -944,9 +895,9 @@ pub const Log = struct { args: anytype, import_kind: ImportKind, err: anyerror, - ) !void { + ) OOM!void { @setCold(true); - return try _addResolveError(log, source, r, allocator, fmt, args, import_kind, false, err); + return try addResolveErrorWithLevel(log, source, r, allocator, fmt, args, import_kind, false, .err, err); } pub fn addResolveErrorWithTextDupe( @@ -957,30 +908,12 @@ pub const Log = struct { comptime fmt: string, args: anytype, import_kind: ImportKind, - ) !void { + ) OOM!void { @setCold(true); - return try _addResolveError(log, source, r, allocator, fmt, args, import_kind, true, error.ModuleNotFound); + return try addResolveErrorWithLevel(log, source, r, allocator, fmt, args, import_kind, true, .err, error.ModuleNotFound); } - pub fn addResolveErrorWithTextDupeMaybeWarn( - log: *Log, - source: ?*const Source, - r: Range, - allocator: std.mem.Allocator, - comptime fmt: string, - args: anytype, - import_kind: ImportKind, - warn: bool, - ) !void { - @setCold(true); - if (warn) { - return try _addResolveError(log, source, r, allocator, fmt, args, import_kind, true, error.ModuleNotFound); - } else { - return try _addResolveWarn(log, source, r, allocator, fmt, args, import_kind, true, error.ModuleNotFound); - } - } - - pub fn addRangeError(log: *Log, source: ?*const Source, r: Range, text: string) !void { + pub fn addRangeError(log: *Log, source: ?*const Source, r: Range, text: string) OOM!void { @setCold(true); log.errors += 1; try log.addMsg(.{ @@ -989,44 +922,55 @@ pub const Log = struct { }); } - pub fn addRangeErrorFmt(log: *Log, source: ?*const Source, r: Range, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { + pub fn addRangeErrorFmt(log: *Log, source: ?*const Source, r: Range, allocator: std.mem.Allocator, comptime text: string, args: anytype) OOM!void { @setCold(true); log.errors += 1; try log.addMsg(.{ .kind = .err, - .data = try rangeData(source, r, allocPrint(allocator, text, args) catch unreachable).cloneLineText(log.clone_line_text, log.msgs.allocator), + .data = try rangeData(source, r, try allocPrint(allocator, text, args)).cloneLineText(log.clone_line_text, log.msgs.allocator), }); } - pub fn addRangeErrorFmtWithNotes(log: *Log, source: ?*const Source, r: Range, allocator: std.mem.Allocator, notes: []Data, comptime fmt: string, args: anytype) !void { + pub fn addRangeErrorFmtWithNotes(log: *Log, source: ?*const Source, r: Range, allocator: std.mem.Allocator, notes: []Data, comptime fmt: string, args: anytype) OOM!void { @setCold(true); log.errors += 1; try log.addMsg(.{ .kind = .err, - .data = try rangeData(source, r, allocPrint(allocator, fmt, args) catch unreachable).cloneLineText(log.clone_line_text, log.msgs.allocator), + .data = try rangeData(source, r, try allocPrint(allocator, fmt, args)).cloneLineText(log.clone_line_text, log.msgs.allocator), .notes = notes, }); } - pub fn addErrorFmtNoLoc(log: *Log, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { - try log.addErrorFmt(null, Loc.Empty, allocator, text, args); - } - - pub fn addErrorFmt(log: *Log, source: ?*const Source, l: Loc, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { + pub fn addErrorFmt(log: *Log, source: ?*const Source, l: Loc, allocator: std.mem.Allocator, comptime text: string, args: anytype) OOM!void { @setCold(true); log.errors += 1; try log.addMsg(.{ .kind = .err, - .data = try rangeData(source, Range{ .loc = l }, allocPrint(allocator, text, args) catch unreachable).cloneLineText(log.clone_line_text, log.msgs.allocator), + .data = try rangeData(source, .{ .loc = l }, try allocPrint(allocator, text, args)).cloneLineText(log.clone_line_text, log.msgs.allocator), }); } - pub fn addZigErrorWithNote(log: *Log, allocator: std.mem.Allocator, err: anyerror, comptime noteFmt: string, args: anytype) !void { + // TODO(dylan-conway): rename and replace `addErrorFmt` + pub fn addErrorFmtOpts(log: *Log, allocator: std.mem.Allocator, comptime fmt: string, args: anytype, opts: AddErrorOptions) OOM!void { + @setCold(true); + log.errors += 1; + try log.addMsg(.{ + .kind = .err, + .data = try rangeData( + opts.source, + .{ .loc = opts.loc, .len = opts.len }, + try allocPrint(allocator, fmt, args), + ).cloneLineText(log.clone_line_text, log.msgs.allocator), + .redact_sensitive_information = opts.redact_sensitive_information, + }); + } + + pub fn addZigErrorWithNote(log: *Log, allocator: std.mem.Allocator, err: anyerror, comptime noteFmt: string, args: anytype) OOM!void { @setCold(true); log.errors += 1; var notes = try allocator.alloc(Data, 1); - notes[0] = rangeData(null, Range.None, allocPrint(allocator, noteFmt, args) catch unreachable); + notes[0] = rangeData(null, Range.None, try allocPrint(allocator, noteFmt, args)); try log.addMsg(.{ .kind = .err, @@ -1035,7 +979,7 @@ pub const Log = struct { }); } - pub fn addRangeWarning(log: *Log, source: ?*const Source, r: Range, text: string) !void { + pub fn addRangeWarning(log: *Log, source: ?*const Source, r: Range, text: string) OOM!void { @setCold(true); if (!Kind.shouldPrint(.warn, log.level)) return; log.warnings += 1; @@ -1045,17 +989,17 @@ pub const Log = struct { }); } - pub fn addWarningFmt(log: *Log, source: ?*const Source, l: Loc, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { + pub fn addWarningFmt(log: *Log, source: ?*const Source, l: Loc, allocator: std.mem.Allocator, comptime text: string, args: anytype) OOM!void { @setCold(true); if (!Kind.shouldPrint(.warn, log.level)) return; log.warnings += 1; try log.addMsg(.{ .kind = .warn, - .data = try rangeData(source, Range{ .loc = l }, allocPrint(allocator, text, args) catch unreachable).cloneLineText(log.clone_line_text, log.msgs.allocator), + .data = try rangeData(source, Range{ .loc = l }, try allocPrint(allocator, text, args)).cloneLineText(log.clone_line_text, log.msgs.allocator), }); } - pub fn addWarningFmtLineCol(log: *Log, filepath: []const u8, line: u32, col: u32, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { + pub fn addWarningFmtLineCol(log: *Log, filepath: []const u8, line: u32, col: u32, allocator: std.mem.Allocator, comptime text: string, args: anytype) OOM!void { @setCold(true); if (!Kind.shouldPrint(.warn, log.level)) return; log.warnings += 1; @@ -1064,24 +1008,24 @@ pub const Log = struct { try log.addMsg(.{ .kind = .warn, - .data = Data.cloneLineText(Data{ - .text = allocPrint(allocator, text, args) catch unreachable, + .data = try Data.cloneLineText(Data{ + .text = try allocPrint(allocator, text, args), .location = Location{ .file = filepath, .line = @intCast(line), .column = @intCast(col), }, - }, log.clone_line_text, allocator) catch unreachable, + }, log.clone_line_text, allocator), }); } - pub fn addRangeWarningFmt(log: *Log, source: ?*const Source, r: Range, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { + pub fn addRangeWarningFmt(log: *Log, source: ?*const Source, r: Range, allocator: std.mem.Allocator, comptime text: string, args: anytype) OOM!void { @setCold(true); if (!Kind.shouldPrint(.warn, log.level)) return; log.warnings += 1; try log.addMsg(.{ .kind = .warn, - .data = try rangeData(source, r, allocPrint(allocator, text, args) catch unreachable).cloneLineText(log.clone_line_text, log.msgs.allocator), + .data = try rangeData(source, r, try allocPrint(allocator, text, args)).cloneLineText(log.clone_line_text, log.msgs.allocator), }); } @@ -1095,27 +1039,27 @@ pub const Log = struct { comptime note_fmt: string, note_args: anytype, note_range: Range, - ) !void { + ) OOM!void { @setCold(true); if (!Kind.shouldPrint(.warn, log.level)) return; log.warnings += 1; var notes = try allocator.alloc(Data, 1); - notes[0] = rangeData(source, note_range, allocPrint(allocator, note_fmt, note_args) catch unreachable); + notes[0] = rangeData(source, note_range, try allocPrint(allocator, note_fmt, note_args)); try log.addMsg(.{ .kind = .warn, - .data = rangeData(source, r, allocPrint(allocator, fmt, args) catch unreachable), + .data = rangeData(source, r, try allocPrint(allocator, fmt, args)), .notes = notes, }); } - pub fn addRangeWarningFmtWithNotes(log: *Log, source: ?*const Source, r: Range, allocator: std.mem.Allocator, notes: []Data, comptime fmt: string, args: anytype) !void { + pub fn addRangeWarningFmtWithNotes(log: *Log, source: ?*const Source, r: Range, allocator: std.mem.Allocator, notes: []Data, comptime fmt: string, args: anytype) OOM!void { @setCold(true); log.warnings += 1; try log.addMsg(.{ .kind = .warn, - .data = try rangeData(source, r, allocPrint(allocator, fmt, args) catch unreachable).cloneLineText(log.clone_line_text, log.msgs.allocator), + .data = try rangeData(source, r, try allocPrint(allocator, fmt, args)).cloneLineText(log.clone_line_text, log.msgs.allocator), .notes = notes, }); } @@ -1130,22 +1074,22 @@ pub const Log = struct { comptime note_fmt: string, note_args: anytype, note_range: Range, - ) !void { + ) OOM!void { @setCold(true); if (!Kind.shouldPrint(.err, log.level)) return; log.errors += 1; var notes = try allocator.alloc(Data, 1); - notes[0] = rangeData(source, note_range, allocPrint(allocator, note_fmt, note_args) catch unreachable); + notes[0] = rangeData(source, note_range, try allocPrint(allocator, note_fmt, note_args)); try log.addMsg(.{ .kind = .err, - .data = rangeData(source, r, allocPrint(allocator, fmt, args) catch unreachable), + .data = rangeData(source, r, try allocPrint(allocator, fmt, args)), .notes = notes, }); } - pub fn addWarning(log: *Log, source: ?*const Source, l: Loc, text: string) !void { + pub fn addWarning(log: *Log, source: ?*const Source, l: Loc, text: string) OOM!void { @setCold(true); if (!Kind.shouldPrint(.warn, log.level)) return; log.warnings += 1; @@ -1155,13 +1099,13 @@ pub const Log = struct { }); } - pub fn addWarningWithNote(log: *Log, source: ?*const Source, l: Loc, allocator: std.mem.Allocator, warn: string, comptime note_fmt: string, note_args: anytype) !void { + pub fn addWarningWithNote(log: *Log, source: ?*const Source, l: Loc, allocator: std.mem.Allocator, warn: string, comptime note_fmt: string, note_args: anytype) OOM!void { @setCold(true); if (!Kind.shouldPrint(.warn, log.level)) return; log.warnings += 1; var notes = try allocator.alloc(Data, 1); - notes[0] = rangeData(source, Range{ .loc = l }, allocPrint(allocator, note_fmt, note_args) catch unreachable); + notes[0] = rangeData(source, Range{ .loc = l }, try allocPrint(allocator, note_fmt, note_args)); try log.addMsg(.{ .kind = .warn, @@ -1170,7 +1114,7 @@ pub const Log = struct { }); } - pub fn addRangeDebug(log: *Log, source: ?*const Source, r: Range, text: string) !void { + pub fn addRangeDebug(log: *Log, source: ?*const Source, r: Range, text: string) OOM!void { @setCold(true); if (!Kind.shouldPrint(.debug, log.level)) return; try log.addMsg(.{ @@ -1179,7 +1123,7 @@ pub const Log = struct { }); } - pub fn addRangeDebugWithNotes(log: *Log, source: ?*const Source, r: Range, text: string, notes: []Data) !void { + pub fn addRangeDebugWithNotes(log: *Log, source: ?*const Source, r: Range, text: string, notes: []Data) OOM!void { @setCold(true); if (!Kind.shouldPrint(.debug, log.level)) return; // log.de += 1; @@ -1190,7 +1134,7 @@ pub const Log = struct { }); } - pub fn addRangeErrorWithNotes(log: *Log, source: ?*const Source, r: Range, text: string, notes: []Data) !void { + pub fn addRangeErrorWithNotes(log: *Log, source: ?*const Source, r: Range, text: string, notes: []Data) OOM!void { @setCold(true); log.errors += 1; try log.addMsg(.{ @@ -1200,7 +1144,7 @@ pub const Log = struct { }); } - pub fn addRangeWarningWithNotes(log: *Log, source: ?*const Source, r: Range, text: string, notes: []Data) !void { + pub fn addRangeWarningWithNotes(log: *Log, source: ?*const Source, r: Range, text: string, notes: []Data) OOM!void { @setCold(true); if (!Kind.shouldPrint(.warn, log.level)) return; log.warnings += 1; @@ -1211,17 +1155,35 @@ pub const Log = struct { }); } - pub inline fn addMsg(self: *Log, msg: Msg) !void { + pub fn addMsg(self: *Log, msg: Msg) OOM!void { try self.msgs.append(msg); } - pub fn addError(self: *Log, _source: ?*const Source, loc: Loc, text: string) !void { + pub fn addError(self: *Log, _source: ?*const Source, loc: Loc, text: string) OOM!void { @setCold(true); self.errors += 1; try self.addMsg(.{ .kind = .err, .data = rangeData(_source, Range{ .loc = loc }, text) }); } - pub fn addSymbolAlreadyDeclaredError(self: *Log, allocator: std.mem.Allocator, source: *const Source, name: string, new_loc: Loc, old_loc: Loc) !void { + const AddErrorOptions = struct { + source: ?*const Source = null, + loc: Loc = Loc.Empty, + len: i32 = 0, + redact_sensitive_information: bool = false, + }; + + // TODO(dylan-conway): rename and replace `addError` + pub fn addErrorOpts(self: *Log, text: string, opts: AddErrorOptions) OOM!void { + @setCold(true); + self.errors += 1; + try self.addMsg(.{ + .kind = .err, + .data = rangeData(opts.source, .{ .loc = opts.loc, .len = opts.len }, text), + .redact_sensitive_information = opts.redact_sensitive_information, + }); + } + + pub fn addSymbolAlreadyDeclaredError(self: *Log, allocator: std.mem.Allocator, source: *const Source, name: string, new_loc: Loc, old_loc: Loc) OOM!void { var notes = try allocator.alloc(Data, 1); notes[0] = rangeData( source, @@ -1239,13 +1201,13 @@ pub const Log = struct { ); } - pub fn printForLogLevel(self: *Log, to: anytype) !void { + pub fn print(self: *Log, to: anytype) !void { return switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| self.printForLogLevelWithEnableAnsiColors(to, enable_ansi_colors), + inline else => |enable_ansi_colors| self.printWithEnableAnsiColors(to, enable_ansi_colors), }; } - pub fn printForLogLevelWithEnableAnsiColors(self: *const Log, to: anytype, comptime enable_ansi_colors: bool) !void { + pub fn printWithEnableAnsiColors(self: *const Log, to: anytype, comptime enable_ansi_colors: bool) !void { var needs_newline = false; if (self.warnings > 0 and self.errors > 0) { // Print warnings at the top @@ -1285,14 +1247,6 @@ pub const Log = struct { if (needs_newline) _ = try to.write("\n"); } - pub fn printForLogLevelColorsRuntime(self: *Log, to: anytype, enable_ansi_colors: bool) !void { - if (enable_ansi_colors) { - return self.printForLogLevelWithEnableAnsiColors(to, true); - } else { - return self.printForLogLevelWithEnableAnsiColors(to, false); - } - } - pub fn toZigException(this: *const Log, allocator: std.mem.Allocator) *js.ZigException.Holder { var holder = try allocator.create(js.ZigException.Holder); holder.* = js.ZigException.Holder.init(); diff --git a/src/main_wasm.zig b/src/main_wasm.zig index d4d0c43dca..3a1cfc5c83 100644 --- a/src/main_wasm.zig +++ b/src/main_wasm.zig @@ -483,7 +483,7 @@ export fn getTests(opts_array: u64) u64 { Output.print("Error: {s}\n", .{@errorName(err)}); - log_.printForLogLevel(Output.writer()) catch unreachable; + log_.print(Output.writer()) catch unreachable; return 0; }; diff --git a/src/router.zig b/src/router.zig index f5316982ed..7fcf314411 100644 --- a/src/router.zig +++ b/src/router.zig @@ -993,7 +993,7 @@ pub const Test = struct { const Resolver = @import("./resolver/resolver.zig").Resolver; var logger = Logger.Log.init(default_allocator); errdefer { - logger.printForLogLevel(Output.errorWriter()) catch {}; + logger.print(Output.errorWriter()) catch {}; } const opts = Options.BundleOptions{ @@ -1049,7 +1049,7 @@ pub const Test = struct { const Resolver = @import("./resolver/resolver.zig").Resolver; var logger = Logger.Log.init(default_allocator); errdefer { - logger.printForLogLevel(Output.errorWriter()) catch {}; + logger.print(Output.errorWriter()) catch {}; } const opts = Options.BundleOptions{ diff --git a/src/string.zig b/src/string.zig index 930ee12dd5..2c694efbdd 100644 --- a/src/string.zig +++ b/src/string.zig @@ -341,7 +341,7 @@ pub const String = extern struct { return bytes; } - pub fn toOwnedSliceReturningAllASCII(this: String, allocator: std.mem.Allocator) !struct { []u8, bool } { + pub fn toOwnedSliceReturningAllASCII(this: String, allocator: std.mem.Allocator) OOM!struct { []u8, bool } { switch (this.tag) { .ZigString => return .{ try this.value.ZigString.toOwnedSlice(allocator), true }, .WTFStringImpl => { diff --git a/src/string_immutable.zig b/src/string_immutable.zig index 5745bd2b91..d62af2b977 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -85,9 +85,6 @@ fn literalLength(comptime T: type, comptime str: string) usize { }; } -// TODO: remove this -pub const toUTF16LiteralZ = toUTF16Literal; - pub const OptionalUsize = std.meta.Int(.unsigned, @bitSizeOf(usize) - 1); pub fn indexOfAny(slice: string, comptime str: []const u8) ?OptionalUsize { switch (comptime str.len) { @@ -217,9 +214,8 @@ pub fn isNPMPackageName(target: string) bool { return !scoped or slash_index > 0 and slash_index + 1 < target.len; } -pub fn startsWithUUID(str: string) bool { - const uuid_len = 36; - if (str.len < uuid_len) return false; +pub fn isUUID(str: string) bool { + if (str.len != uuid_len) return false; for (0..8) |i| { switch (str[i]) { '0'...'9', 'a'...'f', 'A'...'F' => {}, @@ -257,6 +253,12 @@ pub fn startsWithUUID(str: string) bool { return true; } +pub const uuid_len = 36; + +pub fn startsWithUUID(str: string) bool { + return isUUID(str[0..@min(str.len, uuid_len)]); +} + /// https://github.com/npm/cli/blob/63d6a732c3c0e9c19fd4d147eaa5cc27c29b168d/node_modules/%40npmcli/redact/lib/matchers.js#L7 /// /\b(npms?_)[a-zA-Z0-9]{36,48}\b/gi /// Returns the length of the secret if one exist. @@ -294,6 +296,128 @@ pub fn startsWithNpmSecret(str: string) u8 { return i; } +fn startsWithRedactedItem(text: string, comptime item: string) ?struct { usize, usize } { + if (!strings.hasPrefixComptime(text, item)) return null; + + var whitespace = false; + var offset: usize = item.len; + while (offset < text.len and std.ascii.isWhitespace(text[offset])) { + offset += 1; + whitespace = true; + } + if (offset == text.len) return null; + const cont = js_lexer.isIdentifierContinue(text[offset]); + + // must be another identifier + if (!whitespace and cont) return null; + + // `null` is not returned after this point. Redact to the next + // newline if anything is unexpected + if (cont) return .{ offset, indexOfChar(text[offset..], '\n') orelse text[offset..].len }; + offset += 1; + + var end = offset; + while (end < text.len and std.ascii.isWhitespace(text[end])) { + end += 1; + } + + if (end == text.len) { + return .{ offset, text[offset..].len }; + } + + switch (text[end]) { + inline '\'', '"', '`' => |q| { + // attempt to find closing + const opening = end; + end += 1; + while (end < text.len) { + switch (text[end]) { + '\\' => { + // skip + end += 1; + end += 1; + }, + q => { + // closing + return .{ opening + 1, (end - 1) - opening }; + }, + else => { + end += 1; + }, + } + } + + const len = strings.indexOfChar(text[offset..], '\n') orelse text[offset..].len; + return .{ offset, len }; + }, + else => { + const len = strings.indexOfChar(text[offset..], '\n') orelse text[offset..].len; + return .{ offset, len }; + }, + } +} + +/// Returns offset and length of first secret found. +pub fn startsWithSecret(str: string) ?struct { usize, usize } { + if (startsWithRedactedItem(str, "_auth")) |auth| { + const offset, const len = auth; + return .{ offset, len }; + } + if (startsWithRedactedItem(str, "_authToken")) |auth_token| { + const offset, const len = auth_token; + return .{ offset, len }; + } + if (startsWithRedactedItem(str, "email")) |email| { + const offset, const len = email; + return .{ offset, len }; + } + if (startsWithRedactedItem(str, "_password")) |password| { + const offset, const len = password; + return .{ offset, len }; + } + if (startsWithRedactedItem(str, "token")) |token| { + const offset, const len = token; + return .{ offset, len }; + } + + if (startsWithUUID(str)) { + return .{ 0, 36 }; + } + + const npm_secret_len = startsWithNpmSecret(str); + if (npm_secret_len > 0) { + return .{ 0, npm_secret_len }; + } + + if (findUrlPassword(str)) |url_pass| { + const offset, const len = url_pass; + return .{ offset, len }; + } + + return null; +} + +pub fn findUrlPassword(text: string) ?struct { usize, usize } { + if (!strings.hasPrefixComptime(text, "http")) return null; + var offset: usize = "http".len; + if (hasPrefixComptime(text[offset..], "://")) { + offset += "://".len; + } else if (hasPrefixComptime(text[offset..], "s://")) { + offset += "s://".len; + } else { + return null; + } + var remain = text[offset..]; + const end = indexOfChar(remain, '\n') orelse remain.len; + remain = remain[0..end]; + const at = indexOfChar(remain, '@') orelse return null; + const colon = indexOfCharNeg(remain[0..at], ':'); + if (colon == -1 or colon == at - 1) return null; + offset += @intCast(colon + 1); + const len: usize = at - @as(usize, @intCast(colon + 1)); + return .{ offset, len }; +} + pub fn indexAnyComptime(target: string, comptime chars: string) ?usize { for (target, 0..) |parent, i| { inline for (chars) |char| { diff --git a/src/toml/toml_lexer.zig b/src/toml/toml_lexer.zig index 4e53e1a2b4..34f3646de1 100644 --- a/src/toml/toml_lexer.zig +++ b/src/toml/toml_lexer.zig @@ -70,6 +70,8 @@ pub const Lexer = struct { has_newline_before: bool = false, + should_redact_logs: bool, + pub inline fn loc(self: *const Lexer) logger.Loc { return logger.usize2Loc(self.start); } @@ -80,12 +82,12 @@ pub const Lexer = struct { // Only add this if there is not already an error. // It is possible that there is a more descriptive error already emitted. if (!self.log.hasErrors()) - self.addError(self.start, "Syntax Error", .{}, true); + self.addError(self.start, "Syntax Error", .{}); return Error.SyntaxError; } - pub fn addError(self: *Lexer, _loc: usize, comptime format: []const u8, args: anytype, _: bool) void { + pub fn addError(self: *Lexer, _loc: usize, comptime format: []const u8, args: anytype) void { @setCold(true); var __loc = logger.usize2Loc(_loc); @@ -93,24 +95,33 @@ pub const Lexer = struct { return; } - self.log.addErrorFmt(&self.source, __loc, self.log.msgs.allocator, format, args) catch unreachable; + self.log.addErrorFmtOpts( + self.log.msgs.allocator, + format, + args, + .{ + .source = &self.source, + .loc = __loc, + .redact_sensitive_information = self.should_redact_logs, + }, + ) catch unreachable; self.prev_error_loc = __loc; } pub fn addDefaultError(self: *Lexer, msg: []const u8) !void { @setCold(true); - self.addError(self.start, "{s}", .{msg}, true); + self.addError(self.start, "{s}", .{msg}); return Error.SyntaxError; } pub fn addSyntaxError(self: *Lexer, _loc: usize, comptime fmt: []const u8, args: anytype) !void { @setCold(true); - self.addError(_loc, fmt, args, false); + self.addError(_loc, fmt, args); return Error.SyntaxError; } - pub fn addRangeError(self: *Lexer, r: logger.Range, comptime format: []const u8, args: anytype, _: bool) !void { + pub fn addRangeError(self: *Lexer, r: logger.Range, comptime format: []const u8, args: anytype) !void { @setCold(true); if (self.prev_error_loc.eql(r.loc)) { @@ -118,12 +129,13 @@ pub const Lexer = struct { } const errorMessage = std.fmt.allocPrint(self.log.msgs.allocator, format, args) catch unreachable; - try self.log.addRangeError(&self.source, r, errorMessage); + try self.log.addErrorOpts(errorMessage, .{ + .source = &self.source, + .loc = r.loc, + .len = r.len, + .redact_sensitive_information = self.should_redact_logs, + }); self.prev_error_loc = r.loc; - - // if (panic) { - // return Error.ParserError; - // } } /// Look ahead at the next n codepoints without advancing the iterator. @@ -924,7 +936,6 @@ pub const Lexer = struct { logger.Range{ .loc = .{ .start = @as(i32, @intCast(octal_start)) }, .len = @as(i32, @intCast(iter.i - octal_start)) }, "Invalid legacy octal literal", .{}, - false, ) catch unreachable; } }, @@ -1036,7 +1047,6 @@ pub const Lexer = struct { .{ .loc = .{ .start = @as(i32, @intCast(start + hex_start)) }, .len = @as(i32, @intCast((iter.i - hex_start))) }, "Unicode escape sequence is out of range", .{}, - true, ); return; } @@ -1132,7 +1142,7 @@ pub const Lexer = struct { } }; - try lexer.addRangeError(lexer.range(), "Unexpected {s}", .{found}, true); + try lexer.addRangeError(lexer.range(), "Unexpected {s}", .{found}); } pub fn expectedString(self: *Lexer, text: string) !void { @@ -1144,7 +1154,7 @@ pub const Lexer = struct { } }; - try self.addRangeError(self.range(), "Expected {s} but found {s}", .{ text, found }, true); + try self.addRangeError(self.range(), "Expected {s} but found {s}", .{ text, found }); } pub fn range(self: *Lexer) logger.Range { @@ -1154,12 +1164,13 @@ pub const Lexer = struct { }; } - pub fn init(log: *logger.Log, source: logger.Source, allocator: std.mem.Allocator) !Lexer { + pub fn init(log: *logger.Log, source: logger.Source, allocator: std.mem.Allocator, redact_logs: bool) !Lexer { var lex = Lexer{ .log = log, .source = source, .prev_error_loc = logger.Loc.Empty, .allocator = allocator, + .should_redact_logs = redact_logs, }; lex.step(); try lex.next(); diff --git a/src/toml/toml_parser.zig b/src/toml/toml_parser.zig index 25d760526a..b569be7495 100644 --- a/src/toml/toml_parser.zig +++ b/src/toml/toml_parser.zig @@ -81,9 +81,9 @@ pub const TOML = struct { log: *logger.Log, allocator: std.mem.Allocator, - pub fn init(allocator: std.mem.Allocator, source_: logger.Source, log: *logger.Log) !TOML { + pub fn init(allocator: std.mem.Allocator, source_: logger.Source, log: *logger.Log, redact_logs: bool) !TOML { return TOML{ - .lexer = try Lexer.init(log, source_, allocator), + .lexer = try Lexer.init(log, source_, allocator, redact_logs), .allocator = allocator, .log = log, }; @@ -166,7 +166,7 @@ pub const TOML = struct { return head; } - pub fn parse(source_: *const logger.Source, log: *logger.Log, allocator: std.mem.Allocator) !Expr { + pub fn parse(source_: *const logger.Source, log: *logger.Log, allocator: std.mem.Allocator, redact_logs: bool) !Expr { switch (source_.contents.len) { // This is to be consisntent with how disabled JS files are handled 0 => { @@ -175,7 +175,7 @@ pub const TOML = struct { else => {}, } - var parser = try TOML.init(allocator, source_.*, log); + var parser = try TOML.init(allocator, source_.*, log, redact_logs); return try parser.runParser(); } diff --git a/test/cli/install/redacted-config-logs.test.ts b/test/cli/install/redacted-config-logs.test.ts new file mode 100644 index 0000000000..68d9a70c5b --- /dev/null +++ b/test/cli/install/redacted-config-logs.test.ts @@ -0,0 +1,102 @@ +import { bunExe, bunEnv, tmpdirSync } from "harness"; +import { write, spawnSync } from "bun"; +import { describe, test, expect } from "bun:test"; +import { join } from "path"; + +describe("redact", async () => { + const tests = [ + { + title: "url password", + bunfig: `install.registry = "https://user:pass@registry.org`, + expected: `"https://user:****@registry.org`, + }, + { + title: "empty url password", + bunfig: `install.registry = "https://user:@registry.org`, + expected: `"https://user:@registry.org`, + }, + { + title: "small string", + bunfig: `l;token = "1"`, + expected: `"*"`, + }, + { + title: "random UUID", + bunfig: 'unre;lated = "f1b0b6b4-4b1b-4b1b-8b1b-4b1b4b1b4b1b"', + expected: '"************************************"', + }, + { + title: "random npm_ secret", + bunfig: 'the;secret = "npm_1234567890abcdefghijklmnopqrstuvwxyz"', + expected: '"****************************************"', + }, + { + title: "random npms_ secret", + bunfig: 'the;secret = "npms_1234567890abcdefghijklmnopqrstuvwxyz"', + expected: "*****************************************", + }, + { + title: "zero length unterminated string", + bunfig: '_authToken = "', + expected: "*", + }, + { + title: "invalid _auth", + npmrc: "//registry.npmjs.org/:_auth = does-not-decode", + expected: "****************", + }, + { + title: "unexpected _auth", + npmrc: "//registry.npmjs.org/:_auth=:secret", + expected: "*******", + }, + { + title: "_auth zero length", + npmrc: "//registry.npmjs.org/:_auth=", + expected: "received an empty string", + }, + { + title: "_auth one length", + npmrc: "//registry.npmjs.org/:_auth=1", + expected: "*", + }, + ]; + + for (const { title, bunfig, npmrc, expected } of tests) { + test(title + (bunfig ? " (bunfig)" : " (npmrc)"), async () => { + const testDir = tmpdirSync(); + await Promise.all([ + write(join(testDir, bunfig ? "bunfig.toml" : ".npmrc"), (bunfig || npmrc)!), + write(join(testDir, "package.json"), "{}"), + ]); + + // once without color + let proc = spawnSync({ + cmd: [bunExe(), "install"], + cwd: testDir, + env: { ...bunEnv, NO_COLOR: "1" }, + stdout: "pipe", + stderr: "pipe", + }); + + let out = proc.stdout.toString(); + let err = proc.stderr.toString(); + expect(proc.exitCode).toBe(+!!bunfig); + expect(err).toContain(expected || "*"); + + // once with color + proc = spawnSync({ + cmd: [bunExe(), "install"], + cwd: testDir, + env: { ...bunEnv, NO_COLOR: undefined, FORCE_COLOR: "1" }, + stdout: "pipe", + stderr: "pipe", + }); + + out = proc.stdout.toString(); + err = proc.stderr.toString(); + expect(proc.exitCode).toBe(+!!bunfig); + expect(err).toContain(expected || "*"); + }); + } +}); diff --git a/test/cli/install/registry/bun-install-registry.test.ts b/test/cli/install/registry/bun-install-registry.test.ts index 4e5f931221..1321410be3 100644 --- a/test/cli/install/registry/bun-install-registry.test.ts +++ b/test/cli/install/registry/bun-install-registry.test.ts @@ -515,7 +515,7 @@ ${Object.keys(opts) dotEnv: { SECRET_AUTH: "" }, }, (stdout: string, stderr: string) => { - expect(stderr).toContain("got an empty string"); + expect(stderr).toContain("received an empty string"); }, ); }); From 71fdb599187de66dec33fb87b0bb0a8a7d72ee9b Mon Sep 17 00:00:00 2001 From: 190n Date: Thu, 31 Oct 2024 21:02:26 -0700 Subject: [PATCH 07/17] Fix napi property methods on non-objects (#14935) --- src/bun.js/bindings/napi.cpp | 73 +++++++++++++++--------------------- test/napi/napi-app/module.js | 5 +++ 2 files changed, 35 insertions(+), 43 deletions(-) diff --git a/src/bun.js/bindings/napi.cpp b/src/bun.js/bindings/napi.cpp index 8ebff08c1b..1032d5e72a 100644 --- a/src/bun.js/bindings/napi.cpp +++ b/src/bun.js/bindings/napi.cpp @@ -598,15 +598,16 @@ extern "C" napi_status napi_set_property(napi_env env, napi_value target, auto globalObject = toJS(env); auto& vm = globalObject->vm(); - auto* object = targetValue.getObject(); + auto scope = DECLARE_CATCH_SCOPE(vm); + auto* object = targetValue.toObject(globalObject); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); auto keyProp = toJS(key); - auto scope = DECLARE_CATCH_SCOPE(vm); PutPropertySlot slot(object, false); Identifier identifier = keyProp.toPropertyKey(globalObject); - RETURN_IF_EXCEPTION(scope, napi_generic_failure); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); JSValue jsValue = toJS(value); @@ -637,15 +638,13 @@ extern "C" napi_status napi_has_property(napi_env env, napi_value object, auto globalObject = toJS(env); auto& vm = globalObject->vm(); - auto* target = toJS(object).getObject(); - if (!target) { - return napi_object_expected; - } + auto scope = DECLARE_CATCH_SCOPE(vm); + auto* target = toJS(object).toObject(globalObject); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); auto keyProp = toJS(key); - auto scope = DECLARE_CATCH_SCOPE(vm); *result = target->hasProperty(globalObject, keyProp.toPropertyKey(globalObject)); - RETURN_IF_EXCEPTION(scope, napi_generic_failure); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); scope.clearException(); return napi_ok; @@ -693,16 +692,14 @@ extern "C" napi_status napi_get_property(napi_env env, napi_value object, auto globalObject = toJS(env); auto& vm = globalObject->vm(); + auto scope = DECLARE_CATCH_SCOPE(vm); - auto* target = toJS(object).getObject(); - if (!target) { - return napi_object_expected; - } + auto* target = toJS(object).toObject(globalObject); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); JSC::EnsureStillAliveScope ensureAlive(target); auto keyProp = toJS(key); JSC::EnsureStillAliveScope ensureAlive2(keyProp); - auto scope = DECLARE_CATCH_SCOPE(vm); *result = toNapi(target->get(globalObject, keyProp.toPropertyKey(globalObject)), globalObject); RETURN_IF_EXCEPTION(scope, napi_pending_exception); @@ -717,16 +714,14 @@ extern "C" napi_status napi_delete_property(napi_env env, napi_value object, auto globalObject = toJS(env); auto& vm = globalObject->vm(); + auto scope = DECLARE_CATCH_SCOPE(vm); - auto* target = toJS(object).getObject(); - if (!target) { - return napi_object_expected; - } + auto* target = toJS(object).toObject(globalObject); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); auto keyProp = toJS(key); - auto scope = DECLARE_CATCH_SCOPE(vm); auto deleteResult = target->deleteProperty(globalObject, keyProp.toPropertyKey(globalObject)); - RETURN_IF_EXCEPTION(scope, napi_generic_failure); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); if (LIKELY(result)) { *result = deleteResult; @@ -746,16 +741,14 @@ extern "C" napi_status napi_has_own_property(napi_env env, napi_value object, auto globalObject = toJS(env); auto& vm = globalObject->vm(); + auto scope = DECLARE_CATCH_SCOPE(vm); - auto* target = toJS(object).getObject(); - if (!target) { - return napi_object_expected; - } + auto* target = toJS(object).toObject(globalObject); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); auto keyProp = toJS(key); - auto scope = DECLARE_CATCH_SCOPE(vm); *result = target->hasOwnProperty(globalObject, JSC::PropertyName(keyProp.toPropertyKey(globalObject))); - RETURN_IF_EXCEPTION(scope, napi_generic_failure); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); scope.clearException(); return napi_ok; @@ -768,11 +761,10 @@ extern "C" napi_status napi_set_named_property(napi_env env, napi_value object, NAPI_PREMABLE auto globalObject = toJS(env); - auto target = toJS(object).getObject(); auto& vm = globalObject->vm(); - if (UNLIKELY(!target)) { - return napi_object_expected; - } + auto scope = DECLARE_CATCH_SCOPE(vm); + auto target = toJS(object).toObject(globalObject); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); if (UNLIKELY(utf8name == nullptr || !*utf8name || !value)) { return napi_invalid_arg; @@ -785,11 +777,10 @@ extern "C" napi_status napi_set_named_property(napi_env env, napi_value object, auto nameStr = WTF::String::fromUTF8({ utf8name, strlen(utf8name) }); auto identifier = JSC::Identifier::fromString(vm, WTFMove(nameStr)); - auto scope = DECLARE_CATCH_SCOPE(vm); PutPropertySlot slot(target, true); target->put(target, globalObject, identifier, jsValue, slot); - RETURN_IF_EXCEPTION(scope, napi_generic_failure); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); scope.clearException(); return napi_ok; } @@ -851,18 +842,16 @@ extern "C" napi_status napi_has_named_property(napi_env env, napi_value object, auto globalObject = toJS(env); auto& vm = globalObject->vm(); + auto scope = DECLARE_CATCH_SCOPE(vm); - JSObject* target = toJS(object).getObject(); - if (UNLIKELY(!target)) { - return napi_object_expected; - } + JSObject* target = toJS(object).toObject(globalObject); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); PROPERTY_NAME_FROM_UTF8(name); - auto scope = DECLARE_CATCH_SCOPE(vm); PropertySlot slot(target, PropertySlot::InternalMethodType::HasProperty); *result = target->getPropertySlot(globalObject, name, slot); - RETURN_IF_EXCEPTION(scope, napi_generic_failure); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); scope.clearException(); return napi_ok; @@ -880,14 +869,12 @@ extern "C" napi_status napi_get_named_property(napi_env env, napi_value object, auto globalObject = toJS(env); auto& vm = globalObject->vm(); - JSObject* target = toJS(object).getObject(); - if (UNLIKELY(!target)) { - return napi_object_expected; - } + auto scope = DECLARE_CATCH_SCOPE(vm); + JSObject* target = toJS(object).toObject(globalObject); + RETURN_IF_EXCEPTION(scope, napi_pending_exception); PROPERTY_NAME_FROM_UTF8(name); - auto scope = DECLARE_CATCH_SCOPE(vm); *result = toNapi(target->get(globalObject, name), globalObject); RETURN_IF_EXCEPTION(scope, napi_pending_exception); diff --git a/test/napi/napi-app/module.js b/test/napi/napi-app/module.js index 3113db58b1..6cb1280cf2 100644 --- a/test/napi/napi-app/module.js +++ b/test/napi/napi-app/module.js @@ -64,6 +64,9 @@ nativeTests.test_get_property = () => { }, }, ), + 5, + "hello", + // TODO(@190n) test null and undefined here on the napi fix branch ]; const keys = [ "foo", @@ -77,6 +80,8 @@ nativeTests.test_get_property = () => { throw new Error("Symbol.toPrimitive"); }, }, + "toString", + "slice", ]; for (const object of objects) { From 4d269995ad52c43bffdb34f1cee73a1c130b1de1 Mon Sep 17 00:00:00 2001 From: Ashcon Partovi Date: Fri, 1 Nov 2024 11:57:47 -0700 Subject: [PATCH 08/17] Run tests from npm packages, `elysia` to start (#14932) --- .buildkite/ci.mjs | 82 +++++++++------ scripts/runner.node.mjs | 215 +++++++++++++++++++++++++++++++++++++--- test/vendor.json | 15 +++ 3 files changed, 269 insertions(+), 43 deletions(-) create mode 100644 test/vendor.json diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index bfe09735cc..d7d704a40c 100644 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -32,6 +32,10 @@ function getCommit() { return getEnv("BUILDKITE_COMMIT"); } +function getCommitMessage() { + return getEnv("BUILDKITE_MESSAGE", false) || ""; +} + function getBranch() { return getEnv("BUILDKITE_BRANCH"); } @@ -83,9 +87,7 @@ async function getBuildIdWithArtifacts() { while (url) { const response = await fetch(`${url}.json`, { - headers: { - "Accept": "application/json", - }, + headers: { "Accept": "application/json" }, }); if (!response.ok) { @@ -113,14 +115,6 @@ async function getBuildIdWithArtifacts() { } } -function isDocumentation(filename) { - return /^(\.vscode|\.github|bench|docs|examples)|\.(md)$/.test(filename); -} - -function isTest(filename) { - return /^test/.test(filename); -} - function toYaml(obj, indent = 0) { const spaces = " ".repeat(indent); let result = ""; @@ -185,13 +179,12 @@ function getPipeline(buildId) { }; const getLabel = platform => { - const { os, arch, baseline } = platform; - + const { os, arch, baseline, release } = platform; + let label = release ? `:${os}: ${release} ${arch}` : `:${os}: ${arch}`; if (baseline) { - return `:${os}: ${arch}-baseline`; + label += `-baseline`; } - - return `:${os}: ${arch}`; + return label; }; // https://buildkite.com/docs/pipelines/command-step#retry-attributes @@ -315,9 +308,9 @@ function getPipeline(buildId) { let name; if (os === "darwin" || os === "windows") { - name = getLabel(platform); + name = getLabel({ ...platform, release }); } else { - name = getLabel({ ...platform, os: distro }); + name = getLabel({ ...platform, os: distro, release }); } let agents; @@ -353,12 +346,19 @@ function getPipeline(buildId) { depends = [`${getKey(platform)}-build-bun`]; } + let retry; + if (os !== "windows") { + // When the runner fails on Windows, Buildkite only detects an exit code of 1. + // Because of this, we don't know if the run was fatal, or soft-failed. + retry = getRetry(); + } + return { key: `${getKey(platform)}-${distro}-${release.replace(/\./g, "")}-test-bun`, label: `${name} - test-bun`, depends_on: depends, agents, - retry: getRetry(), + retry, cancel_on_build_failing: isMergeQueue(), soft_fail: isMainBranch(), parallelism, @@ -435,29 +435,51 @@ async function main() { console.log(" - Repository:", getRepository()); console.log(" - Branch:", getBranch()); console.log(" - Commit:", getCommit()); + console.log(" - Commit Message:", getCommitMessage()); console.log(" - Is Main Branch:", isMainBranch()); console.log(" - Is Merge Queue:", isMergeQueue()); console.log(" - Is Pull Request:", isPullRequest()); - let buildId; const changedFiles = await getChangedFiles(); if (changedFiles) { console.log( `Found ${changedFiles.length} changed files: \n${changedFiles.map(filename => ` - ${filename}`).join("\n")}`, ); + } - if (changedFiles.every(filename => isDocumentation(filename))) { - console.log("Since changed files are only documentation, skipping..."); - return; + const isDocumentationFile = filename => /^(\.vscode|\.github|bench|docs|examples)|\.(md)$/i.test(filename); + + const isSkip = () => { + const message = getCommitMessage(); + if (/\[(skip ci|no ci|ci skip|ci no)\]/i.test(message)) { + return true; } + return changedFiles && changedFiles.every(filename => isDocumentationFile(filename)); + }; - if (changedFiles.every(filename => isTest(filename) || isDocumentation(filename))) { - buildId = await getBuildIdWithArtifacts(); - if (buildId) { - console.log("Since changed files are only tests, using build artifacts from previous build...", buildId); - } else { - console.log("Changed files are only tests, but could not find previous build artifacts..."); - } + if (isSkip()) { + console.log("Skipping CI due to commit message or changed files..."); + return; + } + + const isTestFile = filename => /^test/i.test(filename) || /runner\.node\.mjs$/i.test(filename); + + const isSkipBuild = () => { + const message = getCommitMessage(); + if (/\[(only tests?|tests? only|skip build|no build|build skip|build no)\]/i.test(message)) { + return true; + } + return changedFiles && changedFiles.every(filename => isTestFile(filename) || isDocumentationFile(filename)); + }; + + let buildId; + if (isSkipBuild()) { + buildId = await getBuildIdWithArtifacts(); + if (buildId) { + console.log("Skipping build due to commit message or changed files..."); + console.log("Using build artifacts from previous build:", buildId); + } else { + console.log("Attempted to skip build, but could not find previous build"); } } diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs index 2f9b069907..af5f02732b 100755 --- a/scripts/runner.node.mjs +++ b/scripts/runner.node.mjs @@ -93,6 +93,10 @@ const { values: options, positionals: filters } = parseArgs({ type: "string", default: undefined, }, + ["vendor"]: { + type: "string", + default: undefined, + }, }, }); @@ -171,13 +175,25 @@ async function runTests() { const tests = getRelevantTests(testsPath); console.log("Running tests:", tests.length); + /** @type {VendorTest[] | undefined} */ + let vendorTests; + let vendorTotal = 0; + if (/true|1|yes|on/i.test(options["vendor"]) || (isCI && typeof options["vendor"] === "undefined")) { + vendorTests = await getVendorTests(cwd); + if (vendorTests.length) { + vendorTotal = vendorTests.reduce((total, { testPaths }) => total + testPaths.length + 1, 0); + console.log("Running vendor tests:", vendorTotal); + } + } + let i = 0; - let total = tests.length + 2; + let total = vendorTotal + tests.length + 2; const results = []; /** * @param {string} title * @param {function} fn + * @returns {Promise} */ const runTest = async (title, fn) => { const label = `${getAnsi("gray")}[${++i}/${total}]${getAnsi("reset")} ${title}`; @@ -188,7 +204,9 @@ async function runTests() { const { ok, error, stdoutPreview } = result; const markdown = formatTestToMarkdown(result); if (markdown) { - reportAnnotationToBuildKite(title, markdown); + const style = title.startsWith("vendor") ? "warning" : "error"; + const priority = title.startsWith("vendor") ? 1 : 5; + reportAnnotationToBuildKite({ label: title, content: markdown, style, priority }); } if (!ok) { @@ -212,6 +230,8 @@ async function runTests() { if (options["bail"] && !result.ok) { process.exit(getExitCode("fail")); } + + return result; }; for (const path of [cwd, testsPath]) { @@ -226,6 +246,42 @@ async function runTests() { } } + if (vendorTests?.length) { + for (const { cwd: vendorPath, packageManager, testRunner, testPaths } of vendorTests) { + if (!testPaths.length) { + continue; + } + + const packageJson = join(relative(cwd, vendorPath), "package.json").replace(/\\/g, "/"); + if (packageManager === "bun") { + const { ok } = await runTest(packageJson, () => spawnBunInstall(execPath, { cwd: vendorPath })); + if (!ok) { + continue; + } + } else { + throw new Error(`Unsupported package manager: ${packageManager}`); + } + + for (const testPath of testPaths) { + const title = join(relative(cwd, vendorPath), testPath).replace(/\\/g, "/"); + + if (testRunner === "bun") { + await runTest(title, () => spawnBunTest(execPath, testPath, { cwd: vendorPath })); + } else if (testRunner === "node") { + const preload = join(import.meta.dirname, "..", "test", "runners", "node.ts"); + await runTest(title, () => + spawnBun(execPath, { + cwd: vendorPath, + args: ["--preload", preload, testPath], + }), + ); + } else { + throw new Error(`Unsupported test runner: ${testRunner}`); + } + } + } + } + const failedTests = results.filter(({ ok }) => !ok); if (isGitHubAction) { reportOutputToGitHubAction("failing_tests_count", failedTests.length); @@ -534,15 +590,18 @@ async function spawnBun(execPath, { args, cwd, timeout, env, stdout, stderr }) { * * @param {string} execPath * @param {string} testPath + * @param {object} [options] + * @param {string} [options.cwd] * @returns {Promise} */ -async function spawnBunTest(execPath, testPath) { +async function spawnBunTest(execPath, testPath, options = { cwd }) { const timeout = getTestTimeout(testPath); const perTestTimeout = Math.ceil(timeout / 2); const isReallyTest = isTestStrict(testPath); + const absPath = join(options["cwd"], testPath); const { ok, error, stdout } = await spawnBun(execPath, { - args: isReallyTest ? ["test", `--timeout=${perTestTimeout}`, testPath] : [testPath], - cwd: cwd, + args: isReallyTest ? ["test", `--timeout=${perTestTimeout}`, absPath] : [absPath], + cwd: options["cwd"], timeout: isReallyTest ? timeout : 30_000, env: { GITHUB_ACTIONS: "true", // always true so annotations are parsed @@ -817,6 +876,14 @@ function isJavaScript(path) { return /\.(c|m)?(j|t)sx?$/.test(basename(path)); } +/** + * @param {string} path + * @returns {boolean} + */ +function isJavaScriptTest(path) { + return isJavaScript(path) && /\.test|spec\./.test(basename(path)); +} + /** * @param {string} path * @returns {boolean} @@ -862,6 +929,121 @@ function getTests(cwd) { return [...getFiles(cwd, "")].sort(); } +/** + * @typedef {object} Vendor + * @property {string} package + * @property {string} repository + * @property {string} tag + * @property {string} [packageManager] + * @property {string} [testPath] + * @property {string} [testRunner] + * @property {boolean | Record} [skipTests] + */ + +/** + * @typedef {object} VendorTest + * @property {string} cwd + * @property {string} packageManager + * @property {string} testRunner + * @property {string[]} testPaths + */ + +/** + * @param {string} cwd + * @returns {Promise} + */ +async function getVendorTests(cwd) { + const vendorPath = join(cwd, "test", "vendor.json"); + if (!existsSync(vendorPath)) { + throw new Error(`Did not find vendor.json: ${vendorPath}`); + } + + /** @type {Vendor[]} */ + const vendors = JSON.parse(readFileSync(vendorPath, "utf-8")).sort( + (a, b) => a.package.localeCompare(b.package) || a.tag.localeCompare(b.tag), + ); + + const shardId = parseInt(options["shard"]); + const maxShards = parseInt(options["max-shards"]); + + /** @type {Vendor[]} */ + let relevantVendors = []; + if (maxShards > 1) { + for (let i = 0; i < vendors.length; i++) { + if (i % maxShards === shardId) { + relevantVendors.push(vendors[i]); + } + } + } else { + relevantVendors = vendors.flat(); + } + + return Promise.all( + relevantVendors.map(async ({ package: name, repository, tag, testPath, testRunner, packageManager, skipTests }) => { + const vendorPath = join(cwd, "vendor", name); + + if (!existsSync(vendorPath)) { + await spawnSafe({ + command: "git", + args: ["clone", "--depth", "1", "--single-branch", repository, vendorPath], + timeout: testTimeout, + cwd, + }); + } + + await spawnSafe({ + command: "git", + args: ["fetch", "--depth", "1", "origin", "tag", tag], + timeout: testTimeout, + cwd: vendorPath, + }); + + const packageJsonPath = join(vendorPath, "package.json"); + if (!existsSync(packageJsonPath)) { + throw new Error(`Vendor '${name}' does not have a package.json: ${packageJsonPath}`); + } + + const testPathPrefix = testPath || "test"; + const testParentPath = join(vendorPath, testPathPrefix); + if (!existsSync(testParentPath)) { + throw new Error(`Vendor '${name}' does not have a test directory: ${testParentPath}`); + } + + const isTest = path => { + if (!isJavaScriptTest(path)) { + return false; + } + + if (typeof skipTests === "boolean") { + return !skipTests; + } + + if (typeof skipTests === "object") { + for (const [glob, reason] of Object.entries(skipTests)) { + const pattern = new RegExp(`^${glob.replace(/\*/g, ".*")}$`); + if (pattern.test(path) && reason) { + return false; + } + } + } + + return true; + }; + + const testPaths = readdirSync(testParentPath, { encoding: "utf-8", recursive: true }) + .filter(filename => isTest(filename)) + .map(filename => join(testPathPrefix, filename)); + + return { + cwd: vendorPath, + packageManager: packageManager || "bun", + testRunner: testRunner || "bun", + testPaths, + }; + }), + ); +} + /** * @param {string} cwd * @returns {string[]} @@ -1525,14 +1707,21 @@ function listArtifactsFromBuildKite(glob, step) { } /** - * @param {string} label - * @param {string} content - * @param {number | undefined} attempt + * @typedef {object} BuildkiteAnnotation + * @property {string} label + * @property {string} content + * @property {"error" | "warning" | "info"} [style] + * @property {number} [priority] + * @property {number} [attempt] */ -function reportAnnotationToBuildKite(label, content, attempt = 0) { + +/** + * @param {BuildkiteAnnotation} annotation + */ +function reportAnnotationToBuildKite({ label, content, style = "error", priority = 3, attempt = 0 }) { const { error, status, signal, stderr } = spawnSync( "buildkite-agent", - ["annotate", "--append", "--style", "error", "--context", `${label}`, "--priority", `${attempt}`], + ["annotate", "--append", "--style", `${style}`, "--context", `${label}`, "--priority", `${priority}`], { input: content, stdio: ["pipe", "ignore", "pipe"], @@ -1551,11 +1740,11 @@ function reportAnnotationToBuildKite(label, content, attempt = 0) { const buildLabel = getBuildLabel(); const buildUrl = getBuildUrl(); const platform = buildUrl ? `${buildLabel}` : buildLabel; - let message = `
${label} - annotation error on ${platform}`; + let errorMessage = `
${label} - annotation error on ${platform}`; if (stderr) { - message += `\n\n\`\`\`terminal\n${escapeCodeBlock(stderr)}\n\`\`\`\n\n
\n\n`; + errorMessage += `\n\n\`\`\`terminal\n${escapeCodeBlock(stderr)}\n\`\`\`\n\n
\n\n`; } - reportAnnotationToBuildKite(`${label}-error`, message, attempt + 1); + reportAnnotationToBuildKite({ label: `${label}-error`, content: errorMessage, attempt: attempt + 1 }); } /** diff --git a/test/vendor.json b/test/vendor.json new file mode 100644 index 0000000000..e6eb7a4e67 --- /dev/null +++ b/test/vendor.json @@ -0,0 +1,15 @@ +[ + { + "package": "elysia", + "repository": "https://github.com/elysiajs/elysia", + "tag": "1.1.24" + }, + { + "package": "uuid", + "repository": "https://github.com/uuidjs/uuid", + "tag": "v10.0.0", + "testRunner": "node", + "testPath": "src/test", + "skipTests": true + } +] From d75488124d537366ae3e583e4376e592a3a41462 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 1 Nov 2024 14:45:19 -0700 Subject: [PATCH 09/17] Inline `process.versions.bun` in `bun build --compile` (#14940) --- src/cli/build_command.zig | 1 + src/compile_target.zig | 2 ++ test/bundler/bundler_compile.test.ts | 15 +++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index a8ea3f0d67..8b02cc40e6 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -36,6 +36,7 @@ pub const BuildCommand = struct { const compile_define_keys = &.{ "process.platform", "process.arch", + "process.versions.bun", }; pub fn exec(ctx: Command.Context) !void { diff --git a/src/compile_target.zig b/src/compile_target.zig index bd060d24bb..242a36c09e 100644 --- a/src/compile_target.zig +++ b/src/compile_target.zig @@ -429,6 +429,8 @@ pub fn defineValues(this: *const CompileTarget) []const []const u8 { .arm64 => "\"arm64\"", else => @compileError("TODO"), }, + + "\"" ++ Global.package_json_version ++ "\"", }; }.values, else => @panic("TODO"), diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index 9771f61703..d4c3610527 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -14,6 +14,21 @@ describe.todoIf(isFlaky && isWindows)("bundler", () => { }, run: { stdout: "Hello, world!" }, }); + itBundled("compile/HelloWorldWithProcessVersionsBun", { + compile: true, + files: { + [`/${process.platform}-${process.arch}.js`]: "module.exports = process.versions.bun;", + "/entry.ts": /* js */ ` + process.exitCode = 1; + process.versions.bun = "bun!"; + if (process.versions.bun === "bun!") throw new Error("fail"); + if (require("./${process.platform}-${process.arch}.js") === "${Bun.version.replaceAll("-debug", "")}") { + process.exitCode = 0; + } + `, + }, + run: { exitCode: 0 }, + }); itBundled("compile/HelloWorldBytecode", { compile: true, bytecode: true, From c89a9582991edd11b175371310ad7019b3aa4ac3 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 1 Nov 2024 15:35:28 -0700 Subject: [PATCH 10/17] Fixes #11754 (#14948) --- src/bun.js/bindings/ImportMetaObject.cpp | 22 ++++++++++++++++++++-- src/bun.js/bindings/ImportMetaObject.h | 2 +- test/js/bun/resolve/import-meta.test.js | 11 ++++++++++- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/bun.js/bindings/ImportMetaObject.cpp b/src/bun.js/bindings/ImportMetaObject.cpp index 686925c781..c3525feb04 100644 --- a/src/bun.js/bindings/ImportMetaObject.cpp +++ b/src/bun.js/bindings/ImportMetaObject.cpp @@ -484,6 +484,24 @@ JSC_DEFINE_CUSTOM_GETTER(jsImportMetaObjectGetter_require, (JSGlobalObject * glo return JSValue::encode(thisObject->requireProperty.getInitializedOnMainThread(thisObject)); } +// https://github.com/oven-sh/bun/issues/11754#issuecomment-2452626172 +// This setter exists mainly to support various libraries doing weird things wrapping the require function. +JSC_DEFINE_CUSTOM_SETTER(jsImportMetaObjectSetter_require, (JSGlobalObject * jsGlobalObject, JSC::EncodedJSValue thisValue, JSC::EncodedJSValue encodedValue, PropertyName propertyName)) +{ + ImportMetaObject* thisObject = jsDynamicCast(JSValue::decode(thisValue)); + if (UNLIKELY(!thisObject)) + return false; + + JSValue value = JSValue::decode(encodedValue); + if (!value.isCell()) { + // TODO: + return true; + } + + thisObject->requireProperty.set(thisObject->vm(), thisObject, value.asCell()); + return true; +} + JSC_DEFINE_CUSTOM_GETTER(jsImportMetaObjectGetter_env, (JSGlobalObject * jsGlobalObject, JSC::EncodedJSValue thisValue, PropertyName propertyName)) { auto* globalObject = jsCast(jsGlobalObject); @@ -497,7 +515,7 @@ static const HashTableValue ImportMetaObjectPrototypeValues[] = { { "file"_s, static_cast(JSC::PropertyAttribute::ReadOnly | JSC::PropertyAttribute::CustomAccessor | PropertyAttribute::DontDelete), NoIntrinsic, { HashTableValue::GetterSetterType, jsImportMetaObjectGetter_file, 0 } }, { "filename"_s, static_cast(JSC::PropertyAttribute::ReadOnly | JSC::PropertyAttribute::CustomAccessor | PropertyAttribute::DontDelete), NoIntrinsic, { HashTableValue::GetterSetterType, jsImportMetaObjectGetter_path, 0 } }, { "path"_s, static_cast(JSC::PropertyAttribute::ReadOnly | JSC::PropertyAttribute::CustomAccessor | PropertyAttribute::DontDelete), NoIntrinsic, { HashTableValue::GetterSetterType, jsImportMetaObjectGetter_path, 0 } }, - { "require"_s, static_cast(JSC::PropertyAttribute::ReadOnly | JSC::PropertyAttribute::CustomAccessor | PropertyAttribute::DontDelete), NoIntrinsic, { HashTableValue::GetterSetterType, jsImportMetaObjectGetter_require, 0 } }, + { "require"_s, static_cast(JSC::PropertyAttribute::CustomAccessor | PropertyAttribute::DontDelete), NoIntrinsic, { HashTableValue::GetterSetterType, jsImportMetaObjectGetter_require, jsImportMetaObjectSetter_require } }, { "resolve"_s, static_cast(JSC::PropertyAttribute::Function | PropertyAttribute::DontDelete), NoIntrinsic, { HashTableValue::NativeFunctionType, functionImportMeta__resolve, 0 } }, { "resolveSync"_s, static_cast(JSC::PropertyAttribute::Function | PropertyAttribute::DontDelete), NoIntrinsic, { HashTableValue::NativeFunctionType, functionImportMeta__resolveSync, 0 } }, { "url"_s, static_cast(JSC::PropertyAttribute::ReadOnly | JSC::PropertyAttribute::CustomAccessor | PropertyAttribute::DontDelete), NoIntrinsic, { HashTableValue::GetterSetterType, jsImportMetaObjectGetter_url, 0 } }, @@ -572,7 +590,7 @@ void ImportMetaObject::finishCreation(VM& vm) Base::finishCreation(vm); ASSERT(inherits(info())); - this->requireProperty.initLater([](const JSC::LazyProperty::Initializer& init) { + this->requireProperty.initLater([](const JSC::LazyProperty::Initializer& init) { ImportMetaObject* meta = jsCast(init.owner); WTF::URL url = isAbsolutePath(meta->url) ? WTF::URL::fileURLWithFileSystemPath(meta->url) : WTF::URL(meta->url); diff --git a/src/bun.js/bindings/ImportMetaObject.h b/src/bun.js/bindings/ImportMetaObject.h index 663df39682..078f8b6b8c 100644 --- a/src/bun.js/bindings/ImportMetaObject.h +++ b/src/bun.js/bindings/ImportMetaObject.h @@ -71,7 +71,7 @@ public: static void analyzeHeap(JSCell*, JSC::HeapAnalyzer&); WTF::String url; - LazyProperty requireProperty; + LazyProperty requireProperty; LazyProperty dirProperty; LazyProperty urlProperty; LazyProperty fileProperty; diff --git a/test/js/bun/resolve/import-meta.test.js b/test/js/bun/resolve/import-meta.test.js index 49b40f5d9d..9ad8a7bc03 100644 --- a/test/js/bun/resolve/import-meta.test.js +++ b/test/js/bun/resolve/import-meta.test.js @@ -1,5 +1,5 @@ import { spawnSync } from "bun"; -import { expect, it } from "bun:test"; +import { expect, it, mock } from "bun:test"; import { bunEnv, bunExe, ospath } from "harness"; import { mkdirSync, rmSync, writeFileSync } from "node:fs"; import Module from "node:module"; @@ -12,6 +12,15 @@ const { path, dir, dirname, filename } = import.meta; const tmpbase = tmpdir() + sep; +it("import.meta.require is settable", () => { + const old = import.meta.require; + const fn = mock(() => "hello"); + import.meta.require = fn; + expect(import.meta.require("hello")).toBe("hello"); + import.meta.require = old; + expect(fn).toHaveBeenCalledTimes(1); +}); + it("import.meta.main", () => { const { exitCode } = spawnSync({ cmd: [bunExe(), "run", join(import.meta.dir, "./main-test-script.js")], From 6e448619d06de254b17ec35f7620779acd216ba2 Mon Sep 17 00:00:00 2001 From: Ciro Spaciari Date: Fri, 1 Nov 2024 17:23:00 -0700 Subject: [PATCH 11/17] fix drain event, drain must be called only after internal buffer is drained --- src/bun.js/api/bun/socket.zig | 29 ++++++++++++++++++++++------- src/bun.js/api/sockets.classes.ts | 7 +++---- src/js/node/net.ts | 26 +++++++++++++++++++------- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/src/bun.js/api/bun/socket.zig b/src/bun.js/api/bun/socket.zig index 7cc8057156..39330e3a45 100644 --- a/src/bun.js/api/bun/socket.zig +++ b/src/bun.js/api/bun/socket.zig @@ -1474,6 +1474,11 @@ fn NewSocket(comptime ssl: bool) type { if (vm.isShuttingDown()) { return; } + + this.internalFlush(); + // is not writable if we have buffered data + if (this.buffered_data_for_node_net.len > 0) return; + vm.eventLoop().enter(); defer vm.eventLoop().exit(); @@ -2363,15 +2368,10 @@ fn NewSocket(comptime ssl: bool) type { }; } - pub fn flush( - this: *This, - _: *JSC.JSGlobalObject, - _: *JSC.CallFrame, - ) JSValue { - JSC.markBinding(@src()); + fn internalFlush(this: *This) JSValue { if (this.buffered_data_for_node_net.len > 0) { const written: usize = @intCast(@max(this.socket.write(this.buffered_data_for_node_net.slice(), false), 0)); - + this.bytes_written += written; if (written > 0) { if (this.buffered_data_for_node_net.len > written) { const remaining = this.buffered_data_for_node_net.slice()[written..]; @@ -2385,6 +2385,15 @@ fn NewSocket(comptime ssl: bool) type { } this.socket.flush(); + } + + pub fn flush( + this: *This, + _: *JSC.JSGlobalObject, + _: *JSC.CallFrame, + ) JSValue { + JSC.markBinding(@src()); + this.internalFlush(); return JSValue.jsUndefined(); } @@ -2706,6 +2715,12 @@ fn NewSocket(comptime ssl: bool) type { ) JSValue { return JSC.JSValue.jsNumber(this.bytes_written + this.buffered_data_for_node_net.len); } + pub fn getBufferedQueueSize( + this: *This, + _: *JSC.JSGlobalObject, + ) JSValue { + return JSC.JSValue.jsNumber(this.buffered_data_for_node_net.len); + } pub fn getALPNProtocol( this: *This, globalObject: *JSC.JSGlobalObject, diff --git a/src/bun.js/api/sockets.classes.ts b/src/bun.js/api/sockets.classes.ts index ee3e60e1df..d50aabf7ef 100644 --- a/src/bun.js/api/sockets.classes.ts +++ b/src/bun.js/api/sockets.classes.ts @@ -83,9 +83,6 @@ function generate(ssl) { alpnProtocol: { getter: "getALPNProtocol", }, - bytesWritten: { - getter: "getBytesWritten", - }, write: { fn: "write", length: 3, @@ -169,7 +166,9 @@ function generate(ssl) { bytesWritten: { getter: "getBytesWritten", }, - + bufferedQueueSize: { + getter: "getBufferedQueueSize", + }, setServername: { fn: "setServername", length: 1, diff --git a/src/js/node/net.ts b/src/js/node/net.ts index 63003ae684..26b83ae81c 100644 --- a/src/js/node/net.ts +++ b/src/js/node/net.ts @@ -241,7 +241,7 @@ const Socket = (function (InternalSocket) { if (callback) { const writeChunk = self._pendingData; - if (socket.$write(writeChunk || "", "utf8")) { + if (!writeChunk || socket.$write(writeChunk || "", self._pendingEncoding || "utf8")) { self._pendingData = self.#writeCallback = null; callback(null); } else { @@ -856,16 +856,28 @@ const Socket = (function (InternalSocket) { if (!socket) { // detached but connected? wait for the socket to be attached this.#writeCallback = callback; - this._pendingEncoding = "buffer"; - this._pendingData = Buffer.from(chunk, encoding); + this._pendingEncoding = encoding; + this._pendingData = chunk; return; } - const success = socket?.$write(chunk, encoding); + const writeResult = socket.$write(chunk, encoding); this[kBytesWritten] = socket.bytesWritten; - if (success) { - callback(); - } else if (this.#writeCallback) { + switch (writeResult) { + case -1: + // dropped + this.#writeCallback = callback; + this._pendingEncoding = encoding; + this._pendingData = chunk; + break; + default: + // written or buffered by the socket + if (socket.bufferedQueueSize === 0) { + callback(); + return; + } + } + if (this.#writeCallback) { callback(new Error("overlapping _write()")); } else { this.#writeCallback = callback; From 5236d974b5e68eda7afc73fbbbe5d526077a33d6 Mon Sep 17 00:00:00 2001 From: Ciro Spaciari Date: Fri, 1 Nov 2024 17:25:59 -0700 Subject: [PATCH 12/17] revert --- src/bun.js/api/bun/socket.zig | 29 +++++++---------------------- src/bun.js/api/sockets.classes.ts | 7 ++++--- src/js/node/net.ts | 26 +++++++------------------- 3 files changed, 18 insertions(+), 44 deletions(-) diff --git a/src/bun.js/api/bun/socket.zig b/src/bun.js/api/bun/socket.zig index 39330e3a45..7cc8057156 100644 --- a/src/bun.js/api/bun/socket.zig +++ b/src/bun.js/api/bun/socket.zig @@ -1474,11 +1474,6 @@ fn NewSocket(comptime ssl: bool) type { if (vm.isShuttingDown()) { return; } - - this.internalFlush(); - // is not writable if we have buffered data - if (this.buffered_data_for_node_net.len > 0) return; - vm.eventLoop().enter(); defer vm.eventLoop().exit(); @@ -2368,10 +2363,15 @@ fn NewSocket(comptime ssl: bool) type { }; } - fn internalFlush(this: *This) JSValue { + pub fn flush( + this: *This, + _: *JSC.JSGlobalObject, + _: *JSC.CallFrame, + ) JSValue { + JSC.markBinding(@src()); if (this.buffered_data_for_node_net.len > 0) { const written: usize = @intCast(@max(this.socket.write(this.buffered_data_for_node_net.slice(), false), 0)); - this.bytes_written += written; + if (written > 0) { if (this.buffered_data_for_node_net.len > written) { const remaining = this.buffered_data_for_node_net.slice()[written..]; @@ -2385,15 +2385,6 @@ fn NewSocket(comptime ssl: bool) type { } this.socket.flush(); - } - - pub fn flush( - this: *This, - _: *JSC.JSGlobalObject, - _: *JSC.CallFrame, - ) JSValue { - JSC.markBinding(@src()); - this.internalFlush(); return JSValue.jsUndefined(); } @@ -2715,12 +2706,6 @@ fn NewSocket(comptime ssl: bool) type { ) JSValue { return JSC.JSValue.jsNumber(this.bytes_written + this.buffered_data_for_node_net.len); } - pub fn getBufferedQueueSize( - this: *This, - _: *JSC.JSGlobalObject, - ) JSValue { - return JSC.JSValue.jsNumber(this.buffered_data_for_node_net.len); - } pub fn getALPNProtocol( this: *This, globalObject: *JSC.JSGlobalObject, diff --git a/src/bun.js/api/sockets.classes.ts b/src/bun.js/api/sockets.classes.ts index d50aabf7ef..ee3e60e1df 100644 --- a/src/bun.js/api/sockets.classes.ts +++ b/src/bun.js/api/sockets.classes.ts @@ -83,6 +83,9 @@ function generate(ssl) { alpnProtocol: { getter: "getALPNProtocol", }, + bytesWritten: { + getter: "getBytesWritten", + }, write: { fn: "write", length: 3, @@ -166,9 +169,7 @@ function generate(ssl) { bytesWritten: { getter: "getBytesWritten", }, - bufferedQueueSize: { - getter: "getBufferedQueueSize", - }, + setServername: { fn: "setServername", length: 1, diff --git a/src/js/node/net.ts b/src/js/node/net.ts index 26b83ae81c..63003ae684 100644 --- a/src/js/node/net.ts +++ b/src/js/node/net.ts @@ -241,7 +241,7 @@ const Socket = (function (InternalSocket) { if (callback) { const writeChunk = self._pendingData; - if (!writeChunk || socket.$write(writeChunk || "", self._pendingEncoding || "utf8")) { + if (socket.$write(writeChunk || "", "utf8")) { self._pendingData = self.#writeCallback = null; callback(null); } else { @@ -856,28 +856,16 @@ const Socket = (function (InternalSocket) { if (!socket) { // detached but connected? wait for the socket to be attached this.#writeCallback = callback; - this._pendingEncoding = encoding; - this._pendingData = chunk; + this._pendingEncoding = "buffer"; + this._pendingData = Buffer.from(chunk, encoding); return; } - const writeResult = socket.$write(chunk, encoding); + const success = socket?.$write(chunk, encoding); this[kBytesWritten] = socket.bytesWritten; - switch (writeResult) { - case -1: - // dropped - this.#writeCallback = callback; - this._pendingEncoding = encoding; - this._pendingData = chunk; - break; - default: - // written or buffered by the socket - if (socket.bufferedQueueSize === 0) { - callback(); - return; - } - } - if (this.#writeCallback) { + if (success) { + callback(); + } else if (this.#writeCallback) { callback(new Error("overlapping _write()")); } else { this.#writeCallback = callback; From ce2afac8278f6e763f7a6caefe6cfc17f4aa9a01 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 1 Nov 2024 18:16:04 -0700 Subject: [PATCH 13/17] Align `"encoding"` option in node fs with node (#14937) --- src/bun.js/api/server.zig | 4 +- src/bun.js/bindings/bindings.cpp | 13 +- src/bun.js/bindings/bindings.zig | 20 +- src/bun.js/bindings/headers.h | 1 - src/bun.js/bindings/headers.zig | 1 - src/bun.js/modules/NodeBufferModule.h | 4 +- src/bun.js/node/node_fs.zig | 497 ++++++-------------------- src/bun.js/node/node_fs_binding.zig | 8 +- src/bun.js/node/node_fs_watcher.zig | 35 +- src/bun.js/node/types.zig | 35 +- src/bun.js/test/expect.zig | 10 +- src/bun.js/test/jest.zig | 2 +- src/bun.js/webcore/streams.zig | 3 +- test/js/bun/http/bun-server.test.ts | 2 +- test/js/node/fs/fs.test.ts | 111 ++++++ 15 files changed, 298 insertions(+), 448 deletions(-) diff --git a/src/bun.js/api/server.zig b/src/bun.js/api/server.zig index 0efe72a66b..b3a1933734 100644 --- a/src/bun.js/api/server.zig +++ b/src/bun.js/api/server.zig @@ -1453,7 +1453,9 @@ pub const ServerConfig = struct { } if (arg.getTruthy(global, "tls")) |tls| { - if (tls.jsType().isArray()) { + if (tls.isFalsey()) { + args.ssl_config = null; + } else if (tls.jsType().isArray()) { var value_iter = tls.arrayIterator(global); if (value_iter.len == 1) { JSC.throwInvalidArguments("tls option expects at least 1 tls object", .{}, global, exception); diff --git a/src/bun.js/bindings/bindings.cpp b/src/bun.js/bindings/bindings.cpp index 13ec88ee17..30288d4a18 100644 --- a/src/bun.js/bindings/bindings.cpp +++ b/src/bun.js/bindings/bindings.cpp @@ -3889,10 +3889,6 @@ bool JSC__JSValue__symbolKeyFor(JSC__JSValue symbolValue_, JSC__JSGlobalObject* return true; } -bool JSC__JSValue__toBoolean(JSC__JSValue JSValue0) -{ - return JSC::JSValue::decode(JSValue0).asBoolean(); -} int32_t JSC__JSValue__toInt32(JSC__JSValue JSValue0) { return JSC::JSValue::decode(JSValue0).asInt32(); @@ -5004,6 +5000,7 @@ enum class BuiltinNamesMap : uint8_t { message, error, defaultKeyword, + encoding, }; static const JSC::Identifier builtinNameMap(JSC::JSGlobalObject* globalObject, unsigned char name) @@ -5065,6 +5062,9 @@ static const JSC::Identifier builtinNameMap(JSC::JSGlobalObject* globalObject, u case BuiltinNamesMap::defaultKeyword: { return vm.propertyNames->defaultKeyword; } + case BuiltinNamesMap::encoding: { + return clientData->builtinNames().encodingPublicName(); + } default: { ASSERT_NOT_REACHED(); return Identifier(); @@ -5100,9 +5100,10 @@ extern "C" JSC__JSValue JSC__JSValue__fastGetOwn(JSC__JSValue JSValue0, JSC__JSG return JSValue::encode({}); } -bool JSC__JSValue__toBooleanSlow(JSC__JSValue JSValue0, JSC__JSGlobalObject* globalObject) +bool JSC__JSValue__toBoolean(JSC__JSValue JSValue0) { - return JSValue::decode(JSValue0).toBoolean(globalObject); + // We count masquerades as undefined as true. + return JSValue::decode(JSValue0).pureToBoolean() != TriState::False; } template diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index f12ae5bb41..fab4918bf3 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -4017,6 +4017,12 @@ pub const JSValue = enum(JSValueReprInt) { return JSC__JSValue__getDirectIndex(this, globalThis, i); } + pub fn isFalsey(this: JSValue) bool { + return !this.toBoolean(); + } + + pub const isTruthy = toBoolean; + const PropertyIteratorFn = *const fn ( globalObject_: *JSGlobalObject, ctx_ptr: ?*anyopaque, @@ -4067,7 +4073,7 @@ pub const JSValue = enum(JSValueReprInt) { pub fn coerce(this: JSValue, comptime T: type, globalThis: *JSC.JSGlobalObject) T { return switch (T) { ZigString => this.getZigString(globalThis), - bool => this.toBooleanSlow(globalThis), + bool => this.toBoolean(), f64 => { if (this.isDouble()) { return this.asDouble(); @@ -5175,6 +5181,7 @@ pub const JSValue = enum(JSValueReprInt) { message, @"error", default, + encoding, pub fn has(property: []const u8) bool { return bun.ComptimeEnumMap(BuiltinName).has(property); @@ -5723,16 +5730,8 @@ pub const JSValue = enum(JSValueReprInt) { return fromPtrAddress(@intFromPtr(addr)); } - pub fn toBooleanSlow(this: JSValue, global: *JSGlobalObject) bool { - return cppFn("toBooleanSlow", .{ this, global }); - } - pub fn toBoolean(this: JSValue) bool { - if (isEmptyOrUndefinedOrNull(this)) { - return false; - } - - return asBoolean(this); + return this != .zero and cppFn("toBoolean", .{this}); } pub fn asBoolean(this: JSValue) bool { @@ -6022,7 +6021,6 @@ pub const JSValue = enum(JSValueReprInt) { "symbolFor", "symbolKeyFor", "toBoolean", - "toBooleanSlow", "toError_", "toInt32", "toInt64", diff --git a/src/bun.js/bindings/headers.h b/src/bun.js/bindings/headers.h index 6d5efdb701..7cc6afdecd 100644 --- a/src/bun.js/bindings/headers.h +++ b/src/bun.js/bindings/headers.h @@ -394,7 +394,6 @@ CPP_DECL bool JSC__JSValue__stringIncludes(JSC__JSValue JSValue0, JSC__JSGlobalO CPP_DECL JSC__JSValue JSC__JSValue__symbolFor(JSC__JSGlobalObject* arg0, ZigString* arg1); CPP_DECL bool JSC__JSValue__symbolKeyFor(JSC__JSValue JSValue0, JSC__JSGlobalObject* arg1, ZigString* arg2); CPP_DECL bool JSC__JSValue__toBoolean(JSC__JSValue JSValue0); -CPP_DECL bool JSC__JSValue__toBooleanSlow(JSC__JSValue JSValue0, JSC__JSGlobalObject* arg1); CPP_DECL JSC__JSValue JSC__JSValue__toError_(JSC__JSValue JSValue0); CPP_DECL int32_t JSC__JSValue__toInt32(JSC__JSValue JSValue0); CPP_DECL int64_t JSC__JSValue__toInt64(JSC__JSValue JSValue0); diff --git a/src/bun.js/bindings/headers.zig b/src/bun.js/bindings/headers.zig index ec358ceb3e..5718a8c39b 100644 --- a/src/bun.js/bindings/headers.zig +++ b/src/bun.js/bindings/headers.zig @@ -289,7 +289,6 @@ pub extern fn JSC__JSValue__stringIncludes(JSValue0: JSC__JSValue, arg1: *bindin pub extern fn JSC__JSValue__symbolFor(arg0: *bindings.JSGlobalObject, arg1: [*c]ZigString) JSC__JSValue; pub extern fn JSC__JSValue__symbolKeyFor(JSValue0: JSC__JSValue, arg1: *bindings.JSGlobalObject, arg2: [*c]ZigString) bool; pub extern fn JSC__JSValue__toBoolean(JSValue0: JSC__JSValue) bool; -pub extern fn JSC__JSValue__toBooleanSlow(JSValue0: JSC__JSValue, arg1: *bindings.JSGlobalObject) bool; pub extern fn JSC__JSValue__toError_(JSValue0: JSC__JSValue) JSC__JSValue; pub extern fn JSC__JSValue__toInt32(JSValue0: JSC__JSValue) i32; pub extern fn JSC__JSValue__toInt64(JSValue0: JSC__JSValue) i64; diff --git a/src/bun.js/modules/NodeBufferModule.h b/src/bun.js/modules/NodeBufferModule.h index a86ebb3a12..e53e29950c 100644 --- a/src/bun.js/modules/NodeBufferModule.h +++ b/src/bun.js/modules/NodeBufferModule.h @@ -211,8 +211,10 @@ DEFINE_NATIVE_MODULE(NodeBuffer) put(JSC::Identifier::fromString(vm, "transcode"_s), transcode); - auto* resolveObjectURL = InternalFunction::createFunctionThatMasqueradesAsUndefined( + auto* resolveObjectURL = JSC::JSFunction::create( vm, globalObject, 1, "resolveObjectURL"_s, + jsFunctionResolveObjectURL, + ImplementationVisibility::Public, NoIntrinsic, jsFunctionResolveObjectURL); put(JSC::Identifier::fromString(vm, "resolveObjectURL"_s), resolveObjectURL); diff --git a/src/bun.js/node/node_fs.zig b/src/bun.js/node/node_fs.zig index 6210f09257..99a497e48e 100644 --- a/src/bun.js/node/node_fs.zig +++ b/src/bun.js/node/node_fs.zig @@ -1985,17 +1985,17 @@ pub const Arguments = struct { const big_int = brk: { if (arguments.next()) |next_val| { if (next_val.isObject()) { - if (next_val.isCallable(ctx.ptr().vm())) break :brk false; + if (next_val.isCallable(ctx.vm())) break :brk false; arguments.eat(); - if (next_val.getOptional(ctx.ptr(), "throwIfNoEntry", bool) catch { + if (next_val.getOptional(ctx, "throwIfNoEntry", bool) catch { path.deinit(); return null; }) |throw_if_no_entry_val| { throw_if_no_entry = throw_if_no_entry_val; } - if (next_val.getOptional(ctx.ptr(), "bigint", bool) catch { + if (next_val.getOptional(ctx, "bigint", bool) catch { path.deinit(); return null; }) |big_int| { @@ -2048,10 +2048,10 @@ pub const Arguments = struct { const big_int = brk: { if (arguments.next()) |next_val| { if (next_val.isObject()) { - if (next_val.isCallable(ctx.ptr().vm())) break :brk false; + if (next_val.isCallable(ctx.vm())) break :brk false; arguments.eat(); - if (next_val.getOptional(ctx.ptr(), "bigint", bool) catch false) |big_int| { + if (next_val.getOptional(ctx, "bigint", bool) catch false) |big_int| { break :brk big_int; } } @@ -2193,7 +2193,7 @@ pub const Arguments = struct { // will automatically be normalized to absolute path. if (next_val.isString()) { arguments.eat(); - var str = next_val.toBunString(ctx.ptr()); + var str = next_val.toBunString(ctx); defer str.deref(); if (str.eqlComptime("dir")) break :link_type .dir; if (str.eqlComptime("file")) break :link_type .file; @@ -2263,18 +2263,23 @@ pub const Arguments = struct { switch (val.jsType()) { JSC.JSValue.JSType.String, JSC.JSValue.JSType.StringObject, JSC.JSValue.JSType.DerivedStringObject => { - encoding = Encoding.fromJS(val, ctx.ptr()) orelse Encoding.utf8; + // Exception handled below. + encoding = Encoding.assert(val, ctx, encoding) catch encoding; }, else => { if (val.isObject()) { - if (val.get(ctx.ptr(), "encoding")) |encoding_| { - encoding = Encoding.fromJS(encoding_, ctx.ptr()) orelse Encoding.utf8; - } + // Exception handled below. + encoding = getEncoding(val, ctx, encoding) catch encoding; } }, } } + if (ctx.hasException()) { + path.deinit(); + return null; + } + return Readlink{ .path = path, .encoding = encoding }; } }; @@ -2315,22 +2320,35 @@ pub const Arguments = struct { switch (val.jsType()) { JSC.JSValue.JSType.String, JSC.JSValue.JSType.StringObject, JSC.JSValue.JSType.DerivedStringObject => { - encoding = Encoding.fromJS(val, ctx.ptr()) orelse Encoding.utf8; + // Exception handled below. + encoding = Encoding.assert(val, ctx, encoding) catch encoding; }, else => { if (val.isObject()) { - if (val.get(ctx.ptr(), "encoding")) |encoding_| { - encoding = Encoding.fromJS(encoding_, ctx.ptr()) orelse Encoding.utf8; - } + // Exception handled below. + encoding = getEncoding(val, ctx, encoding) catch encoding; } }, } } + if (ctx.hasException()) { + path.deinit(); + return null; + } + return Realpath{ .path = path, .encoding = encoding }; } }; + fn getEncoding(object: JSC.JSValue, globalObject: *JSC.JSGlobalObject, default: Encoding) !Encoding { + if (object.fastGet(globalObject, .encoding)) |value| { + return Encoding.assert(value, globalObject, default); + } + + return default; + } + pub const Unlink = struct { path: PathLike, @@ -2411,14 +2429,14 @@ pub const Arguments = struct { arguments.eat(); if (val.isObject()) { - if (val.getOptional(ctx.ptr(), "recursive", bool) catch { + if (val.getOptional(ctx, "recursive", bool) catch { path.deinit(); return null; }) |boolean| { recursive = boolean; } - if (val.getOptional(ctx.ptr(), "force", bool) catch { + if (val.getOptional(ctx, "force", bool) catch { path.deinit(); return null; }) |boolean| { @@ -2483,14 +2501,14 @@ pub const Arguments = struct { arguments.eat(); if (val.isObject()) { - if (val.getOptional(ctx.ptr(), "recursive", bool) catch { + if (val.getOptional(ctx, "recursive", bool) catch { path.deinit(); return null; }) |boolean| { recursive = boolean; } - if (val.get(ctx.ptr(), "mode")) |mode_| { + if (val.get(ctx, "mode")) |mode_| { mode = JSC.Node.modeFromJS(ctx, mode_, exception) orelse mode; if (exception.* != null) return null; } @@ -2547,18 +2565,22 @@ pub const Arguments = struct { switch (val.jsType()) { JSC.JSValue.JSType.String, JSC.JSValue.JSType.StringObject, JSC.JSValue.JSType.DerivedStringObject => { - encoding = Encoding.fromJS(val, ctx.ptr()) orelse Encoding.utf8; + // exception handled below. + encoding = Encoding.assert(val, ctx, encoding) catch encoding; }, else => { if (val.isObject()) { - if (val.get(ctx.ptr(), "encoding")) |encoding_| { - encoding = Encoding.fromJS(encoding_, ctx.ptr()) orelse Encoding.utf8; - } + encoding = getEncoding(val, ctx, encoding) catch encoding; } }, } } + if (ctx.hasException()) { + prefix.deinit(); + return null; + } + return MkdirTemp{ .prefix = prefix, .encoding = encoding, @@ -2618,22 +2640,28 @@ pub const Arguments = struct { switch (val.jsType()) { JSC.JSValue.JSType.String, JSC.JSValue.JSType.StringObject, JSC.JSValue.JSType.DerivedStringObject => { - encoding = Encoding.fromJS(val, ctx.ptr()) orelse Encoding.utf8; + encoding = Encoding.assert(val, ctx, encoding) catch { + path.deinit(); + return null; + }; }, else => { if (val.isObject()) { - if (val.get(ctx.ptr(), "encoding")) |encoding_| { - encoding = Encoding.fromJS(encoding_, ctx.ptr()) orelse Encoding.utf8; + encoding = getEncoding(val, ctx, encoding) catch encoding; + + if (ctx.hasException()) { + path.deinit(); + return null; } - if (val.getOptional(ctx.ptr(), "recursive", bool) catch { + if (val.getOptional(ctx, "recursive", bool) catch { path.deinit(); return null; }) |recursive_| { recursive = recursive_; } - if (val.getOptional(ctx.ptr(), "withFileTypes", bool) catch { + if (val.getOptional(ctx, "withFileTypes", bool) catch { path.deinit(); return null; }) |with_file_types_| { @@ -2727,12 +2755,12 @@ pub const Arguments = struct { arguments.eat(); if (val.isObject()) { - if (val.getTruthy(ctx.ptr(), "flags")) |flags_| { + if (val.getTruthy(ctx, "flags")) |flags_| { flags = FileSystemFlags.fromJS(ctx, flags_, exception) orelse flags; if (exception.* != null) return null; } - if (val.getTruthy(ctx.ptr(), "mode")) |mode_| { + if (val.getTruthy(ctx, "mode")) |mode_| { mode = JSC.Node.modeFromJS(ctx, mode_, exception) orelse mode; if (exception.* != null) return null; } @@ -2934,7 +2962,7 @@ pub const Arguments = struct { if (exception.* != null) return null; const buffer_value = arguments.next(); - const buffer = StringOrBuffer.fromJS(ctx.ptr(), bun.default_allocator, buffer_value orelse { + const buffer = StringOrBuffer.fromJS(ctx, bun.default_allocator, buffer_value orelse { if (exception.* == null) { JSC.throwInvalidArguments( "data is required", @@ -2981,7 +3009,10 @@ pub const Arguments = struct { } if (current.isString()) { - args.encoding = Encoding.fromJS(current, ctx.ptr()) orelse Encoding.utf8; + args.encoding = Encoding.assert(current, ctx, args.encoding) catch { + args.deinit(); + return null; + }; arguments.eat(); } }, @@ -3058,7 +3089,7 @@ pub const Arguments = struct { if (exception.* != null) return null; const buffer_value = arguments.next(); - const buffer: Buffer = Buffer.fromJS(ctx.ptr(), buffer_value orelse { + const buffer: Buffer = Buffer.fromJS(ctx, buffer_value orelse { if (exception.* == null) { JSC.throwInvalidArguments( "buffer is required", @@ -3115,20 +3146,20 @@ pub const Arguments = struct { } } } else if (current.isObject()) { - if (current.getTruthy(ctx.ptr(), "offset")) |num| { + if (current.getTruthy(ctx, "offset")) |num| { if (num.isNumber() or num.isBigInt()) { args.offset = num.to(u52); } } - if (current.getTruthy(ctx.ptr(), "length")) |num| { + if (current.getTruthy(ctx, "length")) |num| { if (num.isNumber() or num.isBigInt()) { args.length = num.to(u52); } defined_length = true; } - if (current.getTruthy(ctx.ptr(), "position")) |num| { + if (current.getTruthy(ctx, "position")) |num| { if (num.isNumber() or num.isBigInt()) { args.position = num.to(i52); } @@ -3194,35 +3225,19 @@ pub const Arguments = struct { if (arguments.next()) |arg| { arguments.eat(); if (arg.isString()) { - encoding = Encoding.fromJS(arg, ctx.ptr()) orelse { - if (exception.* == null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } + encoding = Encoding.assert(arg, ctx, encoding) catch { + path.deinit(); return null; }; } else if (arg.isObject()) { - if (arg.get(ctx.ptr(), "encoding")) |encoding_| { - if (!encoding_.isUndefinedOrNull()) { - encoding = Encoding.fromJS(encoding_, ctx.ptr()) orelse { - if (exception.* == null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; - } + encoding = getEncoding(arg, ctx, encoding) catch encoding; + + if (ctx.hasException()) { + path.deinit(); + return null; } - if (arg.getTruthy(ctx.ptr(), "flag")) |flag_| { + if (arg.getTruthy(ctx, "flag")) |flag_| { flag = FileSystemFlags.fromJS(ctx, flag_, exception) orelse { if (exception.* == null) { JSC.throwInvalidArguments( @@ -3235,6 +3250,11 @@ pub const Arguments = struct { return null; }; } + + if (ctx.hasException()) { + path.deinit(); + return null; + } } } @@ -3314,35 +3334,16 @@ pub const Arguments = struct { if (arguments.next()) |arg| { arguments.eat(); if (arg.isString()) { - encoding = Encoding.fromJS(arg, ctx.ptr()) orelse { - defer file.deinit(); - if (exception.* == null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; + encoding = Encoding.assert(arg, ctx, encoding) catch encoding; } else if (arg.isObject()) { - if (arg.getTruthy(ctx.ptr(), "encoding")) |encoding_| { - encoding = Encoding.fromJS(encoding_, ctx.ptr()) orelse { - defer file.deinit(); - if (exception.* == null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; + encoding = getEncoding(arg, ctx, encoding) catch encoding; + + if (ctx.hasException()) { + file.deinit(); + return null; } - if (arg.getTruthy(ctx.ptr(), "flag")) |flag_| { + if (arg.getTruthy(ctx, "flag")) |flag_| { flag = FileSystemFlags.fromJS(ctx, flag_, exception) orelse { defer file.deinit(); if (exception.* == null) { @@ -3357,7 +3358,7 @@ pub const Arguments = struct { }; } - if (arg.getTruthy(ctx.ptr(), "mode")) |mode_| { + if (arg.getTruthy(ctx, "mode")) |mode_| { mode = JSC.Node.modeFromJS(ctx, mode_, exception) orelse { defer file.deinit(); if (exception.* == null) { @@ -3374,7 +3375,12 @@ pub const Arguments = struct { } } - const data = StringOrBuffer.fromJSWithEncodingMaybeAsync(ctx.ptr(), bun.default_allocator, data_value, encoding, arguments.will_be_async) orelse { + if (ctx.hasException()) { + file.deinit(); + return null; + } + + const data = StringOrBuffer.fromJSWithEncodingMaybeAsync(ctx, bun.default_allocator, data_value, encoding, arguments.will_be_async) orelse { defer file.deinit(); if (exception.* == null) { JSC.throwInvalidArguments( @@ -3433,35 +3439,18 @@ pub const Arguments = struct { if (arguments.next()) |arg| { arguments.eat(); if (arg.isString()) { - encoding = Encoding.fromJS(arg, ctx.ptr()) orelse { - if (exception.* == null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; + encoding = Encoding.assert(arg, ctx, encoding) catch encoding; } else if (arg.isObject()) { - if (arg.get(ctx.ptr(), "encoding")) |encoding_| { - if (!encoding_.isUndefinedOrNull()) { - encoding = Encoding.fromJS(encoding_, ctx.ptr()) orelse { - if (exception.* == null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; - } + if (getEncoding(arg, ctx)) |encoding_| { + encoding = encoding_; } - if (arg.get(ctx.ptr(), "bufferSize")) |buffer_size_| { + if (ctx.hasException()) { + path.deinit(); + return null; + } + + if (arg.get(ctx, "bufferSize")) |buffer_size_| { buffer_size = buffer_size_.toInt32(); if (buffer_size < 0) { if (exception.* == null) { @@ -3478,6 +3467,11 @@ pub const Arguments = struct { } } + if (ctx.hasException()) { + path.deinit(); + return null; + } + return OpenDir{ .path = path, .encoding = encoding, @@ -3571,275 +3565,6 @@ pub const Arguments = struct { } }; - pub const CreateReadStream = struct { - file: PathOrFileDescriptor, - flags: FileSystemFlags = FileSystemFlags.r, - encoding: Encoding = Encoding.utf8, - mode: Mode = default_permission, - autoClose: bool = true, - emitClose: bool = true, - start: i32 = 0, - end: i32 = std.math.maxInt(i32), - highwater_mark: u32 = 64 * 1024, - global_object: *JSC.JSGlobalObject, - - pub fn deinit(this: CreateReadStream) void { - this.file.deinit(); - } - - pub fn copyToState(this: CreateReadStream, state: *JSC.Node.Readable.State) void { - state.encoding = this.encoding; - state.highwater_mark = this.highwater_mark; - state.start = this.start; - state.end = this.end; - } - - pub fn fromJS(ctx: JSC.C.JSContextRef, arguments: *ArgumentsSlice, exception: JSC.C.ExceptionRef) ?CreateReadStream { - const path = PathLike.fromJS(ctx, arguments, exception); - if (exception.* != null) return null; - if (path == null) arguments.eat(); - - var stream = CreateReadStream{ - .file = undefined, - .global_object = ctx.ptr(), - }; - var fd = FileDescriptor.invalid; - - if (arguments.next()) |arg| { - arguments.eat(); - if (arg.isString()) { - stream.encoding = Encoding.fromJS(arg, ctx.ptr()) orelse { - if (exception.* != null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; - } else if (arg.isObject()) { - if (arg.get(ctx.ptr(), "mode")) |mode_| { - stream.mode = JSC.Node.modeFromJS(ctx, mode_, exception) orelse { - if (exception.* != null) { - JSC.throwInvalidArguments( - "Invalid mode", - .{}, - ctx, - exception, - ); - } - return null; - }; - } - - if (arg.get(ctx.ptr(), "encoding")) |encoding| { - stream.encoding = Encoding.fromJS(encoding, ctx.ptr()) orelse { - if (exception.* != null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; - } - - if (arg.getTruthy(ctx.ptr(), "flags")) |flags| { - stream.flags = FileSystemFlags.fromJS(ctx, flags, exception) orelse { - if (exception.* == null) { - JSC.throwInvalidArguments( - "Invalid flags", - .{}, - ctx, - exception, - ); - } - return null; - }; - } - - if (arg.get(ctx.ptr(), "fd")) |flags| { - fd = JSC.Node.fileDescriptorFromJS(ctx, flags, exception) orelse { - if (exception.* != null) { - JSC.throwInvalidArguments( - "Invalid file descriptor", - .{}, - ctx, - exception, - ); - } - return null; - }; - } - - if (arg.get(ctx.ptr(), "autoClose")) |autoClose| { - stream.autoClose = autoClose.toBoolean(); - } - - if (arg.get(ctx.ptr(), "emitClose")) |emitClose| { - stream.emitClose = emitClose.toBoolean(); - } - - if (arg.get(ctx.ptr(), "start")) |start| { - stream.start = start.coerce(i32, ctx); - } - - if (arg.get(ctx.ptr(), "end")) |end| { - stream.end = end.coerce(i32, ctx); - } - - if (arg.get(ctx.ptr(), "highWaterMark")) |highwaterMark| { - stream.highwater_mark = highwaterMark.toU32(); - } - } - } - - if (fd.isValid()) { - stream.file = .{ .fd = fd }; - } else if (path) |path_| { - stream.file = .{ .path = path_ }; - } else { - JSC.throwInvalidArguments("Missing path or file descriptor", .{}, ctx, exception); - return null; - } - return stream; - } - }; - - pub const CreateWriteStream = struct { - file: PathOrFileDescriptor, - flags: FileSystemFlags = FileSystemFlags.w, - encoding: Encoding = Encoding.utf8, - mode: Mode = default_permission, - autoClose: bool = true, - emitClose: bool = true, - start: i32 = 0, - highwater_mark: u32 = 256 * 1024, - global_object: *JSC.JSGlobalObject, - - pub fn deinit(this: @This()) void { - this.file.deinit(); - } - - pub fn copyToState(this: CreateWriteStream, state: *JSC.Node.Writable.State) void { - state.encoding = this.encoding; - state.highwater_mark = this.highwater_mark; - state.start = this.start; - state.emit_close = this.emitClose; - } - - pub fn fromJS(ctx: JSC.C.JSContextRef, arguments: *ArgumentsSlice, exception: JSC.C.ExceptionRef) ?CreateWriteStream { - const path = PathLike.fromJS(ctx, arguments, exception); - if (exception.* != null) return null; - if (path == null) arguments.eat(); - - var stream = CreateWriteStream{ - .file = undefined, - .global_object = ctx.ptr(), - }; - var fd: FileDescriptor = bun.invalid_fd; - - if (arguments.next()) |arg| { - arguments.eat(); - if (arg.isString()) { - stream.encoding = Encoding.fromJS(arg, ctx.ptr()) orelse { - if (exception.* != null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; - } else if (arg.isObject()) { - if (arg.get(ctx.ptr(), "mode")) |mode_| { - stream.mode = JSC.Node.modeFromJS(ctx, mode_, exception) orelse { - if (exception.* != null) { - JSC.throwInvalidArguments( - "Invalid mode", - .{}, - ctx, - exception, - ); - } - return null; - }; - } - - if (arg.get(ctx.ptr(), "encoding")) |encoding| { - stream.encoding = Encoding.fromJS(encoding, ctx.ptr()) orelse { - if (exception.* != null) { - JSC.throwInvalidArguments( - "Invalid encoding", - .{}, - ctx, - exception, - ); - } - return null; - }; - } - - if (arg.getTruthy(ctx.ptr(), "flags")) |flags| { - stream.flags = FileSystemFlags.fromJS(ctx, flags, exception) orelse { - if (exception.* == null) { - JSC.throwInvalidArguments( - "Invalid flags", - .{}, - ctx, - exception, - ); - } - return null; - }; - } - - if (arg.get(ctx.ptr(), "fd")) |flags| { - fd = JSC.Node.fileDescriptorFromJS(ctx, flags, exception) orelse { - if (exception.* != null) { - JSC.throwInvalidArguments( - "Invalid file descriptor", - .{}, - ctx, - exception, - ); - } - return null; - }; - } - - if (arg.get(ctx.ptr(), "autoClose")) |autoClose| { - stream.autoClose = autoClose.toBoolean(); - } - - if (arg.get(ctx.ptr(), "emitClose")) |emitClose| { - stream.emitClose = emitClose.toBoolean(); - } - - if (arg.get(ctx.ptr(), "start")) |start| { - stream.start = start.toInt32(); - } - } - } - - if (fd != bun.invalid_fd) { - stream.file = .{ .fd = fd }; - } else if (path) |path_| { - stream.file = .{ .path = path_ }; - } else { - JSC.throwInvalidArguments("Missing path or file descriptor", .{}, ctx, exception); - return null; - } - return stream; - } - }; - pub const FdataSync = struct { fd: FileDescriptor, @@ -6617,14 +6342,6 @@ pub const NodeFS = struct { return args.createFSWatcher(); } - pub fn createReadStream(_: *NodeFS, _: Arguments.CreateReadStream, comptime _: Flavor) Maybe(Return.CreateReadStream) { - return Maybe(Return.CreateReadStream).todo(); - } - - pub fn createWriteStream(_: *NodeFS, _: Arguments.CreateWriteStream, comptime _: Flavor) Maybe(Return.CreateWriteStream) { - return Maybe(Return.CreateWriteStream).todo(); - } - /// This function is `cpSync`, but only if you pass `{ recursive: ..., force: ..., errorOnExist: ..., mode: ... }' /// The other options like `filter` use a JS fallback, see `src/js/internal/fs/cp.ts` pub fn cp(this: *NodeFS, args: Arguments.Cp, comptime flavor: Flavor) Maybe(Return.Cp) { diff --git a/src/bun.js/node/node_fs_binding.zig b/src/bun.js/node/node_fs_binding.zig index 4f10e96b8a..c0dc296e5d 100644 --- a/src/bun.js/node/node_fs_binding.zig +++ b/src/bun.js/node/node_fs_binding.zig @@ -60,6 +60,8 @@ fn callSync(comptime FunctionEnum: NodeFSFunctionEnum) NodeFSFunction { if (exception1 != .zero) { globalObject.throwValue(exception1); return .zero; + } else if (globalObject.hasException()) { + return .zero; } var result = Function( &this.node_fs, @@ -110,13 +112,13 @@ fn call(comptime FunctionEnum: NodeFSFunctionEnum) NodeFSFunction { if (exception1 != .zero) { globalObject.throwValue(exception1); - + slice.deinit(); + return .zero; + } else if (globalObject.hasException()) { slice.deinit(); return .zero; } - // TODO: handle globalObject.throwValue - const Task = @field(JSC.Node.Async, @tagName(FunctionEnum)); if (comptime FunctionEnum == .cp) { return Task.create(globalObject, this, args, globalObject.bunVM(), slice.arena); diff --git a/src/bun.js/node/node_fs_watcher.zig b/src/bun.js/node/node_fs_watcher.zig index 209d4c85f9..8eeb4acd95 100644 --- a/src/bun.js/node/node_fs_watcher.zig +++ b/src/bun.js/node/node_fs_watcher.zig @@ -353,6 +353,8 @@ pub const FSWatcher = struct { } return null; }; + var should_deinit_path = true; + defer if (should_deinit_path) path.deinit(); if (exception.* != null) return null; var listener: JSC.JSValue = .zero; @@ -365,7 +367,7 @@ pub const FSWatcher = struct { // options if (options_or_callable.isObject()) { - if (options_or_callable.get(ctx, "persistent")) |persistent_| { + if (options_or_callable.getTruthy(ctx, "persistent")) |persistent_| { if (!persistent_.isBoolean()) { JSC.throwInvalidArguments( "persistent must be a boolean.", @@ -378,7 +380,7 @@ pub const FSWatcher = struct { persistent = persistent_.toBoolean(); } - if (options_or_callable.get(ctx, "verbose")) |verbose_| { + if (options_or_callable.getTruthy(ctx, "verbose")) |verbose_| { if (!verbose_.isBoolean()) { JSC.throwInvalidArguments( "verbose must be a boolean.", @@ -391,30 +393,11 @@ pub const FSWatcher = struct { verbose = verbose_.toBoolean(); } - if (options_or_callable.get(ctx, "encoding")) |encoding_| { - if (!encoding_.isString()) { - JSC.throwInvalidArguments( - "encoding must be a string.", - .{}, - ctx, - exception, - ); - return null; - } - if (JSC.Node.Encoding.fromJS(encoding_, ctx.ptr())) |node_encoding| { - encoding = node_encoding; - } else { - JSC.throwInvalidArguments( - "invalid encoding.", - .{}, - ctx, - exception, - ); - return null; - } + if (options_or_callable.fastGet(ctx, .encoding)) |encoding_| { + encoding = JSC.Node.Encoding.assert(encoding_, ctx, encoding) catch return null; } - if (options_or_callable.get(ctx, "recursive")) |recursive_| { + if (options_or_callable.getTruthy(ctx, "recursive")) |recursive_| { if (!recursive_.isBoolean()) { JSC.throwInvalidArguments( "recursive must be a boolean.", @@ -428,7 +411,7 @@ pub const FSWatcher = struct { } // abort signal - if (options_or_callable.get(ctx, "signal")) |signal_| { + if (options_or_callable.getTruthy(ctx, "signal")) |signal_| { if (JSC.AbortSignal.fromJS(signal_)) |signal_obj| { //Keep it alive signal_.ensureStillAlive(); @@ -466,6 +449,8 @@ pub const FSWatcher = struct { return null; } + should_deinit_path = false; + return Arguments{ .path = path, .listener = listener, diff --git a/src/bun.js/node/types.zig b/src/bun.js/node/types.zig index 8ff71ffc01..65f584f85a 100644 --- a/src/bun.js/node/types.zig +++ b/src/bun.js/node/types.zig @@ -669,7 +669,6 @@ pub const Encoding = enum(u8) { }; } - /// Caller must verify the value is a string pub fn fromJS(value: JSC.JSValue, global: *JSC.JSGlobalObject) ?Encoding { return map.fromJSCaseInsensitive(global, value); } @@ -679,6 +678,40 @@ pub const Encoding = enum(u8) { return strings.inMapCaseInsensitive(slice, map); } + pub fn assert(value: JSC.JSValue, globalObject: *JSC.JSGlobalObject, default: Encoding) !Encoding { + if (value.isFalsey()) { + return default; + } + + if (!value.isString()) { + throwEncodingError(globalObject, value); + return error.JSError; + } + + return fromJSWithDefaultOnEmpty(value, globalObject, default) orelse { + throwEncodingError(globalObject, value); + return error.JSError; + }; + } + + pub fn fromJSWithDefaultOnEmpty(value: JSC.JSValue, globalObject: *JSC.JSGlobalObject, default: Encoding) ?Encoding { + const str = bun.String.tryFromJS(value, globalObject) orelse return null; + defer str.deref(); + if (str.isEmpty()) { + return default; + } + return str.inMapCaseInsensitive(Encoding.map); + } + + pub fn throwEncodingError(globalObject: *JSC.JSGlobalObject, value: JSC.JSValue) void { + globalObject.ERR_INVALID_ARG_VALUE( + "encoding '{}' is an invalid encoding", + .{ + value.fmtString(globalObject), + }, + ).throw(); + } + pub fn encodeWithSize(encoding: Encoding, globalObject: *JSC.JSGlobalObject, comptime size: usize, input: *const [size]u8) JSC.JSValue { switch (encoding) { .base64 => { diff --git a/src/bun.js/test/expect.zig b/src/bun.js/test/expect.zig index d1c42a9750..1d13dd0172 100644 --- a/src/bun.js/test/expect.zig +++ b/src/bun.js/test/expect.zig @@ -1493,7 +1493,7 @@ pub const Expect = struct { const not = this.flags.not; var pass = false; - const truthy = value.toBooleanSlow(globalThis); + const truthy = value.toBoolean(); if (truthy) pass = true; if (not) pass = !pass; @@ -1649,7 +1649,7 @@ pub const Expect = struct { const not = this.flags.not; var pass = false; - const truthy = value.toBooleanSlow(globalThis); + const truthy = value.toBoolean(); if (!truthy) pass = true; if (not) pass = !pass; @@ -2439,7 +2439,7 @@ pub const Expect = struct { // TODO: REMOVE THIS GETTER! Expose a binding to call .test on the RegExp object directly. if (expected_value.get(globalThis, "test")) |test_fn| { const matches = test_fn.call(globalThis, expected_value, &.{received_message}) catch |err| globalThis.takeException(err); - if (!matches.toBooleanSlow(globalThis)) return .undefined; + if (!matches.toBoolean()) return .undefined; } this.throw(globalThis, signature, "\n\nExpected pattern: not {any}\nReceived message: {any}\n", .{ @@ -2524,7 +2524,7 @@ pub const Expect = struct { // TODO: REMOVE THIS GETTER! Expose a binding to call .test on the RegExp object directly. if (expected_value.get(globalThis, "test")) |test_fn| { const matches = test_fn.call(globalThis, expected_value, &.{received_message}) catch |err| globalThis.takeException(err); - if (matches.toBooleanSlow(globalThis)) return .undefined; + if (matches.toBoolean()) return .undefined; } } @@ -4699,7 +4699,7 @@ pub const Expect = struct { const is_valid = valid: { if (result.isObject()) { if (result.get(globalThis, "pass")) |pass_value| { - pass = pass_value.toBooleanSlow(globalThis); + pass = pass_value.toBoolean(); if (globalThis.hasException()) return false; if (result.fastGet(globalThis, .message)) |message_value| { diff --git a/src/bun.js/test/jest.zig b/src/bun.js/test/jest.zig index ace7a2172a..6c3d092489 100644 --- a/src/bun.js/test/jest.zig +++ b/src/bun.js/test/jest.zig @@ -1853,7 +1853,7 @@ inline fn createIfScope( } const name = ZigString.static(property); - const value = args[0].toBooleanSlow(globalThis); + const value = args[0].toBoolean(); const truthy_falsey = comptime switch (tag) { .pass => .{ Scope.skip, Scope.call }, diff --git a/src/bun.js/webcore/streams.zig b/src/bun.js/webcore/streams.zig index 9c2f589f37..7e9addca18 100644 --- a/src/bun.js/webcore/streams.zig +++ b/src/bun.js/webcore/streams.zig @@ -2958,9 +2958,10 @@ pub fn ReadableStreamSource( } pub fn updateRef(this: *ReadableStreamSourceType, globalObject: *JSGlobalObject, callFrame: *JSC.CallFrame) JSC.JSValue { + _ = globalObject; // autofix JSC.markBinding(@src()); this.this_jsvalue = callFrame.this(); - const ref_or_unref = callFrame.argument(0).toBooleanSlow(globalObject); + const ref_or_unref = callFrame.argument(0).toBoolean(); this.setRef(ref_or_unref); return .undefined; diff --git a/test/js/bun/http/bun-server.test.ts b/test/js/bun/http/bun-server.test.ts index f6ebd8a57a..8cc8eb20ff 100644 --- a/test/js/bun/http/bun-server.test.ts +++ b/test/js/bun/http/bun-server.test.ts @@ -100,7 +100,7 @@ describe("Server", () => { }); test("should not allow Bun.serve with invalid tls option", () => { - [1, "string", true, Symbol("symbol"), false].forEach(value => { + [1, "string", true, Symbol("symbol")].forEach(value => { expect(() => { using server = Bun.serve({ //@ts-ignore diff --git a/test/js/node/fs/fs.test.ts b/test/js/node/fs/fs.test.ts index 81570275e9..6913d38191 100644 --- a/test/js/node/fs/fs.test.ts +++ b/test/js/node/fs/fs.test.ts @@ -96,6 +96,117 @@ it("writing to 1, 2 are possible", () => { expect(fs.writeSync(2, Buffer.from("\nhello-stderr-test\n"))).toBe(19); }); +describe("test-fs-assert-encoding-error", () => { + const testPath = join(tmpdirSync(), "assert-encoding-error"); + const options = "test"; + const expectedError = expect.objectContaining({ + code: "ERR_INVALID_ARG_VALUE", + name: "TypeError", + }); + + it("readFile throws on invalid encoding", () => { + expect(() => { + fs.readFile(testPath, options, () => {}); + }).toThrow(expectedError); + }); + + it("readFileSync throws on invalid encoding", () => { + expect(() => { + fs.readFileSync(testPath, options); + }).toThrow(expectedError); + }); + + it("readdir throws on invalid encoding", () => { + expect(() => { + fs.readdir(testPath, options, () => {}); + }).toThrow(expectedError); + }); + + it("readdirSync throws on invalid encoding", () => { + expect(() => { + fs.readdirSync(testPath, options); + }).toThrow(expectedError); + }); + + it("readlink throws on invalid encoding", () => { + expect(() => { + fs.readlink(testPath, options, () => {}); + }).toThrow(expectedError); + }); + + it("readlinkSync throws on invalid encoding", () => { + expect(() => { + fs.readlinkSync(testPath, options); + }).toThrow(expectedError); + }); + + it("writeFile throws on invalid encoding", () => { + expect(() => { + fs.writeFile(testPath, "data", options, () => {}); + }).toThrow(expectedError); + }); + + it("writeFileSync throws on invalid encoding", () => { + expect(() => { + fs.writeFileSync(testPath, "data", options); + }).toThrow(expectedError); + }); + + it("appendFile throws on invalid encoding", () => { + expect(() => { + fs.appendFile(testPath, "data", options, () => {}); + }).toThrow(expectedError); + }); + + it("appendFileSync throws on invalid encoding", () => { + expect(() => { + fs.appendFileSync(testPath, "data", options); + }).toThrow(expectedError); + }); + + it("watch throws on invalid encoding", () => { + expect(() => { + fs.watch(testPath, options, () => {}); + }).toThrow(expectedError); + }); + + it("realpath throws on invalid encoding", () => { + expect(() => { + fs.realpath(testPath, options, () => {}); + }).toThrow(expectedError); + }); + + it("realpathSync throws on invalid encoding", () => { + expect(() => { + fs.realpathSync(testPath, options); + }).toThrow(expectedError); + }); + + it("mkdtemp throws on invalid encoding", () => { + expect(() => { + fs.mkdtemp(testPath, options, () => {}); + }).toThrow(expectedError); + }); + + it("mkdtempSync throws on invalid encoding", () => { + expect(() => { + fs.mkdtempSync(testPath, options); + }).toThrow(expectedError); + }); + + it.todo("ReadStream throws on invalid encoding", () => { + expect(() => { + fs.ReadStream(testPath, options); + }).toThrow(expectedError); + }); + + it.todo("WriteStream throws on invalid encoding", () => { + expect(() => { + fs.WriteStream(testPath, options); + }).toThrow(expectedError); + }); +}); + // TODO: port node.js tests for these it("fs.readv returns object", async done => { const fd = await promisify(fs.open)(import.meta.path, "r"); From 6914c5e32cfef538658b7c90756931388c717f80 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 1 Nov 2024 18:38:01 -0700 Subject: [PATCH 14/17] Fixes #13816 (#13906) Co-authored-by: pfg Co-authored-by: Ryan Gonzalez Co-authored-by: Ben Grant Co-authored-by: Dave Caruso --- CONTRIBUTING.md | 2 +- cmake/Options.cmake | 2 +- src/bun.js/bindings/ErrorCode.ts | 1 + src/js/builtins/ProcessObjectInternals.ts | 5 ++ src/js/builtins/ReadableStream.ts | 17 +++- .../ReadableStreamDefaultController.ts | 5 +- .../builtins/ReadableStreamDefaultReader.ts | 5 +- src/js/builtins/ReadableStreamInternals.ts | 38 +++++--- test/js/third_party/prompts/prompts.test.ts | 6 +- test/js/web/streams/streams.test.js | 88 +++++++++++++++++++ 10 files changed, 148 insertions(+), 21 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e143d69342..9feff27121 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -290,7 +290,7 @@ $ xcode-select --install Bun defaults to linking `libatomic` statically, as not all systems have it. If you are building on a distro that does not have a static libatomic available, you can run the following command to enable dynamic linking: ```bash -$ bun setup -DUSE_STATIC_LIBATOMIC=OFF +$ bun run build -DUSE_STATIC_LIBATOMIC=OFF ``` The built version of Bun may not work on other systems if compiled this way. diff --git a/cmake/Options.cmake b/cmake/Options.cmake index 36137c50cb..726a94a4b4 100644 --- a/cmake/Options.cmake +++ b/cmake/Options.cmake @@ -138,7 +138,7 @@ if(CMAKE_HOST_LINUX AND NOT WIN32 AND NOT APPLE) OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET ) - if(LINUX_DISTRO MATCHES "NAME=\"(Arch|Manjaro|Artix) Linux\"|NAME=\"openSUSE Tumbleweed\"") + if(LINUX_DISTRO MATCHES "NAME=\"(Arch|Manjaro|Artix) Linux( ARM)?\"|NAME=\"openSUSE Tumbleweed\"") set(DEFAULT_STATIC_LIBATOMIC OFF) endif() endif() diff --git a/src/bun.js/bindings/ErrorCode.ts b/src/bun.js/bindings/ErrorCode.ts index 692ad34a3e..ed8185e191 100644 --- a/src/bun.js/bindings/ErrorCode.ts +++ b/src/bun.js/bindings/ErrorCode.ts @@ -45,6 +45,7 @@ export default [ ["ERR_BUFFER_OUT_OF_BOUNDS", RangeError, "RangeError"], ["ERR_UNKNOWN_SIGNAL", TypeError, "TypeError"], ["ERR_SOCKET_BAD_PORT", RangeError, "RangeError"], + ["ERR_STREAM_RELEASE_LOCK", Error, "AbortError"], // Bun-specific ["ERR_FORMDATA_PARSE_ERROR", TypeError, "TypeError"], diff --git a/src/js/builtins/ProcessObjectInternals.ts b/src/js/builtins/ProcessObjectInternals.ts index b62f3028b3..506596a53f 100644 --- a/src/js/builtins/ProcessObjectInternals.ts +++ b/src/js/builtins/ProcessObjectInternals.ts @@ -195,6 +195,10 @@ export function getStdinStream(fd) { } } } catch (err) { + if (err?.code === "ERR_STREAM_RELEASE_LOCK") { + // Not a bug. Happens in unref(). + return; + } stream.destroy(err); } } @@ -212,6 +216,7 @@ export function getStdinStream(fd) { $debug('on("resume");'); ref(); stream._undestroy(); + stream_destroyed = false; }); stream._readableState.reading = false; diff --git a/src/js/builtins/ReadableStream.ts b/src/js/builtins/ReadableStream.ts index ed45aaab37..6e7e2d5951 100644 --- a/src/js/builtins/ReadableStream.ts +++ b/src/js/builtins/ReadableStream.ts @@ -108,6 +108,7 @@ export function initializeReadableStream( $linkTimeConstant; export function readableStreamToArray(stream: ReadableStream): Promise { + if (!$isReadableStream(stream)) throw $ERR_INVALID_ARG_TYPE("stream", "ReadableStream", typeof stream); // this is a direct stream var underlyingSource = $getByIdDirectPrivate(stream, "underlyingSource"); if (underlyingSource !== undefined) { @@ -119,6 +120,7 @@ export function readableStreamToArray(stream: ReadableStream): Promise { + if (!$isReadableStream(stream)) throw $ERR_INVALID_ARG_TYPE("stream", "ReadableStream", typeof stream); // this is a direct stream var underlyingSource = $getByIdDirectPrivate(stream, "underlyingSource"); if (underlyingSource !== undefined) { @@ -137,6 +139,7 @@ export function readableStreamToText(stream: ReadableStream): Promise { $linkTimeConstant; export function readableStreamToArrayBuffer(stream: ReadableStream): Promise | ArrayBuffer { + if (!$isReadableStream(stream)) throw $ERR_INVALID_ARG_TYPE("stream", "ReadableStream", typeof stream); // this is a direct stream var underlyingSource = $getByIdDirectPrivate(stream, "underlyingSource"); if (underlyingSource !== undefined) { @@ -216,6 +219,7 @@ export function readableStreamToArrayBuffer(stream: ReadableStream) $linkTimeConstant; export function readableStreamToBytes(stream: ReadableStream): Promise | Uint8Array { + if (!$isReadableStream(stream)) throw $ERR_INVALID_ARG_TYPE("stream", "ReadableStream", typeof stream); // this is a direct stream var underlyingSource = $getByIdDirectPrivate(stream, "underlyingSource"); @@ -297,6 +301,7 @@ export function readableStreamToFormData( stream: ReadableStream, contentType: string | ArrayBuffer | ArrayBufferView, ): Promise { + if (!$isReadableStream(stream)) throw $ERR_INVALID_ARG_TYPE("stream", "ReadableStream", typeof stream); if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); return Bun.readableStreamToBlob(stream).then(blob => { return FormData.from(blob, contentType); @@ -305,6 +310,7 @@ export function readableStreamToFormData( $linkTimeConstant; export function readableStreamToJSON(stream: ReadableStream): unknown { + if (!$isReadableStream(stream)) throw $ERR_INVALID_ARG_TYPE("stream", "ReadableStream", typeof stream); if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); let result = $tryUseReadableStreamBufferedFastPath(stream, "json"); if (result) { @@ -326,6 +332,7 @@ export function readableStreamToJSON(stream: ReadableStream): unknown { $linkTimeConstant; export function readableStreamToBlob(stream: ReadableStream): Promise { + if (!$isReadableStream(stream)) throw $ERR_INVALID_ARG_TYPE("stream", "ReadableStream", typeof stream); if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); return ( @@ -422,7 +429,15 @@ export function pipeThrough(this, streams, options) { if ($isWritableStreamLocked(internalWritable)) throw $makeTypeError("WritableStream is locked"); - $readableStreamPipeToWritableStream(this, internalWritable, preventClose, preventAbort, preventCancel, signal); + const promise = $readableStreamPipeToWritableStream( + this, + internalWritable, + preventClose, + preventAbort, + preventCancel, + signal, + ); + $markPromiseAsHandled(promise); return readable; } diff --git a/src/js/builtins/ReadableStreamDefaultController.ts b/src/js/builtins/ReadableStreamDefaultController.ts index 7c1c9ace77..6a04addc33 100644 --- a/src/js/builtins/ReadableStreamDefaultController.ts +++ b/src/js/builtins/ReadableStreamDefaultController.ts @@ -33,8 +33,9 @@ export function initializeReadableStreamDefaultController(this, stream, underlyi export function enqueue(this, chunk) { if (!$isReadableStreamDefaultController(this)) throw $makeThisTypeError("ReadableStreamDefaultController", "enqueue"); - if (!$readableStreamDefaultControllerCanCloseOrEnqueue(this)) - throw new TypeError("ReadableStreamDefaultController is not in a state where chunk can be enqueued"); + if (!$readableStreamDefaultControllerCanCloseOrEnqueue(this)) { + throw $ERR_INVALID_STATE("ReadableStreamDefaultController is not in a state where chunk can be enqueued"); + } return $readableStreamDefaultControllerEnqueue(this, chunk); } diff --git a/src/js/builtins/ReadableStreamDefaultReader.ts b/src/js/builtins/ReadableStreamDefaultReader.ts index 2ff8e385f0..9ddb3e3f38 100644 --- a/src/js/builtins/ReadableStreamDefaultReader.ts +++ b/src/js/builtins/ReadableStreamDefaultReader.ts @@ -172,10 +172,7 @@ export function releaseLock(this) { if (!$getByIdDirectPrivate(this, "ownerReadableStream")) return; - if ($getByIdDirectPrivate(this, "readRequests")?.isNotEmpty()) - throw new TypeError("There are still pending read requests, cannot release the lock"); - - $readableStreamReaderGenericRelease(this); + $readableStreamDefaultReaderRelease(this); } $getter; diff --git a/src/js/builtins/ReadableStreamInternals.ts b/src/js/builtins/ReadableStreamInternals.ts index 7f95f39ee9..72b42a2c76 100644 --- a/src/js/builtins/ReadableStreamInternals.ts +++ b/src/js/builtins/ReadableStreamInternals.ts @@ -331,7 +331,10 @@ export function pipeToDoReadWrite(pipeState) { pipeState.pendingReadPromiseCapability.resolve.$call(undefined, canWrite); if (!canWrite) return; - pipeState.pendingWritePromise = $writableStreamDefaultWriterWrite(pipeState.writer, result.value); + pipeState.pendingWritePromise = $writableStreamDefaultWriterWrite(pipeState.writer, result.value).$then( + undefined, + () => {}, + ); }, e => { pipeState.pendingReadPromiseCapability.resolve.$call(undefined, false); @@ -396,7 +399,7 @@ export function pipeToClosingMustBePropagatedForward(pipeState) { action(); return; } - $getByIdDirectPrivate(pipeState.reader, "closedPromiseCapability").promise.$then(action, undefined); + $getByIdDirectPrivate(pipeState.reader, "closedPromiseCapability").promise.$then(action, () => {}); } export function pipeToClosingMustBePropagatedBackward(pipeState) { @@ -1367,20 +1370,18 @@ export function readableStreamError(stream, error) { if (!reader) return; + $getByIdDirectPrivate(reader, "closedPromiseCapability").reject.$call(undefined, error); + const promise = $getByIdDirectPrivate(reader, "closedPromiseCapability").promise; + $markPromiseAsHandled(promise); + if ($isReadableStreamDefaultReader(reader)) { - const requests = $getByIdDirectPrivate(reader, "readRequests"); - $putByIdDirectPrivate(reader, "readRequests", $createFIFO()); - for (var request = requests.shift(); request; request = requests.shift()) $rejectPromise(request, error); + $readableStreamDefaultReaderErrorReadRequests(reader, error); } else { $assert($isReadableStreamBYOBReader(reader)); const requests = $getByIdDirectPrivate(reader, "readIntoRequests"); $putByIdDirectPrivate(reader, "readIntoRequests", $createFIFO()); for (var request = requests.shift(); request; request = requests.shift()) $rejectPromise(request, error); } - - $getByIdDirectPrivate(reader, "closedPromiseCapability").reject.$call(undefined, error); - const promise = $getByIdDirectPrivate(reader, "closedPromiseCapability").promise; - $markPromiseAsHandled(promise); } export function readableStreamDefaultControllerShouldCallPull(controller) { @@ -1608,6 +1609,15 @@ export function isReadableStreamDisturbed(stream) { return stream.$disturbed; } +$visibility = "Private"; +export function readableStreamDefaultReaderRelease(reader) { + $readableStreamReaderGenericRelease(reader); + $readableStreamDefaultReaderErrorReadRequests( + reader, + $ERR_STREAM_RELEASE_LOCK("Stream reader cancelled via releaseLock()"), + ); +} + $visibility = "Private"; export function readableStreamReaderGenericRelease(reader) { $assert(!!$getByIdDirectPrivate(reader, "ownerReadableStream")); @@ -1616,11 +1626,11 @@ export function readableStreamReaderGenericRelease(reader) { if ($getByIdDirectPrivate($getByIdDirectPrivate(reader, "ownerReadableStream"), "state") === $streamReadable) $getByIdDirectPrivate(reader, "closedPromiseCapability").reject.$call( undefined, - $makeTypeError("releasing lock of reader whose stream is still in readable state"), + $ERR_STREAM_RELEASE_LOCK("Stream reader cancelled via releaseLock()"), ); else $putByIdDirectPrivate(reader, "closedPromiseCapability", { - promise: $newHandledRejectedPromise($makeTypeError("reader released lock")), + promise: $newHandledRejectedPromise($ERR_STREAM_RELEASE_LOCK("Stream reader cancelled via releaseLock()")), }); const promise = $getByIdDirectPrivate(reader, "closedPromiseCapability").promise; @@ -1636,6 +1646,12 @@ export function readableStreamReaderGenericRelease(reader) { $putByIdDirectPrivate(reader, "ownerReadableStream", undefined); } +export function readableStreamDefaultReaderErrorReadRequests(reader, error) { + const requests = $getByIdDirectPrivate(reader, "readRequests"); + $putByIdDirectPrivate(reader, "readRequests", $createFIFO()); + for (var request = requests.shift(); request; request = requests.shift()) $rejectPromise(request, error); +} + export function readableStreamDefaultControllerCanCloseOrEnqueue(controller) { if ($getByIdDirectPrivate(controller, "closeRequested")) { return false; diff --git a/test/js/third_party/prompts/prompts.test.ts b/test/js/third_party/prompts/prompts.test.ts index 4d58ee36af..00765fe76d 100644 --- a/test/js/third_party/prompts/prompts.test.ts +++ b/test/js/third_party/prompts/prompts.test.ts @@ -9,7 +9,11 @@ test("works with prompts", async () => { stdin: "pipe", }); - await Bun.sleep(100); + const reader = child.stdout.getReader(); + + await reader.read(); + reader.releaseLock(); + child.stdin.write("dylan\n"); await Bun.sleep(100); child.stdin.write("999\n"); diff --git a/test/js/web/streams/streams.test.js b/test/js/web/streams/streams.test.js index 4f769a5420..1caa6eb7ae 100644 --- a/test/js/web/streams/streams.test.js +++ b/test/js/web/streams/streams.test.js @@ -756,6 +756,55 @@ it("ReadableStream for empty file closes immediately", async () => { expect(chunks.length).toBe(0); }); +it("ReadableStream errors the stream on pull rejection", async () => { + let stream = new ReadableStream({ + pull(controller) { + return Promise.reject("pull rejected"); + }, + }); + + let reader = stream.getReader(); + let closed = reader.closed.catch(err => `closed: ${err}`); + let read = reader.read().catch(err => `read: ${err}`); + expect(await Promise.race([closed, read])).toBe("closed: pull rejected"); + expect(await read).toBe("read: pull rejected"); +}); + +it("ReadableStream rejects pending reads when the lock is released", async () => { + let { resolve, promise } = Promise.withResolvers(); + let stream = new ReadableStream({ + async pull(controller) { + controller.enqueue("123"); + await promise; + controller.enqueue("456"); + controller.close(); + }, + }); + + let reader = stream.getReader(); + expect((await reader.read()).value).toBe("123"); + + let read = reader.read(); + reader.releaseLock(); + expect(read).rejects.toThrow( + expect.objectContaining({ + name: "AbortError", + code: "ERR_STREAM_RELEASE_LOCK", + }), + ); + expect(reader.closed).rejects.toThrow( + expect.objectContaining({ + name: "AbortError", + code: "ERR_STREAM_RELEASE_LOCK", + }), + ); + + resolve(); + + reader = stream.getReader(); + expect((await reader.read()).value).toBe("456"); +}); + it("new Response(stream).arrayBuffer() (bytes)", async () => { var queue = [Buffer.from("abdefgh")]; var stream = new ReadableStream({ @@ -1053,3 +1102,42 @@ it("fs.createReadStream(filename) should be able to break inside async loop", as expect(true).toBe(true); } }); + +it("pipeTo doesn't cause unhandled rejections on readable errors", async () => { + // https://github.com/WebKit/WebKit/blob/3a75b5d2de94aa396a99b454ac47f3be9e0dc726/LayoutTests/streams/pipeTo-unhandled-promise.html + let unhandledRejectionCaught = false; + + const catchUnhandledRejection = () => { + unhandledRejectionCaught = true; + }; + process.on("unhandledRejection", catchUnhandledRejection); + + const writable = new WritableStream(); + const readable = new ReadableStream({ start: c => c.error("error") }); + readable.pipeTo(writable).catch(() => {}); + + await Bun.sleep(15); + + process.off("unhandledRejection", catchUnhandledRejection); + + expect(unhandledRejectionCaught).toBe(false); +}); + +it("pipeThrough doesn't cause unhandled rejections on readable errors", async () => { + let unhandledRejectionCaught = false; + + const catchUnhandledRejection = () => { + unhandledRejectionCaught = true; + }; + process.on("unhandledRejection", catchUnhandledRejection); + + const readable = new ReadableStream({ start: c => c.error("error") }); + const ts = new TransformStream(); + readable.pipeThrough(ts); + + await Bun.sleep(15); + + process.off("unhandledRejection", catchUnhandledRejection); + + expect(unhandledRejectionCaught).toBe(false); +}); From 85fd471d9d1dcad138de8c4deb04cbdae0b6ba29 Mon Sep 17 00:00:00 2001 From: Ciro Spaciari Date: Fri, 1 Nov 2024 19:43:42 -0700 Subject: [PATCH 15/17] fix(net) fix bytesWritten drain (#14949) --- src/bun.js/api/bun/socket.zig | 25 ++++++++++++++++++------- src/bun.js/api/sockets.classes.ts | 4 ---- src/js/node/net.ts | 8 ++++---- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/bun.js/api/bun/socket.zig b/src/bun.js/api/bun/socket.zig index 7cc8057156..0651256cd5 100644 --- a/src/bun.js/api/bun/socket.zig +++ b/src/bun.js/api/bun/socket.zig @@ -1474,6 +1474,12 @@ fn NewSocket(comptime ssl: bool) type { if (vm.isShuttingDown()) { return; } + this.ref(); + defer this.deref(); + this.internalFlush(); + // is not writable if we have buffered data or if we are already detached + if (this.buffered_data_for_node_net.len > 0 or this.socket.isDetached()) return; + vm.eventLoop().enter(); defer vm.eventLoop().exit(); @@ -2363,15 +2369,10 @@ fn NewSocket(comptime ssl: bool) type { }; } - pub fn flush( - this: *This, - _: *JSC.JSGlobalObject, - _: *JSC.CallFrame, - ) JSValue { - JSC.markBinding(@src()); + fn internalFlush(this: *This) void { if (this.buffered_data_for_node_net.len > 0) { const written: usize = @intCast(@max(this.socket.write(this.buffered_data_for_node_net.slice(), false), 0)); - + this.bytes_written += written; if (written > 0) { if (this.buffered_data_for_node_net.len > written) { const remaining = this.buffered_data_for_node_net.slice()[written..]; @@ -2385,6 +2386,15 @@ fn NewSocket(comptime ssl: bool) type { } this.socket.flush(); + } + + pub fn flush( + this: *This, + _: *JSC.JSGlobalObject, + _: *JSC.CallFrame, + ) JSValue { + JSC.markBinding(@src()); + this.internalFlush(); return JSValue.jsUndefined(); } @@ -2706,6 +2716,7 @@ fn NewSocket(comptime ssl: bool) type { ) JSValue { return JSC.JSValue.jsNumber(this.bytes_written + this.buffered_data_for_node_net.len); } + pub fn getALPNProtocol( this: *This, globalObject: *JSC.JSGlobalObject, diff --git a/src/bun.js/api/sockets.classes.ts b/src/bun.js/api/sockets.classes.ts index ee3e60e1df..a3a06da9d8 100644 --- a/src/bun.js/api/sockets.classes.ts +++ b/src/bun.js/api/sockets.classes.ts @@ -83,9 +83,6 @@ function generate(ssl) { alpnProtocol: { getter: "getALPNProtocol", }, - bytesWritten: { - getter: "getBytesWritten", - }, write: { fn: "write", length: 3, @@ -169,7 +166,6 @@ function generate(ssl) { bytesWritten: { getter: "getBytesWritten", }, - setServername: { fn: "setServername", length: 1, diff --git a/src/js/node/net.ts b/src/js/node/net.ts index 63003ae684..fd80b0783b 100644 --- a/src/js/node/net.ts +++ b/src/js/node/net.ts @@ -241,7 +241,7 @@ const Socket = (function (InternalSocket) { if (callback) { const writeChunk = self._pendingData; - if (socket.$write(writeChunk || "", "utf8")) { + if (!writeChunk || socket.$write(writeChunk || "", self._pendingEncoding || "utf8")) { self._pendingData = self.#writeCallback = null; callback(null); } else { @@ -856,12 +856,12 @@ const Socket = (function (InternalSocket) { if (!socket) { // detached but connected? wait for the socket to be attached this.#writeCallback = callback; - this._pendingEncoding = "buffer"; - this._pendingData = Buffer.from(chunk, encoding); + this._pendingEncoding = encoding; + this._pendingData = chunk; return; } - const success = socket?.$write(chunk, encoding); + const success = socket.$write(chunk, encoding); this[kBytesWritten] = socket.bytesWritten; if (success) { callback(); From 5e5e7c60f1004c79a3f87e1f5470ee19d57f10d0 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 1 Nov 2024 20:41:20 -0700 Subject: [PATCH 16/17] Fix release build --- src/cli.zig | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/cli.zig b/src/cli.zig index d6a863a97e..19f64c55e1 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -769,9 +769,11 @@ pub const Arguments = struct { ctx.bundler_options.transform_only = args.flag("--no-bundle"); ctx.bundler_options.bytecode = args.flag("--bytecode"); - if (args.flag("--app")) { - ctx.bundler_options.bake = true; - ctx.bundler_options.bake_debug_dump_server = args.flag("--debug-dump-server-files"); + if (comptime FeatureFlags.bake) { + if (args.flag("--app")) { + ctx.bundler_options.bake = true; + ctx.bundler_options.bake_debug_dump_server = args.flag("--debug-dump-server-files"); + } } // TODO: support --format=esm @@ -933,19 +935,21 @@ pub const Arguments = struct { ctx.bundler_options.asset_naming = try strings.concat(allocator, &.{ "./", bun.strings.removeLeadingDotSlash(asset_naming) }); } - if (args.flag("--server-components")) { - if (!bun.FeatureFlags.cli_server_components) { - // TODO: i want to disable this in non-canary - // but i also want to have tests that can run for PRs - } - ctx.bundler_options.server_components = true; - if (opts.target) |target| { - if (!bun.options.Target.from(target).isServerSide()) { - bun.Output.errGeneric("Cannot use client-side --target={s} with --server-components", .{@tagName(target)}); - Global.crash(); + if (comptime FeatureFlags.bake) { + if (args.flag("--server-components")) { + if (!bun.FeatureFlags.cli_server_components) { + // TODO: i want to disable this in non-canary + // but i also want to have tests that can run for PRs + } + ctx.bundler_options.server_components = true; + if (opts.target) |target| { + if (!bun.options.Target.from(target).isServerSide()) { + bun.Output.errGeneric("Cannot use client-side --target={s} with --server-components", .{@tagName(target)}); + Global.crash(); + } + } else { + opts.target = .bun; } - } else { - opts.target = .bun; } } From 497fa59bf02dc5374ca306d5eaaf9ae03585272b Mon Sep 17 00:00:00 2001 From: pfg Date: Sat, 2 Nov 2024 17:13:31 -0700 Subject: [PATCH 17/17] Fix #14865 (#14953) --- src/js/thirdparty/node-fetch.ts | 5 ++--- test/regression/issue/014865.test.ts | 8 ++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 test/regression/issue/014865.test.ts diff --git a/src/js/thirdparty/node-fetch.ts b/src/js/thirdparty/node-fetch.ts index d2b5831690..d0676ee418 100644 --- a/src/js/thirdparty/node-fetch.ts +++ b/src/js/thirdparty/node-fetch.ts @@ -126,7 +126,7 @@ var ResponsePrototype = Response.prototype; const kUrl = Symbol("kUrl"); class Request extends WebRequest { - [kUrl]: string; + [kUrl]?: string; constructor(input, init) { // node-fetch is relaxed with the URL, for example, it allows "/" as a valid URL. @@ -137,12 +137,11 @@ class Request extends WebRequest { this[kUrl] = input; } else { super(input, init); - this[kUrl] = input.url; } } get url() { - return this[kUrl]; + return this[kUrl] ?? super.url; } } diff --git a/test/regression/issue/014865.test.ts b/test/regression/issue/014865.test.ts new file mode 100644 index 0000000000..37e6b1bfbb --- /dev/null +++ b/test/regression/issue/014865.test.ts @@ -0,0 +1,8 @@ +import { test, expect } from "bun:test"; +import { Request } from "node-fetch"; + +test("node fetch Request URL field is set even with a valid URL", () => { + expect(new Request("/").url).toBe("/"); + expect(new Request("https://bun.sh/").url).toBe("https://bun.sh/"); + expect(new Request(new URL("https://bun.sh/")).url).toBe("https://bun.sh/"); +});