diff --git a/bench/snippets/react-dom-render.bun.js b/bench/snippets/react-dom-render.bun.js index a9ac5d97de..b13508d75d 100644 --- a/bench/snippets/react-dom-render.bun.js +++ b/bench/snippets/react-dom-render.bun.js @@ -26,6 +26,11 @@ group("new Response(stream).arrayBuffer()", () => { bench("react-dom/server.bun", async () => await new Response(await renderToReadableStreamBun()).arrayBuffer()); }); +group("new Response(stream).bytes()", () => { + bench("react-dom/server.browser", async () => await new Response(await renderToReadableStream()).bytes()); + bench("react-dom/server.bun", async () => await new Response(await renderToReadableStreamBun()).bytes()); +}); + group("new Response(stream).blob()", () => { bench("react-dom/server.browser", async () => await new Response(await renderToReadableStream()).blob()); bench("react-dom/server.bun", async () => await new Response(await renderToReadableStreamBun()).blob()); diff --git a/docs/api/binary-data.md b/docs/api/binary-data.md index efb294fc42..864ef9db0e 100644 --- a/docs/api/binary-data.md +++ b/docs/api/binary-data.md @@ -886,15 +886,25 @@ new Response(stream).arrayBuffer(); Bun.readableStreamToArrayBuffer(stream); ``` +#### To `Uint8Array` + +```ts +// with Response +new Response(stream).bytes(); + +// with Bun function +Bun.readableStreamToBytes(stream); +``` + #### To `TypedArray` ```ts // with Response const buf = await new Response(stream).arrayBuffer(); -new Uint8Array(buf); +new Int8Array(buf); // with Bun function -new Uint8Array(Bun.readableStreamToArrayBuffer(stream)); +new Int8Array(Bun.readableStreamToArrayBuffer(stream)); ``` #### To `DataView` diff --git a/docs/api/file-io.md b/docs/api/file-io.md index f1f12f5eb4..206abfe475 100644 --- a/docs/api/file-io.md +++ b/docs/api/file-io.md @@ -28,6 +28,7 @@ const foo = Bun.file("foo.txt"); await foo.text(); // contents as a string await foo.stream(); // contents as ReadableStream await foo.arrayBuffer(); // contents as ArrayBuffer +await foo.bytes(); // contents as Uint8Array ``` File references can also be created using numerical [file descriptors](https://en.wikipedia.org/wiki/File_descriptor) or `file://` URLs. diff --git a/docs/api/utils.md b/docs/api/utils.md index 0a541288b5..05034b2f68 100644 --- a/docs/api/utils.md +++ b/docs/api/utils.md @@ -275,6 +275,7 @@ Bun.stringWidth("\u001b[31mhello\u001b[0m", { countAnsiEscapeCodes: true }); // ``` This is useful for: + - Aligning text in a terminal - Quickly checking if a string contains ANSI escape codes - Measuring the width of a string in a terminal @@ -372,7 +373,6 @@ npm/string-width 95,000 chars ansi+emoji+ascii 3.68 s/iter (3.66 s {% /details %} - TypeScript definition: ```ts @@ -400,7 +400,6 @@ namespace Bun { } ``` - ## `Bun.fileURLToPath()` @@ -603,6 +602,9 @@ stream; // => ReadableStream await Bun.readableStreamToArrayBuffer(stream); // => ArrayBuffer +await Bun.readableStreamToBytes(stream); +// => Uint8Array + await Bun.readableStreamToBlob(stream); // => Blob diff --git a/docs/bundler/index.md b/docs/bundler/index.md index 4229e18a40..780e401343 100644 --- a/docs/bundler/index.md +++ b/docs/bundler/index.md @@ -1096,6 +1096,7 @@ const build = await Bun.build({ for (const output of build.outputs) { await output.arrayBuffer(); // => ArrayBuffer + await output.bytes(); // => Uint8Array await output.text(); // string } ``` diff --git a/docs/guides/read-file/arraybuffer.md b/docs/guides/read-file/arraybuffer.md index 149b08d8eb..d64165d5f3 100644 --- a/docs/guides/read-file/arraybuffer.md +++ b/docs/guides/read-file/arraybuffer.md @@ -13,11 +13,11 @@ const buffer = await file.arrayBuffer(); --- -The binary content in the `ArrayBuffer` can then be read as a typed array, such as `Uint8Array`. +The binary content in the `ArrayBuffer` can then be read as a typed array, such as `Int8Array`. For `Uint8Array`, use [`.bytes()`](./uint8array). ```ts const buffer = await file.arrayBuffer(); -const bytes = new Uint8Array(buffer); +const bytes = new Int8Array(buffer); bytes[0]; bytes.length; diff --git a/docs/guides/read-file/uint8array.md b/docs/guides/read-file/uint8array.md index 1afcaa7971..f1c853c65d 100644 --- a/docs/guides/read-file/uint8array.md +++ b/docs/guides/read-file/uint8array.md @@ -4,14 +4,13 @@ name: Read a file to a Uint8Array The `Bun.file()` function accepts a path and returns a `BunFile` instance. The `BunFile` class extends `Blob` and allows you to lazily read the file in a variety of formats. -To read the file into a `Uint8Array` instance, retrieve the contents of the `BunFile` as an `ArrayBuffer` with `.arrayBuffer()`, then pass it into the `Uint8Array` constructor. +To read the file into a `Uint8Array` instance, retrieve the contents of the `BunFile` with `.bytes()`. ```ts const path = "/path/to/package.json"; const file = Bun.file(path); -const arrBuffer = await file.arrayBuffer(); -const byteArray = new Uint8Array(arrBuffer); +const byteArray = await file.bytes(); byteArray[0]; // first byteArray byteArray.length; // length of byteArray diff --git a/docs/guides/streams/node-readable-to-uint8array.md b/docs/guides/streams/node-readable-to-uint8array.md new file mode 100644 index 0000000000..e354e21700 --- /dev/null +++ b/docs/guides/streams/node-readable-to-uint8array.md @@ -0,0 +1,11 @@ +--- +name: Convert a Node.js Readable to an Uint8Array +--- + +To convert a Node.js `Readable` stream to an `Uint8Array` in Bun, you can create a new `Response` object with the stream as the body, then use `bytes()` to read the stream into an `Uint8Array`. + +```ts +import { Readable } from "stream"; +const stream = Readable.from(["Hello, ", "world!"]); +const buf = await new Response(stream).bytes(); +``` diff --git a/docs/guides/streams/to-typedarray.md b/docs/guides/streams/to-typedarray.md index faa18e4ad2..b6e2b9955f 100644 --- a/docs/guides/streams/to-typedarray.md +++ b/docs/guides/streams/to-typedarray.md @@ -10,6 +10,13 @@ const buf = await Bun.readableStreamToArrayBuffer(stream); const uint8 = new Uint8Array(buf); ``` +Additionally, there is a convenience method to convert to `Uint8Array` directly. + +```ts +const stream = new ReadableStream(); +const uint8 = await Bun.readableStreamToBytes(stream); +``` + --- See [Docs > API > Utils](/docs/api/utils#bun-readablestreamto) for documentation on Bun's other `ReadableStream` conversion functions. diff --git a/packages/bun-polyfills/src/modules/bun.ts b/packages/bun-polyfills/src/modules/bun.ts index aa21bb0e2f..2d5efc7372 100644 --- a/packages/bun-polyfills/src/modules/bun.ts +++ b/packages/bun-polyfills/src/modules/bun.ts @@ -410,6 +410,21 @@ export const readableStreamToArrayBuffer = ((stream: ReadableStream): Uint8Array | Promise => { + return (async () => { + const sink = new ArrayBufferSink(); + sink.start({ asUint8Array: true }); + const reader = stream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + sink.write(value); + } + return sink.end() as Uint8Array; + })(); +}) satisfies typeof Bun.readableStreamToBytes; + export const readableStreamToText = (async (stream: ReadableStream) => { let result = ''; const reader = stream.pipeThrough(new TextDecoderStream()).getReader(); ReadableStreamDefaultReader @@ -445,16 +460,19 @@ export const readableStreamToJSON = (async (stream: ReadableStream< } }) satisfies typeof Bun.readableStreamToJSON; -export const concatArrayBuffers = ((buffers) => { +export const concatArrayBuffers = ((buffers, maxLength = Infinity, asUint8Array = false) => { let size = 0; for (const chunk of buffers) size += chunk.byteLength; + size = Math.min(size, maxLength); const buffer = new ArrayBuffer(size); const view = new Uint8Array(buffer); let offset = 0; for (const chunk of buffers) { + if (offset > size) break; view.set(new Uint8Array(chunk instanceof ArrayBuffer || chunk instanceof SharedArrayBuffer ? chunk : chunk.buffer), offset); offset += chunk.byteLength; } + if (asUint8Array) return view; return buffer; }) satisfies typeof Bun.concatArrayBuffers; diff --git a/packages/bun-types/bun.d.ts b/packages/bun-types/bun.d.ts index d23c2b4b8c..ab78f47ddf 100644 --- a/packages/bun-types/bun.d.ts +++ b/packages/bun-types/bun.d.ts @@ -410,6 +410,19 @@ declare module "bun" { */ arrayBuffer(): ArrayBuffer; + /** + * Read from stdout as an Uint8Array + * + * @returns Stdout as an Uint8Array + * @example + * + * ```ts + * const output = await $`echo hello`; + * console.log(output.bytes()); // Uint8Array { byteLength: 6 } + * ``` + */ + bytes(): Uint8Array; + /** * Read from stdout as a Blob * @@ -688,7 +701,17 @@ declare module "bun" { * This function is faster because it uses uninitialized memory when copying. Since the entire * length of the buffer is known, it is safe to use uninitialized memory. */ - function concatArrayBuffers(buffers: Array): ArrayBuffer; + function concatArrayBuffers(buffers: Array, maxLength?: number): ArrayBuffer; + function concatArrayBuffers( + buffers: Array, + maxLength: number, + asUint8Array: false, + ): ArrayBuffer; + function concatArrayBuffers( + buffers: Array, + maxLength: number, + asUint8Array: true, + ): Uint8Array; /** * Consume all data from a {@link ReadableStream} until it closes or errors. @@ -705,6 +728,21 @@ declare module "bun" { stream: ReadableStream, ): Promise | ArrayBuffer; + /** + * Consume all data from a {@link ReadableStream} until it closes or errors. + * + * Concatenate the chunks into a single {@link ArrayBuffer}. + * + * Each chunk must be a TypedArray or an ArrayBuffer. If you need to support + * chunks of different types, consider {@link readableStreamToBlob} + * + * @param stream The stream to consume. + * @returns A promise that resolves with the concatenated chunks or the concatenated chunks as a {@link Uint8Array}. + */ + function readableStreamToBytes( + stream: ReadableStream, + ): Promise | Uint8Array; + /** * Consume all data from a {@link ReadableStream} until it closes or errors. * diff --git a/packages/bun-types/test/globals.test.ts b/packages/bun-types/test/globals.test.ts index 0cecbf0fad..c324ad18f9 100644 --- a/packages/bun-types/test/globals.test.ts +++ b/packages/bun-types/test/globals.test.ts @@ -6,6 +6,7 @@ import { expectAssignable, expectType } from "./utilities.test"; // FileBlob expectType>(Bun.file("index.test-d.ts").stream()); expectType>(Bun.file("index.test-d.ts").arrayBuffer()); +expectType>(Bun.file("index.test-d.ts").bytes()); expectType>(Bun.file("index.test-d.ts").text()); expectType(Bun.file("index.test-d.ts").size); diff --git a/src/bun.js/api/BunObject.zig b/src/bun.js/api/BunObject.zig index 0c08b037c1..14e846af0f 100644 --- a/src/bun.js/api/BunObject.zig +++ b/src/bun.js/api/BunObject.zig @@ -2418,7 +2418,7 @@ pub const Crypto = struct { return output_buf.value; } else { // Clone to GC-managed memory - return JSC.ArrayBuffer.create(globalThis, output_digest_slice[0..len], .Buffer); + return JSC.ArrayBuffer.createBuffer(globalThis, output_digest_slice[0..len]); } } @@ -2590,7 +2590,7 @@ pub const Crypto = struct { return output_buf.value; } else { // Clone to GC-managed memory - return JSC.ArrayBuffer.create(globalThis, result, .Buffer); + return JSC.ArrayBuffer.createBuffer(globalThis, result); } } @@ -2695,7 +2695,7 @@ pub const Crypto = struct { var out: [Algorithm.digest_length]u8 = undefined; h.final(&out); // Clone to GC-managed memory - return JSC.ArrayBuffer.create(globalThis, &out, .Buffer); + return JSC.ArrayBuffer.createBuffer(globalThis, &out); } } diff --git a/src/bun.js/api/brotli.zig b/src/bun.js/api/brotli.zig index dacd5dd6d5..05f4198d4d 100644 --- a/src/bun.js/api/brotli.zig +++ b/src/bun.js/api/brotli.zig @@ -92,7 +92,7 @@ pub const BrotliEncoder = struct { defer this.output_lock.unlock(); defer this.output.clearRetainingCapacity(); - return JSC.ArrayBuffer.create(this.globalThis, this.output.items, .Buffer); + return JSC.ArrayBuffer.createBuffer(this.globalThis, this.output.items); } pub fn runFromJSThread(this: *BrotliEncoder) void { @@ -383,7 +383,7 @@ pub const BrotliDecoder = struct { defer this.output_lock.unlock(); defer this.output.clearRetainingCapacity(); - return JSC.ArrayBuffer.create(this.globalThis, this.output.items, .Buffer); + return JSC.ArrayBuffer.createBuffer(this.globalThis, this.output.items); } pub fn runFromJSThread(this: *BrotliDecoder) void { diff --git a/src/bun.js/api/server.zig b/src/bun.js/api/server.zig index 7d719d814b..e635abd1a9 100644 --- a/src/bun.js/api/server.zig +++ b/src/bun.js/api/server.zig @@ -4064,10 +4064,9 @@ pub const ServerWebSocket = struct { fn binaryToJS(this: *const ServerWebSocket, globalThis: *JSC.JSGlobalObject, data: []const u8) JSC.JSValue { return switch (this.flags.binary_type) { - .Buffer => JSC.ArrayBuffer.create( + .Buffer => JSC.ArrayBuffer.createBuffer( globalThis, data, - .Buffer, ), .Uint8Array => JSC.ArrayBuffer.create( globalThis, diff --git a/src/bun.js/base.zig b/src/bun.js/base.zig index 25856d4684..3ce1a7304d 100644 --- a/src/bun.js/base.zig +++ b/src/bun.js/base.zig @@ -312,7 +312,7 @@ pub const ArrayBuffer = extern struct { return buffer_value; } - extern fn ArrayBuffer__fromSharedMemfd(fd: i64, globalObject: *JSC.JSGlobalObject, byte_offset: usize, byte_length: usize, total_size: usize) JSC.JSValue; + extern fn ArrayBuffer__fromSharedMemfd(fd: i64, globalObject: *JSC.JSGlobalObject, byte_offset: usize, byte_length: usize, total_size: usize, JSC.JSValue.JSType) JSC.JSValue; pub const toArrayBufferFromSharedMemfd = ArrayBuffer__fromSharedMemfd; pub fn toJSBufferFromMemfd(fd: bun.FileDescriptor, globalObject: *JSC.JSGlobalObject) JSC.JSValue { @@ -384,11 +384,10 @@ pub const ArrayBuffer = extern struct { return Stream{ .pos = 0, .buf = this.slice() }; } - pub fn create(globalThis: *JSC.JSGlobalObject, bytes: []const u8, comptime kind: BinaryType) JSValue { + pub fn create(globalThis: *JSC.JSGlobalObject, bytes: []const u8, comptime kind: JSValue.JSType) JSValue { JSC.markBinding(@src()); return switch (comptime kind) { .Uint8Array => Bun__createUint8ArrayForCopy(globalThis, bytes.ptr, bytes.len, false), - .Buffer => Bun__createUint8ArrayForCopy(globalThis, bytes.ptr, bytes.len, true), .ArrayBuffer => Bun__createArrayBufferForCopy(globalThis, bytes.ptr, bytes.len), else => @compileError("Not implemented yet"), }; diff --git a/src/bun.js/bindings/BunObject.cpp b/src/bun.js/bindings/BunObject.cpp index 87f3c0d784..09aab6d1dc 100644 --- a/src/bun.js/bindings/BunObject.cpp +++ b/src/bun.js/bindings/BunObject.cpp @@ -50,7 +50,7 @@ static JSValue constructEnvObject(VM& vm, JSObject* object) return jsCast(object->globalObject())->processEnvObject(); } -static inline JSC::EncodedJSValue flattenArrayOfBuffersIntoArrayBuffer(JSGlobalObject* lexicalGlobalObject, JSValue arrayValue) +static inline JSC::EncodedJSValue flattenArrayOfBuffersIntoArrayBufferOrUint8Array(JSGlobalObject* lexicalGlobalObject, JSValue arrayValue, size_t maxLength, bool asUint8Array) { auto& vm = lexicalGlobalObject->vm(); @@ -120,6 +120,7 @@ static inline JSC::EncodedJSValue flattenArrayOfBuffersIntoArrayBuffer(JSGlobalO return JSValue::encode(jsUndefined()); } } + byteLength = std::min(byteLength, maxLength); if (byteLength == 0) { RELEASE_AND_RETURN(throwScope, JSValue::encode(JSC::JSArrayBuffer::create(vm, lexicalGlobalObject->arrayBufferStructure(), JSC::ArrayBuffer::create(static_cast(0), 1)))); @@ -173,22 +174,46 @@ static inline JSC::EncodedJSValue flattenArrayOfBuffersIntoArrayBuffer(JSGlobalO } } + if (asUint8Array) { + auto uint8array = JSC::JSUint8Array::create(lexicalGlobalObject, lexicalGlobalObject->m_typedArrayUint8.get(lexicalGlobalObject), WTFMove(buffer), 0, byteLength); + return JSValue::encode(uint8array); + } + RELEASE_AND_RETURN(throwScope, JSValue::encode(JSC::JSArrayBuffer::create(vm, lexicalGlobalObject->arrayBufferStructure(), WTFMove(buffer)))); } JSC_DEFINE_HOST_FUNCTION(functionConcatTypedArrays, (JSGlobalObject * globalObject, JSC::CallFrame* callFrame)) { auto& vm = globalObject->vm(); + auto throwScope = DECLARE_THROW_SCOPE(vm); if (UNLIKELY(callFrame->argumentCount() < 1)) { - auto throwScope = DECLARE_THROW_SCOPE(vm); throwTypeError(globalObject, throwScope, "Expected at least one argument"_s); return JSValue::encode(jsUndefined()); } auto arrayValue = callFrame->uncheckedArgument(0); - return flattenArrayOfBuffersIntoArrayBuffer(globalObject, arrayValue); + size_t maxLength = std::numeric_limits::max(); + auto arg1 = callFrame->argument(1); + if (!arg1.isUndefined() && arg1.isNumber()) { + double number = arg1.toNumber(globalObject); + if (std::isnan(number) || number < 0) { + throwRangeError(globalObject, throwScope, "Maximum length must be >= 0"_s); + return {}; + } + if (!std::isinf(number)) { + maxLength = arg1.toUInt32(globalObject); + } + } + + bool asUint8Array = false; + auto arg2 = callFrame->argument(2); + if (!arg2.isUndefined()) { + asUint8Array = arg2.toBoolean(globalObject); + } + + return flattenArrayOfBuffersIntoArrayBufferOrUint8Array(globalObject, arrayValue, maxLength, asUint8Array); } JSC_DECLARE_HOST_FUNCTION(functionConcatTypedArrays); @@ -525,7 +550,7 @@ JSC_DEFINE_HOST_FUNCTION(functionFileURLToPath, (JSC::JSGlobalObject * globalObj allocUnsafe BunObject_callback_allocUnsafe DontDelete|Function 1 argv BunObject_getter_wrap_argv DontDelete|PropertyCallback build BunObject_callback_build DontDelete|Function 1 - concatArrayBuffers functionConcatTypedArrays DontDelete|Function 1 + concatArrayBuffers functionConcatTypedArrays DontDelete|Function 3 connect BunObject_callback_connect DontDelete|Function 1 cwd BunObject_getter_wrap_cwd DontEnum|DontDelete|PropertyCallback deepEquals functionBunDeepEquals DontDelete|Function 2 @@ -561,6 +586,7 @@ JSC_DEFINE_HOST_FUNCTION(functionFileURLToPath, (JSC::JSGlobalObject * globalObj plugin constructPluginObject ReadOnly|DontDelete|PropertyCallback readableStreamToArray JSBuiltin Builtin|Function 1 readableStreamToArrayBuffer JSBuiltin Builtin|Function 1 + readableStreamToBytes JSBuiltin Builtin|Function 1 readableStreamToBlob JSBuiltin Builtin|Function 1 readableStreamToFormData JSBuiltin Builtin|Function 1 readableStreamToJSON JSBuiltin Builtin|Function 1 @@ -630,6 +656,7 @@ public: #define bunObjectReadableStreamToArrayCodeGenerator WebCore::readableStreamReadableStreamToArrayCodeGenerator #define bunObjectReadableStreamToArrayBufferCodeGenerator WebCore::readableStreamReadableStreamToArrayBufferCodeGenerator +#define bunObjectReadableStreamToBytesCodeGenerator WebCore::readableStreamReadableStreamToBytesCodeGenerator #define bunObjectReadableStreamToBlobCodeGenerator WebCore::readableStreamReadableStreamToBlobCodeGenerator #define bunObjectReadableStreamToFormDataCodeGenerator WebCore::readableStreamReadableStreamToFormDataCodeGenerator #define bunObjectReadableStreamToJSONCodeGenerator WebCore::readableStreamReadableStreamToJSONCodeGenerator @@ -639,6 +666,7 @@ public: #undef bunObjectReadableStreamToArrayCodeGenerator #undef bunObjectReadableStreamToArrayBufferCodeGenerator +#undef bunObjectReadableStreamToBytesCodeGenerator #undef bunObjectReadableStreamToBlobCodeGenerator #undef bunObjectReadableStreamToFormDataCodeGenerator #undef bunObjectReadableStreamToJSONCodeGenerator diff --git a/src/bun.js/bindings/ZigGlobalObject.cpp b/src/bun.js/bindings/ZigGlobalObject.cpp index d4d2c30268..c89cbc7216 100644 --- a/src/bun.js/bindings/ZigGlobalObject.cpp +++ b/src/bun.js/bindings/ZigGlobalObject.cpp @@ -1530,7 +1530,7 @@ JSC_DEFINE_HOST_FUNCTION(functionReportError, return JSC::JSValue::encode(JSC::jsUndefined()); } -extern "C" JSC__JSValue ArrayBuffer__fromSharedMemfd(int64_t fd, JSC::JSGlobalObject* globalObject, size_t byteOffset, size_t byteLength, size_t totalLength) +extern "C" JSC__JSValue ArrayBuffer__fromSharedMemfd(int64_t fd, JSC::JSGlobalObject* globalObject, size_t byteOffset, size_t byteLength, size_t totalLength, JSC::JSType type) { // Windows doesn't have mmap @@ -1546,13 +1546,23 @@ extern "C" JSC__JSValue ArrayBuffer__fromSharedMemfd(int64_t fd, JSC::JSGlobalOb munmap(ptr, totalLength); })); - Structure* structure = globalObject->arrayBufferStructure(JSC::ArrayBufferSharingMode::Default); - - if (UNLIKELY(!structure)) { - return JSC::JSValue::encode(JSC::JSValue {}); + if (type == JSC::Uint8ArrayType) { + auto uint8array = JSC::JSUint8Array::create(globalObject, globalObject->m_typedArrayUint8.get(globalObject), WTFMove(buffer), 0, byteLength); + return JSValue::encode(uint8array); } - return JSValue::encode(JSC::JSArrayBuffer::create(globalObject->vm(), structure, WTFMove(buffer))); + if (type == JSC::ArrayBufferType) { + + Structure* structure = globalObject->arrayBufferStructure(JSC::ArrayBufferSharingMode::Default); + + if (UNLIKELY(!structure)) { + return JSC::JSValue::encode(JSC::JSValue {}); + } + + return JSValue::encode(JSC::JSArrayBuffer::create(globalObject->vm(), structure, WTFMove(buffer))); + } else { + RELEASE_ASSERT_NOT_REACHED(); + } #else return JSC::JSValue::encode(JSC::JSValue {}); #endif @@ -2053,6 +2063,46 @@ extern "C" JSC__JSValue ZigGlobalObject__readableStreamToArrayBuffer(Zig::Global return ZigGlobalObject__readableStreamToArrayBufferBody(reinterpret_cast(globalObject), readableStreamValue); } +extern "C" JSC__JSValue ZigGlobalObject__readableStreamToBytes(Zig::GlobalObject* globalObject, JSC__JSValue readableStreamValue); +extern "C" JSC__JSValue ZigGlobalObject__readableStreamToBytes(Zig::GlobalObject* globalObject, JSC__JSValue readableStreamValue) +{ + auto& vm = globalObject->vm(); + + auto throwScope = DECLARE_THROW_SCOPE(vm); + + auto* function = globalObject->m_readableStreamToBytes.get(); + if (!function) { + function = JSFunction::create(vm, static_cast(readableStreamReadableStreamToBytesCodeGenerator(vm)), globalObject); + globalObject->m_readableStreamToBytes.set(vm, globalObject, function); + } + + JSC::MarkedArgumentBuffer arguments = JSC::MarkedArgumentBuffer(); + arguments.append(JSValue::decode(readableStreamValue)); + + auto callData = JSC::getCallData(function); + JSValue result = call(globalObject, function, callData, JSC::jsUndefined(), arguments); + + JSC::JSObject* object = result.getObject(); + + if (UNLIKELY(!result || result.isUndefinedOrNull())) + return JSValue::encode(result); + + if (UNLIKELY(!object)) { + auto throwScope = DECLARE_THROW_SCOPE(vm); + throwTypeError(globalObject, throwScope, "Expected object"_s); + return JSValue::encode(jsUndefined()); + } + + JSC::JSPromise* promise = JSC::jsDynamicCast(object); + if (UNLIKELY(!promise)) { + auto throwScope = DECLARE_THROW_SCOPE(vm); + throwTypeError(globalObject, throwScope, "Expected promise"_s); + return JSValue::encode(jsUndefined()); + } + + RELEASE_AND_RETURN(throwScope, JSC::JSValue::encode(promise)); +} + extern "C" JSC__JSValue ZigGlobalObject__readableStreamToText(Zig::GlobalObject* globalObject, JSC__JSValue readableStreamValue); extern "C" JSC__JSValue ZigGlobalObject__readableStreamToText(Zig::GlobalObject* globalObject, JSC__JSValue readableStreamValue) { @@ -2152,6 +2202,21 @@ JSC_DEFINE_HOST_FUNCTION(functionReadableStreamToArrayBuffer, (JSGlobalObject * return ZigGlobalObject__readableStreamToArrayBufferBody(reinterpret_cast(globalObject), JSValue::encode(readableStreamValue)); } +JSC_DECLARE_HOST_FUNCTION(functionReadableStreamToBytes); +JSC_DEFINE_HOST_FUNCTION(functionReadableStreamToBytes, (JSGlobalObject * globalObject, JSC::CallFrame* callFrame)) +{ + auto& vm = globalObject->vm(); + + if (UNLIKELY(callFrame->argumentCount() < 1)) { + auto throwScope = DECLARE_THROW_SCOPE(vm); + throwTypeError(globalObject, throwScope, "Expected at least one argument"_s); + return JSValue::encode(jsUndefined()); + } + + auto readableStreamValue = callFrame->uncheckedArgument(0); + return ZigGlobalObject__readableStreamToBytes(reinterpret_cast(globalObject), JSValue::encode(readableStreamValue)); +} + JSC_DEFINE_HOST_FUNCTION(jsFunctionPerformMicrotask, (JSGlobalObject * globalObject, CallFrame* callframe)) { auto& vm = globalObject->vm(); @@ -3346,6 +3411,7 @@ void GlobalObject::visitChildrenImpl(JSCell* cell, Visitor& visitor) visitor.append(thisObject->m_assignToStream); visitor.append(thisObject->m_readableStreamToArrayBuffer); visitor.append(thisObject->m_readableStreamToArrayBufferResolve); + visitor.append(thisObject->m_readableStreamToBytes); visitor.append(thisObject->m_readableStreamToBlob); visitor.append(thisObject->m_readableStreamToJSON); visitor.append(thisObject->m_readableStreamToText); diff --git a/src/bun.js/bindings/ZigGlobalObject.h b/src/bun.js/bindings/ZigGlobalObject.h index 353749846a..035c5d4f2e 100644 --- a/src/bun.js/bindings/ZigGlobalObject.h +++ b/src/bun.js/bindings/ZigGlobalObject.h @@ -391,6 +391,7 @@ public: mutable WriteBarrier m_assignToStream; mutable WriteBarrier m_readableStreamToArrayBuffer; mutable WriteBarrier m_readableStreamToArrayBufferResolve; + mutable WriteBarrier m_readableStreamToBytes; mutable WriteBarrier m_readableStreamToBlob; mutable WriteBarrier m_readableStreamToJSON; mutable WriteBarrier m_readableStreamToText; diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index 2637700dc6..52657e5178 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -3068,6 +3068,7 @@ pub const JSGlobalObject = extern struct { } extern fn ZigGlobalObject__readableStreamToArrayBuffer(*JSGlobalObject, JSValue) JSValue; + extern fn ZigGlobalObject__readableStreamToBytes(*JSGlobalObject, JSValue) JSValue; extern fn ZigGlobalObject__readableStreamToText(*JSGlobalObject, JSValue) JSValue; extern fn ZigGlobalObject__readableStreamToJSON(*JSGlobalObject, JSValue) JSValue; extern fn ZigGlobalObject__readableStreamToFormData(*JSGlobalObject, JSValue, JSValue) JSValue; @@ -3078,6 +3079,11 @@ pub const JSGlobalObject = extern struct { return ZigGlobalObject__readableStreamToArrayBuffer(this, value); } + pub fn readableStreamToBytes(this: *JSGlobalObject, value: JSValue) JSValue { + if (comptime is_bindgen) unreachable; + return ZigGlobalObject__readableStreamToBytes(this, value); + } + pub fn readableStreamToText(this: *JSGlobalObject, value: JSValue) JSValue { if (comptime is_bindgen) unreachable; return ZigGlobalObject__readableStreamToText(this, value); diff --git a/src/bun.js/webcore/blob.zig b/src/bun.js/webcore/blob.zig index bfd8e51970..9789aadfda 100644 --- a/src/bun.js/webcore/blob.zig +++ b/src/bun.js/webcore/blob.zig @@ -2905,6 +2905,17 @@ pub const Blob = struct { return promisified(this.toArrayBuffer(globalThis, .clone), globalThis); } + pub fn getBytes( + this: *Blob, + globalThis: *JSC.JSGlobalObject, + _: *JSC.CallFrame, + ) callconv(.C) JSValue { + const store = this.store; + if (store) |st| st.ref(); + defer if (store) |st| st.deref(); + return promisified(this.toUint8Array(globalThis, .clone), globalThis); + } + pub fn getFormData( this: *Blob, globalThis: *JSC.JSGlobalObject, @@ -3808,6 +3819,14 @@ pub const Blob = struct { } pub fn toArrayBufferWithBytes(this: *Blob, global: *JSGlobalObject, buf: []u8, comptime lifetime: Lifetime) JSValue { + return toArrayBufferViewWithBytes(this, global, buf, lifetime, .ArrayBuffer); + } + + pub fn toUint8ArrayWithBytes(this: *Blob, global: *JSGlobalObject, buf: []u8, comptime lifetime: Lifetime) JSValue { + return toArrayBufferViewWithBytes(this, global, buf, lifetime, .Uint8Array); + } + + pub fn toArrayBufferViewWithBytes(this: *Blob, global: *JSGlobalObject, buf: []u8, comptime lifetime: Lifetime, comptime TypedArrayView: JSC.JSValue.JSType) JSValue { switch (comptime lifetime) { .clone => { if (comptime Environment.isLinux) { @@ -3829,6 +3848,7 @@ pub const Blob = struct { byteOffset, byteLength, allocated_slice.len, + TypedArrayView, ); bloblog("toArrayBuffer COW clone({d}, {d}) = {d}", .{ byteOffset, byteLength, @intFromBool(result != .zero) }); @@ -3840,11 +3860,11 @@ pub const Blob = struct { } } } - return JSC.ArrayBuffer.create(global, buf, .ArrayBuffer); + return JSC.ArrayBuffer.create(global, buf, TypedArrayView); }, .share => { this.store.?.ref(); - return JSC.ArrayBuffer.fromBytes(buf, .ArrayBuffer).toJSWithContext( + return JSC.ArrayBuffer.fromBytes(buf, TypedArrayView).toJSWithContext( global, this.store.?, JSC.BlobArrayBuffer_deallocator, @@ -3854,7 +3874,7 @@ pub const Blob = struct { .transfer => { const store = this.store.?; this.transfer(); - return JSC.ArrayBuffer.fromBytes(buf, .ArrayBuffer).toJSWithContext( + return JSC.ArrayBuffer.fromBytes(buf, TypedArrayView).toJSWithContext( global, store, JSC.BlobArrayBuffer_deallocator, @@ -3862,7 +3882,7 @@ pub const Blob = struct { ); }, .temporary => { - return JSC.ArrayBuffer.fromBytes(buf, .ArrayBuffer).toJS( + return JSC.ArrayBuffer.fromBytes(buf, TypedArrayView).toJS( global, null, ); @@ -3872,15 +3892,28 @@ pub const Blob = struct { pub fn toArrayBuffer(this: *Blob, global: *JSGlobalObject, comptime lifetime: Lifetime) JSValue { bloblog("toArrayBuffer", .{}); + return toArrayBufferView(this, global, lifetime, .ArrayBuffer); + } + + pub fn toUint8Array(this: *Blob, global: *JSGlobalObject, comptime lifetime: Lifetime) JSValue { + bloblog("toUin8Array", .{}); + return toArrayBufferView(this, global, lifetime, .Uint8Array); + } + + pub fn toArrayBufferView(this: *Blob, global: *JSGlobalObject, comptime lifetime: Lifetime, comptime TypedArrayView: JSC.JSValue.JSType) JSValue { + const WithBytesFn = comptime if (TypedArrayView == .Uint8Array) + toUint8ArrayWithBytes + else + toArrayBufferWithBytes; if (this.needsToReadFile()) { - return this.doReadFile(toArrayBufferWithBytes, global); + return this.doReadFile(WithBytesFn, global); } const view_ = this.sharedView(); if (view_.len == 0) - return JSC.ArrayBuffer.create(global, "", .ArrayBuffer); + return JSC.ArrayBuffer.create(global, "", TypedArrayView); - return toArrayBufferWithBytes(this, global, @constCast(view_), lifetime); + return WithBytesFn(this, global, @constCast(view_), lifetime); } pub fn toFormData(this: *Blob, global: *JSGlobalObject, comptime lifetime: Lifetime) JSValue { @@ -4291,8 +4324,16 @@ pub const AnyBlob = union(enum) { } pub fn toArrayBuffer(this: *AnyBlob, global: *JSGlobalObject, comptime lifetime: JSC.WebCore.Lifetime) JSValue { + return this.toArrayBufferView(global, lifetime, .ArrayBuffer); + } + + pub fn toUint8Array(this: *AnyBlob, global: *JSGlobalObject, comptime lifetime: JSC.WebCore.Lifetime) JSValue { + return this.toArrayBufferView(global, lifetime, .Uint8Array); + } + + pub fn toArrayBufferView(this: *AnyBlob, global: *JSGlobalObject, comptime lifetime: JSC.WebCore.Lifetime, comptime TypedArrayView: JSC.JSValue.JSType) JSValue { switch (this.*) { - .Blob => return this.Blob.toArrayBuffer(global, lifetime), + .Blob => return this.Blob.toArrayBufferView(global, lifetime, TypedArrayView), // .InlineBlob => { // if (this.InlineBlob.len == 0) { // return JSC.ArrayBuffer.create(global, "", .ArrayBuffer); @@ -4308,14 +4349,14 @@ pub const AnyBlob = union(enum) { // }, .InternalBlob => { if (this.InternalBlob.bytes.items.len == 0) { - return JSC.ArrayBuffer.create(global, "", .ArrayBuffer); + return JSC.ArrayBuffer.create(global, "", TypedArrayView); } const bytes = this.InternalBlob.toOwnedSlice(); this.* = .{ .Blob = .{} }; const value = JSC.ArrayBuffer.fromBytes( bytes, - .ArrayBuffer, + TypedArrayView, ); return value.toJS(global, null); }, @@ -4328,12 +4369,12 @@ pub const AnyBlob = union(enum) { if (out_bytes.isAllocated()) { const value = JSC.ArrayBuffer.fromBytes( @constCast(out_bytes.slice()), - .ArrayBuffer, + TypedArrayView, ); return value.toJS(global, null); } - return JSC.ArrayBuffer.create(global, out_bytes.slice(), .ArrayBuffer); + return JSC.ArrayBuffer.create(global, out_bytes.slice(), TypedArrayView); }, } } diff --git a/src/bun.js/webcore/body.zig b/src/bun.js/webcore/body.zig index 71a7a5a93d..cebbcedc8c 100644 --- a/src/bun.js/webcore/body.zig +++ b/src/bun.js/webcore/body.zig @@ -193,10 +193,11 @@ pub const Body = struct { value.action = action; if (value.readable.get()) |readable| handle_stream: { switch (action) { - .getFormData, .getText, .getJSON, .getBlob, .getArrayBuffer => { + .getFormData, .getText, .getJSON, .getBlob, .getArrayBuffer, .getBytes => { value.promise = switch (action) { .getJSON => globalThis.readableStreamToJSON(readable.value), .getArrayBuffer => globalThis.readableStreamToArrayBuffer(readable.value), + .getBytes => globalThis.readableStreamToBytes(readable.value), .getText => globalThis.readableStreamToText(readable.value), .getBlob => globalThis.readableStreamToBlob(readable.value), .getFormData => |form_data| brk: { @@ -256,6 +257,7 @@ pub const Body = struct { getText: void, getJSON: void, getArrayBuffer: void, + getBytes: void, getBlob: void, getFormData: ?*bun.FormData.AsyncFormData, }; @@ -664,9 +666,12 @@ pub const Body = struct { }, .getArrayBuffer => { var blob = new.useAsAnyBlobAllowNonUTF8String(); - // toArrayBuffer checks for non-UTF8 strings promise.resolve(global, blob.toArrayBuffer(global, .transfer)); }, + .getBytes => { + var blob = new.useAsAnyBlobAllowNonUTF8String(); + promise.resolve(global, blob.toUint8Array(global, .transfer)); + }, .getFormData => inner: { var blob = new.useAsAnyBlob(); defer blob.detach(); @@ -1059,6 +1064,29 @@ pub fn BodyMixin(comptime Type: type) type { return JSC.JSPromise.wrap(globalObject, blob.toArrayBuffer(globalObject, .transfer)); } + pub fn getBytes( + this: *Type, + globalObject: *JSC.JSGlobalObject, + callframe: *JSC.CallFrame, + ) callconv(.C) JSC.JSValue { + var value: *Body.Value = this.getBodyValue(); + + if (value.* == .Used) { + return handleBodyAlreadyUsed(globalObject); + } + + if (value.* == .Locked) { + if (value.Locked.isDisturbed(Type, globalObject, callframe.this())) { + return handleBodyAlreadyUsed(globalObject); + } + return value.Locked.setPromise(globalObject, .{ .getBytes = {} }); + } + + // toArrayBuffer in AnyBlob checks for non-UTF8 strings + var blob: AnyBlob = value.useAsAnyBlobAllowNonUTF8String(); + return JSC.JSPromise.wrap(globalObject, blob.toUint8Array(globalObject, .transfer)); + } + pub fn getFormData( this: *Type, globalObject: *JSC.JSGlobalObject, diff --git a/src/bun.js/webcore/request.zig b/src/bun.js/webcore/request.zig index 0b9a9e9b37..117f0a6e97 100644 --- a/src/bun.js/webcore/request.zig +++ b/src/bun.js/webcore/request.zig @@ -77,6 +77,7 @@ pub const Request = struct { pub usingnamespace JSC.Codegen.JSRequest; pub const getText = RequestMixin.getText; + pub const getBytes = RequestMixin.getBytes; pub const getBody = RequestMixin.getBody; pub const getBodyUsed = RequestMixin.getBodyUsed; pub const getJSON = RequestMixin.getJSON; diff --git a/src/bun.js/webcore/response.classes.ts b/src/bun.js/webcore/response.classes.ts index ed4c5c14e6..96f92121a4 100644 --- a/src/bun.js/webcore/response.classes.ts +++ b/src/bun.js/webcore/response.classes.ts @@ -13,6 +13,7 @@ export default [ proto: { text: { fn: "getText" }, json: { fn: "getJSON" }, + bytes: { fn: "getBytes" }, body: { getter: "getBody", cache: true }, arrayBuffer: { fn: "getArrayBuffer" }, formData: { fn: "getFormData" }, @@ -90,6 +91,7 @@ export default [ text: { fn: "getText" }, json: { fn: "getJSON" }, + bytes: { fn: "getBytes" }, arrayBuffer: { fn: "getArrayBuffer" }, blob: { fn: "getBlob" }, clone: { fn: "doClone", length: 1 }, @@ -140,6 +142,9 @@ export default [ formData: { fn: "getFormData" }, exists: { fn: "getExists", length: 0 }, + // Non-standard, but consistent! + bytes: { fn: "getBytes" }, + type: { getter: "getType", }, diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig index 2be5bcf7b1..23d1e08a84 100644 --- a/src/bun.js/webcore/response.zig +++ b/src/bun.js/webcore/response.zig @@ -70,6 +70,7 @@ pub const Response = struct { pub const getText = ResponseMixin.getText; pub const getBody = ResponseMixin.getBody; + pub const getBytes = ResponseMixin.getBytes; pub const getBodyUsed = ResponseMixin.getBodyUsed; pub const getJSON = ResponseMixin.getJSON; pub const getArrayBuffer = ResponseMixin.getArrayBuffer; diff --git a/src/js/builtins/ReadableStream.ts b/src/js/builtins/ReadableStream.ts index 7b07761715..16726eeff0 100644 --- a/src/js/builtins/ReadableStream.ts +++ b/src/js/builtins/ReadableStream.ts @@ -112,7 +112,6 @@ export function readableStreamToArray(stream: ReadableStream): Promise { if (underlyingSource !== undefined) { return $readableStreamToTextDirect(stream, underlyingSource); } - return $readableStreamIntoText(stream); } @@ -133,19 +131,38 @@ export function readableStreamToArrayBuffer(stream: ReadableStream) var underlyingSource = $getByIdDirectPrivate(stream, "underlyingSource"); if (underlyingSource !== undefined) { - return $readableStreamToArrayBufferDirect(stream, underlyingSource); + return $readableStreamToArrayBufferDirect(stream, underlyingSource, false); } var result = Bun.readableStreamToArray(stream); if ($isPromise(result)) { // `result` is an InternalPromise, which doesn't have a `.then` method // but `.then` isn't user-overridable, so we can use it safely. - return result.then(Bun.concatArrayBuffers); + return result.then(x => Bun.concatArrayBuffers(x)); } return Bun.concatArrayBuffers(result); } +$linkTimeConstant; +export function readableStreamToBytes(stream: ReadableStream): Promise | Uint8Array { + // this is a direct stream + var underlyingSource = $getByIdDirectPrivate(stream, "underlyingSource"); + + if (underlyingSource !== undefined) { + return $readableStreamToArrayBufferDirect(stream, underlyingSource, true); + } + + var result = Bun.readableStreamToArray(stream); + if ($isPromise(result)) { + // `result` is an InternalPromise, which doesn't have a `.then` method + // but `.then` isn't user-overridable, so we can use it safely. + return result.then(x => Bun.concatArrayBuffers(x, Infinity, true)); + } + + return Bun.concatArrayBuffers(result, Infinity, true); +} + $linkTimeConstant; export function readableStreamToFormData( stream: ReadableStream, diff --git a/src/js/builtins/ReadableStreamInternals.ts b/src/js/builtins/ReadableStreamInternals.ts index 44651b98aa..2fdc118f34 100644 --- a/src/js/builtins/ReadableStreamInternals.ts +++ b/src/js/builtins/ReadableStreamInternals.ts @@ -1862,11 +1862,11 @@ export function readableStreamIntoText(stream) { return closer.promise.$then($withoutUTF8BOM); } -export function readableStreamToArrayBufferDirect(stream, underlyingSource) { +export function readableStreamToArrayBufferDirect(stream, underlyingSource, asUint8Array) { var sink = new Bun.ArrayBufferSink(); $putByIdDirectPrivate(stream, "underlyingSource", undefined); var highWaterMark = $getByIdDirectPrivate(stream, "highWaterMark"); - sink.start(highWaterMark ? { highWaterMark } : {}); + sink.start({ highWaterMark, asUint8Array }); var capability = $newPromiseCapability(Promise); var ended = false; var pull = underlyingSource.pull; diff --git a/src/js/builtins/shell.ts b/src/js/builtins/shell.ts index 2e8429402d..81b37c5b57 100644 --- a/src/js/builtins/shell.ts +++ b/src/js/builtins/shell.ts @@ -50,6 +50,10 @@ export function createBunShellTemplateFunction(ShellInterpreter) { return this.#output!.arrayBuffer(); } + bytes() { + return this.#output!.bytes(); + } + blob() { return this.#output!.blob(); } @@ -77,6 +81,10 @@ export function createBunShellTemplateFunction(ShellInterpreter) { return this.stdout.buffer; } + bytes() { + return new Uint8Array(this.arrayBuffer()); + } + blob() { return new Blob([this.stdout]); } @@ -219,6 +227,10 @@ export function createBunShellTemplateFunction(ShellInterpreter) { return stdout.buffer; } + async bytes() { + return this.arrayBuffer().then(x => new Uint8Array(x)); + } + async blob() { const { stdout } = (await this.#quiet()) as ShellOutput; return new Blob([stdout]); diff --git a/src/js/node/stream.consumers.ts b/src/js/node/stream.consumers.ts index 4df51d1bf0..d56c456bb4 100644 --- a/src/js/node/stream.consumers.ts +++ b/src/js/node/stream.consumers.ts @@ -1,5 +1,6 @@ // Hardcoded module "node:stream/consumers" / "readable-stream/consumer" const arrayBuffer = Bun.readableStreamToArrayBuffer; +const bytes = Bun.readableStreamToBytes; const text = Bun.readableStreamToText; const json = stream => Bun.readableStreamToText(stream).then(JSON.parse); @@ -11,6 +12,7 @@ const blob = Bun.readableStreamToBlob; export default { arrayBuffer, + bytes, text, json, buffer, diff --git a/test/exports/bun-exports.bun-v0.6.11.json b/test/exports/bun-exports.bun-v0.6.11.json index 37062461b7..7a876ecdde 100644 --- a/test/exports/bun-exports.bun-v0.6.11.json +++ b/test/exports/bun-exports.bun-v0.6.11.json @@ -7645,6 +7645,7 @@ "plugin": "function", "readableStreamToArray": "function", "readableStreamToArrayBuffer": "function", + "readableStreamToBytes": "function", "readableStreamToBlob": "function", "readableStreamToJSON": "function", "readableStreamToText": "function", diff --git a/test/js/bun/http/async-iterator-stream.test.ts b/test/js/bun/http/async-iterator-stream.test.ts index 6691dce88a..e2784d8eb6 100644 --- a/test/js/bun/http/async-iterator-stream.test.ts +++ b/test/js/bun/http/async-iterator-stream.test.ts @@ -204,7 +204,7 @@ describe("Streaming body via", () => { ["Response", () => new Response(bodyInit)], ["Request", () => new Request({ "url": "https://example.com", body: bodyInit })], ]) { - for (let method of ["arrayBuffer", "text"]) { + for (let method of ["arrayBuffer", "bytes", "text"]) { test(`${label}(${method})`, async () => { const result = await constructFn()[method](); expect(Buffer.from(result)).toEqual(Buffer.from(expected)); diff --git a/test/js/bun/shell/bunshell-instance.test.ts b/test/js/bun/shell/bunshell-instance.test.ts index 496f336132..12f7ee2f76 100644 --- a/test/js/bun/shell/bunshell-instance.test.ts +++ b/test/js/bun/shell/bunshell-instance.test.ts @@ -46,6 +46,10 @@ test("$.arrayBuffer", async () => { expect(await $`echo hello`.arrayBuffer()).toEqual(new TextEncoder().encode("hello\n").buffer); }); +test("$.bytes", async () => { + expect(await $`echo hello`.bytes()).toEqual(new TextEncoder().encode("hello\n")); +}); + test("$.blob", async () => { expect(await $`echo hello`.blob()).toEqual(new Blob([new TextEncoder().encode("hello\n")])); }); diff --git a/test/js/bun/stream/direct-readable-stream.test.tsx b/test/js/bun/stream/direct-readable-stream.test.tsx index edcb7dedeb..02852b6e8a 100644 --- a/test/js/bun/stream/direct-readable-stream.test.tsx +++ b/test/js/bun/stream/direct-readable-stream.test.tsx @@ -1,6 +1,7 @@ import { concatArrayBuffers, readableStreamToArray, + readableStreamToBytes, readableStreamToArrayBuffer, readableStreamToBlob, readableStreamToText, @@ -188,6 +189,15 @@ describe("ReactDOM", () => { expect(text.replaceAll("", "")).toBe(inputString); gc(); }); + it("readableStreamToBytes(stream)", async () => { + const stream = await renderToReadableStream(reactElement); + gc(); + const uint8 = await readableStreamToBytes(stream); + const text = new TextDecoder().decode(uint8); + gc(); + expect(text.replaceAll("", "")).toBe(inputString); + gc(); + }); it("for await (chunk of stream)", async () => { const stream = await renderToReadableStream(reactElement); gc(); diff --git a/test/js/bun/util/concat.test.js b/test/js/bun/util/concat.test.js index 0cea303fe9..0a5bc23edf 100644 --- a/test/js/bun/util/concat.test.js +++ b/test/js/bun/util/concat.test.js @@ -18,7 +18,7 @@ describe("concat", () => { } function concatToString(chunks) { - return Array.from(new Uint8Array(concatArrayBuffers(chunks))).join(""); + return Array.from(concatArrayBuffers(chunks, Infinity, true)).join(""); } function polyfillToString(chunks) { @@ -40,4 +40,16 @@ describe("concat", () => { polyfillToString([Uint8Array.from([123]), Uint8Array.from([456])]), ); }); + + it("can be trimmed to a max length", () => { + const a = Uint8Array.from([1, 2, 3]); + const b = Uint8Array.from([4, 5, 6]); + expect(concatArrayBuffers([a, b], 4, true)).toEqual(Uint8Array.from([1, 2, 3, 4])); + }); + + it("can be trimmed to a max length (ArrayBuffer)", () => { + const a = Uint8Array.from([1, 2, 3]); + const b = Uint8Array.from([4, 5, 6]); + expect(concatArrayBuffers([a, b], 4)).toEqual(Uint8Array.from([1, 2, 3, 4]).buffer); + }); }); diff --git a/test/js/deno/harness.ts b/test/js/deno/harness.ts index 8bca01b3ba..d5673b4a2b 100644 --- a/test/js/deno/harness.ts +++ b/test/js/deno/harness.ts @@ -273,7 +273,7 @@ export function createDenoTest(path: string) { // https://deno.land/std@0.171.0/bytes/concat.ts const concat = (...buffers: Uint8Array[]): Uint8Array => { - return new Uint8Array(concatArrayBuffers(buffers)); + return concatArrayBuffers(buffers, Infinity, true); }; // https://deno.land/api@v1.31.1?s=Deno.readTextFile diff --git a/test/js/web/fetch/body-stream.test.ts b/test/js/web/fetch/body-stream.test.ts index 8f76755289..57ce950be8 100644 --- a/test/js/web/fetch/body-stream.test.ts +++ b/test/js/web/fetch/body-stream.test.ts @@ -7,6 +7,7 @@ var port = 0; { const BodyMixin = [ Request.prototype.arrayBuffer, + Request.prototype.bytes, Request.prototype.blob, Request.prototype.text, Request.prototype.json, @@ -284,9 +285,7 @@ describe("reader", function () { gc(); const expectedHash = - huge instanceof Blob - ? Bun.SHA1.hash(new Uint8Array(await huge.arrayBuffer()), "base64") - : Bun.SHA1.hash(huge, "base64"); + huge instanceof Blob ? Bun.SHA1.hash(await huge.bytes(), "base64") : Bun.SHA1.hash(huge, "base64"); const expectedSize = huge instanceof Blob ? huge.size : huge.byteLength; const out = await runInServer( @@ -347,7 +346,7 @@ describe("reader", function () { const response = await pendingResponse; huge = undefined; expect(response.status).toBe(200); - const response_body = new Uint8Array(await response.arrayBuffer()); + const response_body = await response.bytes(); expect(response_body.byteLength).toBe(expectedSize); expect(Bun.SHA1.hash(response_body, "base64")).toBe(expectedHash); @@ -375,9 +374,7 @@ describe("reader", function () { gc(); const expectedHash = - huge instanceof Blob - ? Bun.SHA1.hash(new Uint8Array(await huge.arrayBuffer()), "base64") - : Bun.SHA1.hash(huge, "base64"); + huge instanceof Blob ? Bun.SHA1.hash(await huge.bytes(), "base64") : Bun.SHA1.hash(huge, "base64"); const expectedSize = huge instanceof Blob ? huge.size : huge.byteLength; const out = await runInServer( @@ -463,7 +460,7 @@ describe("reader", function () { }); huge = undefined; expect(response.status).toBe(200); - const response_body = new Uint8Array(await response.arrayBuffer()); + const response_body = await response.bytes(); expect(response_body.byteLength).toBe(expectedSize); expect(Bun.SHA1.hash(response_body, "base64")).toBe(expectedHash); diff --git a/test/js/web/fetch/body.test.ts b/test/js/web/fetch/body.test.ts index 4f5be65618..a26b078450 100644 --- a/test/js/web/fetch/body.test.ts +++ b/test/js/web/fetch/body.test.ts @@ -247,6 +247,17 @@ for (const { body, fn } of bodyTypes) { expect(await fn(string).arrayBuffer()).toStrictEqual(buffer.buffer); }); }); + describe("bytes()", () => { + test("undefined", async () => { + expect(await fn().bytes()).toStrictEqual(new Uint8Array(0)); + }); + test("null", async () => { + expect(await fn(null).bytes()).toStrictEqual(new Uint8Array(0)); + }); + test(`"${string}"`, async () => { + expect(await fn(string).bytes()).toStrictEqual(new Uint8Array(buffer)); + }); + }); describe("text()", () => { test("undefined", async () => { expect(await fn().text()).toBe(""); @@ -607,7 +618,7 @@ for (const { body, fn } of bodyTypes) { }); describe("new Response()", () => { - ["text", "arrayBuffer", "blob"].map(method => { + ["text", "arrayBuffer", "bytes", "blob"].map(method => { test(method, async () => { const result = new Response(); expect(result).toHaveProperty("bodyUsed", false); @@ -620,7 +631,7 @@ for (const { body, fn } of bodyTypes) { }); describe('new Request(url, {method: "POST" })', () => { - ["text", "arrayBuffer", "blob"].map(method => { + ["text", "arrayBuffer", "bytes", "blob"].map(method => { test(method, async () => { const result = new Request("https://example.com", { method: "POST" }); expect(result).toHaveProperty("bodyUsed", false); @@ -633,7 +644,7 @@ for (const { body, fn } of bodyTypes) { }); describe("new Request(url)", () => { - ["text", "arrayBuffer", "blob"].map(method => { + ["text", "arrayBuffer", "bytes", "blob"].map(method => { test(method, async () => { const result = new Request("https://example.com"); expect(result).toHaveProperty("bodyUsed", false); diff --git a/test/js/web/fetch/fetch.test.ts b/test/js/web/fetch/fetch.test.ts index 9ce6137d17..d8e6859230 100644 --- a/test/js/web/fetch/fetch.test.ts +++ b/test/js/web/fetch/fetch.test.ts @@ -750,6 +750,29 @@ function testBlobInterface(blobbyConstructor: { (..._: any[]): any }, hasBlobFn? if (withGC) gc(); }); + it(`${jsonObject.hello === true ? "latin1" : "utf16"} bytes${withGC ? " (with gc) " : ""}`, async () => { + if (withGC) gc(); + + var response = blobbyConstructor(JSON.stringify(jsonObject)); + if (withGC) gc(); + + const bytes = new TextEncoder().encode(JSON.stringify(jsonObject)); + if (withGC) gc(); + + const compare = await response.bytes(); + if (withGC) gc(); + + withoutAggressiveGC(() => { + for (let i = 0; i < compare.length; i++) { + if (withGC) gc(); + + expect(compare[i]).toBe(bytes[i]); + if (withGC) gc(); + } + }); + if (withGC) gc(); + }); + it(`${jsonObject.hello === true ? "latin1" : "utf16"} arrayBuffer -> arrayBuffer${ withGC ? " (with gc) " : "" }`, async () => { @@ -775,6 +798,31 @@ function testBlobInterface(blobbyConstructor: { (..._: any[]): any }, hasBlobFn? if (withGC) gc(); }); + it(`${jsonObject.hello === true ? "latin1" : "utf16"} arrayBuffer -> bytes${ + withGC ? " (with gc) " : "" + }`, async () => { + if (withGC) gc(); + + var response = blobbyConstructor(new TextEncoder().encode(JSON.stringify(jsonObject))); + if (withGC) gc(); + + const bytes = new TextEncoder().encode(JSON.stringify(jsonObject)); + if (withGC) gc(); + + const compare = await response.bytes(); + if (withGC) gc(); + + withoutAggressiveGC(() => { + for (let i = 0; i < compare.length; i++) { + if (withGC) gc(); + + expect(compare[i]).toBe(bytes[i]); + if (withGC) gc(); + } + }); + if (withGC) gc(); + }); + hasBlobFn && it(`${jsonObject.hello === true ? "latin1" : "utf16"} blob${withGC ? " (with gc) " : ""}`, async () => { if (withGC) gc(); @@ -828,7 +876,7 @@ describe("Bun.file", () => { expect(size).toBe(Infinity); }); - const method = ["arrayBuffer", "text", "json"] as const; + const method = ["arrayBuffer", "text", "json", "bytes"] as const; function forEachMethod(fn: (m: (typeof method)[number]) => any, skip?: AnyFunction) { for (const m of method) { (skip ? it.skip : it)(m, fn(m)); diff --git a/test/js/web/streams/streams.test.js b/test/js/web/streams/streams.test.js index a578123bb9..740399230c 100644 --- a/test/js/web/streams/streams.test.js +++ b/test/js/web/streams/streams.test.js @@ -1,4 +1,11 @@ -import { file, readableStreamToArrayBuffer, readableStreamToArray, readableStreamToText, ArrayBufferSink } from "bun"; +import { + file, + readableStreamToArrayBuffer, + readableStreamToBytes, + readableStreamToArray, + readableStreamToText, + ArrayBufferSink, +} from "bun"; import { expect, it, beforeEach, afterEach, describe, test } from "bun:test"; import { mkfifo } from "mkfifo"; import { realpathSync, unlinkSync, writeFileSync } from "node:fs"; @@ -622,6 +629,42 @@ it("readableStreamToArrayBuffer (default)", async () => { expect(new TextDecoder().decode(new Uint8Array(buffer))).toBe("abdefgh"); }); +it("readableStreamToBytes (bytes)", async () => { + var queue = [Buffer.from("abdefgh")]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + if (chunk) { + controller.enqueue(chunk); + } else { + controller.close(); + } + }, + cancel() {}, + type: "bytes", + }); + const buffer = await readableStreamToBytes(stream); + expect(new TextDecoder().decode(new Uint8Array(buffer))).toBe("abdefgh"); +}); + +it("readableStreamToBytes (default)", async () => { + var queue = [Buffer.from("abdefgh")]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + if (chunk) { + controller.enqueue(chunk); + } else { + controller.close(); + } + }, + cancel() {}, + }); + + const buffer = await readableStreamToBytes(stream); + expect(new TextDecoder().decode(new Uint8Array(buffer))).toBe("abdefgh"); +}); + it("ReadableStream for Blob", async () => { var blob = new Blob(["abdefgh", "ijklmnop"]); expect(await blob.text()).toBe("abdefghijklmnop"); @@ -728,7 +771,7 @@ it("new Response(stream).arrayBuffer() (bytes)", async () => { type: "bytes", }); const buffer = await new Response(stream).arrayBuffer(); - expect(new TextDecoder().decode(new Uint8Array(buffer))).toBe("abdefgh"); + expect(new TextDecoder().decode(buffer)).toBe("abdefgh"); }); it("new Response(stream).arrayBuffer() (default)", async () => { @@ -745,9 +788,92 @@ it("new Response(stream).arrayBuffer() (default)", async () => { cancel() {}, }); const buffer = await new Response(stream).arrayBuffer(); + expect(new TextDecoder().decode(buffer)).toBe("abdefgh"); +}); + +it("new Response(stream).arrayBuffer() (direct)", async () => { + var queue = [Buffer.from("abdefgh")]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + controller.write(chunk); + controller.close(); + }, + cancel() {}, + type: "direct", + }); + const buffer = await new Response(stream).arrayBuffer(); expect(new TextDecoder().decode(new Uint8Array(buffer))).toBe("abdefgh"); }); +it("new Response(stream).bytes() (bytes)", async () => { + var queue = [Buffer.from("abdefgh")]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + if (chunk) { + controller.enqueue(chunk); + } else { + controller.close(); + } + }, + cancel() {}, + type: "bytes", + }); + const buffer = await new Response(stream).bytes(); + expect(new TextDecoder().decode(buffer)).toBe("abdefgh"); +}); + +it("new Response(stream).bytes() (default)", async () => { + var queue = [Buffer.from("abdefgh")]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + if (chunk) { + controller.enqueue(chunk); + } else { + controller.close(); + } + }, + cancel() {}, + }); + const buffer = await new Response(stream).bytes(); + expect(new TextDecoder().decode(buffer)).toBe("abdefgh"); +}); + +it("new Response(stream).bytes() (direct)", async () => { + var queue = [Buffer.from("abdefgh")]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + controller.write(chunk); + controller.close(); + }, + cancel() {}, + type: "direct", + }); + const buffer = await new Response(stream).bytes(); + expect(new TextDecoder().decode(buffer)).toBe("abdefgh"); +}); + +it("new Response(stream).text() (bytes)", async () => { + var queue = [Buffer.from("abdefgh")]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + if (chunk) { + controller.enqueue(chunk); + } else { + controller.close(); + } + }, + cancel() {}, + type: "bytes", + }); + const text = await new Response(stream).text(); + expect(text).toBe("abdefgh"); +}); + it("new Response(stream).text() (default)", async () => { var queue = [Buffer.from("abdefgh")]; var stream = new ReadableStream({ @@ -765,6 +891,39 @@ it("new Response(stream).text() (default)", async () => { expect(text).toBe("abdefgh"); }); +it("new Response(stream).text() (direct)", async () => { + var queue = [Buffer.from("abdefgh")]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + controller.write(chunk); + controller.close(); + }, + cancel() {}, + type: "direct", + }); + const text = await new Response(stream).text(); + expect(text).toBe("abdefgh"); +}); + +it("new Response(stream).json() (bytes)", async () => { + var queue = [Buffer.from(JSON.stringify({ hello: true }))]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + if (chunk) { + controller.enqueue(chunk); + } else { + controller.close(); + } + }, + cancel() {}, + type: "bytes", + }); + const json = await new Response(stream).json(); + expect(json.hello).toBe(true); +}); + it("new Response(stream).json() (default)", async () => { var queue = [Buffer.from(JSON.stringify({ hello: true }))]; var stream = new ReadableStream({ @@ -782,6 +941,40 @@ it("new Response(stream).json() (default)", async () => { expect(json.hello).toBe(true); }); +it("new Response(stream).json() (direct)", async () => { + var queue = [Buffer.from(JSON.stringify({ hello: true }))]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + controller.write(chunk); + controller.close(); + }, + cancel() {}, + type: "direct", + }); + const json = await new Response(stream).json(); + expect(json.hello).toBe(true); +}); + +it("new Response(stream).blob() (bytes)", async () => { + var queue = [Buffer.from(JSON.stringify({ hello: true }))]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + if (chunk) { + controller.enqueue(chunk); + } else { + controller.close(); + } + }, + cancel() {}, + type: "bytes", + }); + const response = new Response(stream); + const blob = await response.blob(); + expect(await blob.text()).toBe('{"hello":true}'); +}); + it("new Response(stream).blob() (default)", async () => { var queue = [Buffer.from(JSON.stringify({ hello: true }))]; var stream = new ReadableStream({ @@ -800,6 +993,22 @@ it("new Response(stream).blob() (default)", async () => { expect(await blob.text()).toBe('{"hello":true}'); }); +it("new Response(stream).blob() (direct)", async () => { + var queue = [Buffer.from(JSON.stringify({ hello: true }))]; + var stream = new ReadableStream({ + pull(controller) { + var chunk = queue.shift(); + controller.write(chunk); + controller.close(); + }, + cancel() {}, + type: "direct", + }); + const response = new Response(stream); + const blob = await response.blob(); + expect(await blob.text()).toBe('{"hello":true}'); +}); + it("Blob.stream() -> new Response(stream).text()", async () => { var blob = new Blob(["abdefgh"]); var stream = blob.stream();