From 57c6a7db35d42b6bc088982c53b68bca630212e5 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 24 Jul 2024 01:30:31 -0700 Subject: [PATCH] libdeflate (#12741) --- .gitmodules | 4 + CMakeLists.txt | 14 + Dockerfile | 25 ++ LICENSE.md | 2 + bench/gzip/bun.js | 39 ++- bench/gzip/bun.lockb | Bin 0 -> 1254 bytes bench/gzip/node.mjs | 11 +- bench/gzip/package.json | 3 + packages/bun-types/bun.d.ts | 27 +- packages/bun-uws/src/PerMessageDeflate.h | 19 +- scripts/all-dependencies.ps1 | 4 + scripts/all-dependencies.sh | 3 +- scripts/build-libdeflate.ps1 | 16 + scripts/build-libdeflate.sh | 10 + scripts/write-versions.sh | 2 + src/bun.js/api/BunObject.zig | 380 ++++++++++++++++------ src/bun.js/bindings/BunProcess.cpp | 4 +- src/bun.js/bindings/headers-handwritten.h | 1 + src/bun.js/javascript.zig | 3 + src/bun.js/node/types.zig | 1 + src/bun.js/rare_data.zig | 11 + src/bun.zig | 1 + src/deps/libdeflate | 1 + src/deps/libdeflate.zig | 149 +++++++++ src/feature_flags.zig | 11 + src/generated_versions_list.zig | 1 + src/http.zig | 137 ++++++-- src/install/extract_tarball.zig | 77 ++++- src/install/install.zig | 2 +- test/js/node/zlib/zlib.test.js | 35 +- test/js/web/fetch/fetch.stream.test.ts | 29 +- 31 files changed, 838 insertions(+), 184 deletions(-) create mode 100755 bench/gzip/bun.lockb create mode 100644 scripts/build-libdeflate.ps1 create mode 100755 scripts/build-libdeflate.sh create mode 160000 src/deps/libdeflate create mode 100644 src/deps/libdeflate.zig diff --git a/.gitmodules b/.gitmodules index 98845d5097..c5069240a4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -82,3 +82,7 @@ url = https://github.com/oven-sh/zig depth = 1 shallow = true fetchRecurseSubmodules = false +[submodule "src/deps/libdeflate"] +path = src/deps/libdeflate +url = https://github.com/ebiggers/libdeflate +ignore = "dirty" diff --git a/CMakeLists.txt b/CMakeLists.txt index e18c6d831a..2045d03341 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -310,6 +310,7 @@ endif() # -- Build Flags -- option(USE_STATIC_SQLITE "Statically link SQLite?" ${DEFAULT_ON_UNLESS_APPLE}) option(USE_CUSTOM_ZLIB "Use Bun's recommended version of zlib" ON) +option(USE_CUSTOM_LIBDEFLATE "Use Bun's recommended version of libdeflate" ON) option(USE_CUSTOM_BORINGSSL "Use Bun's recommended version of BoringSSL" ON) option(USE_CUSTOM_LIBARCHIVE "Use Bun's recommended version of libarchive" ON) option(USE_CUSTOM_MIMALLOC "Use Bun's recommended version of Mimalloc" ON) @@ -1358,6 +1359,19 @@ else() target_link_libraries(${bun} PRIVATE LibArchive::LibArchive) endif() +if(USE_CUSTOM_LIBDEFLATE) + include_directories(${BUN_DEPS_DIR}/libdeflate) + + if(WIN32) + target_link_libraries(${bun} PRIVATE "${BUN_DEPS_OUT_DIR}/deflate.lib") + else() + target_link_libraries(${bun} PRIVATE "${BUN_DEPS_OUT_DIR}/libdeflate.a") + endif() +else() + find_package(LibDeflate REQUIRED) + target_link_libraries(${bun} PRIVATE LibDeflate::LibDeflate) +endif() + if(USE_CUSTOM_MIMALLOC) include_directories(${BUN_DEPS_DIR}/mimalloc/include) diff --git a/Dockerfile b/Dockerfile index 6fa2ddf48b..7b707f03e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -263,6 +263,27 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ && bash ./scripts/build-zlib.sh && rm -rf src/deps/zlib scripts +FROM bun-base as libdeflate + +ARG BUN_DIR +ARG CPU_TARGET +ENV CPU_TARGET=${CPU_TARGET} +ARG CCACHE_DIR=/ccache +ENV CCACHE_DIR=${CCACHE_DIR} + +COPY Makefile ${BUN_DIR}/Makefile +COPY CMakeLists.txt ${BUN_DIR}/CMakeLists.txt +COPY scripts ${BUN_DIR}/scripts +COPY src/deps/libdeflate ${BUN_DIR}/src/deps/libdeflate +COPY package.json bun.lockb Makefile .gitmodules ${BUN_DIR}/ + +WORKDIR $BUN_DIR + +RUN --mount=type=cache,target=${CCACHE_DIR} \ + cd $BUN_DIR \ + && bash ./scripts/build-libdeflate.sh && rm -rf src/deps/libdeflate scripts + + FROM bun-base as libarchive ARG BUN_DIR @@ -412,6 +433,9 @@ COPY src ${BUN_DIR}/src COPY CMakeLists.txt ${BUN_DIR}/CMakeLists.txt COPY src/deps/boringssl/include ${BUN_DIR}/src/deps/boringssl/include +# for uWebSockets +COPY src/deps/libdeflate ${BUN_DIR}/src/deps/libdeflate + ARG CCACHE_DIR=/ccache ENV CCACHE_DIR=${CCACHE_DIR} @@ -516,6 +540,7 @@ COPY src/symbols.dyn src/linker.lds ${BUN_DIR}/src/ COPY CMakeLists.txt ${BUN_DIR}/CMakeLists.txt COPY --from=zlib ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/ +COPY --from=libdeflate ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/ COPY --from=libarchive ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/ COPY --from=boringssl ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/ COPY --from=lolhtml ${BUN_DEPS_OUT_DIR}/* ${BUN_DEPS_OUT_DIR}/ diff --git a/LICENSE.md b/LICENSE.md index 719bf08d76..4cc901b7bc 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -34,6 +34,8 @@ Bun statically links these libraries: | [`c-ares`](https://github.com/c-ares/c-ares) | MIT licensed | | [`libicu`](https://github.com/unicode-org/icu) 72 | [license here](https://github.com/unicode-org/icu/blob/main/icu4c/LICENSE) | | [`libbase64`](https://github.com/aklomp/base64/blob/master/LICENSE) | BSD 2-Clause | +| [`libuv`](https://github.com/libuv/libuv) (on Windows) | MIT | +| [`libdeflate`](https://github.com/ebiggers/libdeflate) | MIT | | A fork of [`uWebsockets`](https://github.com/jarred-sumner/uwebsockets) | Apache 2.0 licensed | | Parts of [Tigerbeetle's IO code](https://github.com/tigerbeetle/tigerbeetle/blob/532c8b70b9142c17e07737ab6d3da68d7500cbca/src/io/windows.zig#L1) | Apache 2.0 licensed | diff --git a/bench/gzip/bun.js b/bench/gzip/bun.js index 1c5cdcaddd..6b69ae1fbb 100644 --- a/bench/gzip/bun.js +++ b/bench/gzip/bun.js @@ -1,20 +1,43 @@ -import { run, bench } from "mitata"; +import { run, bench, group } from "mitata"; import { gzipSync, gunzipSync } from "bun"; -const data = new TextEncoder().encode("Hello World!".repeat(9999)); +const data = await Bun.file(require.resolve("@babel/standalone/babel.min.js")).arrayBuffer(); const compressed = gzipSync(data); -bench(`roundtrip - "Hello World!".repeat(9999))`, () => { - gunzipSync(gzipSync(data)); +const libraries = ["zlib"]; +if (Bun.semver.satisfies(Bun.version.replaceAll("-debug", ""), ">=1.1.21")) { + libraries.push("libdeflate"); +} +const options = { library: undefined }; +const benchFn = (name, fn) => { + if (libraries.length > 1) { + group(name, () => { + for (const library of libraries) { + bench(library, () => { + options.library = library; + fn(); + }); + } + }); + } else { + options.library = libraries[0]; + bench(name, () => { + fn(); + }); + } +}; + +benchFn(`roundtrip - @babel/standalone/babel.min.js`, () => { + gunzipSync(gzipSync(data, options), options); }); -bench(`gzipSync("Hello World!".repeat(9999)))`, () => { - gzipSync(data); +benchFn(`gzipSync(@babel/standalone/babel.min.js`, () => { + gzipSync(data, options); }); -bench(`gunzipSync("Hello World!".repeat(9999)))`, () => { - gunzipSync(compressed); +benchFn(`gunzipSync(@babel/standalone/babel.min.js`, () => { + gunzipSync(compressed, options); }); await run(); diff --git a/bench/gzip/bun.lockb b/bench/gzip/bun.lockb new file mode 100755 index 0000000000000000000000000000000000000000..96feac42873a3135230446e8084058ca8334a8a9 GIT binary patch literal 1254 zcmY#Z)GsYA(of3F(@)JSQ%EY!;{sycoc!eMw9K4T-L(9o+{6;yG6OCq1_p)&jeLAw zt2WizKHi?}b11vla)JI_p1kg~EfvBBTi57J3u6H)0s@d)5a57NaJm7?uYxIHNJ`C1 z&VY&vGB7mssh?w*^~y6=y1vSI#;a;mMyMRQ6wD%^*$l2g^Vxy41Q2tPU?9j1$nHSp z|KtCMEJ=d7IF$m;V`PAsi)=K@oP_OfoaWDbnR!C>?c%6{mkKHT(*j&COY1M~@Y3CL zG-K`8+exRdJGJ_THMoD?_Ueki)H91r9$u+KColRi@9xV)HVmib&V-+^k}~{GxPy47K{Y XNE&s`^^8pP3=Q;3(yJihKKMug_SeLD literal 0 HcmV?d00001 diff --git a/bench/gzip/node.mjs b/bench/gzip/node.mjs index 0d6ea51249..d7a1abade7 100644 --- a/bench/gzip/node.mjs +++ b/bench/gzip/node.mjs @@ -1,19 +1,22 @@ import { run, bench } from "mitata"; import { gzipSync, gunzipSync } from "zlib"; +import { createRequire } from "module"; +import { readFileSync } from "fs"; -const data = new TextEncoder().encode("Hello World!".repeat(9999)); +const require = createRequire(import.meta.url); +const data = readFileSync(require.resolve("@babel/standalone/babel.min.js")); const compressed = gzipSync(data); -bench(`roundtrip - "Hello World!".repeat(9999))`, () => { +bench(`roundtrip - @babel/standalone/babel.min.js)`, () => { gunzipSync(gzipSync(data)); }); -bench(`gzipSync("Hello World!".repeat(9999)))`, () => { +bench(`gzipSync(@babel/standalone/babel.min.js))`, () => { gzipSync(data); }); -bench(`gunzipSync("Hello World!".repeat(9999)))`, () => { +bench(`gunzipSync(@babel/standalone/babel.min.js))`, () => { gunzipSync(compressed); }); diff --git a/bench/gzip/package.json b/bench/gzip/package.json index f5c377686b..49e6c3a890 100644 --- a/bench/gzip/package.json +++ b/bench/gzip/package.json @@ -7,5 +7,8 @@ "bench:node": "$NODE node.mjs", "bench:deno": "$DENO run -A --unstable deno.js", "bench": "bun run bench:bun && bun run bench:node && bun run bench:deno" + }, + "dependencies": { + "@babel/standalone": "7.24.10" } } diff --git a/packages/bun-types/bun.d.ts b/packages/bun-types/bun.d.ts index 8307d88ec1..6e7309a89d 100644 --- a/packages/bun-types/bun.d.ts +++ b/packages/bun-types/bun.d.ts @@ -3481,6 +3481,13 @@ declare module "bun" { * Filtered data consists mostly of small values with a somewhat random distribution. */ strategy?: number; + + library?: "zlib"; + } + + interface LibdeflateCompressionOptions { + level?: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; + library?: "libdeflate"; } /** @@ -3489,26 +3496,38 @@ declare module "bun" { * @param options Compression options to use * @returns The output buffer with the compressed data */ - function deflateSync(data: Uint8Array | string | ArrayBuffer, options?: ZlibCompressionOptions): Uint8Array; + function deflateSync( + data: Uint8Array | string | ArrayBuffer, + options?: ZlibCompressionOptions | LibdeflateCompressionOptions, + ): Uint8Array; /** * Compresses a chunk of data with `zlib` GZIP algorithm. * @param data The buffer of data to compress * @param options Compression options to use * @returns The output buffer with the compressed data */ - function gzipSync(data: Uint8Array | string | ArrayBuffer, options?: ZlibCompressionOptions): Uint8Array; + function gzipSync( + data: Uint8Array | string | ArrayBuffer, + options?: ZlibCompressionOptions | LibdeflateCompressionOptions, + ): Uint8Array; /** * Decompresses a chunk of data with `zlib` INFLATE algorithm. * @param data The buffer of data to decompress * @returns The output buffer with the decompressed data */ - function inflateSync(data: Uint8Array | string | ArrayBuffer): Uint8Array; + function inflateSync( + data: Uint8Array | string | ArrayBuffer, + options?: ZlibCompressionOptions | LibdeflateCompressionOptions, + ): Uint8Array; /** * Decompresses a chunk of data with `zlib` GUNZIP algorithm. * @param data The buffer of data to decompress * @returns The output buffer with the decompressed data */ - function gunzipSync(data: Uint8Array | string | ArrayBuffer): Uint8Array; + function gunzipSync( + data: Uint8Array | string | ArrayBuffer, + options?: ZlibCompressionOptions | LibdeflateCompressionOptions, + ): Uint8Array; type Target = /** diff --git a/packages/bun-uws/src/PerMessageDeflate.h b/packages/bun-uws/src/PerMessageDeflate.h index 17832c7165..e4ebaf0ac5 100644 --- a/packages/bun-uws/src/PerMessageDeflate.h +++ b/packages/bun-uws/src/PerMessageDeflate.h @@ -20,6 +20,8 @@ #ifndef UWS_PERMESSAGEDEFLATE_H #define UWS_PERMESSAGEDEFLATE_H +#define UWS_USE_LIBDEFLATE 1 + #include #include @@ -134,6 +136,9 @@ struct ZlibContext { struct DeflationStream { z_stream deflationStream = {}; +#ifdef UWS_USE_LIBDEFLATE + unsigned char reset_buffer[4096 + 1]; +#endif DeflationStream(CompressOptions compressOptions) { @@ -154,13 +159,11 @@ struct DeflationStream { /* Run a fast path in case of shared_compressor */ if (reset) { size_t written = 0; - static unsigned char buf[1024 + 1]; - - written = libdeflate_deflate_compress(zlibContext->compressor, raw.data(), raw.length(), buf, 1024); + written = libdeflate_deflate_compress(zlibContext->compressor, raw.data(), raw.length(), reset_buffer, 4096); if (written) { - memcpy(&buf[written], "\x00", 1); - return std::string_view((char *) buf, written + 1); + memcpy(&reset_buffer[written], "\x00", 1); + return std::string_view((char *) reset_buffer, written + 1); } } #endif @@ -214,6 +217,9 @@ struct DeflationStream { struct InflationStream { z_stream inflationStream = {}; +#ifdef UWS_USE_LIBDEFLATE + char buf[4096]; +#endif InflationStream(CompressOptions compressOptions) { /* Inflation windowBits are the top 8 bits of the 16 bit compressOptions */ @@ -230,13 +236,12 @@ struct InflationStream { #ifdef UWS_USE_LIBDEFLATE /* Try fast path first */ size_t written = 0; - static char buf[1024]; /* We have to pad 9 bytes and restore those bytes when done since 9 is more than 6 of next WebSocket message */ char tmp[9]; memcpy(tmp, (char *) compressed.data() + compressed.length(), 9); memcpy((char *) compressed.data() + compressed.length(), "\x00\x00\xff\xff\x01\x00\x00\xff\xff", 9); - libdeflate_result res = libdeflate_deflate_decompress(zlibContext->decompressor, compressed.data(), compressed.length() + 9, buf, 1024, &written); + libdeflate_result res = libdeflate_deflate_decompress(zlibContext->decompressor, compressed.data(), compressed.length() + 9, buf, 4096, &written); memcpy((char *) compressed.data() + compressed.length(), tmp, 9); if (res == 0) { diff --git a/scripts/all-dependencies.ps1 b/scripts/all-dependencies.ps1 index 23838d1a99..24ac5513d8 100755 --- a/scripts/all-dependencies.ps1 +++ b/scripts/all-dependencies.ps1 @@ -79,6 +79,10 @@ Build-Dependency ` -Script "lshpack" ` -Outputs @("lshpack.lib") +Build-Dependency ` + -Script "libdeflate" ` + -Outputs @("deflate.lib") + if (!($Script:DidAnything)) { Write-Host "(run with -Force to rebuild all)" } diff --git a/scripts/all-dependencies.sh b/scripts/all-dependencies.sh index e3ddd6c476..50a22fe8f8 100755 --- a/scripts/all-dependencies.sh +++ b/scripts/all-dependencies.sh @@ -3,7 +3,7 @@ set -eo pipefail source "$(dirname -- "${BASH_SOURCE[0]}")/env.sh" if [[ "$CI" ]]; then - $(dirname -- "${BASH_SOURCE[0]}")/update-submodules.sh + $(dirname -- "${BASH_SOURCE[0]}")/update-submodules.sh fi FORCE= @@ -92,6 +92,7 @@ dep mimalloc mimalloc libmimalloc.a libmimalloc.o dep tinycc tinycc libtcc.a dep zlib zlib libz.a dep zstd zstd libzstd.a +dep libdeflate libdeflate libdeflate.a dep ls-hpack lshpack liblshpack.a if [ "$BUILT_ANY" -eq 0 ]; then diff --git a/scripts/build-libdeflate.ps1 b/scripts/build-libdeflate.ps1 new file mode 100644 index 0000000000..1d9b9b957b --- /dev/null +++ b/scripts/build-libdeflate.ps1 @@ -0,0 +1,16 @@ +$ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash +. (Join-Path $PSScriptRoot "env.ps1") + +Push-Location (Join-Path $BUN_DEPS_DIR 'libdeflate') +try { + Remove-Item CMakeCache.txt, CMakeFiles, build -Recurse -ErrorAction SilentlyContinue + mkdir -Force build + + Run cmake -S "." -B build @CMAKE_FLAGS -DLIBDEFLATE_BUILD_STATIC_LIB=ON -DLIBDEFLATE_BUILD_SHARED_LIB=OFF -DLIBDEFLATE_BUILD_GZIP=OFF + Run cmake --build build --clean-first --config Release + + # In https://github.com/ebiggers/libdeflate/releases/tag/v1.20, it's outputting libdeflate.a even on Windows + Copy-Item build/deflatestatic.lib $BUN_DEPS_OUT_DIR/deflate.lib + Write-Host "-> deflate.lib" +} finally { Pop-Location } + diff --git a/scripts/build-libdeflate.sh b/scripts/build-libdeflate.sh new file mode 100755 index 0000000000..0bfd2cb565 --- /dev/null +++ b/scripts/build-libdeflate.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -exo pipefail +source $(dirname -- "${BASH_SOURCE[0]}")/env.sh + +mkdir -p $BUN_DEPS_OUT_DIR +cd $BUN_DEPS_DIR/libdeflate +rm -rf build CMakeCache.txt CMakeFiles +cmake "${CMAKE_FLAGS[@]}" -DLIBDEFLATE_BUILD_STATIC_LIB=ON -DLIBDEFLATE_BUILD_SHARED_LIB=OFF -DLIBDEFLATE_BUILD_GZIP=OFF -B build -S . -G Ninja +ninja libdeflate.a -C build +cp build/libdeflate.a $BUN_DEPS_OUT_DIR/libdeflate.a diff --git a/scripts/write-versions.sh b/scripts/write-versions.sh index 74bc29c68c..389acf702c 100755 --- a/scripts/write-versions.sh +++ b/scripts/write-versions.sh @@ -12,6 +12,7 @@ TINYCC=$(git rev-parse HEAD:./src/deps/tinycc) C_ARES=$(git rev-parse HEAD:./src/deps/c-ares) ZSTD=$(git rev-parse HEAD:./src/deps/zstd) LSHPACK=$(git rev-parse HEAD:./src/deps/ls-hpack) +LIBDEFLATE=$(git rev-parse HEAD:./src/deps/libdeflate) rm -rf src/generated_versions_list.zig echo "// AUTO-GENERATED FILE. Created via .scripts/write-versions.sh" >src/generated_versions_list.zig @@ -26,6 +27,7 @@ echo "pub const zlib = \"$ZLIB_VERSION\";" >>src/generated_versions_list.zig echo "pub const tinycc = \"$TINYCC\";" >>src/generated_versions_list.zig echo "pub const lolhtml = \"$LOLHTML\";" >>src/generated_versions_list.zig echo "pub const c_ares = \"$C_ARES\";" >>src/generated_versions_list.zig +echo "pub const libdeflate = \"$LIBDEFLATE\";" >>src/generated_versions_list.zig echo "pub const zstd = \"$ZSTD\";" >>src/generated_versions_list.zig echo "pub const lshpack = \"$LSHPACK\";" >>src/generated_versions_list.zig echo "" >>src/generated_versions_list.zig diff --git a/src/bun.js/api/BunObject.zig b/src/bun.js/api/BunObject.zig index ce7a0f7b73..5748039d20 100644 --- a/src/bun.js/api/BunObject.zig +++ b/src/bun.js/api/BunObject.zig @@ -12,14 +12,14 @@ pub const BunObject = struct { pub const allocUnsafe = toJSCallback(Bun.allocUnsafe); pub const build = toJSCallback(Bun.JSBundler.buildFn); pub const connect = toJSCallback(JSC.wrapStaticMethod(JSC.API.Listener, "connect", false)); - pub const deflateSync = toJSCallback(JSC.wrapStaticMethod(JSZlib, "deflateSync", true)); + pub const deflateSync = toJSCallback(JSZlib.deflateSync); pub const file = toJSCallback(WebCore.Blob.constructBunFile); pub const gc = toJSCallback(Bun.runGC); pub const generateHeapSnapshot = toJSCallback(Bun.generateHeapSnapshot); - pub const gunzipSync = toJSCallback(JSC.wrapStaticMethod(JSZlib, "gunzipSync", true)); - pub const gzipSync = toJSCallback(JSC.wrapStaticMethod(JSZlib, "gzipSync", true)); + pub const gunzipSync = toJSCallback(JSZlib.gunzipSync); + pub const gzipSync = toJSCallback(JSZlib.gzipSync); pub const indexOfLine = toJSCallback(Bun.indexOfLine); - pub const inflateSync = toJSCallback(JSC.wrapStaticMethod(JSZlib, "inflateSync", true)); + pub const inflateSync = toJSCallback(JSZlib.inflateSync); pub const jest = toJSCallback(@import("../test/jest.zig").Jest.call); pub const listen = toJSCallback(JSC.wrapStaticMethod(JSC.API.Listener, "listen", false)); pub const udpSocket = toJSCallback(JSC.wrapStaticMethod(JSC.API.UDPSocket, "udpSocket", false)); @@ -4647,27 +4647,220 @@ pub const JSZlib = struct { reader.list.deinit(reader.allocator); reader.deinit(); } - + export fn global_deallocator(_: ?*anyopaque, ctx: ?*anyopaque) void { + comptime assert(bun.use_mimalloc); + bun.Mimalloc.mi_free(ctx); + } export fn compressor_deallocator(_: ?*anyopaque, ctx: ?*anyopaque) void { var compressor: *zlib.ZlibCompressorArrayList = bun.cast(*zlib.ZlibCompressorArrayList, ctx.?); compressor.list.deinit(compressor.allocator); compressor.deinit(); } + const Library = enum { + zlib, + libdeflate, + + pub const map = bun.ComptimeEnumMap(Library); + }; + + // This has to be `inline` due to the callframe. + inline fn getOptions(globalThis: *JSGlobalObject, callframe: *JSC.CallFrame) ?struct { JSC.Node.StringOrBuffer, ?JSValue } { + const arguments = callframe.arguments(2).slice(); + const buffer_value = if (arguments.len > 0) arguments[0] else JSC.JSValue.jsUndefined(); + const options_val: ?JSValue = + if (arguments.len > 1 and arguments[1].isObject()) + arguments[1] + else if (arguments.len > 1 and !arguments[1].isUndefined()) { + globalThis.throwInvalidArguments("Expected options to be an object", .{}); + return null; + } else null; + + if (JSC.Node.StringOrBuffer.fromJS(globalThis, bun.default_allocator, buffer_value)) |buffer| { + return .{ buffer, options_val }; + } + + globalThis.throwInvalidArguments("Expected buffer to be a string or buffer", .{}); + return null; + } + pub fn gzipSync( globalThis: *JSGlobalObject, - buffer: JSC.Node.StringOrBuffer, - options_val_: ?JSValue, + callframe: *JSC.CallFrame, ) JSValue { - return gzipOrDeflateSync(globalThis, buffer, options_val_, true); + const buffer, const options_val = getOptions(globalThis, callframe) orelse return .zero; + defer buffer.deinit(); + return gzipOrDeflateSync(globalThis, buffer, options_val, true); + } + + pub fn inflateSync( + globalThis: *JSGlobalObject, + callframe: *JSC.CallFrame, + ) JSValue { + const buffer, const options_val = getOptions(globalThis, callframe) orelse return .zero; + defer buffer.deinit(); + return gunzipOrInflateSync(globalThis, buffer, options_val, false); } pub fn deflateSync( + globalThis: *JSGlobalObject, + callframe: *JSC.CallFrame, + ) JSValue { + const buffer, const options_val = getOptions(globalThis, callframe) orelse return .zero; + defer buffer.deinit(); + return gzipOrDeflateSync(globalThis, buffer, options_val, false); + } + + pub fn gunzipSync( + globalThis: *JSGlobalObject, + callframe: *JSC.CallFrame, + ) JSValue { + const buffer, const options_val = getOptions(globalThis, callframe) orelse return .zero; + defer buffer.deinit(); + return gunzipOrInflateSync(globalThis, buffer, options_val, true); + } + + pub fn gunzipOrInflateSync( globalThis: *JSGlobalObject, buffer: JSC.Node.StringOrBuffer, options_val_: ?JSValue, + is_gzip: bool, ) JSValue { - return gzipOrDeflateSync(globalThis, buffer, options_val_, false); + var opts = zlib.Options{ + .gzip = is_gzip, + .windowBits = if (is_gzip) 31 else -15, + }; + + var library: Library = .zlib; + if (options_val_) |options_val| { + if (options_val.get(globalThis, "windowBits")) |window| { + opts.windowBits = window.coerce(i32, globalThis); + library = .zlib; + } + + if (options_val.get(globalThis, "level")) |level| { + opts.level = level.coerce(i32, globalThis); + } + + if (options_val.get(globalThis, "memLevel")) |memLevel| { + opts.memLevel = memLevel.coerce(i32, globalThis); + library = .zlib; + } + + if (options_val.get(globalThis, "strategy")) |strategy| { + opts.strategy = strategy.coerce(i32, globalThis); + library = .zlib; + } + + if (options_val.getTruthy(globalThis, "library")) |library_value| { + if (!library_value.isString()) { + globalThis.throwInvalidArguments("Expected library to be a string", .{}); + return .zero; + } + + library = Library.map.fromJS(globalThis, library_value) orelse { + globalThis.throwInvalidArguments("Expected library to be one of 'zlib' or 'libdeflate'", .{}); + return .zero; + }; + } + } + + if (globalThis.hasException()) return .zero; + + const compressed = buffer.slice(); + const allocator = JSC.VirtualMachine.get().allocator; + + var list = brk: { + if (is_gzip and compressed.len > 64) { + // 0 1 2 3 4 5 6 7 + // +---+---+---+---+---+---+---+---+ + // | CRC32 | ISIZE | + // +---+---+---+---+---+---+---+---+ + const estimated_size: u32 = @bitCast(compressed[compressed.len - 4 ..][0..4].*); + // If it's > 256 MB, let's rely on dynamic allocation to minimize the risk of OOM. + if (estimated_size > 0 and estimated_size < 256 * 1024 * 1024) { + break :brk std.ArrayListUnmanaged(u8).initCapacity(allocator, @max(estimated_size, 64)) catch { + globalThis.throwOutOfMemory(); + return .zero; + }; + } + } + + break :brk std.ArrayListUnmanaged(u8).initCapacity(allocator, if (compressed.len > 512) compressed.len else 32) catch { + globalThis.throwOutOfMemory(); + return .zero; + }; + }; + + switch (library) { + .zlib => { + var reader = zlib.ZlibReaderArrayList.initWithOptions(compressed, &list, allocator, .{ + .windowBits = opts.windowBits, + .level = opts.level, + }) catch |err| { + list.deinit(allocator); + if (err == error.InvalidArgument) { + globalThis.throw("Zlib error: Invalid argument", .{}); + return .zero; + } + + globalThis.throwError(err, "Zlib error"); + return .zero; + }; + + reader.readAll() catch { + defer reader.deinit(); + globalThis.throwValue(ZigString.init(reader.errorMessage() orelse "Zlib returned an error").toErrorInstance(globalThis)); + return .zero; + }; + reader.list = .{ .items = reader.list.items }; + reader.list.capacity = reader.list.items.len; + reader.list_ptr = &reader.list; + + var array_buffer = JSC.ArrayBuffer.fromBytes(reader.list.items, .Uint8Array); + return array_buffer.toJSWithContext(globalThis, reader, reader_deallocator, null); + }, + .libdeflate => { + var decompressor: *bun.libdeflate.Decompressor = bun.libdeflate.Decompressor.alloc() orelse { + list.deinit(allocator); + globalThis.throwOutOfMemory(); + return .zero; + }; + defer decompressor.deinit(); + while (true) { + const result = decompressor.decompress(compressed, list.allocatedSlice(), if (is_gzip) .gzip else .deflate); + + list.items.len = result.written; + + if (result.status == .insufficient_space) { + if (list.capacity > 1024 * 1024 * 1024) { + list.deinit(allocator); + globalThis.throwOutOfMemory(); + return .zero; + } + + list.ensureTotalCapacity(allocator, list.capacity * 2) catch { + list.deinit(allocator); + globalThis.throwOutOfMemory(); + return .zero; + }; + continue; + } + + if (result.status == .success) { + list.items.len = result.written; + break; + } + + list.deinit(allocator); + globalThis.throw("libdeflate returned an error: {s}", .{@tagName(result.status)}); + return .zero; + } + + var array_buffer = JSC.ArrayBuffer.fromBytes(list.items, .Uint8Array); + return array_buffer.toJSWithContext(globalThis, list.items.ptr, global_deallocator, null); + }, + } } pub fn gzipOrDeflateSync( @@ -4676,107 +4869,112 @@ pub const JSZlib = struct { options_val_: ?JSValue, is_gzip: bool, ) JSValue { - var opts = zlib.Options{ .gzip = is_gzip }; + var level: ?i32 = null; + var library: Library = .zlib; + var windowBits: i32 = 0; + if (options_val_) |options_val| { - if (options_val.isObject()) { - if (options_val.get(globalThis, "windowBits")) |window| { - opts.windowBits = window.coerce(i32, globalThis); + if (options_val.get(globalThis, "windowBits")) |window| { + windowBits = window.coerce(i32, globalThis); + library = .zlib; + } + + if (options_val.getTruthy(globalThis, "library")) |library_value| { + if (!library_value.isString()) { + globalThis.throwInvalidArguments("Expected library to be a string", .{}); + return .zero; } - if (options_val.get(globalThis, "level")) |level| { - opts.level = level.coerce(i32, globalThis); - } + library = Library.map.fromJS(globalThis, library_value) orelse { + globalThis.throwInvalidArguments("Expected library to be one of 'zlib' or 'libdeflate'", .{}); + return .zero; + }; + } - if (options_val.get(globalThis, "memLevel")) |memLevel| { - opts.memLevel = memLevel.coerce(i32, globalThis); - } - - if (options_val.get(globalThis, "strategy")) |strategy| { - opts.strategy = strategy.coerce(i32, globalThis); - } + if (options_val.get(globalThis, "level")) |level_value| { + level = level_value.coerce(i32, globalThis); + if (globalThis.hasException()) return .zero; } } + if (globalThis.hasException()) return .zero; + const compressed = buffer.slice(); - const allocator = JSC.VirtualMachine.get().allocator; - var list = std.ArrayListUnmanaged(u8).initCapacity(allocator, if (compressed.len > 512) compressed.len else 32) catch unreachable; - var reader = zlib.ZlibCompressorArrayList.init(compressed, &list, allocator, opts) catch |err| { - if (err == error.InvalidArgument) { - return JSC.toInvalidArguments("Invalid buffer", .{}, globalThis); - } + const allocator = bun.default_allocator; - return JSC.toInvalidArguments("Unexpected", .{}, globalThis); - }; + switch (library) { + .zlib => { + var list = std.ArrayListUnmanaged(u8).initCapacity( + allocator, + if (compressed.len > 512) compressed.len else 32, + ) catch { + globalThis.throwOutOfMemory(); + return .zero; + }; - reader.readAll() catch { - defer reader.deinit(); - globalThis.throwValue(ZigString.init(reader.errorMessage() orelse "Zlib returned an error").toErrorInstance(globalThis)); - return .zero; - }; - reader.list = .{ .items = reader.list.toOwnedSlice(allocator) catch @panic("TODO") }; - reader.list.capacity = reader.list.items.len; - reader.list_ptr = &reader.list; + var reader = zlib.ZlibCompressorArrayList.init(compressed, &list, allocator, .{ + .windowBits = 15, + .gzip = is_gzip, + .level = level orelse 6, + }) catch |err| { + defer list.deinit(allocator); + if (err == error.InvalidArgument) { + globalThis.throw("Zlib error: Invalid argument", .{}); + return .zero; + } - var array_buffer = JSC.ArrayBuffer.fromBytes(reader.list.items, .Uint8Array); - return array_buffer.toJSWithContext(globalThis, reader, reader_deallocator, null); - } + globalThis.throwError(err, "Zlib error"); + return .zero; + }; - pub fn inflateSync( - globalThis: *JSGlobalObject, - buffer: JSC.Node.StringOrBuffer, - ) JSValue { - const compressed = buffer.slice(); - const allocator = JSC.VirtualMachine.get().allocator; - var list = std.ArrayListUnmanaged(u8).initCapacity(allocator, if (compressed.len > 512) compressed.len else 32) catch unreachable; - var reader = zlib.ZlibReaderArrayList.initWithOptions(compressed, &list, allocator, .{ - .windowBits = -15, - }) catch |err| { - if (err == error.InvalidArgument) { - return JSC.toInvalidArguments("Invalid buffer", .{}, globalThis); - } + reader.readAll() catch { + defer reader.deinit(); + globalThis.throwValue(ZigString.init(reader.errorMessage() orelse "Zlib returned an error").toErrorInstance(globalThis)); + return .zero; + }; + reader.list = .{ .items = reader.list.toOwnedSlice(allocator) catch @panic("TODO") }; + reader.list.capacity = reader.list.items.len; + reader.list_ptr = &reader.list; - return JSC.toInvalidArguments("Unexpected", .{}, globalThis); - }; + var array_buffer = JSC.ArrayBuffer.fromBytes(reader.list.items, .Uint8Array); + return array_buffer.toJSWithContext(globalThis, reader, reader_deallocator, null); + }, + .libdeflate => { + var compressor: *bun.libdeflate.Compressor = bun.libdeflate.Compressor.alloc(level orelse 6) orelse { + globalThis.throwOutOfMemory(); + return .zero; + }; + const encoding: bun.libdeflate.Encoding = if (is_gzip) .gzip else .deflate; + defer compressor.deinit(); - reader.readAll() catch { - defer reader.deinit(); - globalThis.throwValue(ZigString.init(reader.errorMessage() orelse "Zlib returned an error").toErrorInstance(globalThis)); - return .zero; - }; - reader.list = .{ .items = reader.list.toOwnedSlice(allocator) catch @panic("TODO") }; - reader.list.capacity = reader.list.items.len; - reader.list_ptr = &reader.list; + var list = std.ArrayListUnmanaged(u8).initCapacity( + allocator, + // This allocation size is unfortunate, but it's not clear how to avoid it with libdeflate. + compressor.maxBytesNeeded(compressed, encoding), + ) catch { + globalThis.throwOutOfMemory(); + return .zero; + }; - var array_buffer = JSC.ArrayBuffer.fromBytes(reader.list.items, .Uint8Array); - return array_buffer.toJSWithContext(globalThis, reader, reader_deallocator, null); - } + while (true) { + const result = compressor.compress(compressed, list.allocatedSlice(), encoding); - pub fn gunzipSync( - globalThis: *JSGlobalObject, - buffer: JSC.Node.StringOrBuffer, - ) JSValue { - const compressed = buffer.slice(); - const allocator = JSC.VirtualMachine.get().allocator; - var list = std.ArrayListUnmanaged(u8).initCapacity(allocator, if (compressed.len > 512) compressed.len else 32) catch unreachable; - var reader = zlib.ZlibReaderArrayList.init(compressed, &list, allocator) catch |err| { - if (err == error.InvalidArgument) { - return JSC.toInvalidArguments("Invalid buffer", .{}, globalThis); - } + list.items.len = result.written; - return JSC.toInvalidArguments("Unexpected", .{}, globalThis); - }; + if (result.status == .success) { + list.items.len = result.written; + break; + } - reader.readAll() catch { - defer reader.deinit(); - globalThis.throwValue(ZigString.init(reader.errorMessage() orelse "Zlib returned an error").toErrorInstance(globalThis)); - return .zero; - }; - reader.list = .{ .items = reader.list.toOwnedSlice(allocator) catch @panic("TODO") }; - reader.list.capacity = reader.list.items.len; - reader.list_ptr = &reader.list; + list.deinit(allocator); + globalThis.throw("libdeflate error: {s}", .{@tagName(result.status)}); + return .zero; + } - var array_buffer = JSC.ArrayBuffer.fromBytes(reader.list.items, .Uint8Array); - return array_buffer.toJSWithContext(globalThis, reader, reader_deallocator, null); + var array_buffer = JSC.ArrayBuffer.fromBytes(list.items, .Uint8Array); + return array_buffer.toJSWithContext(globalThis, list.items.ptr, global_deallocator, null); + }, + } } }; diff --git a/src/bun.js/bindings/BunProcess.cpp b/src/bun.js/bindings/BunProcess.cpp index 3b02306650..a2e0dfc3c6 100644 --- a/src/bun.js/bindings/BunProcess.cpp +++ b/src/bun.js/bindings/BunProcess.cpp @@ -143,7 +143,7 @@ static JSValue constructPlatform(VM& vm, JSObject* processObject) static JSValue constructVersions(VM& vm, JSObject* processObject) { auto* globalObject = processObject->globalObject(); - JSC::JSObject* object = JSC::constructEmptyObject(globalObject, globalObject->objectPrototype(), 23); + JSC::JSObject* object = JSC::constructEmptyObject(globalObject, globalObject->objectPrototype(), 24); object->putDirect(vm, JSC::Identifier::fromString(vm, "node"_s), JSC::JSValue(JSC::jsOwnedString(vm, makeAtomString(ASCIILiteral::fromLiteralUnsafe(REPORTED_NODEJS_VERSION))))); @@ -176,6 +176,8 @@ static JSValue constructVersions(VM& vm, JSObject* processObject) JSC::JSValue(JSC::jsString(vm, makeString(ASCIILiteral::fromLiteralUnsafe(Bun__versions_lolhtml)))), 0); object->putDirect(vm, JSC::Identifier::fromString(vm, "ares"_s), JSC::JSValue(JSC::jsString(vm, makeString(ASCIILiteral::fromLiteralUnsafe(Bun__versions_c_ares)))), 0); + object->putDirect(vm, JSC::Identifier::fromString(vm, "libdeflate"_s), + JSC::JSValue(JSC::jsString(vm, makeString(ASCIILiteral::fromLiteralUnsafe(Bun__versions_libdeflate)))), 0); object->putDirect(vm, JSC::Identifier::fromString(vm, "usockets"_s), JSC::JSValue(JSC::jsString(vm, makeString(ASCIILiteral::fromLiteralUnsafe(Bun__versions_usockets)))), 0); object->putDirect(vm, JSC::Identifier::fromString(vm, "lshpack"_s), diff --git a/src/bun.js/bindings/headers-handwritten.h b/src/bun.js/bindings/headers-handwritten.h index e8068957f4..4b6fa3b0bd 100644 --- a/src/bun.js/bindings/headers-handwritten.h +++ b/src/bun.js/bindings/headers-handwritten.h @@ -348,6 +348,7 @@ extern "C" const char* Bun__versions_mimalloc; extern "C" const char* Bun__versions_picohttpparser; extern "C" const char* Bun__versions_uws; extern "C" const char* Bun__versions_webkit; +extern "C" const char* Bun__versions_libdeflate; extern "C" const char* Bun__versions_zig; extern "C" const char* Bun__versions_zlib; extern "C" const char* Bun__versions_tinycc; diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 177193e6b4..c1cea3acfa 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -2398,6 +2398,9 @@ pub const VirtualMachine = struct { // TODO: pub fn deinit(this: *VirtualMachine) void { this.source_mappings.deinit(); + if (this.rare_data) |rare_data| { + rare_data.deinit(); + } this.has_terminated = true; } diff --git a/src/bun.js/node/types.zig b/src/bun.js/node/types.zig index e7591cfd53..adf5bb2184 100644 --- a/src/bun.js/node/types.zig +++ b/src/bun.js/node/types.zig @@ -2102,6 +2102,7 @@ pub const Process = struct { pub export const Bun__versions_tinycc: [*:0]const u8 = bun.Global.versions.tinycc; pub export const Bun__versions_lolhtml: [*:0]const u8 = bun.Global.versions.lolhtml; pub export const Bun__versions_c_ares: [*:0]const u8 = bun.Global.versions.c_ares; + pub export const Bun__versions_libdeflate: [*:0]const u8 = bun.Global.versions.libdeflate; pub export const Bun__versions_usockets: [*:0]const u8 = bun.Environment.git_sha; pub export const Bun__version_sha: [*:0]const u8 = bun.Environment.git_sha; pub export const Bun__versions_lshpack: [*:0]const u8 = bun.Global.versions.lshpack; diff --git a/src/bun.js/rare_data.zig b/src/bun.js/rare_data.zig index 5782200373..1a65e3e287 100644 --- a/src/bun.js/rare_data.zig +++ b/src/bun.js/rare_data.zig @@ -392,3 +392,14 @@ pub fn nodeFSStatWatcherScheduler(rare: *RareData, vm: *JSC.VirtualMachine) *Sta return rare.node_fs_stat_watcher_scheduler.?; }; } + +pub fn deinit(this: *RareData) void { + if (this.temp_pipe_read_buffer) |pipe| { + this.temp_pipe_read_buffer = null; + bun.default_allocator.destroy(pipe); + } + + if (this.boring_ssl_engine) |engine| { + _ = bun.BoringSSL.ENGINE_free(engine); + } +} diff --git a/src/bun.zig b/src/bun.zig index 14bb4cc27c..c6c2d5592a 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -3614,3 +3614,4 @@ pub fn memmove(output: []u8, input: []const u8) void { } pub const hmac = @import("./hmac.zig"); +pub const libdeflate = @import("./deps/libdeflate.zig"); diff --git a/src/deps/libdeflate b/src/deps/libdeflate new file mode 160000 index 0000000000..dc76454a39 --- /dev/null +++ b/src/deps/libdeflate @@ -0,0 +1 @@ +Subproject commit dc76454a39e7e83b68c3704b6e3784654f8d5ac5 diff --git a/src/deps/libdeflate.zig b/src/deps/libdeflate.zig new file mode 100644 index 0000000000..d38d6dcb9f --- /dev/null +++ b/src/deps/libdeflate.zig @@ -0,0 +1,149 @@ +const std = @import("std"); +const bun = @import("root").bun; +pub const Options = extern struct { + sizeof_options: usize = @sizeOf(Options), + malloc_func: ?*const fn (usize) callconv(.C) ?*anyopaque = @import("std").mem.zeroes(?*const fn (usize) callconv(.C) ?*anyopaque), + free_func: ?*const fn (?*anyopaque) callconv(.C) void = @import("std").mem.zeroes(?*const fn (?*anyopaque) callconv(.C) void), +}; +pub extern fn libdeflate_alloc_compressor(compression_level: c_int) ?*Compressor; +pub extern fn libdeflate_alloc_compressor_ex(compression_level: c_int, options: ?*const Options) ?*Compressor; +pub extern fn libdeflate_deflate_compress(compressor: *Compressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize) usize; +pub extern fn libdeflate_deflate_compress_bound(compressor: *Compressor, in_nbytes: usize) usize; +pub extern fn libdeflate_zlib_compress(compressor: *Compressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize) usize; +pub extern fn libdeflate_zlib_compress_bound(compressor: *Compressor, in_nbytes: usize) usize; +pub extern fn libdeflate_gzip_compress(compressor: *Compressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize) usize; +pub extern fn libdeflate_gzip_compress_bound(compressor: *Compressor, in_nbytes: usize) usize; +pub extern fn libdeflate_free_compressor(compressor: *Compressor) void; + +fn load_once() void { + libdeflate_set_memory_allocator(bun.Mimalloc.mi_malloc, bun.Mimalloc.mi_free); +} + +var loaded_once = std.once(load_once); + +pub fn load() void { + loaded_once.call(); +} + +pub const Compressor = opaque { + pub fn alloc(compression_level: c_int) ?*Compressor { + return libdeflate_alloc_compressor(compression_level); + } + + pub fn alloc_ex(compression_level: c_int, options: ?*const Options) ?*Compressor { + return libdeflate_alloc_compressor_ex(compression_level, options); + } + + pub fn deinit(this: *Compressor) void { + return libdeflate_free_compressor(this); + } + + /// Compresses `input` into `output` and returns the number of bytes written. + pub fn inflate(this: *Compressor, input: []const u8, output: []u8) Result { + const written = libdeflate_deflate_compress(this, input.ptr, input.len, output.ptr, output.len); + return Result{ .read = input.len, .written = written, .status = Status.success }; + } + + pub fn maxBytesNeeded(this: *Compressor, input: []const u8, encoding: Encoding) usize { + return switch (encoding) { + Encoding.deflate => return libdeflate_deflate_compress_bound(this, input.len), + Encoding.zlib => return libdeflate_zlib_compress_bound(this, input.len), + Encoding.gzip => return libdeflate_gzip_compress_bound(this, input.len), + }; + } + + pub fn compress(this: *Compressor, input: []const u8, output: []u8, encoding: Encoding) Result { + switch (encoding) { + Encoding.deflate => return this.inflate(input, output), + Encoding.zlib => return this.zlib(input, output), + Encoding.gzip => return this.gzip(input, output), + } + } + + pub fn zlib(this: *Compressor, input: []const u8, output: []u8) Result { + const result = libdeflate_zlib_compress(this, input.ptr, input.len, output.ptr, output.len); + return Result{ .read = input.len, .written = result, .status = Status.success }; + } + + pub fn gzip(this: *Compressor, input: []const u8, output: []u8) Result { + const result = libdeflate_gzip_compress(this, input.ptr, input.len, output.ptr, output.len); + return Result{ .read = input.len, .written = result, .status = Status.success }; + } +}; + +pub const Decompressor = opaque { + pub fn alloc() ?*Decompressor { + return libdeflate_alloc_decompressor(); + } + + pub fn deinit(this: *Decompressor) void { + return libdeflate_free_decompressor(this); + } + + pub fn deflate(this: *Decompressor, input: []const u8, output: []u8) Result { + var actual_in_bytes_ret: usize = input.len; + var actual_out_bytes_ret: usize = output.len; + const result = libdeflate_deflate_decompress_ex(this, input.ptr, input.len, output.ptr, output.len, &actual_in_bytes_ret, &actual_out_bytes_ret); + return Result{ .read = actual_in_bytes_ret, .written = actual_out_bytes_ret, .status = result }; + } + + pub fn zlib(this: *Decompressor, input: []const u8, output: []u8) Result { + var actual_in_bytes_ret: usize = input.len; + var actual_out_bytes_ret: usize = output.len; + const result = libdeflate_zlib_decompress_ex(this, input.ptr, input.len, output.ptr, output.len, &actual_in_bytes_ret, &actual_out_bytes_ret); + return Result{ .read = actual_in_bytes_ret, .written = actual_out_bytes_ret, .status = result }; + } + + pub fn gzip(this: *Decompressor, input: []const u8, output: []u8) Result { + var actual_in_bytes_ret: usize = input.len; + var actual_out_bytes_ret: usize = output.len; + const result = libdeflate_gzip_decompress_ex(this, input.ptr, input.len, output.ptr, output.len, &actual_in_bytes_ret, &actual_out_bytes_ret); + return Result{ .read = actual_in_bytes_ret, .written = actual_out_bytes_ret, .status = result }; + } + + pub fn decompress(this: *Decompressor, input: []const u8, output: []u8, encoding: Encoding) Result { + switch (encoding) { + Encoding.deflate => return this.deflate(input, output), + Encoding.zlib => return this.zlib(input, output), + Encoding.gzip => return this.gzip(input, output), + } + } +}; + +pub const Result = struct { + read: usize, + written: usize, + status: Status, +}; + +pub const Encoding = enum { + deflate, + zlib, + gzip, +}; + +pub extern fn libdeflate_alloc_decompressor() ?*Decompressor; +pub extern fn libdeflate_alloc_decompressor_ex(options: ?*const Options) ?*Decompressor; +pub const LIBDEFLATE_SUCCESS = 0; +pub const LIBDEFLATE_BAD_DATA = 1; +pub const LIBDEFLATE_SHORT_OUTPUT = 2; +pub const LIBDEFLATE_INSUFFICIENT_SPACE = 3; +pub const Status = enum(c_uint) { + success = LIBDEFLATE_SUCCESS, + bad_data = LIBDEFLATE_BAD_DATA, + short_output = LIBDEFLATE_SHORT_OUTPUT, + insufficient_space = LIBDEFLATE_INSUFFICIENT_SPACE, +}; +pub extern fn libdeflate_deflate_decompress(decompressor: *Decompressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize, actual_out_nbytes_ret: *usize) Status; +pub extern fn libdeflate_deflate_decompress_ex(decompressor: *Decompressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize, actual_in_nbytes_ret: *usize, actual_out_nbytes_ret: *usize) Status; +pub extern fn libdeflate_zlib_decompress(decompressor: *Decompressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize, actual_out_nbytes_ret: *usize) Status; +pub extern fn libdeflate_zlib_decompress_ex(decompressor: *Decompressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize, actual_in_nbytes_ret: *usize, actual_out_nbytes_ret: *usize) Status; +pub extern fn libdeflate_gzip_decompress(decompressor: *Decompressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize, actual_out_nbytes_ret: *usize) Status; +pub extern fn libdeflate_gzip_decompress_ex(decompressor: *Decompressor, in: ?*const anyopaque, in_nbytes: usize, out: ?*anyopaque, out_nbytes_avail: usize, actual_in_nbytes_ret: *usize, actual_out_nbytes_ret: *usize) Status; +pub extern fn libdeflate_free_decompressor(decompressor: *Decompressor) void; +pub extern fn libdeflate_adler32(adler: u32, buffer: ?*const anyopaque, len: usize) u32; +pub extern fn libdeflate_crc32(crc: u32, buffer: ?*const anyopaque, len: usize) u32; +pub extern fn libdeflate_set_memory_allocator(malloc_func: ?*const fn (usize) callconv(.C) ?*anyopaque, free_func: ?*const fn (?*anyopaque) callconv(.C) void) void; +pub const libdeflate_compressor = Compressor; +pub const libdeflate_options = Options; +pub const libdeflate_decompressor = Decompressor; diff --git a/src/feature_flags.zig b/src/feature_flags.zig index 4e25f5442f..76b843e61c 100644 --- a/src/feature_flags.zig +++ b/src/feature_flags.zig @@ -184,3 +184,14 @@ pub const postgresql = env.is_canary or env.isDebug; // TODO: fix Windows-only test failures in fetch-preconnect.test.ts pub const is_fetch_preconnect_supported = env.isPosix; + +pub const libdeflate_supported = env.isNative; + +// Mostly exists as a way to turn it off later, if necessary. +pub fn isLibdeflateEnabled() bool { + if (!libdeflate_supported) { + return false; + } + + return !bun.getRuntimeFeatureFlag("BUN_FEATURE_FLAG_NO_LIBDEFLATE"); +} diff --git a/src/generated_versions_list.zig b/src/generated_versions_list.zig index 82773c4c8e..56658e0e50 100644 --- a/src/generated_versions_list.zig +++ b/src/generated_versions_list.zig @@ -10,5 +10,6 @@ pub const zlib = "886098f3f339617b4243b286f5ed364b9989e245"; pub const tinycc = "ab631362d839333660a265d3084d8ff060b96753"; pub const lolhtml = "8d4c273ded322193d017042d1f48df2766b0f88b"; pub const c_ares = "d1722e6e8acaf10eb73fa995798a9cd421d9f85e"; +pub const libdeflate = "dc76454a39e7e83b68c3704b6e3784654f8d5ac5"; pub const zstd = "794ea1b0afca0f020f4e57b6732332231fb23c70"; pub const lshpack = "3d0f1fc1d6e66a642e7a98c55deb38aa986eb4b0"; diff --git a/src/http.zig b/src/http.zig index 99635e0451..a9e22f8553 100644 --- a/src/http.zig +++ b/src/http.zig @@ -777,6 +777,8 @@ pub const HTTPThread = struct { has_awoken: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), timer: std.time.Timer, + lazy_libdeflater: ?*LibdeflateState = null, + const ShutdownMessage = struct { async_http_id: u32, is_tls: bool, @@ -784,6 +786,23 @@ pub const HTTPThread = struct { const threadlog = Output.scoped(.HTTPThread, true); + pub const LibdeflateState = struct { + decompressor: *bun.libdeflate.Decompressor = undefined, + shared_buffer: [512 * 1024]u8 = undefined, + + pub usingnamespace bun.New(@This()); + }; + + pub fn deflater(this: *@This()) *LibdeflateState { + if (this.lazy_libdeflater == null) { + this.lazy_libdeflater = LibdeflateState.new(.{ + .decompressor = bun.libdeflate.Decompressor.alloc() orelse bun.outOfMemory(), + }); + } + + return this.lazy_libdeflater.?; + } + fn initOnce() void { http_thread = .{ .loop = undefined, @@ -795,7 +814,7 @@ pub const HTTPThread = struct { }, .timer = std.time.Timer.start() catch unreachable, }; - + bun.libdeflate.load(); const thread = std.Thread.spawn( .{ .stack_size = bun.default_thread_stack_size, @@ -1432,6 +1451,7 @@ pub const InternalState = struct { received_last_chunk: bool = false, did_set_content_encoding: bool = false, is_redirect_pending: bool = false, + is_libdeflate_fast_path_disabled: bool = false, resend_request_body_on_redirect: bool = false, transfer_encoding: Encoding = Encoding.identity, encoding: Encoding = Encoding.identity, @@ -1520,40 +1540,93 @@ pub const InternalState = struct { return this.received_last_chunk; } - fn decompressBytes(this: *InternalState, buffer: []const u8, body_out_str: *MutableString) !void { - log("Decompressing {d} bytes\n", .{buffer.len}); - + fn decompressBytes(this: *InternalState, buffer: []const u8, body_out_str: *MutableString, is_final_chunk: bool) !void { defer this.compressed_body.reset(); var gzip_timer: std.time.Timer = undefined; if (extremely_verbose) gzip_timer = std.time.Timer.start() catch @panic("Timer failure"); - try this.decompressor.updateBuffers(this.encoding, buffer, body_out_str); - this.decompressor.readAll(this.isDone()) catch |err| { - if (this.isDone() or error.ShortRead != err) { - Output.prettyErrorln("Decompression error: {s}", .{bun.asByteSlice(@errorName(err))}); - Output.flush(); - return err; + var still_needs_to_decompress = true; + + if (FeatureFlags.isLibdeflateEnabled()) { + // Fast-path: use libdeflate + if (is_final_chunk and !this.is_libdeflate_fast_path_disabled and this.encoding.canUseLibDeflate() and this.isDone()) libdeflate: { + this.is_libdeflate_fast_path_disabled = true; + + log("Decompressing {d} bytes with libdeflate\n", .{buffer.len}); + var deflater = http_thread.deflater(); + + // gzip stores the size of the uncompressed data in the last 4 bytes of the stream + // But it's only valid if the stream is less than 4.7 GB, since it's 4 bytes. + // If we know that the stream is going to be larger than our + // pre-allocated buffer, then let's dynamically allocate the exact + // size. + if (this.encoding == Encoding.gzip and buffer.len > 16 and buffer.len < 1024 * 1024 * 1024) { + const estimated_size: u32 = @bitCast(buffer[buffer.len - 4 ..][0..4].*); + // Since this is arbtirary input from the internet, let's set an upper bound of 32 MB for the allocation size. + if (estimated_size > deflater.shared_buffer.len and estimated_size < 32 * 1024 * 1024) { + try body_out_str.list.ensureTotalCapacityPrecise(body_out_str.allocator, estimated_size); + const result = deflater.decompressor.decompress(buffer, body_out_str.list.allocatedSlice(), .gzip); + + if (result.status == .success) { + body_out_str.list.items.len = result.written; + still_needs_to_decompress = false; + } + + break :libdeflate; + } + } + + const result = deflater.decompressor.decompress(buffer, &deflater.shared_buffer, switch (this.encoding) { + .gzip => .gzip, + .deflate => .deflate, + else => unreachable, + }); + + if (result.status == .success) { + try body_out_str.list.ensureTotalCapacityPrecise(body_out_str.allocator, result.written); + body_out_str.list.appendSliceAssumeCapacity(deflater.shared_buffer[0..result.written]); + still_needs_to_decompress = false; + } } - }; + } + + // Slow path, or brotli: use the .decompressor + if (still_needs_to_decompress) { + log("Decompressing {d} bytes\n", .{buffer.len}); + if (body_out_str.list.capacity == 0) { + const min = @min(@ceil(@as(f64, @floatFromInt(buffer.len)) * 1.5), @as(f64, 1024 * 1024 * 2)); + try body_out_str.growBy(@max(@as(usize, @intFromFloat(min)), 32)); + } + + try this.decompressor.updateBuffers(this.encoding, buffer, body_out_str); + + this.decompressor.readAll(this.isDone()) catch |err| { + if (this.isDone() or error.ShortRead != err) { + Output.prettyErrorln("Decompression error: {s}", .{bun.asByteSlice(@errorName(err))}); + Output.flush(); + return err; + } + }; + } if (extremely_verbose) this.gzip_elapsed = gzip_timer.read(); } - fn decompress(this: *InternalState, buffer: MutableString, body_out_str: *MutableString) !void { - try this.decompressBytes(buffer.list.items, body_out_str); + fn decompress(this: *InternalState, buffer: MutableString, body_out_str: *MutableString, is_final_chunk: bool) !void { + try this.decompressBytes(buffer.list.items, body_out_str, is_final_chunk); } - pub fn processBodyBuffer(this: *InternalState, buffer: MutableString) !bool { + pub fn processBodyBuffer(this: *InternalState, buffer: MutableString, is_final_chunk: bool) !bool { if (this.is_redirect_pending) return false; var body_out_str = this.body_out_str.?; switch (this.encoding) { Encoding.brotli, Encoding.gzip, Encoding.deflate => { - try this.decompress(buffer, body_out_str); + try this.decompress(buffer, body_out_str, is_final_chunk); }, else => { if (!body_out_str.owns(buffer.list.items)) { @@ -1696,6 +1769,13 @@ pub const Encoding = enum { brotli, chunked, + pub fn canUseLibDeflate(this: Encoding) bool { + return switch (this) { + .gzip, .deflate => true, + else => false, + }; + } + pub fn isCompressed(this: Encoding) bool { return switch (this) { .brotli, .gzip, .deflate => true, @@ -3328,14 +3408,7 @@ fn handleResponseBodyFromSinglePacket(this: *HTTPClient, incoming_data: []const if (this.state.is_redirect_pending) return; if (this.state.encoding.isCompressed()) { - var body_buffer = this.state.body_out_str.?; - if (body_buffer.list.capacity == 0) { - const min = @min(@ceil(@as(f64, @floatFromInt(incoming_data.len)) * 1.5), @as(f64, 1024 * 1024 * 2)); - try body_buffer.growBy(@max(@as(usize, @intFromFloat(min)), 32)); - } - - // assert(!body_buffer.owns(b)); - try this.state.decompressBytes(incoming_data, body_buffer); + try this.state.decompressBytes(incoming_data, this.state.body_out_str.?, true); } else { try this.state.getBodyBuffer().appendSliceExact(incoming_data); } @@ -3383,7 +3456,12 @@ fn handleResponseBodyFromMultiplePackets(this: *HTTPClient, incoming_data: []con // done or streaming const is_done = content_length != null and this.state.total_body_received >= content_length.?; if (is_done or this.signals.get(.body_streaming) or content_length == null) { - const processed = try this.state.processBodyBuffer(buffer.*); + const is_final_chunk = is_done; + const processed = try this.state.processBodyBuffer(buffer.*, is_final_chunk); + + // We can only use the libdeflate fast path when we are not streaming + // If we ever call processBodyBuffer again, it cannot go through the fast path. + this.state.is_libdeflate_fast_path_disabled = true; if (this.progress_node) |progress| { progress.activate(); @@ -3448,7 +3526,9 @@ fn handleResponseBodyChunkedEncodingFromMultiplePackets( } // streaming chunks if (this.signals.get(.body_streaming)) { - return try this.state.processBodyBuffer(buffer); + // If we're streaming, we cannot use the libdeflate fast path + this.state.is_libdeflate_fast_path_disabled = true; + return try this.state.processBodyBuffer(buffer, false); } return false; @@ -3458,6 +3538,7 @@ fn handleResponseBodyChunkedEncodingFromMultiplePackets( this.state.received_last_chunk = true; _ = try this.state.processBodyBuffer( buffer, + true, ); if (this.progress_node) |progress| { @@ -3526,7 +3607,10 @@ fn handleResponseBodyChunkedEncodingFromSinglePacket( // streaming chunks if (this.signals.get(.body_streaming)) { - return try this.state.processBodyBuffer(body_buffer.*); + // If we're streaming, we cannot use the libdeflate fast path + this.state.is_libdeflate_fast_path_disabled = true; + + return try this.state.processBodyBuffer(body_buffer.*, true); } return false; @@ -3534,7 +3618,6 @@ fn handleResponseBodyChunkedEncodingFromSinglePacket( // Done else => { this.state.received_last_chunk = true; - try this.handleResponseBodyFromSinglePacket(buffer); assert(this.state.body_out_str.?.list.items.ptr != buffer.ptr); if (this.progress_node) |progress| { diff --git a/src/install/extract_tarball.zig b/src/install/extract_tarball.zig index fc740cb479..85e01170d8 100644 --- a/src/install/extract_tarball.zig +++ b/src/install/extract_tarball.zig @@ -198,28 +198,70 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD defer extract_destination.close(); - if (PackageManager.verbose_install) { - Output.prettyErrorln("[{s}] Start extracting {s}", .{ name, tmpname }); - Output.flush(); - } - const Archive = @import("../libarchive/libarchive.zig").Archive; const Zlib = @import("../zlib.zig"); var zlib_pool = Npm.Registry.BodyPool.get(default_allocator); zlib_pool.data.reset(); defer Npm.Registry.BodyPool.release(zlib_pool); - var zlib_entry = try Zlib.ZlibReaderArrayList.init(tgz_bytes, &zlib_pool.data.list, default_allocator); - zlib_entry.readAll() catch |err| { - this.package_manager.log.addErrorFmt( - null, - logger.Loc.Empty, - this.package_manager.allocator, - "{s} decompressing \"{s}\" to \"{}\"", - .{ @errorName(err), name, bun.fmt.fmtPath(u8, std.mem.span(tmpname), .{}) }, - ) catch unreachable; - return error.InstallFailed; - }; + var esimated_output_size: usize = 0; + + const time_started_for_verbose_logs: u64 = if (PackageManager.verbose_install) bun.getRoughTickCount().ns() else 0; + + { + // Last 4 bytes of a gzip-compressed file are the uncompressed size. + if (tgz_bytes.len > 16) { + // If the file claims to be larger than 16 bytes and smaller than 64 MB, we'll preallocate the buffer. + // If it's larger than that, we'll do it incrementally. We want to avoid OOMing. + const last_4_bytes: u32 = @bitCast(tgz_bytes[tgz_bytes.len - 4 ..][0..4].*); + if (last_4_bytes > 16 and last_4_bytes < 64 * 1024 * 1024) { + // It's okay if this fails. We will just allocate as we go and that will error if we run out of memory. + esimated_output_size = last_4_bytes; + if (zlib_pool.data.list.capacity == 0) { + zlib_pool.data.list.ensureTotalCapacityPrecise(zlib_pool.data.allocator, last_4_bytes) catch {}; + } else { + zlib_pool.data.ensureUnusedCapacity(last_4_bytes) catch {}; + } + } + } + } + + var needs_to_decompress = true; + if (bun.FeatureFlags.isLibdeflateEnabled() and zlib_pool.data.list.capacity > 16 and esimated_output_size > 0) use_libdeflate: { + const decompressor = bun.libdeflate.Decompressor.alloc() orelse break :use_libdeflate; + defer decompressor.deinit(); + + const result = decompressor.gzip(tgz_bytes, zlib_pool.data.list.allocatedSlice()); + + if (result.status == .success) { + zlib_pool.data.list.items.len = result.written; + needs_to_decompress = false; + } + + // If libdeflate fails for any reason, fallback to zlib. + } + + if (needs_to_decompress) { + zlib_pool.data.list.clearRetainingCapacity(); + var zlib_entry = try Zlib.ZlibReaderArrayList.init(tgz_bytes, &zlib_pool.data.list, default_allocator); + zlib_entry.readAll() catch |err| { + this.package_manager.log.addErrorFmt( + null, + logger.Loc.Empty, + this.package_manager.allocator, + "{s} decompressing \"{s}\" to \"{}\"", + .{ @errorName(err), name, bun.fmt.fmtPath(u8, std.mem.span(tmpname), .{}) }, + ) catch unreachable; + return error.InstallFailed; + }; + } + + if (PackageManager.verbose_install) { + const decompressing_ended_at: u64 = bun.getRoughTickCount().ns(); + const elapsed = decompressing_ended_at - time_started_for_verbose_logs; + Output.prettyErrorln("[{s}] Extract {s} (decompressed {} tgz file in {})", .{ name, tmpname, bun.fmt.size(tgz_bytes.len), bun.fmt.fmtDuration(elapsed) }); + } + switch (this.resolution.tag) { .github => { const DirnameReader = struct { @@ -278,7 +320,8 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD } if (PackageManager.verbose_install) { - Output.prettyErrorln("[{s}] Extracted", .{name}); + const elapsed = bun.getRoughTickCount().ns() - time_started_for_verbose_logs; + Output.prettyErrorln("[{s}] Extracted to {s} ({})", .{ name, tmpname, bun.fmt.fmtDuration(elapsed) }); Output.flush(); } } diff --git a/src/install/install.zig b/src/install/install.zig index 89d94fc801..4b71015595 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -6485,7 +6485,7 @@ pub const PackageManager = struct { if (comptime log_level.isVerbose()) { Output.prettyError(" ", .{}); Output.printElapsed(@as(f64, @floatCast(@as(f64, @floatFromInt(task.http.elapsed)) / std.time.ns_per_ms))); - Output.prettyError("Downloaded {s} tarball\n", .{extract.name.slice()}); + Output.prettyError(" Downloaded {s} tarball\n", .{extract.name.slice()}); Output.flush(); } diff --git a/test/js/node/zlib/zlib.test.js b/test/js/node/zlib/zlib.test.js index 35bd2ffdc5..40f9670856 100644 --- a/test/js/node/zlib/zlib.test.js +++ b/test/js/node/zlib/zlib.test.js @@ -9,28 +9,35 @@ import { tmpdirSync } from "harness"; import * as stream from "node:stream"; describe("zlib", () => { - it("should be able to deflate and inflate", () => { - const data = new TextEncoder().encode("Hello World!".repeat(1)); - const compressed = deflateSync(data); - const decompressed = inflateSync(compressed); - expect(decompressed.join("")).toBe(data.join("")); - }); + for (let library of ["zlib", "libdeflate"]) { + for (let outputLibrary of ["zlib", "libdeflate"]) { + describe(`${library} -> ${outputLibrary}`, () => { + it("should be able to deflate and inflate", () => { + const data = new TextEncoder().encode("Hello World!".repeat(1)); + const compressed = deflateSync(data, { library }); + console.log(compressed); + const decompressed = inflateSync(compressed, { library: outputLibrary }); + expect(decompressed.join("")).toBe(data.join("")); + }); - it("should be able to gzip and gunzip", () => { - const data = new TextEncoder().encode("Hello World!".repeat(1)); - const compressed = gzipSync(data); - const decompressed = gunzipSync(compressed); - expect(decompressed.join("")).toBe(data.join("")); - }); + it("should be able to gzip and gunzip", () => { + const data = new TextEncoder().encode("Hello World!".repeat(1)); + const compressed = gzipSync(data, { library }); + const decompressed = gunzipSync(compressed, { library: outputLibrary }); + expect(decompressed.join("")).toBe(data.join("")); + }); + }); + } + } it("should throw on invalid raw deflate data", () => { const data = new TextEncoder().encode("Hello World!".repeat(1)); - expect(() => inflateSync(data)).toThrow(new Error("invalid stored block lengths")); + expect(() => inflateSync(data, { library: "zlib" })).toThrow(new Error("invalid stored block lengths")); }); it("should throw on invalid gzip data", () => { const data = new TextEncoder().encode("Hello World!".repeat(1)); - expect(() => gunzipSync(data)).toThrow(new Error("incorrect header check")); + expect(() => gunzipSync(data, { library: "zlib" })).toThrow(new Error("incorrect header check")); }); }); diff --git a/test/js/web/fetch/fetch.stream.test.ts b/test/js/web/fetch/fetch.stream.test.ts index eac19feffc..7d7908f6a3 100644 --- a/test/js/web/fetch/fetch.stream.test.ts +++ b/test/js/web/fetch/fetch.stream.test.ts @@ -652,24 +652,28 @@ describe("fetch() with streaming", () => { } } - type CompressionType = "no" | "gzip" | "deflate" | "br" | "deflate_with_headers"; - type TestType = { headers: Record; compression: CompressionType; skip?: boolean }; - const types: Array = [ + const types = [ { headers: {}, compression: "no" }, { headers: { "Content-Encoding": "gzip" }, compression: "gzip" }, + { headers: { "Content-Encoding": "gzip" }, compression: "gzip-libdeflate" }, { headers: { "Content-Encoding": "deflate" }, compression: "deflate" }, + { headers: { "Content-Encoding": "deflate" }, compression: "deflate-libdeflate" }, { headers: { "Content-Encoding": "deflate" }, compression: "deflate_with_headers" }, - // { headers: { "Content-Encoding": "br" }, compression: "br", skip: true }, // not implemented yet - ]; + { headers: { "Content-Encoding": "br" }, compression: "br" }, + ] as const; - function compress(compression: CompressionType, data: Uint8Array) { + function compress(compression, data: Uint8Array) { switch (compression) { + case "gzip-libdeflate": case "gzip": - return Bun.gzipSync(data); + return Bun.gzipSync(data, { library: compression === "gzip-libdeflate" ? "libdeflate" : "zlib" }); + case "deflate-libdeflate": case "deflate": - return Bun.deflateSync(data); + return Bun.deflateSync(data, { library: compression === "deflate-libdeflate" ? "libdeflate" : "zlib" }); case "deflate_with_headers": return zlib.deflateSync(data); + case "br": + return zlib.brotliCompressSync(data); default: return data; } @@ -1186,7 +1190,14 @@ describe("fetch() with streaming", () => { gcTick(false); expect(buffer.toString("utf8")).toBe("unreachable"); } catch (err) { - expect((err as Error).name).toBe("ZlibError"); + if (compression === "br") { + expect((err as Error).name).toBe("BrotliDecompressionError"); + } else if (compression === "deflate-libdeflate") { + // Since the compressed data is different, the error ends up different. + expect((err as Error).name).toBe("ShortRead"); + } else { + expect((err as Error).name).toBe("ZlibError"); + } } } });