diff --git a/misctools/compression.zig b/misctools/compression.zig new file mode 100644 index 0000000000..f7aa3853d4 --- /dev/null +++ b/misctools/compression.zig @@ -0,0 +1,293 @@ +/// Demo app testing the macOS libcompression bindings. +const std = @import("std"); +const CompressionFramework = struct { + var handle: ?*anyopaque = null; + pub fn load() !void { + handle = std.os.darwin.dlopen("libcompression.dylib", 1); + + if (handle == null) + return error.@"failed to load Compression.framework"; + + compression_encode_scratch_buffer_size = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_encode_scratch_buffer_size").?)); + compression_encode_buffer = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_encode_buffer").?)); + compression_decode_scratch_buffer_size = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_decode_scratch_buffer_size").?)); + compression_decode_buffer = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_decode_buffer").?)); + compression_stream_init = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_stream_init").?)); + compression_stream_process = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_stream_process").?)); + compression_stream_destroy = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_stream_destroy").?)); + } + + pub const compression_algorithm = enum(c_uint) { + LZ4 = 256, + ZLIB = 0x205, + LZMA = 774, + LZ4_RAW = 257, + BROTLI = 2818, + LZFSE = 2049, + LZBITMAP = 1794, + + pub fn fromName(name: []const u8) ?compression_algorithm { + if (std.mem.endsWith(u8, name, ".br")) { + return .BROTLI; + } else if (std.mem.endsWith(u8, name, ".lz4")) { + return .LZ4; + } else if (std.mem.endsWith(u8, name, ".lzma")) { + return .LZMA; + } else if (std.mem.endsWith(u8, name, ".lzfse")) { + return .LZFSE; + } else if (std.mem.endsWith(u8, name, ".zlib") or std.mem.endsWith(u8, name, ".gz")) { + return .ZLIB; + } else { + return null; + } + } + }; + const compression_encode_scratch_buffer_size_type = fn (algorithm: compression_algorithm) callconv(.C) usize; + const compression_encode_buffer_type = fn (noalias dst_buffer: [*]u8, dst_size: usize, noalias src_buffer: ?[*]const u8, src_size: usize, noalias scratch_buffer: ?*anyopaque, algorithm: compression_algorithm) callconv(.C) usize; + const compression_decode_scratch_buffer_size_type = fn (algorithm: compression_algorithm) callconv(.C) usize; + const compression_decode_buffer_type = fn (noalias dst_buffer: [*]u8, dst_size: usize, noalias src_buffer: ?[*]const u8, src_size: usize, noalias scratch_buffer: ?*anyopaque, algorithm: compression_algorithm) callconv(.C) usize; + + const compression_stream_init_type = fn (stream: *compression_stream, operation: compression_stream_operation, algorithm: compression_algorithm) callconv(.C) compression_status; + const compression_stream_process_type = fn (stream: *compression_stream, flags: c_int) callconv(.C) compression_status; + const compression_stream_destroy_type = fn (stream: *compression_stream) callconv(.C) compression_status; + + var compression_encode_scratch_buffer_size: *const compression_encode_scratch_buffer_size_type = undefined; + var compression_encode_buffer: *const compression_encode_buffer_type = undefined; + var compression_decode_scratch_buffer_size: *const compression_decode_scratch_buffer_size_type = undefined; + var compression_decode_buffer: *const compression_decode_buffer_type = undefined; + + var compression_stream_init: *const compression_stream_init_type = undefined; + var compression_stream_process: *const compression_stream_process_type = undefined; + var compression_stream_destroy: *const compression_stream_destroy_type = undefined; + pub const compression_stream = extern struct { + dst_ptr: ?[*]u8 = null, + dst_size: usize = 0, + src_ptr: ?[*]const u8 = null, + src_size: usize = 0, + state: ?*anyopaque = null, + + pub fn init(src: []const u8, operation: compression_stream_operation, algorithm: compression_algorithm) !compression_stream { + var stream = compression_stream{ + .src_ptr = src.ptr, + .src_size = src.len, + .dst_ptr = null, + .dst_size = 0, + }; + + switch (compression_stream_init(&stream, operation, algorithm)) { + .OK => {}, + .ERROR => return error.@"failed to initialize compression stream", + .END => return error.@"compression stream init returned END", + } + + return stream; + } + + pub fn process(stream: *compression_stream, data: []const u8, is_done: bool, comptime Iterator: type, iter: *Iterator) !StreamResult { + stream.src_ptr = data.ptr; + stream.src_size = data.len; + + const initial_dest = try iter.wrote(0); + stream.dst_ptr = initial_dest.ptr; + stream.dst_size = initial_dest.len; + + var total_written: usize = 0; + while (true) { + var flags: c_int = 0; + if (stream.src_size == 0 and is_done) { + flags = COMPRESSION_STREAM_FINALIZE; + } else if (stream.src_size == 0) { + return .{ + .progress = .{ + .read = data.len - stream.src_size, + .wrote = total_written, + }, + }; + } + + const prev_size = stream.dst_size; + const rc = compression_stream_process(stream, flags); + const wrote = prev_size - stream.dst_size; + switch (rc) { + .OK => { + const new_buffer = try iter.wrote(wrote); + stream.dst_ptr = new_buffer.ptr; + stream.dst_size = new_buffer.len; + total_written += wrote; + }, + .END => { + _ = try iter.wrote(wrote); + total_written += wrote; + + return .{ + .done = .{ + .read = data.len - stream.src_size, + .wrote = total_written, + }, + }; + }, + .ERROR => { + return .{ + .err = .{ + .err = error.@"failed to process compression stream", + .read = data.len - stream.src_size, + .wrote = total_written, + }, + }; + }, + } + } + } + }; + pub const COMPRESSION_STREAM_ENCODE: c_int = 0; + pub const COMPRESSION_STREAM_DECODE: c_int = 1; + pub const compression_stream_operation = enum(c_uint) { + ENCODE = 0, + DECODE = 1, + }; + pub const COMPRESSION_STREAM_FINALIZE: c_int = 1; + pub const compression_stream_flags = c_uint; + pub const compression_status = enum(c_int) { + OK = 0, + ERROR = -1, + END = 1, + }; + + const StreamResult = union(enum) { + done: struct { + read: usize = 0, + wrote: usize = 0, + }, + err: struct { + read: usize = 0, + wrote: usize = 0, + err: anyerror, + }, + progress: struct { + read: usize = 0, + wrote: usize = 0, + }, + }; + + pub fn compress(data: []const u8, algorithm: compression_algorithm, is_done: bool, writer: anytype) !StreamResult { + var scratch_buffer: [64 * 1024]u8 = undefined; + + const scratch_buffer_size = compression_encode_scratch_buffer_size(algorithm); + if (scratch_buffer_size >= scratch_buffer.len) { + std.debug.panic("scratch buffer size is too small {d}", .{scratch_buffer_size}); + } + + var stream = try compression_stream.init(data, .ENCODE, algorithm); + + defer _ = compression_stream_destroy(&stream); + const Iterator = struct { + writer: @TypeOf(writer), + scratch_buffer: []u8, + pub fn wrote(this: *@This(), w: usize) ![]u8 { + try this.writer.writeAll(this.scratch_buffer[0..w]); + return this.scratch_buffer; + } + }; + + var iter = Iterator{ + .writer = writer, + .scratch_buffer = &scratch_buffer, + }; + + return try stream.process(data, is_done, Iterator, &iter); + } + + pub fn decompress(data: []const u8, algorithm: compression_algorithm, is_done: bool, writer: anytype) !StreamResult { + var scratch_buffer: [64 * 1024]u8 = undefined; + + const scratch_buffer_size = compression_decode_scratch_buffer_size(algorithm); + if (scratch_buffer_size >= scratch_buffer.len) { + std.debug.panic("scratch buffer size is too small {d}", .{scratch_buffer_size}); + } + + var stream = try compression_stream.init(data, .DECODE, algorithm); + defer _ = compression_stream_destroy(&stream); + + const Iterator = struct { + writer: @TypeOf(writer), + scratch_buffer: []u8, + pub fn wrote(this: *@This(), w: usize) ![]u8 { + try this.writer.writeAll(this.scratch_buffer[0..w]); + return this.scratch_buffer; + } + }; + + var iter = Iterator{ + .writer = writer, + .scratch_buffer = &scratch_buffer, + }; + + return try stream.process(data, is_done, Iterator, &iter); + } +}; + +pub fn main() anyerror!void { + try CompressionFramework.load(); + + var args = std.process.args(); + const argv0 = args.next() orelse ""; + + const first = args.next() orelse ""; + const second = args.next() orelse ""; + var algorithm: ?CompressionFramework.compression_algorithm = null; + var operation: ?CompressionFramework.compression_stream_operation = null; + + if (CompressionFramework.compression_algorithm.fromName(first)) |a| { + algorithm = a; + operation = .DECODE; + } else if (CompressionFramework.compression_algorithm.fromName(second)) |o| { + algorithm = o; + operation = .ENCODE; + } + + if (algorithm == null or operation == null) { + try std.io.getStdErr().writer().print("to compress: {s} ./file ./out.{{br,gz,lz4,lzfse}}\nto decompress: {s} ./out.{{br,gz,lz4,lzfse}} ./out\n", .{ argv0, argv0 }); + std.os.exit(1); + } + + var output_file: std.fs.File = undefined; + var input_file: std.fs.File = undefined; + + if (second.len == 0) { + output_file = std.io.getStdOut(); + } else { + output_file = try std.fs.cwd().createFile(second, .{ + .truncate = true, + }); + } + + if (first.len == 0) { + input_file = std.io.getStdIn(); + } else { + input_file = try std.fs.cwd().openFile(first, .{}); + } + + var writer = std.io.BufferedWriter(64 * 1024, @TypeOf(output_file.writer())){ + .unbuffered_writer = output_file.writer(), + }; + + const input_bytes = try input_file.readToEndAlloc(std.heap.c_allocator, std.math.maxInt(usize)); + + if (operation == .ENCODE) { + switch (try CompressionFramework.compress(input_bytes, algorithm.?, true, writer.writer())) { + .err => |err| { + return err.err; + }, + else => {}, + } + } else { + switch (try CompressionFramework.decompress(input_bytes, algorithm.?, true, writer.writer())) { + .err => |err| { + return err.err; + }, + else => {}, + } + } + + try writer.flush(); +} diff --git a/src/brotli.zig b/src/brotli.zig new file mode 100644 index 0000000000..8d50d8b242 --- /dev/null +++ b/src/brotli.zig @@ -0,0 +1,169 @@ +const bun = @import("root").bun; +const std = @import("std"); +const c = @import("./deps/brotli_decoder.zig"); +const BrotliDecoder = c.BrotliDecoder; + +const mimalloc = bun.Mimalloc; + +pub fn hasBrotli() bool { + return BrotliDecoder.initializeBrotli(); +} + +const BrotliAllocator = struct { + pub fn alloc(_: ?*anyopaque, len: usize) callconv(.C) *anyopaque { + if (comptime bun.is_heap_breakdown_enabled) { + const zone = bun.HeapBreakdown.malloc_zone_t.get(BrotliAllocator); + return zone.malloc_zone_malloc(len).?; + } + + return mimalloc.mi_malloc(len) orelse unreachable; + } + + pub fn free(_: ?*anyopaque, data: *anyopaque) callconv(.C) void { + if (comptime bun.is_heap_breakdown_enabled) { + const zone = bun.HeapBreakdown.malloc_zone_t.get(BrotliAllocator); + zone.malloc_zone_free(data); + return; + } + + mimalloc.mi_free(data); + } +}; + +pub const Options = struct { + pub const Params = std.enums.EnumFieldStruct(c.BrotliDecoderParameter, bool, false); + + params: Params = Params{ + .LARGE_WINDOW = true, + .DISABLE_RING_BUFFER_REALLOCATION = false, + }, +}; + +pub const BrotliReaderArrayList = struct { + pub const State = enum { + Uninitialized, + Inflating, + End, + Error, + }; + + input: []const u8, + list: std.ArrayListUnmanaged(u8), + list_allocator: std.mem.Allocator, + list_ptr: *std.ArrayListUnmanaged(u8), + brotli: *BrotliDecoder, + state: State = State.Uninitialized, + total_out: usize = 0, + total_in: usize = 0, + + pub usingnamespace bun.New(BrotliReaderArrayList); + + pub fn initWithOptions(input: []const u8, list: *std.ArrayListUnmanaged(u8), allocator: std.mem.Allocator, options: Options) !*BrotliReaderArrayList { + if (!BrotliDecoder.initializeBrotli()) { + return error.BrotliFailedToLoad; + } + + var brotli = BrotliDecoder.createInstance(&BrotliAllocator.alloc, &BrotliAllocator.free, null) orelse return error.BrotliFailedToCreateInstance; + if (options.params.LARGE_WINDOW) + _ = brotli.setParameter(c.BrotliDecoderParameter.LARGE_WINDOW, 1); + if (options.params.DISABLE_RING_BUFFER_REALLOCATION) + _ = brotli.setParameter(c.BrotliDecoderParameter.DISABLE_RING_BUFFER_REALLOCATION, 1); + + std.debug.assert(list.items.ptr != input.ptr); + + return BrotliReaderArrayList.new( + .{ + .input = input, + .list_ptr = list, + .list = list.*, + .list_allocator = allocator, + .brotli = brotli, + }, + ); + } + + pub fn end(this: *BrotliReaderArrayList) void { + this.state = .End; + } + + pub fn readAll(this: *BrotliReaderArrayList, is_done: bool) !void { + defer { + this.list_ptr.* = this.list; + } + + if (this.state == .End or this.state == .Error) { + return; + } + + std.debug.assert(this.list.items.ptr != this.input.ptr); + + while (this.state == State.Uninitialized or this.state == State.Inflating) { + var unused_capacity = this.list.unusedCapacitySlice(); + + if (unused_capacity.len < 4096) { + try this.list.ensureUnusedCapacity(this.list_allocator, 4096); + unused_capacity = this.list.unusedCapacitySlice(); + } + + std.debug.assert(unused_capacity.len > 0); + + var next_in = this.input[this.total_in..]; + + var in_remaining = next_in.len; + var out_remaining = unused_capacity.len; + + // https://github.com/google/brotli/blob/fef82ea10435abb1500b615b1b2c6175d429ec6c/go/cbrotli/reader.go#L15-L27 + const result = this.brotli.decompressStream( + &in_remaining, + @ptrCast(&next_in), + &out_remaining, + @ptrCast(&unused_capacity.ptr), + null, + ); + + const bytes_written = unused_capacity.len -| out_remaining; + const bytes_read = next_in.len -| in_remaining; + + this.list.items.len += bytes_written; + this.total_in += bytes_read; + + switch (result) { + .success => { + if (comptime bun.Environment.allow_assert) { + std.debug.assert(this.brotli.isFinished()); + } + + this.end(); + return; + }, + .err => { + this.state = .Error; + if (comptime bun.Environment.allow_assert) { + const code = this.brotli.getErrorCode(); + bun.Output.debugWarn("Brotli error: {s} ({d})", .{ @tagName(code), @intFromEnum(code) }); + } + + return error.BrotliDecompressionError; + }, + + .needs_more_input => { + this.state = .Inflating; + if (is_done) { + this.state = .Error; + } + + return error.ShortRead; + }, + .needs_more_output => { + try this.list.ensureTotalCapacity(this.list_allocator, this.list.capacity + 4096); + this.state = .Inflating; + }, + } + } + } + + pub fn deinit(this: *BrotliReaderArrayList) void { + this.brotli.destroyInstance(); + this.destroy(); + } +}; diff --git a/src/bun.zig b/src/bun.zig index 8f3705d3e1..6fef4c252c 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -2794,7 +2794,7 @@ pub noinline fn outOfMemory() noreturn { pub const is_heap_breakdown_enabled = Environment.allow_assert and Environment.isMac; -const HeapBreakdown = if (is_heap_breakdown_enabled) @import("./heap_breakdown.zig") else struct {}; +pub const HeapBreakdown = if (is_heap_breakdown_enabled) @import("./heap_breakdown.zig") else struct {}; /// Globally-allocate a value on the heap. /// @@ -3010,3 +3010,9 @@ pub const S = if (Environment.isWindows) windows.libuv.S else std.os.S; /// Deprecated! pub const trait = @import("./trait.zig"); + +pub const brotli = @import("./brotli.zig"); + +/// macOS-only libcompression +/// It supports brotli without needing to link to libbrotlidec +pub const CompressionFramework = @import("./deps/libcompression.zig").CompressionFramework; diff --git a/src/c.zig b/src/c.zig index 285528fa52..ed9ba6fca0 100644 --- a/src/c.zig +++ b/src/c.zig @@ -380,7 +380,7 @@ const LazyStatus = enum { failed, }; -fn _dlsym(handle: ?*anyopaque, name: [:0]const u8) ?*anyopaque { +pub fn _dlsym(handle: ?*anyopaque, name: [:0]const u8) ?*anyopaque { if (comptime Environment.isWindows) { return bun.windows.GetProcAddressA(handle, name); } else if (comptime Environment.isMac or Environment.isLinux) { diff --git a/src/deps/brotli_decoder.zig b/src/deps/brotli_decoder.zig new file mode 100644 index 0000000000..1a69ef2b4c --- /dev/null +++ b/src/deps/brotli_decoder.zig @@ -0,0 +1,322 @@ +const bun = @import("root").bun; +const std = @import("std"); + +pub const brotli_alloc_func = ?*const fn (?*anyopaque, usize) callconv(.C) ?*anyopaque; +pub const brotli_free_func = ?*const fn (?*anyopaque, *anyopaque) callconv(.C) void; +pub const struct_BrotliSharedDictionaryStruct = opaque {}; +pub const BrotliSharedDictionary = struct_BrotliSharedDictionaryStruct; +pub const BROTLI_SHARED_DICTIONARY_RAW: c_int = 0; +pub const BROTLI_SHARED_DICTIONARY_SERIALIZED: c_int = 1; +pub const enum_BrotliSharedDictionaryType = c_uint; +pub const BrotliSharedDictionaryType = enum_BrotliSharedDictionaryType; +// pub extern fn BrotliSharedDictionaryCreateInstance(alloc_func: brotli_alloc_func, free_func: brotli_free_func, @"opaque": ?*anyopaque) ?*BrotliSharedDictionary; +// pub extern fn BrotliSharedDictionaryDestroyInstance(dict: ?*BrotliSharedDictionary) void; +// pub extern fn BrotliSharedDictionaryAttach(dict: ?*BrotliSharedDictionary, @"type": BrotliSharedDictionaryType, data_size: usize, data: [*]const u8) c_int; +pub const BrotliDecoder = opaque { + const BrotliDecoderSetParameterFnType = fn (state: *BrotliDecoder, param: BrotliDecoderParameter, value: u32) callconv(.C) c_int; + const BrotliDecoderAttachDictionaryFnType = fn (state: *BrotliDecoder, @"type": BrotliSharedDictionaryType, data_size: usize, data: [*]const u8) callconv(.C) c_int; + const BrotliDecoderCreateInstanceFnType = fn (alloc_func: brotli_alloc_func, free_func: brotli_free_func, @"opaque": ?*anyopaque) callconv(.C) ?*BrotliDecoder; + const BrotliDecoderDestroyInstanceFnType = fn (state: *BrotliDecoder) callconv(.C) void; + const BrotliDecoderDecompressFnType = fn (encoded_size: usize, encoded_buffer: [*]const u8, decoded_size: *usize, decoded_buffer: [*]u8) callconv(.C) BrotliDecoderResult; + const BrotliDecoderDecompressStreamFnType = fn (state: *BrotliDecoder, available_in: *usize, next_in: *?[*]const u8, available_out: *usize, next_out: *?[*]u8, total_out: ?*usize) callconv(.C) BrotliDecoderResult; + const BrotliDecoderHasMoreOutputFnType = fn (state: *const BrotliDecoder) callconv(.C) c_int; + const BrotliDecoderTakeOutputFnType = fn (state: *BrotliDecoder, size: *usize) callconv(.C) ?[*]const u8; + const BrotliDecoderIsUsedFnType = fn (state: *const BrotliDecoder) callconv(.C) c_int; + const BrotliDecoderIsFinishedFnType = fn (state: *const BrotliDecoder) callconv(.C) c_int; + const BrotliDecoderGetErrorCodeFnType = fn (state: *const BrotliDecoder) callconv(.C) BrotliDecoderErrorCode; + const BrotliDecoderErrorStringFnType = fn (c: BrotliDecoderErrorCode) callconv(.C) ?[*:0]const u8; + const BrotliDecoderVersionFnType = fn () callconv(.C) u32; + const BrotliDecoderSetMetadataCallbacks = fn (state: *BrotliDecoder, start_func: brotli_decoder_metadata_start_func, chunk_func: brotli_decoder_metadata_chunk_func, @"opaque": ?*anyopaque) callconv(.C) void; + const brotli_decoder_metadata_start_func = ?*const fn (?*anyopaque, usize) callconv(.C) void; + const brotli_decoder_metadata_chunk_func = ?*const fn (?*anyopaque, [*]const u8, usize) callconv(.C) void; + + var brotli_handle: ?*anyopaque = null; + + fn loadBrotli() ?*anyopaque { + if (brotli_handle != null) { + return brotli_handle; + } + + brotli_handle = bun.C.dlopen("brotlidec", 1) orelse brk: { + var arena = std.heap.ArenaAllocator.init(bun.default_allocator); + defer arena.deinit(); + const output = std.ChildProcess.run(.{ + .allocator = arena.allocator(), + .argv = &.{ + "pkg-config", + "--libs", + "libbrotlidec", + }, + .max_output_bytes = 8192, + }) catch break :brk null; + if (!(output.term == .Exited and output.term.Exited == 0)) { + break :brk null; + } + + if (!bun.strings.hasPrefixComptime(output.stdout, "-L")) { + break :brk null; + } + + var lib_path = output.stdout[2..]; + if (lib_path.len == 0) { + break :brk null; + } + + if (bun.strings.indexOfChar(lib_path, ' ')) |i| { + lib_path = lib_path[0..i]; + } + + const absolute = std.fmt.allocPrintZ( + arena.allocator(), + "{s}" ++ std.fs.path.sep_str ++ "libbrotlidec." ++ + if (bun.Environment.isWindows) + "dll" + else if (bun.Environment.isLinux) + "so" + else + "dylib", + .{ + bun.strings.withoutTrailingSlash(lib_path), + }, + ) catch break :brk null; + + break :brk bun.C.dlopen(absolute, 1); + }; + return brotli_handle; + } + + pub fn hasLoaded() bool { + return did_load_brotli orelse false; + } + + var BrotliDecoderSetParameter: ?*const BrotliDecoderSetParameterFnType = null; + var BrotliDecoderAttachDictionary: ?*const BrotliDecoderAttachDictionaryFnType = null; + var BrotliDecoderCreateInstance: ?*const BrotliDecoderCreateInstanceFnType = null; + var BrotliDecoderDestroyInstance: ?*const BrotliDecoderDestroyInstanceFnType = null; + var BrotliDecoderDecompress: ?*const BrotliDecoderDecompressFnType = null; + var BrotliDecoderDecompressStream: ?*const BrotliDecoderDecompressStreamFnType = null; + var BrotliDecoderHasMoreOutput: ?*const BrotliDecoderHasMoreOutputFnType = null; + var BrotliDecoderTakeOutput: ?*const BrotliDecoderTakeOutputFnType = null; + var BrotliDecoderIsUsed: ?*const BrotliDecoderIsUsedFnType = null; + var BrotliDecoderIsFinished: ?*const BrotliDecoderIsFinishedFnType = null; + var BrotliDecoderGetErrorCode: ?*const BrotliDecoderGetErrorCodeFnType = null; + var BrotliDecoderErrorString: ?*const BrotliDecoderErrorStringFnType = null; + var BrotliDecoderVersion: ?*const BrotliDecoderVersionFnType = null; + var did_load_brotli: ?bool = null; + + pub fn setParameter(state: *BrotliDecoder, param: BrotliDecoderParameter, value: u32) callconv(.C) c_int { + return BrotliDecoderSetParameter.?(state, param, value); + } + + pub fn attachDictionary(state: *BrotliDecoder, @"type": BrotliSharedDictionaryType, data: []const u8) callconv(.C) c_int { + return BrotliDecoderAttachDictionary.?(state, @"type", data.len, data.ptr); + } + + pub fn createInstance(alloc_func: brotli_alloc_func, free_func: brotli_free_func, @"opaque": ?*anyopaque) callconv(.C) ?*BrotliDecoder { + return BrotliDecoderCreateInstance.?(alloc_func, free_func, @"opaque"); + } + + pub fn destroyInstance(state: *BrotliDecoder) callconv(.C) void { + return BrotliDecoderDestroyInstance.?(state); + } + + pub fn decompress(encoded: []const u8, decoded: *[]u8) callconv(.C) BrotliDecoderResult { + return BrotliDecoderDecompress.?(encoded.len, encoded.ptr, &decoded.len, decoded.ptr); + } + + pub fn decompressStream(state: *BrotliDecoder, available_in: *usize, next_in: *?[*]const u8, available_out: *usize, next_out: *?[*]u8, total_out: ?*usize) callconv(.C) BrotliDecoderResult { + return BrotliDecoderDecompressStream.?( + state, + available_in, + next_in, + available_out, + next_out, + total_out, + ); + } + + pub fn hasMoreOutput(state: *const BrotliDecoder) callconv(.C) bool { + return BrotliDecoderHasMoreOutput.?(state) != 0; + } + + pub fn takeOutput(state: *BrotliDecoder) callconv(.C) []const u8 { + var max_size: usize = std.math.maxInt(usize); + const ptr = BrotliDecoderTakeOutput.?(state, &max_size) orelse return ""; + return ptr[0..max_size]; + } + + pub fn isUsed(state: *const BrotliDecoder) callconv(.C) bool { + return BrotliDecoderIsUsed.?(state) != 0; + } + + pub fn isFinished(state: *const BrotliDecoder) callconv(.C) bool { + return BrotliDecoderIsFinished.?(state) != 0; + } + + pub fn getErrorCode(state: *const BrotliDecoder) callconv(.C) BrotliDecoderErrorCode { + return BrotliDecoderGetErrorCode.?(state); + } + + pub fn errorString(c: BrotliDecoderErrorCode) callconv(.C) [:0]const u8 { + return bun.sliceTo(BrotliDecoderErrorString.?(c) orelse "", 0); + } + + pub fn version() callconv(.C) u32 { + return BrotliDecoderVersion.?(); + } + + pub fn initializeBrotli() bool { + if (did_load_brotli) |did| { + return did; + } + + defer { + if (comptime bun.Environment.isDebug) { + if (did_load_brotli) |did| { + if (!did) { + bun.Output.debugWarn("failed to load Brotli", .{}); + } + } + } + } + + const handle = loadBrotli() orelse { + did_load_brotli = false; + return false; + }; + + BrotliDecoderSetParameter = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderSetParameter") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderAttachDictionary = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderAttachDictionary") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderCreateInstance = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderCreateInstance") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderDestroyInstance = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderDestroyInstance") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderDecompress = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderDecompress") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderDecompressStream = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderDecompressStream") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderHasMoreOutput = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderHasMoreOutput") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderTakeOutput = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderTakeOutput") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderIsUsed = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderIsUsed") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderIsFinished = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderIsFinished") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderGetErrorCode = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderGetErrorCode") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderErrorString = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderErrorString") orelse { + did_load_brotli = false; + return false; + })); + BrotliDecoderVersion = @alignCast(@ptrCast(bun.C._dlsym(handle, "BrotliDecoderVersion") orelse { + did_load_brotli = false; + return false; + })); + + return true; + } +}; +pub const BrotliDecoderResult = enum(c_uint) { + err = 0, + success = 1, + needs_more_input = 2, + needs_more_output = 3, +}; +pub const BROTLI_DECODER_NO_ERROR: c_int = 0; +pub const BROTLI_DECODER_SUCCESS: c_int = 1; +pub const BROTLI_DECODER_NEEDS_MORE_INPUT: c_int = 2; +pub const BROTLI_DECODER_NEEDS_MORE_OUTPUT: c_int = 3; +pub const BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE: c_int = -1; +pub const BROTLI_DECODER_ERROR_FORMAT_RESERVED: c_int = -2; +pub const BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE: c_int = -3; +pub const BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET: c_int = -4; +pub const BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME: c_int = -5; +pub const BROTLI_DECODER_ERROR_FORMAT_CL_SPACE: c_int = -6; +pub const BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE: c_int = -7; +pub const BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT: c_int = -8; +pub const BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1: c_int = -9; +pub const BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2: c_int = -10; +pub const BROTLI_DECODER_ERROR_FORMAT_TRANSFORM: c_int = -11; +pub const BROTLI_DECODER_ERROR_FORMAT_DICTIONARY: c_int = -12; +pub const BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: c_int = -13; +pub const BROTLI_DECODER_ERROR_FORMAT_PADDING_1: c_int = -14; +pub const BROTLI_DECODER_ERROR_FORMAT_PADDING_2: c_int = -15; +pub const BROTLI_DECODER_ERROR_FORMAT_DISTANCE: c_int = -16; +pub const BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY: c_int = -18; +pub const BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET: c_int = -19; +pub const BROTLI_DECODER_ERROR_INVALID_ARGUMENTS: c_int = -20; +pub const BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES: c_int = -21; +pub const BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS: c_int = -22; +pub const BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP: c_int = -25; +pub const BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1: c_int = -26; +pub const BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2: c_int = -27; +pub const BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES: c_int = -30; +pub const BROTLI_DECODER_ERROR_UNREACHABLE: c_int = -31; +pub const BrotliDecoderErrorCode = enum(c_int) { + FORMAT_EXUBERANT_NIBBLE = -1, + FORMAT_RESERVED = -2, + FORMAT_EXUBERANT_META_NIBBLE = -3, + FORMAT_SIMPLE_HUFFMAN_ALPHABET = -4, + FORMAT_SIMPLE_HUFFMAN_SAME = -5, + FORMAT_CL_SPACE = -6, + FORMAT_HUFFMAN_SPACE = -7, + FORMAT_CONTEXT_MAP_REPEAT = -8, + FORMAT_BLOCK_LENGTH_1 = -9, + FORMAT_BLOCK_LENGTH_2 = -10, + FORMAT_TRANSFORM = -11, + FORMAT_DICTIONARY = -12, + FORMAT_WINDOW_BITS = -13, + FORMAT_PADDING_1 = -14, + FORMAT_PADDING_2 = -15, + FORMAT_DISTANCE = -16, + COMPOUND_DICTIONARY = -18, + DICTIONARY_NOT_SET = -19, + INVALID_ARGUMENTS = -20, + ALLOC_CONTEXT_MODES = -21, + ALLOC_TREE_GROUPS = -22, + ALLOC_CONTEXT_MAP = -25, + ALLOC_RING_BUFFER_1 = -26, + ALLOC_RING_BUFFER_2 = -27, + ALLOC_BLOCK_TYPE_TREES = -30, + UNREACHABLE = -31, +}; +pub const BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION: c_int = 0; +pub const BROTLI_DECODER_PARAM_LARGE_WINDOW: c_int = 1; +pub const BrotliDecoderParameter = enum(c_uint) { + DISABLE_RING_BUFFER_REALLOCATION = 0, + LARGE_WINDOW = 1, +}; + +pub const BROTLI_UINT32_MAX = ~@import("std").zig.c_translation.cast(u32, @as(c_int, 0)); +pub const BROTLI_SIZE_MAX = ~@import("std").zig.c_translation.cast(usize, @as(c_int, 0)); +pub const SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH = @as(c_int, 4); +pub const SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH = @as(c_int, 31); +pub const SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS = @as(c_int, 64); +pub const SHARED_BROTLI_MAX_COMPOUND_DICTS = @as(c_int, 15); +pub const BROTLI_LAST_ERROR_CODE = BROTLI_DECODER_ERROR_UNREACHABLE; +pub const BrotliSharedDictionaryStruct = struct_BrotliSharedDictionaryStruct; diff --git a/src/deps/libcompression.zig b/src/deps/libcompression.zig new file mode 100644 index 0000000000..7809b465fd --- /dev/null +++ b/src/deps/libcompression.zig @@ -0,0 +1,400 @@ +const std = @import("std"); +const bun = @import("root").bun; + +fn macOSOnly() noreturn { + @panic("CompressionFramework is only available on macOS. This code should not be reachable."); +} + +/// https://developer.apple.com/documentation/compression?language=objc +/// We only use this for Brotli on macOS, to avoid linking in libbrotli. +/// +/// Note: this doesn't seem to work for gzip. +pub const CompressionFramework = struct { + var handle: ?*anyopaque = null; + + pub fn isAvailable() bool { + if (comptime !bun.Environment.isMac) { + return false; + } + const cached_value = struct { + pub var value: ?bool = null; + }; + + if (cached_value.value == null) { + if (bun.getenvZ("BUN_DISABLE_COMPRESSION_FRAMEWORK") != null) { + cached_value.value = false; + return false; + } + + cached_value.value = CompressionFramework.load(); + } + + return cached_value.value.?; + } + + pub fn load() bool { + if (comptime !bun.Environment.isMac) { + return false; + } + if (handle != null) { + return true; + } + handle = std.os.darwin.dlopen("libcompression.dylib", 1); + + if (handle == null) + return false; + + compression_encode_scratch_buffer_size = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_encode_scratch_buffer_size") orelse return false)); + compression_encode_buffer = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_encode_buffer") orelse return false)); + compression_decode_scratch_buffer_size = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_decode_scratch_buffer_size") orelse return false)); + compression_decode_buffer = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_decode_buffer") orelse return false)); + compression_stream_init = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_stream_init") orelse return false)); + compression_stream_process = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_stream_process") orelse return false)); + compression_stream_destroy = @alignCast(@ptrCast(std.c.dlsym(handle, "compression_stream_destroy") orelse return false)); + + return true; + } + + pub const compression_algorithm = enum(c_uint) { + LZ4 = 256, + ZLIB = 0x205, + LZMA = 774, + LZ4_RAW = 257, + BROTLI = 2818, + LZFSE = 2049, + LZBITMAP = 1794, + + pub fn fromName(name: []const u8) ?compression_algorithm { + if (std.mem.endsWith(u8, name, ".br")) { + return .BROTLI; + } else if (std.mem.endsWith(u8, name, ".lz4")) { + return .LZ4; + } else if (std.mem.endsWith(u8, name, ".lzma")) { + return .LZMA; + } else if (std.mem.endsWith(u8, name, ".lzfse")) { + return .LZFSE; + } else if (std.mem.endsWith(u8, name, ".zlib") or std.mem.endsWith(u8, name, ".gz")) { + return .ZLIB; + } else { + return null; + } + } + }; + const compression_encode_scratch_buffer_size_type = fn (algorithm: compression_algorithm) callconv(.C) usize; + const compression_encode_buffer_type = fn (noalias dst_buffer: [*]u8, dst_size: usize, noalias src_buffer: ?[*]const u8, src_size: usize, noalias scratch_buffer: ?*anyopaque, algorithm: compression_algorithm) callconv(.C) usize; + const compression_decode_scratch_buffer_size_type = fn (algorithm: compression_algorithm) callconv(.C) usize; + const compression_decode_buffer_type = fn (noalias dst_buffer: [*]u8, dst_size: usize, noalias src_buffer: ?[*]const u8, src_size: usize, noalias scratch_buffer: ?*anyopaque, algorithm: compression_algorithm) callconv(.C) usize; + + const compression_stream_init_type = fn (stream: *compression_stream, operation: compression_stream_operation, algorithm: compression_algorithm) callconv(.C) compression_status; + const compression_stream_process_type = fn (stream: *compression_stream, flags: c_int) callconv(.C) compression_status; + const compression_stream_destroy_type = fn (stream: *compression_stream) callconv(.C) compression_status; + + var compression_encode_scratch_buffer_size: *const compression_encode_scratch_buffer_size_type = undefined; + var compression_encode_buffer: *const compression_encode_buffer_type = undefined; + var compression_decode_scratch_buffer_size: *const compression_decode_scratch_buffer_size_type = undefined; + var compression_decode_buffer: *const compression_decode_buffer_type = undefined; + + var compression_stream_init: *const compression_stream_init_type = undefined; + var compression_stream_process: *const compression_stream_process_type = undefined; + var compression_stream_destroy: *const compression_stream_destroy_type = undefined; + pub const compression_stream = extern struct { + dst_ptr: ?[*]u8 = null, + dst_size: usize = 0, + src_ptr: ?[*]const u8 = null, + src_size: usize = 0, + state: ?*anyopaque = null, + + pub fn init(src: []const u8, operation: compression_stream_operation, algorithm: compression_algorithm) !compression_stream { + var stream = compression_stream{ + .src_ptr = src.ptr, + .src_size = src.len, + .dst_ptr = null, + .dst_size = 0, + }; + + switch (compression_stream_init(&stream, operation, algorithm)) { + .OK => {}, + .ERROR => return error.@"failed to initialize compression stream", + .END => return error.@"compression stream init returned END", + } + + return stream; + } + + pub fn deinit(this: *compression_stream) void { + if (comptime !bun.Environment.isMac) { + macOSOnly(); + } + + _ = compression_stream_destroy(this); + } + + pub fn process(stream: *compression_stream, data: []const u8, is_done: bool, comptime Iterator: type, iter: *Iterator) !StreamResult { + if (comptime !bun.Environment.isMac) { + macOSOnly(); + } + stream.src_ptr = data.ptr; + stream.src_size = data.len; + + const initial_dest = try iter.wrote(0); + stream.dst_ptr = initial_dest.ptr; + stream.dst_size = initial_dest.len; + + var total_written: usize = 0; + while (true) { + var flags: c_int = 0; + if (stream.src_size == 0 and is_done) { + flags = COMPRESSION_STREAM_FINALIZE; + } else if (stream.src_size == 0) { + return .{ + .progress = .{ + .read = data.len - stream.src_size, + .wrote = total_written, + }, + }; + } + + const prev_size = stream.dst_size; + const rc = compression_stream_process(stream, flags); + const wrote = prev_size - stream.dst_size; + switch (rc) { + .OK => { + const new_buffer = try iter.wrote(wrote); + stream.dst_ptr = new_buffer.ptr; + stream.dst_size = new_buffer.len; + total_written += wrote; + }, + .END => { + _ = try iter.wrote(wrote); + total_written += wrote; + + return .{ + .done = .{ + .read = data.len - stream.src_size, + .wrote = total_written, + }, + }; + }, + .ERROR => { + return .{ + .err = .{ + .err = error.@"failed to process compression stream", + .read = data.len - stream.src_size, + .wrote = total_written, + }, + }; + }, + } + } + } + }; + pub const COMPRESSION_STREAM_ENCODE: c_int = 0; + pub const COMPRESSION_STREAM_DECODE: c_int = 1; + pub const compression_stream_operation = enum(c_uint) { + ENCODE = 0, + DECODE = 1, + }; + pub const COMPRESSION_STREAM_FINALIZE: c_int = 1; + pub const compression_stream_flags = c_uint; + pub const compression_status = enum(c_int) { + OK = 0, + ERROR = -1, + END = 1, + }; + + const StreamResult = union(enum) { + done: struct { + read: usize = 0, + wrote: usize = 0, + }, + err: struct { + read: usize = 0, + wrote: usize = 0, + err: anyerror, + }, + progress: struct { + read: usize = 0, + wrote: usize = 0, + }, + }; + + pub fn compress(data: []const u8, algorithm: compression_algorithm, is_done: bool, writer: anytype) !StreamResult { + if (comptime !bun.Environment.isMac) { + macOSOnly(); + } + + var scratch_buffer: [64 * 1024]u8 = undefined; + + const scratch_buffer_size = compression_encode_scratch_buffer_size(algorithm); + if (scratch_buffer_size >= scratch_buffer.len) { + std.debug.panic("scratch buffer size is too small {d}", .{scratch_buffer_size}); + } + + var stream = try compression_stream.init(data, .ENCODE, algorithm); + + defer _ = compression_stream_destroy(&stream); + const Iterator = struct { + writer: @TypeOf(writer), + scratch_buffer: []u8, + pub fn wrote(this: *@This(), w: usize) ![]u8 { + try this.writer.writeAll(this.scratch_buffer[0..w]); + return this.scratch_buffer; + } + }; + + var iter = Iterator{ + .writer = writer, + .scratch_buffer = &scratch_buffer, + }; + + return try stream.process(data, is_done, Iterator, &iter); + } + + pub fn decompress(data: []const u8, algorithm: compression_algorithm, is_done: bool, writer: anytype) !StreamResult { + if (comptime !bun.Environment.isMac) { + macOSOnly(); + } + + var scratch_buffer: [64 * 1024]u8 = undefined; + + const scratch_buffer_size = compression_decode_scratch_buffer_size(algorithm); + if (scratch_buffer_size >= scratch_buffer.len) { + std.debug.panic("scratch buffer size is too small {d}", .{scratch_buffer_size}); + } + + var stream = try compression_stream.init(data, .DECODE, algorithm); + defer _ = compression_stream_destroy(&stream); + + const Iterator = struct { + writer: @TypeOf(writer), + scratch_buffer: []u8, + pub fn wrote(this: *@This(), w: usize) ![]u8 { + try this.writer.writeAll(this.scratch_buffer[0..w]); + return this.scratch_buffer; + } + }; + + var iter = Iterator{ + .writer = writer, + .scratch_buffer = &scratch_buffer, + }; + + return try stream.process(data, is_done, Iterator, &iter); + } + + pub const DecompressionArrayList = struct { + pub const State = enum { + Uninitialized, + Inflating, + End, + Error, + }; + + input: []const u8, + list: std.ArrayListUnmanaged(u8), + list_allocator: std.mem.Allocator, + list_ptr: *std.ArrayListUnmanaged(u8), + stream: CompressionFramework.compression_stream, + state: State = State.Uninitialized, + total_out: usize = 0, + total_in: usize = 0, + total_read: usize = 0, + + pub usingnamespace bun.New(DecompressionArrayList); + + pub fn initWithOptions(input: []const u8, list: *std.ArrayListUnmanaged(u8), allocator: std.mem.Allocator, algorithm: CompressionFramework.compression_algorithm) !*DecompressionArrayList { + if (comptime !bun.Environment.isMac) { + macOSOnly(); + } + + if (!CompressionFramework.load()) { + return error.CompressionFrameworkFailedToLoad; + } + + const stream = try CompressionFramework.compression_stream.init(input, .DECODE, algorithm); + std.debug.assert(list.items.ptr != input.ptr); + + return DecompressionArrayList.new( + .{ + .input = input, + .list_ptr = list, + .list = list.*, + .list_allocator = allocator, + .stream = stream, + }, + ); + } + + pub fn deinit(this: *DecompressionArrayList) void { + this.stream.deinit(); + this.destroy(); + } + + pub fn readAll(this: *DecompressionArrayList, is_done: bool) !void { + if (this.state == State.End or this.state == State.Error or this.input.len == 0) { + return; + } + defer this.list_ptr.* = this.list; + + var scratch_buffer = this.list.unusedCapacitySlice(); + + if (scratch_buffer.len < 4096) { + try this.list.ensureUnusedCapacity(this.list_allocator, 4096); + scratch_buffer = this.list.unusedCapacitySlice(); + } + + const Iterator = struct { + list: *std.ArrayListUnmanaged(u8), + scratch_buffer: []u8, + list_allocator: std.mem.Allocator, + pub fn wrote(i: *@This(), w: usize) ![]u8 { + i.list.items.len += w; + i.scratch_buffer = i.list.unusedCapacitySlice(); + + if (i.scratch_buffer.len < 4096) { + try i.list.ensureUnusedCapacity(i.list_allocator, 4096); + } + + i.scratch_buffer = i.list.unusedCapacitySlice(); + + return i.scratch_buffer; + } + }; + + var iter = Iterator{ + .list = &this.list, + .list_allocator = this.list_allocator, + .scratch_buffer = scratch_buffer, + }; + + const result = try CompressionFramework.compression_stream.process(&this.stream, this.input, is_done, Iterator, &iter); + switch (result) { + .done => |done| { + this.total_out += done.wrote; + this.total_in += done.read; + + this.state = State.End; + return; + }, + .err => |*err| { + this.state = State.Error; + this.total_out += err.wrote; + this.total_in += err.read; + + return err.err; + }, + .progress => |*progress| { + this.total_out += progress.wrote; + this.total_in += progress.read; + this.total_read += progress.read; + + if (progress.read < this.input.len) { + return error.ShortRead; + } + + return; + }, + } + } + }; +}; diff --git a/src/heap_breakdown.zig b/src/heap_breakdown.zig index 3c4a836c05..c9bff94879 100644 --- a/src/heap_breakdown.zig +++ b/src/heap_breakdown.zig @@ -3,27 +3,10 @@ const std = @import("std"); const HeapBreakdown = @This(); pub fn allocator(comptime T: type) std.mem.Allocator { - const Holder = struct { - pub var zone_t: std.atomic.Value(?*malloc_zone_t) = std.atomic.Value(?*malloc_zone_t).init(null); - pub var zone_t_lock: bun.Lock = bun.Lock.init(); - }; - const zone = Holder.zone_t.load(.Monotonic) orelse brk: { - Holder.zone_t_lock.lock(); - defer Holder.zone_t_lock.unlock(); - - if (Holder.zone_t.load(.Monotonic)) |z| { - break :brk z; - } - - const z = malloc_zone_t.create(T); - Holder.zone_t.store(z, .Monotonic); - break :brk z; - }; - - return zone.getAllocator(); + return malloc_zone_t.get(T).getAllocator(); } -const malloc_zone_t = opaque { +pub const malloc_zone_t = opaque { const Allocator = std.mem.Allocator; const vm_size_t = usize; @@ -48,6 +31,25 @@ const malloc_zone_t = opaque { pub extern fn malloc_get_zone_name(zone: *malloc_zone_t) ?[*:0]const u8; pub extern fn malloc_zone_pressure_relief(zone: *malloc_zone_t, goal: usize) usize; + pub fn get(comptime T: type) *malloc_zone_t { + const Holder = struct { + pub var zone_t: std.atomic.Value(?*malloc_zone_t) = std.atomic.Value(?*malloc_zone_t).init(null); + pub var zone_t_lock: bun.Lock = bun.Lock.init(); + }; + return Holder.zone_t.load(.Monotonic) orelse brk: { + Holder.zone_t_lock.lock(); + defer Holder.zone_t_lock.unlock(); + + if (Holder.zone_t.load(.Monotonic)) |z| { + break :brk z; + } + + const z = malloc_zone_t.create(T); + Holder.zone_t.store(z, .Monotonic); + break :brk z; + }; + } + fn alignedAlloc(zone: *malloc_zone_t, len: usize, alignment: usize) ?[*]u8 { // The posix_memalign only accepts alignment values that are a // multiple of the pointer size diff --git a/src/http.zig b/src/http.zig index e8fda65c5b..60ef4b4a7c 100644 --- a/src/http.zig +++ b/src/http.zig @@ -21,6 +21,7 @@ const Api = @import("./api/schema.zig").Api; const Lock = @import("./lock.zig").Lock; const HTTPClient = @This(); const Zlib = @import("./zlib.zig"); +const Brotli = bun.brotli; const StringBuilder = @import("./string_builder.zig"); const ThreadPool = bun.ThreadPool; const ObjectPool = @import("./pool.zig").ObjectPool; @@ -45,7 +46,7 @@ var dead_socket = @as(*DeadSocket, @ptrFromInt(1)); //TODO: this needs to be freed when Worker Threads are implemented var socket_async_http_abort_tracker = std.AutoArrayHashMap(u32, *uws.Socket).init(bun.default_allocator); var async_http_id: std.atomic.Value(u32) = std.atomic.Value(u32).init(0); - +const MAX_REDIRECT_URL_LENGTH = 128 * 1024; const print_every = 0; var print_every_i: usize = 0; @@ -114,6 +115,16 @@ pub const HTTPRequestBody = union(enum) { } }; +pub fn canUseBrotli() bool { + if (Environment.isMac) { + if (bun.CompressionFramework.isAvailable()) { + return true; + } + } + + return bun.brotli.hasBrotli(); +} + pub const Sendfile = struct { fd: bun.FileDescriptor, remain: usize = 0, @@ -1194,6 +1205,130 @@ pub const CertificateInfo = struct { } }; +const Decompressor = union(enum) { + zlib: *Zlib.ZlibReaderArrayList, + brotli: *Brotli.BrotliReaderArrayList, + CompressionFramework: *bun.CompressionFramework.DecompressionArrayList, + none: void, + + pub fn deinit(this: *Decompressor) void { + switch (this.*) { + inline .brotli, .CompressionFramework, .zlib => |that| { + that.deinit(); + this.* = .{ .none = {} }; + }, + .none => {}, + } + } + + pub fn updateBuffers(this: *Decompressor, encoding: Encoding, buffer: []const u8, body_out_str: *MutableString) !void { + if (!encoding.isCompressed()) { + return; + } + + if (this.* == .none) { + switch (encoding) { + .gzip, .deflate => { + this.* = .{ + .zlib = try Zlib.ZlibReaderArrayList.initWithOptionsAndListAllocator( + buffer, + &body_out_str.list, + body_out_str.allocator, + default_allocator, + .{ + // zlib.MAX_WBITS = 15 + // to (de-)compress deflate format, use wbits = -zlib.MAX_WBITS + // to (de-)compress deflate format with headers we use wbits = 0 (we can detect the first byte using 120) + // to (de-)compress gzip format, use wbits = zlib.MAX_WBITS | 16 + .windowBits = if (encoding == Encoding.gzip) Zlib.MAX_WBITS | 16 else (if (buffer.len > 1 and buffer[0] == 120) 0 else -Zlib.MAX_WBITS), + }, + ), + }; + return; + }, + .brotli => { + if (bun.CompressionFramework.isAvailable()) { + this.* = .{ + .CompressionFramework = try bun.CompressionFramework.DecompressionArrayList.initWithOptions( + buffer, + &body_out_str.list, + body_out_str.allocator, + .BROTLI, + ), + }; + } else { + this.* = .{ + .brotli = try Brotli.BrotliReaderArrayList.initWithOptions( + buffer, + &body_out_str.list, + body_out_str.allocator, + .{}, + ), + }; + } + + return; + }, + else => @panic("Invalid encoding. This code should not be reachable"), + } + } + + switch (this.*) { + .zlib => |reader| { + std.debug.assert(reader.zlib.avail_in == 0); + reader.zlib.next_in = buffer.ptr; + reader.zlib.avail_in = @as(u32, @truncate(buffer.len)); + + const initial = body_out_str.list.items.len; + body_out_str.list.expandToCapacity(); + if (body_out_str.list.capacity == initial) { + try body_out_str.list.ensureUnusedCapacity(body_out_str.allocator, 4096); + body_out_str.list.expandToCapacity(); + } + reader.list = body_out_str.list; + reader.zlib.next_out = @ptrCast(&body_out_str.list.items[initial]); + reader.zlib.avail_out = @as(u32, @truncate(body_out_str.list.capacity - initial)); + // we reset the total out so we can track how much we decompressed this time + reader.zlib.total_out = @truncate(initial); + }, + .brotli => |reader| { + reader.input = buffer; + reader.total_in = @as(u32, @truncate(buffer.len)); + + const initial = body_out_str.list.items.len; + reader.list = body_out_str.list; + reader.total_out = @truncate(initial); + }, + .CompressionFramework => |reader| { + if (comptime !Environment.isMac) { + @panic("CompressionFramework is not supported on this platform. This code should not be reachable"); + } + + reader.input = buffer; + reader.total_in = @as(u32, @truncate(buffer.len)); + + const initial = body_out_str.list.items.len; + + reader.list = body_out_str.list; + reader.total_out = @truncate(initial); + }, + else => @panic("Invalid encoding. This code should not be reachable"), + } + } + + pub fn readAll(this: *Decompressor, is_done: bool) !void { + switch (this.*) { + .zlib => |zlib| try zlib.readAll(), + .brotli => |brotli| try brotli.readAll(is_done), + .CompressionFramework => |framework| if (!bun.Environment.isMac) + unreachable + else + try framework.readAll(is_done), + .none => {}, + } + } +}; + pub const InternalState = struct { response_message_buffer: MutableString = undefined, /// pending response is the temporary storage for the response headers, url and status code @@ -1214,7 +1349,7 @@ pub const InternalState = struct { encoding: Encoding = Encoding.identity, content_encoding_i: u8 = std.math.maxInt(u8), chunked_decoder: picohttp.phr_chunked_decoder = .{}, - zlib_reader: ?*Zlib.ZlibReaderArrayList = null, + decompressor: Decompressor = .{ .none = {} }, stage: Stage = Stage.pending, /// This is owned by the user and should not be freed here body_out_str: ?*MutableString = null, @@ -1251,10 +1386,7 @@ pub const InternalState = struct { const body_msg = this.body_out_str; if (body_msg) |body| body.reset(); - if (this.zlib_reader) |reader| { - this.zlib_reader = null; - reader.deinit(); - } + this.decompressor.deinit(); // just in case we check and free to avoid leaks if (this.cloned_metadata != null) { @@ -1279,14 +1411,11 @@ pub const InternalState = struct { } pub fn getBodyBuffer(this: *InternalState) *MutableString { - switch (this.encoding) { - Encoding.gzip, Encoding.deflate => { - return &this.compressed_body; - }, - else => { - return this.body_out_str.?; - }, + if (this.encoding.isCompressed()) { + return &this.compressed_body; } + + return this.body_out_str.?; } fn isDone(this: *InternalState) bool { @@ -1302,54 +1431,19 @@ pub const InternalState = struct { return this.received_last_chunk; } - fn decompressConst(this: *InternalState, buffer: []const u8, body_out_str: *MutableString) !void { + fn decompressBytes(this: *InternalState, buffer: []const u8, body_out_str: *MutableString) !void { log("Decompressing {d} bytes\n", .{buffer.len}); - std.debug.assert(!body_out_str.owns(buffer)); + defer this.compressed_body.reset(); var gzip_timer: std.time.Timer = undefined; if (extremely_verbose) gzip_timer = std.time.Timer.start() catch @panic("Timer failure"); - var reader: *Zlib.ZlibReaderArrayList = undefined; - if (this.zlib_reader) |current_reader| { - std.debug.assert(current_reader.zlib.avail_in == 0); - reader = current_reader; - reader.zlib.next_in = buffer.ptr; - reader.zlib.avail_in = @as(u32, @truncate(buffer.len)); - - const initial = body_out_str.list.items.len; - body_out_str.list.expandToCapacity(); - if (body_out_str.list.capacity == initial) { - try body_out_str.list.ensureUnusedCapacity(body_out_str.allocator, 4096); - body_out_str.list.expandToCapacity(); - } - reader.list = body_out_str.list; - reader.zlib.next_out = @ptrCast(&body_out_str.list.items[initial]); - reader.zlib.avail_out = @as(u32, @truncate(body_out_str.list.capacity - initial)); - // we reset the total out so we can track how much we decompressed this time - reader.zlib.total_out = @truncate(initial); - } else { - reader = try Zlib.ZlibReaderArrayList.initWithOptionsAndListAllocator( - buffer, - &body_out_str.list, - body_out_str.allocator, - default_allocator, - .{ - // TODO: add br support today we support gzip and deflate only - // zlib.MAX_WBITS = 15 - // to (de-)compress deflate format, use wbits = -zlib.MAX_WBITS - // to (de-)compress deflate format with headers we use wbits = 0 (we can detect the first byte using 120) - // to (de-)compress gzip format, use wbits = zlib.MAX_WBITS | 16 - .windowBits = if (this.encoding == Encoding.gzip) Zlib.MAX_WBITS | 16 else (if (buffer.len > 1 and buffer[0] == 120) 0 else -Zlib.MAX_WBITS), - }, - ); - this.zlib_reader = reader; - } - - reader.readAll() catch |err| { + try this.decompressor.updateBuffers(this.encoding, buffer, body_out_str); + this.decompressor.readAll(this.isDone()) catch |err| { if (this.isDone() or error.ShortRead != err) { - Output.prettyErrorln("Zlib error: {s}", .{bun.asByteSlice(@errorName(err))}); + Output.prettyErrorln("Decompression error: {s}", .{bun.asByteSlice(@errorName(err))}); Output.flush(); return err; } @@ -1360,14 +1454,14 @@ pub const InternalState = struct { } fn decompress(this: *InternalState, buffer: MutableString, body_out_str: *MutableString) !void { - try this.decompressConst(buffer.list.items, body_out_str); + try this.decompressBytes(buffer.list.items, body_out_str); } pub fn processBodyBuffer(this: *InternalState, buffer: MutableString) !usize { var body_out_str = this.body_out_str.?; switch (this.encoding) { - Encoding.gzip, Encoding.deflate => { + Encoding.brotli, Encoding.gzip, Encoding.deflate => { try this.decompress(buffer, body_out_str); }, else => { @@ -1397,7 +1491,7 @@ verbose: bool = Environment.isTest, remaining_redirect_count: i8 = default_redirect_count, allow_retry: bool = false, redirect_type: FetchRedirect = FetchRedirect.follow, -redirect: ?*URLBufferPool.Node = null, +redirect: []u8 = &.{}, timeout: usize = 0, progress_node: ?*std.Progress.Node = null, received_keep_alive: bool = false, @@ -1445,9 +1539,9 @@ pub fn init( } pub fn deinit(this: *HTTPClient) void { - if (this.redirect) |redirect| { - redirect.release(); - this.redirect = null; + if (this.redirect.len > 0) { + bun.default_allocator.free(this.redirect); + this.redirect = &.{}; } if (this.proxy_authorization) |auth| { this.allocator.free(auth); @@ -1522,8 +1616,7 @@ pub const Encoding = enum { pub fn isCompressed(this: Encoding) bool { return switch (this) { - // we don't support brotli yet - .gzip, .deflate => true, + .brotli, .gzip, .deflate => true, else => false, }; } @@ -1536,8 +1629,10 @@ const connection_closing_header = picohttp.Header{ .name = "Connection", .value const accept_header = picohttp.Header{ .name = "Accept", .value = "*/*" }; const accept_encoding_no_compression = "identity"; -const accept_encoding_compression = "gzip, deflate"; +const accept_encoding_compression = "gzip, deflate, br"; +const accept_encoding_compression_no_brotli = "gzip, deflate"; const accept_encoding_header_compression = picohttp.Header{ .name = "Accept-Encoding", .value = accept_encoding_compression }; +const accept_encoding_header_compression_no_brotli = picohttp.Header{ .name = "Accept-Encoding", .value = accept_encoding_compression_no_brotli }; const accept_encoding_header_no_compression = picohttp.Header{ .name = "Accept-Encoding", .value = accept_encoding_no_compression }; const accept_encoding_header = if (FeatureFlags.disable_compression_in_http_client) @@ -2052,7 +2147,12 @@ pub fn buildRequest(this: *HTTPClient, body_len: usize) picohttp.Request { } if (!override_accept_encoding and !this.disable_decompression) { - request_headers_buf[header_count] = accept_encoding_header; + if (canUseBrotli()) { + request_headers_buf[header_count] = accept_encoding_header; + } else { + request_headers_buf[header_count] = accept_encoding_header_compression_no_brotli; + } + header_count += 1; } @@ -2573,15 +2673,8 @@ pub fn onData(this: *HTTPClient, comptime is_ssl: bool, incoming_data: []const u return; } - var deferred_redirect: ?*URLBufferPool.Node = null; const should_continue = this.handleResponseMetadata( &response, - // If there are multiple consecutive redirects - // and the redirect differs in hostname - // the new URL buffer may point to invalid memory after - // this function is called - // That matters because for Keep Alive, the hostname must point to valid memory - &deferred_redirect, ) catch |err| { if (err == error.Redirect) { this.state.response_message_buffer.deinit(); @@ -2601,11 +2694,6 @@ pub fn onData(this: *HTTPClient, comptime is_ssl: bool, incoming_data: []const u socket.close(0, null); } - if (deferred_redirect) |redirect| { - std.debug.assert(redirect != this.redirect); - // connected_url no longer points to valid memory - redirect.release(); - } this.connected_url = URL{}; this.doRedirect(); return; @@ -3041,7 +3129,8 @@ fn handleResponseBodyFromSinglePacket(this: *HTTPClient, incoming_data: []const try body_buffer.growBy(@max(@as(usize, @intFromFloat(min)), 32)); } - try this.state.decompressConst(incoming_data, body_buffer); + // std.debug.assert(!body_buffer.owns(b)); + try this.state.decompressBytes(incoming_data, body_buffer); } else { try this.state.getBodyBuffer().appendSliceExact(incoming_data); } @@ -3269,7 +3358,6 @@ const ShouldContinue = enum { pub fn handleResponseMetadata( this: *HTTPClient, response: *picohttp.Response, - deferred_redirect: *?*URLBufferPool.Node, ) !ShouldContinue { var location: string = ""; var pretend_304 = false; @@ -3298,6 +3386,9 @@ pub fn handleResponseMetadata( } else if (strings.eqlComptime(header.value, "deflate")) { this.state.encoding = Encoding.deflate; this.state.content_encoding_i = @as(u8, @truncate(header_i)); + } else if (strings.eqlComptime(header.value, "br")) { + this.state.encoding = Encoding.brotli; + this.state.content_encoding_i = @as(u8, @truncate(header_i)); } } }, @@ -3310,6 +3401,10 @@ pub fn handleResponseMetadata( if (!this.disable_decompression) { this.state.transfer_encoding = Encoding.deflate; } + } else if (strings.eqlComptime(header.value, "br")) { + if (!this.disable_decompression) { + this.state.transfer_encoding = Encoding.brotli; + } } else if (strings.eqlComptime(header.value, "identity")) { this.state.transfer_encoding = Encoding.identity; } else if (strings.eqlComptime(header.value, "chunked")) { @@ -3391,89 +3486,118 @@ pub fn handleResponseMetadata( 302, 301, 307, 308, 303 => { var is_same_origin = true; - if (strings.indexOf(location, "://")) |i| { - var url_buf = URLBufferPool.get(default_allocator); + { + var url_arena = std.heap.ArenaAllocator.init(bun.default_allocator); + defer url_arena.deinit(); + var fba = std.heap.stackFallback(4096, url_arena.allocator()); + const url_allocator = fba.get(); + if (strings.indexOf(location, "://")) |i| { + var string_builder = bun.StringBuilder{}; - const is_protocol_relative = i == 0; - const protocol_name = if (is_protocol_relative) this.url.displayProtocol() else location[0..i]; - const is_http = strings.eqlComptime(protocol_name, "http"); - if (is_http or strings.eqlComptime(protocol_name, "https")) {} else { - return error.UnsupportedRedirectProtocol; - } - - if ((protocol_name.len * @as(usize, @intFromBool(is_protocol_relative))) + location.len > url_buf.data.len) { - return error.RedirectURLTooLong; - } - - deferred_redirect.* = this.redirect; - var url_buf_len = location.len; - if (is_protocol_relative) { - if (is_http) { - url_buf.data[0.."http".len].* = "http".*; - bun.copy(u8, url_buf.data["http".len..], location); - url_buf_len += "http".len; - } else { - url_buf.data[0.."https".len].* = "https".*; - bun.copy(u8, url_buf.data["https".len..], location); - url_buf_len += "https".len; + const is_protocol_relative = i == 0; + const protocol_name = if (is_protocol_relative) this.url.displayProtocol() else location[0..i]; + const is_http = strings.eqlComptime(protocol_name, "http"); + if (is_http or strings.eqlComptime(protocol_name, "https")) {} else { + return error.UnsupportedRedirectProtocol; } + + if ((protocol_name.len * @as(usize, @intFromBool(is_protocol_relative))) + location.len > MAX_REDIRECT_URL_LENGTH) { + return error.RedirectURLTooLong; + } + + string_builder.count(location); + + if (is_protocol_relative) { + if (is_http) { + string_builder.count("http"); + } else { + string_builder.count("https"); + } + } + + try string_builder.allocate(url_allocator); + + if (is_protocol_relative) { + if (is_http) { + _ = string_builder.append("http"); + } else { + _ = string_builder.append("https"); + } + } + + _ = string_builder.append(location); + + if (comptime Environment.allow_assert) + std.debug.assert(string_builder.cap == string_builder.len); + + const normalized_url = JSC.URL.hrefFromString(bun.String.fromBytes(string_builder.allocatedSlice())); + defer normalized_url.deref(); + const normalized_url_str = try normalized_url.toOwnedSlice(bun.default_allocator); + + const new_url = URL.parse(normalized_url_str); + is_same_origin = strings.eqlCaseInsensitiveASCII(strings.withoutTrailingSlash(new_url.origin), strings.withoutTrailingSlash(this.url.origin), true); + this.url = new_url; + this.redirect = normalized_url_str; + } else if (strings.hasPrefixComptime(location, "//")) { + var string_builder = bun.StringBuilder{}; + + const protocol_name = this.url.displayProtocol(); + + if (protocol_name.len + 1 + location.len > MAX_REDIRECT_URL_LENGTH) { + return error.RedirectURLTooLong; + } + + const is_http = strings.eqlComptime(protocol_name, "http"); + + if (is_http) { + string_builder.count("http:"); + } else { + string_builder.count("https:"); + } + + string_builder.count(location); + + try string_builder.allocate(url_allocator); + + if (is_http) { + _ = string_builder.append("http:"); + } else { + _ = string_builder.append("https:"); + } + + _ = string_builder.append(location); + + if (comptime Environment.allow_assert) + std.debug.assert(string_builder.cap == string_builder.len); + + const normalized_url = JSC.URL.hrefFromString(bun.String.fromBytes(string_builder.allocatedSlice())); + defer normalized_url.deref(); + const normalized_url_str = try normalized_url.toOwnedSlice(bun.default_allocator); + + const new_url = URL.parse(normalized_url_str); + is_same_origin = strings.eqlCaseInsensitiveASCII(strings.withoutTrailingSlash(new_url.origin), strings.withoutTrailingSlash(this.url.origin), true); + this.url = new_url; + this.redirect = normalized_url_str; } else { - bun.copy(u8, &url_buf.data, location); + const original_url = this.url; + + const new_url_ = bun.JSC.URL.join( + bun.String.fromBytes(original_url.href), + bun.String.fromBytes(location), + ); + defer new_url_.deref(); + + if (new_url_.isEmpty()) { + return error.InvalidRedirectURL; + } + + const new_url = new_url_.toOwnedSlice(bun.default_allocator) catch { + return error.RedirectURLTooLong; + }; + this.url = URL.parse(new_url); + is_same_origin = strings.eqlCaseInsensitiveASCII(strings.withoutTrailingSlash(this.url.origin), strings.withoutTrailingSlash(original_url.origin), true); + this.redirect = new_url; } - - const new_url = URL.parse(url_buf.data[0..url_buf_len]); - is_same_origin = strings.eqlCaseInsensitiveASCII(strings.withoutTrailingSlash(new_url.origin), strings.withoutTrailingSlash(this.url.origin), true); - this.url = new_url; - this.redirect = url_buf; - } else if (strings.hasPrefixComptime(location, "//")) { - var url_buf = URLBufferPool.get(default_allocator); - - const protocol_name = this.url.displayProtocol(); - - if (protocol_name.len + 1 + location.len > url_buf.data.len) { - return error.RedirectURLTooLong; - } - - deferred_redirect.* = this.redirect; - var url_buf_len = location.len; - - if (strings.eqlComptime(protocol_name, "http")) { - url_buf.data[0.."http:".len].* = "http:".*; - bun.copy(u8, url_buf.data["http:".len..], location); - url_buf_len += "http:".len; - } else { - url_buf.data[0.."https:".len].* = "https:".*; - bun.copy(u8, url_buf.data["https:".len..], location); - url_buf_len += "https:".len; - } - - const new_url = URL.parse(url_buf.data[0..url_buf_len]); - is_same_origin = strings.eqlCaseInsensitiveASCII(strings.withoutTrailingSlash(new_url.origin), strings.withoutTrailingSlash(this.url.origin), true); - this.url = new_url; - this.redirect = url_buf; - } else { - var url_buf = URLBufferPool.get(default_allocator); - var fba = std.heap.FixedBufferAllocator.init(&url_buf.data); - const original_url = this.url; - - const new_url_ = bun.JSC.URL.join( - bun.String.fromUTF8(original_url.href), - bun.String.fromUTF8(location), - ); - defer new_url_.deref(); - - if (new_url_.isEmpty()) { - return error.InvalidRedirectURL; - } - - const new_url = new_url_.toOwnedSlice(fba.allocator()) catch { - return error.RedirectURLTooLong; - }; - this.url = URL.parse(new_url); - - is_same_origin = strings.eqlCaseInsensitiveASCII(strings.withoutTrailingSlash(this.url.origin), strings.withoutTrailingSlash(original_url.origin), true); - deferred_redirect.* = this.redirect; - this.redirect = url_buf; } // If one of the following is true diff --git a/src/output.zig b/src/output.zig index 0b55793599..7d729e8621 100644 --- a/src/output.zig +++ b/src/output.zig @@ -733,8 +733,10 @@ pub inline fn warn(comptime fmt: []const u8, args: anytype) void { /// Print a yellow warning message, only in debug mode pub inline fn debugWarn(comptime fmt: []const u8, args: anytype) void { - if (Environment.isDebug) + if (Environment.isDebug) { prettyErrorln("debug warn: " ++ fmt, args); + flush(); + } } /// Print a red error message. The first argument takes an `error_name` value, which can be either diff --git a/src/zlib.zig b/src/zlib.zig index 4bccb4422b..1be5855759 100644 --- a/src/zlib.zig +++ b/src/zlib.zig @@ -316,6 +316,27 @@ pub const ZlibError = error{ ShortRead, }; +const ZlibAllocator = struct { + pub fn alloc(_: *anyopaque, items: uInt, len: uInt) callconv(.C) *anyopaque { + if (comptime bun.is_heap_breakdown_enabled) { + const zone = bun.HeapBreakdown.malloc_zone_t.get(ZlibAllocator); + return zone.malloc_zone_calloc(items, len).?; + } + + return mimalloc.mi_calloc(items, len) orelse unreachable; + } + + pub fn free(_: *anyopaque, data: *anyopaque) callconv(.C) void { + if (comptime bun.is_heap_breakdown_enabled) { + const zone = bun.HeapBreakdown.malloc_zone_t.get(ZlibAllocator); + zone.malloc_zone_free(data); + return; + } + + mimalloc.mi_free(data); + } +}; + pub const ZlibReaderArrayList = struct { const ZlibReader = ZlibReaderArrayList; @@ -334,14 +355,6 @@ pub const ZlibReaderArrayList = struct { allocator: std.mem.Allocator, state: State = State.Uninitialized, - pub fn alloc(_: *anyopaque, items: uInt, len: uInt) callconv(.C) *anyopaque { - return mimalloc.mi_malloc(items * len) orelse unreachable; - } - - pub fn free(_: *anyopaque, data: *anyopaque) callconv(.C) void { - mimalloc.mi_free(data); - } - pub fn deinit(this: *ZlibReader) void { var allocator = this.allocator; this.end(); @@ -393,8 +406,8 @@ pub const ZlibReaderArrayList = struct { .total_out = @truncate(zlib_reader.list.items.len), .err_msg = null, - .alloc_func = ZlibReader.alloc, - .free_func = ZlibReader.free, + .alloc_func = ZlibAllocator.alloc, + .free_func = ZlibAllocator.free, .internal_state = null, .user_data = zlib_reader, diff --git a/test/js/web/fetch/fetch.test.ts b/test/js/web/fetch/fetch.test.ts index edc6bd6b8f..8a6a815791 100644 --- a/test/js/web/fetch/fetch.test.ts +++ b/test/js/web/fetch/fetch.test.ts @@ -1747,7 +1747,8 @@ describe("should handle relative location in the redirect, issue#5635", () => { }); }); -it("should throw RedirectURLTooLong when location is too long", async () => { +it("should allow very long redirect URLS", async () => { + const Location = "/" + "B".repeat(32 * 1024); const server = Bun.serve({ port: 0, async fetch(request: Request) { @@ -1756,7 +1757,7 @@ it("should throw RedirectURLTooLong when location is too long", async () => { if (url.pathname == "/redirect") { return new Response("redirecting", { headers: { - "Location": "B".repeat(8193), + Location, }, status: 302, }); @@ -1767,17 +1768,10 @@ it("should throw RedirectURLTooLong when location is too long", async () => { }, }); - let err = undefined; - try { - gc(); - const resp = await fetch(`http://${server.hostname}:${server.port}/redirect`); - } catch (error) { - gc(); - err = error; - } - expect(err).not.toBeUndefined(); - expect(err).toBeInstanceOf(Error); - expect(err.code).toStrictEqual("RedirectURLTooLong"); + const { url, status } = await fetch(`http://${server.hostname}:${server.port}/redirect`); + + expect(url).toBe(`http://${server.hostname}:${server.port}${Location}`); + expect(status).toBe(404); server.stop(true); });