diff --git a/.gitmodules b/.gitmodules index e8b448aa40..c2c534daeb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -74,4 +74,11 @@ fetchRecurseSubmodules = false url = https://github.com/aklomp/base64.git ignore = dirty depth = 1 - shallow = true \ No newline at end of file + shallow = true +[submodule "src/deps/brotli"] + path = src/deps/brotli + url = https://github.com/google/brotli.git + ignore = dirty + depth = 1 + shallow = true + fetchRecurseSubmodules = false \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index acc7728266..43257fd25a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -316,6 +316,28 @@ WORKDIR $BUN_DIR RUN cd $BUN_DIR && \ make base64 && rm -rf src/deps/base64 Makefile +FROM bun-base as brotli + +ARG DEBIAN_FRONTEND +ARG GITHUB_WORKSPACE +ARG ZIG_PATH +# Directory extracts to "bun-webkit" +ARG WEBKIT_DIR +ARG BUN_RELEASE_DIR +ARG BUN_DEPS_OUT_DIR +ARG BUN_DIR +ARG CPU_TARGET +ENV CPU_TARGET=${CPU_TARGET} + +COPY Makefile ${BUN_DIR}/Makefile +COPY src/deps/brotli ${BUN_DIR}/src/deps/brotli + +WORKDIR $BUN_DIR + +RUN cd $BUN_DIR && \ + make brotli && rm -rf src/deps/brotli Makefile + + FROM bun-base as picohttp ARG DEBIAN_FRONTEND @@ -578,6 +600,7 @@ ENV LIB_ICU_PATH=${WEBKIT_DIR}/lib COPY --from=zlib ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/ COPY --from=base64 ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/ +COPY --from=brotli ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/ COPY --from=libarchive ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/ COPY --from=boringssl ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/ COPY --from=lolhtml ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/ diff --git a/Makefile b/Makefile index 888d102ea1..647d2b97ce 100644 --- a/Makefile +++ b/Makefile @@ -462,6 +462,9 @@ ARCHIVE_FILES_WITHOUT_LIBCRYPTO = $(MINIMUM_ARCHIVE_FILES) \ -lusockets \ -lcares \ -lzstd \ + -lbrotlicommon \ + -lbrotlienc \ + -lbrotlidec \ $(BUN_DEPS_OUT_DIR)/libuwsockets.o ARCHIVE_FILES = $(ARCHIVE_FILES_WITHOUT_LIBCRYPTO) @@ -1366,6 +1369,20 @@ mimalloc: && ninja; cp $(BUN_DEPS_DIR)/mimalloc/$(MIMALLOC_INPUT_PATH) $(BUN_DEPS_OUT_DIR)/$(MIMALLOC_FILE) +.PHONY: brotli +brotli: + rm -rf $(BUN_DEPS_DIR)/brotli/CMakeCache* $(BUN_DEPS_DIR)/brotli/CMakeFiles && (cd src/deps/brotli && make clean || echo "") + cd $(BUN_DEPS_DIR)/brotli; \ + cmake $(CMAKE_FLAGS) \ + -DCMAKE_BUILD_TYPE=Release \ + -DBROTLI_DISABLE_TESTS=ON \ + -DBROTLI_BUNDLED_MODE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_C_FLAGS="$(CFLAGS)" \ + -DCMAKE_CXX_FLAGS="$(CFLAGS)" \ + -GNinja \ + . \ + && ninja && cp libbrotli*.a $(BUN_DEPS_OUT_DIR); mimalloc-wasm: cd $(BUN_DEPS_DIR)/mimalloc; emcmake cmake -DMI_BUILD_SHARED=OFF -DMI_BUILD_STATIC=ON -DMI_BUILD_TESTS=OFF -DMI_BUILD_OBJECT=ON ${MIMALLOC_OVERRIDE_FLAG} -DMI_USE_CXX=ON .; emmake make; @@ -1879,8 +1896,7 @@ cold-jsc-start: misctools/cold-jsc-start.cpp -o cold-jsc-start .PHONY: vendor-without-npm -vendor-without-npm: node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp zlib boringssl libarchive lolhtml sqlite usockets uws tinycc c-ares zstd base64 - +vendor-without-npm: node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp zlib boringssl libarchive lolhtml sqlite usockets uws tinycc c-ares zstd base64 brotli .PHONY: vendor-without-check vendor-without-check: npm-install vendor-without-npm diff --git a/src/deps/brotli b/src/deps/brotli new file mode 160000 index 0000000000..dd3eb162b0 --- /dev/null +++ b/src/deps/brotli @@ -0,0 +1 @@ +Subproject commit dd3eb162b071dbceb227d29bd523a37aff712219 diff --git a/src/deps/brotli.zig b/src/deps/brotli.zig new file mode 100644 index 0000000000..387b27d820 --- /dev/null +++ b/src/deps/brotli.zig @@ -0,0 +1,136 @@ +pub const brotli_alloc_func = ?*const fn (?*anyopaque, usize) callconv(.C) ?*anyopaque; +pub const brotli_free_func = ?*const fn (?*anyopaque, ?*anyopaque) callconv(.C) void; +pub const struct_BrotliSharedDictionaryStruct = opaque {}; +pub const BrotliSharedDictionary = struct_BrotliSharedDictionaryStruct; +pub const BROTLI_SHARED_DICTIONARY_RAW: c_int = 0; +pub const BROTLI_SHARED_DICTIONARY_SERIALIZED: c_int = 1; +pub const enum_BrotliSharedDictionaryType = c_uint; +pub const BrotliSharedDictionaryType = enum_BrotliSharedDictionaryType; +pub extern fn BrotliSharedDictionaryCreateInstance(alloc_func: brotli_alloc_func, free_func: brotli_free_func, @"opaque": ?*anyopaque) ?*BrotliSharedDictionary; +pub extern fn BrotliSharedDictionaryDestroyInstance(dict: ?*BrotliSharedDictionary) void; +pub extern fn BrotliSharedDictionaryAttach(dict: ?*BrotliSharedDictionary, @"type": BrotliSharedDictionaryType, data_size: usize, data: [*c]const u8) c_int; +pub const struct_BrotliDecoderStateStruct = opaque {}; +pub const BrotliDecoderState = struct_BrotliDecoderStateStruct; +pub const BROTLI_DECODER_RESULT_ERROR: c_int = 0; +pub const BROTLI_DECODER_RESULT_SUCCESS: c_int = 1; +pub const BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: c_int = 2; +pub const BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: c_int = 3; +pub const BrotliDecoderResult = c_uint; +pub const BROTLI_DECODER_NO_ERROR: c_int = 0; +pub const BROTLI_DECODER_SUCCESS: c_int = 1; +pub const BROTLI_DECODER_NEEDS_MORE_INPUT: c_int = 2; +pub const BROTLI_DECODER_NEEDS_MORE_OUTPUT: c_int = 3; +pub const BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_NIBBLE: c_int = -1; +pub const BROTLI_DECODER_ERROR_FORMAT_RESERVED: c_int = -2; +pub const BROTLI_DECODER_ERROR_FORMAT_EXUBERANT_META_NIBBLE: c_int = -3; +pub const BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_ALPHABET: c_int = -4; +pub const BROTLI_DECODER_ERROR_FORMAT_SIMPLE_HUFFMAN_SAME: c_int = -5; +pub const BROTLI_DECODER_ERROR_FORMAT_CL_SPACE: c_int = -6; +pub const BROTLI_DECODER_ERROR_FORMAT_HUFFMAN_SPACE: c_int = -7; +pub const BROTLI_DECODER_ERROR_FORMAT_CONTEXT_MAP_REPEAT: c_int = -8; +pub const BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_1: c_int = -9; +pub const BROTLI_DECODER_ERROR_FORMAT_BLOCK_LENGTH_2: c_int = -10; +pub const BROTLI_DECODER_ERROR_FORMAT_TRANSFORM: c_int = -11; +pub const BROTLI_DECODER_ERROR_FORMAT_DICTIONARY: c_int = -12; +pub const BROTLI_DECODER_ERROR_FORMAT_WINDOW_BITS: c_int = -13; +pub const BROTLI_DECODER_ERROR_FORMAT_PADDING_1: c_int = -14; +pub const BROTLI_DECODER_ERROR_FORMAT_PADDING_2: c_int = -15; +pub const BROTLI_DECODER_ERROR_FORMAT_DISTANCE: c_int = -16; +pub const BROTLI_DECODER_ERROR_COMPOUND_DICTIONARY: c_int = -18; +pub const BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET: c_int = -19; +pub const BROTLI_DECODER_ERROR_INVALID_ARGUMENTS: c_int = -20; +pub const BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MODES: c_int = -21; +pub const BROTLI_DECODER_ERROR_ALLOC_TREE_GROUPS: c_int = -22; +pub const BROTLI_DECODER_ERROR_ALLOC_CONTEXT_MAP: c_int = -25; +pub const BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1: c_int = -26; +pub const BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2: c_int = -27; +pub const BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES: c_int = -30; +pub const BROTLI_DECODER_ERROR_UNREACHABLE: c_int = -31; +pub const BrotliDecoderErrorCode = c_int; +pub const BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION: c_int = 0; +pub const BROTLI_DECODER_PARAM_LARGE_WINDOW: c_int = 1; +pub const enum_BrotliDecoderParameter = c_uint; +pub const BrotliDecoderParameter = enum_BrotliDecoderParameter; +pub extern fn BrotliDecoderSetParameter(state: ?*BrotliDecoderState, param: BrotliDecoderParameter, value: u32) c_int; +pub extern fn BrotliDecoderAttachDictionary(state: ?*BrotliDecoderState, @"type": BrotliSharedDictionaryType, data_size: usize, data: [*c]const u8) c_int; +pub extern fn BrotliDecoderCreateInstance(alloc_func: brotli_alloc_func, free_func: brotli_free_func, @"opaque": ?*anyopaque) ?*BrotliDecoderState; +pub extern fn BrotliDecoderDestroyInstance(state: ?*BrotliDecoderState) void; +pub extern fn BrotliDecoderDecompress(encoded_size: usize, encoded_buffer: [*c]const u8, decoded_size: [*c]usize, decoded_buffer: [*c]u8) BrotliDecoderResult; +pub extern fn BrotliDecoderDecompressStream(state: ?*BrotliDecoderState, available_in: [*c]usize, next_in: [*c][*c]const u8, available_out: [*c]usize, next_out: [*c][*c]u8, total_out: [*c]usize) BrotliDecoderResult; +pub extern fn BrotliDecoderHasMoreOutput(state: ?*const BrotliDecoderState) c_int; +pub extern fn BrotliDecoderTakeOutput(state: ?*BrotliDecoderState, size: [*c]usize) [*c]const u8; +pub extern fn BrotliDecoderIsUsed(state: ?*const BrotliDecoderState) c_int; +pub extern fn BrotliDecoderIsFinished(state: ?*const BrotliDecoderState) c_int; +pub extern fn BrotliDecoderGetErrorCode(state: ?*const BrotliDecoderState) BrotliDecoderErrorCode; +pub extern fn BrotliDecoderErrorString(c: BrotliDecoderErrorCode) [*c]const u8; +pub extern fn BrotliDecoderVersion() u32; +pub const brotli_decoder_metadata_start_func = ?*const fn (?*anyopaque, usize) callconv(.C) void; +pub const brotli_decoder_metadata_chunk_func = ?*const fn (?*anyopaque, [*c]const u8, usize) callconv(.C) void; +pub extern fn BrotliDecoderSetMetadataCallbacks(state: ?*BrotliDecoderState, start_func: brotli_decoder_metadata_start_func, chunk_func: brotli_decoder_metadata_chunk_func, @"opaque": ?*anyopaque) void; +pub const BROTLI_TRUE = @as(c_int, 1); +pub const BROTLI_FALSE = @as(c_int, 0); +pub const BROTLI_UINT32_MAX = ~@import("std").zig.c_translation.cast(u32, @as(c_int, 0)); +pub const BROTLI_SIZE_MAX = ~@import("std").zig.c_translation.cast(usize, @as(c_int, 0)); +pub const SHARED_BROTLI_MIN_DICTIONARY_WORD_LENGTH = @as(c_int, 4); +pub const SHARED_BROTLI_MAX_DICTIONARY_WORD_LENGTH = @as(c_int, 31); +pub const SHARED_BROTLI_NUM_DICTIONARY_CONTEXTS = @as(c_int, 64); +pub const SHARED_BROTLI_MAX_COMPOUND_DICTS = @as(c_int, 15); +pub const BROTLI_LAST_ERROR_CODE = BROTLI_DECODER_ERROR_UNREACHABLE; +pub const BrotliSharedDictionaryStruct = struct_BrotliSharedDictionaryStruct; +pub const BrotliDecoderStateStruct = struct_BrotliDecoderStateStruct; + +pub const BROTLI_MODE_GENERIC: c_int = 0; +pub const BROTLI_MODE_TEXT: c_int = 1; +pub const BROTLI_MODE_FONT: c_int = 2; +pub const enum_BrotliEncoderMode = c_uint; +pub const BrotliEncoderMode = enum_BrotliEncoderMode; +pub const BROTLI_OPERATION_PROCESS: c_int = 0; +pub const BROTLI_OPERATION_FLUSH: c_int = 1; +pub const BROTLI_OPERATION_FINISH: c_int = 2; +pub const BROTLI_OPERATION_EMIT_METADATA: c_int = 3; +pub const enum_BrotliEncoderOperation = c_uint; +pub const BrotliEncoderOperation = enum_BrotliEncoderOperation; +pub const BROTLI_PARAM_MODE: c_int = 0; +pub const BROTLI_PARAM_QUALITY: c_int = 1; +pub const BROTLI_PARAM_LGWIN: c_int = 2; +pub const BROTLI_PARAM_LGBLOCK: c_int = 3; +pub const BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING: c_int = 4; +pub const BROTLI_PARAM_SIZE_HINT: c_int = 5; +pub const BROTLI_PARAM_LARGE_WINDOW: c_int = 6; +pub const BROTLI_PARAM_NPOSTFIX: c_int = 7; +pub const BROTLI_PARAM_NDIRECT: c_int = 8; +pub const BROTLI_PARAM_STREAM_OFFSET: c_int = 9; +pub const enum_BrotliEncoderParameter = c_uint; +pub const BrotliEncoderParameter = enum_BrotliEncoderParameter; +pub const struct_BrotliEncoderStateStruct = opaque {}; +pub const BrotliEncoderState = struct_BrotliEncoderStateStruct; +pub extern fn BrotliEncoderSetParameter(state: ?*BrotliEncoderState, param: BrotliEncoderParameter, value: u32) c_int; +pub extern fn BrotliEncoderCreateInstance(alloc_func: brotli_alloc_func, free_func: brotli_free_func, ctx: ?*anyopaque) ?*BrotliEncoderState; +pub extern fn BrotliEncoderDestroyInstance(state: ?*BrotliEncoderState) void; +pub const struct_BrotliEncoderPreparedDictionaryStruct = opaque {}; +pub const BrotliEncoderPreparedDictionary = struct_BrotliEncoderPreparedDictionaryStruct; +pub extern fn BrotliEncoderPrepareDictionary(@"type": BrotliSharedDictionaryType, data_size: usize, data: [*c]const u8, quality: c_int, alloc_func: brotli_alloc_func, free_func: brotli_free_func, @"opaque": ?*anyopaque) ?*BrotliEncoderPreparedDictionary; +pub extern fn BrotliEncoderDestroyPreparedDictionary(dictionary: ?*BrotliEncoderPreparedDictionary) void; +pub extern fn BrotliEncoderAttachPreparedDictionary(state: ?*BrotliEncoderState, dictionary: ?*const BrotliEncoderPreparedDictionary) c_int; +pub extern fn BrotliEncoderMaxCompressedSize(input_size: usize) usize; +pub extern fn BrotliEncoderCompress(quality: c_int, lgwin: c_int, mode: BrotliEncoderMode, input_size: usize, input_buffer: [*]const u8, encoded_size: *usize, encoded_buffer: [*]u8) c_int; +pub extern fn BrotliEncoderCompressStream(state: *BrotliEncoderState, op: BrotliEncoderOperation, available_in: *usize, next_in: *[*]const u8, available_out: *usize, next_out: *?[*]u8, total_out: *usize) c_int; +pub extern fn BrotliEncoderIsFinished(state: *BrotliEncoderState) c_int; +pub extern fn BrotliEncoderHasMoreOutput(state: *BrotliEncoderState) c_int; +pub extern fn BrotliEncoderTakeOutput(state: *BrotliEncoderState, size: [*c]usize) [*c]const u8; +pub extern fn BrotliEncoderEstimatePeakMemoryUsage(quality: c_int, lgwin: c_int, input_size: usize) usize; +pub extern fn BrotliEncoderGetPreparedDictionarySize(dictionary: ?*const BrotliEncoderPreparedDictionary) usize; +pub extern fn BrotliEncoderVersion() u32; +pub const BROTLI_ENC_ENCODE_H_ = ""; +pub const BROTLI_MIN_WINDOW_BITS = @as(c_int, 10); +pub const BROTLI_MAX_WINDOW_BITS = @as(c_int, 24); +pub const BROTLI_LARGE_MAX_WINDOW_BITS = @as(c_int, 30); +pub const BROTLI_MIN_INPUT_BLOCK_BITS = @as(c_int, 16); +pub const BROTLI_MAX_INPUT_BLOCK_BITS = @as(c_int, 24); +pub const BROTLI_MIN_QUALITY = @as(c_int, 0); +pub const BROTLI_MAX_QUALITY = @as(c_int, 11); +pub const BROTLI_DEFAULT_QUALITY = @as(c_int, 11); +pub const BROTLI_DEFAULT_WINDOW = @as(c_int, 22); +pub const BROTLI_DEFAULT_MODE = BROTLI_MODE_GENERIC; +pub const BrotliEncoderStateStruct = struct_BrotliEncoderStateStruct; +pub const BrotliEncoderPreparedDictionaryStruct = struct_BrotliEncoderPreparedDictionaryStruct;