From 3abe18ea1b50cb34baca5d53ea70b3cccc244467 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 9 Jun 2025 14:37:31 +0000 Subject: [PATCH] Checkpoint before follow-up message --- src/bun.js/api/JSBundler.zig | 9 + src/bundler/Chunk.zig | 92 ++++++ src/bundler/LinkerContext.zig | 7 +- src/bundler/bundle_v2.zig | 3 + .../generateChunksInParallel.zig | 12 + src/cli.zig | 17 ++ src/cli/build_command.zig | 1 + src/compression.zig | 28 ++ src/options.zig | 2 + src/transpiler.zig | 9 + test/bundler/bundler_compression.test.ts | 287 ++++++++++++++++++ 11 files changed, 466 insertions(+), 1 deletion(-) create mode 100644 src/compression.zig create mode 100644 test/bundler/bundler_compression.test.ts diff --git a/src/bun.js/api/JSBundler.zig b/src/bun.js/api/JSBundler.zig index 1ea0de22c4..b6a9bf1663 100644 --- a/src/bun.js/api/JSBundler.zig +++ b/src/bun.js/api/JSBundler.zig @@ -19,6 +19,7 @@ const logger = bun.logger; const Loader = options.Loader; const Target = options.Target; const Index = @import("../../ast/base.zig").Index; +const compression = @import("../../compression.zig"); const debug = bun.Output.scoped(.Transpiler, false); @@ -26,6 +27,7 @@ pub const JSBundler = struct { const OwnedString = bun.MutableString; pub const Config = struct { + output_compression: compression.OutputCompression = .none, target: Target = Target.browser, entry_points: bun.StringSet = bun.StringSet.init(bun.default_allocator), hot: bool = false, @@ -266,6 +268,13 @@ pub const JSBundler = struct { } } + if (try config.getOptional(globalThis, "gz", ZigString.Slice)) |compression_slice| { + defer compression_slice.deinit(); + this.output_compression = compression.OutputCompression.fromString(compression_slice.slice()) orelse { + return globalThis.throwInvalidArguments("Invalid compression type: \"{s}\". Must be 'gzip' or 'brotli'", .{compression_slice.slice()}); + }; + } + if (try config.getArray(globalThis, "entrypoints") orelse try config.getArray(globalThis, "entryPoints")) |entry_points| { var iter = entry_points.arrayIterator(globalThis); while (iter.next()) |entry_point| { diff --git a/src/bundler/Chunk.zig b/src/bundler/Chunk.zig index 97388a8f5c..0fa610d3c8 100644 --- a/src/bundler/Chunk.zig +++ b/src/bundler/Chunk.zig @@ -129,6 +129,21 @@ pub const Chunk = struct { display_size: ?*usize, enable_source_map_shifts: bool, ) !CodeResult { + // Apply compression if needed + if (linker_graph.linker.options.output_compression.canCompress() and chunk.content != .css) { + return try this.codeWithCompression( + allocator_to_use, + parse_graph, + linker_graph, + import_prefix, + chunk, + chunks, + display_size, + enable_source_map_shifts, + linker_graph.linker.options.output_compression, + ); + } + return switch (enable_source_map_shifts) { inline else => |source_map_shifts| this.codeWithSourceMapShifts( allocator_to_use, @@ -143,6 +158,83 @@ pub const Chunk = struct { }; } + pub fn codeWithCompression( + this: *IntermediateOutput, + allocator_to_use: ?std.mem.Allocator, + graph: *const Graph, + linker_graph: *const LinkerGraph, + import_prefix: []const u8, + chunk: *Chunk, + chunks: []Chunk, + display_size: ?*usize, + enable_source_map_shifts: bool, + output_compression: bundler.compression.OutputCompression, + ) !CodeResult { + // First get the uncompressed result + const result_uncompressed = try switch (enable_source_map_shifts) { + inline else => |source_map_shifts| this.codeWithSourceMapShifts( + allocator_to_use, + graph, + linker_graph, + import_prefix, + chunk, + chunks, + display_size, + source_map_shifts, + ), + }; + + // Don't compress empty files + if (result_uncompressed.buffer.len == 0) { + return result_uncompressed; + } + + const allocator = allocator_to_use orelse allocatorForSize(result_uncompressed.buffer.len); + + switch (output_compression) { + .none => return result_uncompressed, + .gzip => { + const zlib = @import("../zlib.zig"); + + var compressed_list = std.ArrayList(u8).init(allocator); + errdefer compressed_list.deinit(); + + var compressor = zlib.ZlibCompressorArrayList.init( + result_uncompressed.buffer, + &compressed_list, + allocator, + .{ + .gzip = true, + .level = 6, + .strategy = 0, + .windowBits = 15, + }, + ) catch |err| { + return err; + }; + defer compressor.deinit(); + + compressor.readAll() catch |err| { + return err; + }; + + // Free the old buffer and replace with compressed + if (allocator != allocator_to_use) { + allocator.free(result_uncompressed.buffer); + } + + return .{ + .buffer = try compressed_list.toOwnedSlice(), + .shifts = result_uncompressed.shifts, + }; + }, + .brotli => { + // TODO: Implement brotli compression + return error.BrotliNotYetImplemented; + }, + } + } + pub fn codeWithSourceMapShifts( this: *IntermediateOutput, allocator_to_use: ?std.mem.Allocator, diff --git a/src/bundler/LinkerContext.zig b/src/bundler/LinkerContext.zig index 1a2d4f3246..1b44bcc10c 100644 --- a/src/bundler/LinkerContext.zig +++ b/src/bundler/LinkerContext.zig @@ -23,7 +23,7 @@ pub const LinkerContext = struct { ambiguous_result_pool: std.ArrayList(MatchImport) = undefined, - loop: EventLoop, + loop: *bundle_v2.EventLoop, /// string buffer containing pre-formatted unique keys unique_key_buf: []u8 = "", @@ -69,6 +69,10 @@ pub const LinkerContext = struct { public_path: []const u8 = "", + /// Used for bake to insert code for dev/production + dev_server: ?*DevServer = null, + output_compression: compression.OutputCompression = .none, + pub const Mode = enum { passthrough, bundle, @@ -2477,3 +2481,4 @@ const WrapKind = bundler.WrapKind; const genericPathWithPrettyInitialized = bundler.genericPathWithPrettyInitialized; const AdditionalFile = bundler.AdditionalFile; const logPartDependencyTree = bundler.logPartDependencyTree; +const compression = @import("../compression.zig"); diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index 60e9fe2199..d7a275a4c3 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -816,6 +816,7 @@ pub const BundleV2 = struct { this.linker.options.target = transpiler.options.target; this.linker.options.output_format = transpiler.options.output_format; this.linker.options.generate_bytecode_cache = transpiler.options.bytecode; + this.linker.options.output_compression = transpiler.options.output_compression; this.linker.dev_server = transpiler.options.dev_server; @@ -1667,6 +1668,7 @@ pub const BundleV2 = struct { transpiler.options.css_chunking = config.css_chunking; transpiler.options.banner = config.banner.slice(); transpiler.options.footer = config.footer.slice(); + transpiler.options.output_compression = config.output_compression; transpiler.configureLinker(); try transpiler.configureDefines(); @@ -4052,3 +4054,4 @@ pub const ParseTask = @import("ParseTask.zig").ParseTask; pub const LinkerContext = @import("LinkerContext.zig").LinkerContext; pub const LinkerGraph = @import("LinkerGraph.zig").LinkerGraph; pub const Graph = @import("Graph.zig").Graph; +const compression = @import("../compression.zig"); diff --git a/src/bundler/linker_context/generateChunksInParallel.zig b/src/bundler/linker_context/generateChunksInParallel.zig index 23a5ba8210..a69a3230dc 100644 --- a/src/bundler/linker_context/generateChunksInParallel.zig +++ b/src/bundler/linker_context/generateChunksInParallel.zig @@ -272,6 +272,18 @@ pub fn generateChunksInParallel(c: *LinkerContext, chunks: []Chunk, comptime is_ chunk.final_rel_path = rel_path; } + // Add compression extension if compression is enabled + if (!is_dev_server and c.options.output_compression != .none) { + for (chunks) |*chunk| { + // Only compress JavaScript chunks (not CSS or HTML) + if (chunk.content == .javascript) { + const compression_ext = c.options.output_compression.extension(); + const new_path = try std.fmt.allocPrint(c.allocator, "{s}{s}", .{ chunk.final_rel_path, compression_ext }); + chunk.final_rel_path = new_path; + } + } + } + if (duplicates_map.count() > 0) { var msg = std.ArrayList(u8).init(bun.default_allocator); errdefer msg.deinit(); diff --git a/src/cli.zig b/src/cli.zig index 562a34fac2..9c8efa759e 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -27,6 +27,7 @@ const transpiler = bun.transpiler; const DotEnv = @import("./env_loader.zig"); const RunCommand_ = @import("./cli/run_command.zig").RunCommand; const FilterRun = @import("./cli/filter_run.zig"); +const compression = @import("./compression.zig"); const fs = @import("fs.zig"); @@ -300,6 +301,7 @@ pub const Arguments = struct { clap.parseParam("--env Inline environment variables into the bundle as process.env.${name}. Defaults to 'disable'. To inline environment variables matching a prefix, use my prefix like 'FOO_PUBLIC_*'.") catch unreachable, clap.parseParam("--windows-hide-console When using --compile targeting Windows, prevent a Command prompt from opening alongside the executable") catch unreachable, clap.parseParam("--windows-icon When using --compile targeting Windows, assign an executable icon") catch unreachable, + clap.parseParam("--gz Compress output files. Options: 'gzip', 'brotli'") catch unreachable, } ++ if (FeatureFlags.bake_debugging_features) [_]ParamType{ clap.parseParam("--debug-dump-server-files When --app is set, dump all server files to disk even when building statically") catch unreachable, clap.parseParam("--debug-no-minify When --app is set, do not minify anything") catch unreachable, @@ -998,6 +1000,19 @@ pub const Arguments = struct { ctx.bundler_options.inline_entrypoint_import_meta_main = true; } + if (args.option("--gz")) |compression_str| { + ctx.bundler_options.output_compression = compression.OutputCompression.fromString(compression_str) orelse { + Output.prettyErrorln("error: Invalid compression type: \"{s}\". Must be 'gzip' or 'brotli'", .{compression_str}); + Global.exit(1); + }; + + // Check if --gz was specified with --compile + if (ctx.bundler_options.compile) { + Output.errGeneric("--gz is not supported with --compile", .{}); + Global.exit(1); + } + } + if (args.flag("--windows-hide-console")) { // --windows-hide-console technically doesnt depend on WinAPI, but since since --windows-icon // does, all of these customization options have been gated to windows-only @@ -1611,6 +1626,8 @@ pub const Command = struct { compile_target: Cli.CompileTarget = .{}, windows_hide_console: bool = false, windows_icon: ?[]const u8 = null, + + output_compression: compression.OutputCompression = .none, }; pub fn create(allocator: std.mem.Allocator, log: *logger.Log, comptime command: Command.Tag) anyerror!Context { diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index 225f50af0a..0dcb8d966c 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -107,6 +107,7 @@ pub const BuildCommand = struct { this_transpiler.options.output_dir = ctx.bundler_options.outdir; this_transpiler.options.output_format = ctx.bundler_options.output_format; + this_transpiler.options.output_compression = ctx.bundler_options.output_compression; if (ctx.bundler_options.output_format == .internal_bake_dev) { this_transpiler.options.tree_shaking = false; diff --git a/src/compression.zig b/src/compression.zig new file mode 100644 index 0000000000..2c6656251e --- /dev/null +++ b/src/compression.zig @@ -0,0 +1,28 @@ +const std = @import("std"); +const bun = @import("root").bun; +const strings = bun.strings; + +pub const OutputCompression = enum { + none, + gzip, + brotli, + + pub fn fromString(str: []const u8) ?OutputCompression { + if (strings.eqlComptime(str, "gzip")) return .gzip; + if (strings.eqlComptime(str, "brotli")) return .brotli; + if (strings.eqlComptime(str, "none")) return .none; + return null; + } + + pub fn extension(self: OutputCompression) []const u8 { + return switch (self) { + .none => "", + .gzip => ".gz", + .brotli => ".br", + }; + } + + pub fn canCompress(self: OutputCompression) bool { + return self != .none; + } +}; diff --git a/src/options.zig b/src/options.zig index d161e41f06..94e8e18b55 100644 --- a/src/options.zig +++ b/src/options.zig @@ -22,6 +22,7 @@ const Analytics = @import("./analytics/analytics_thread.zig"); const MacroRemap = @import("./resolver/package_json.zig").MacroMap; const DotEnv = @import("./env_loader.zig"); const PackageJSON = @import("./resolver/package_json.zig").PackageJSON; +const compression = @import("./compression.zig"); pub const defines = @import("./defines.zig"); pub const Define = defines.Define; @@ -1765,6 +1766,7 @@ pub const BundleOptions = struct { ignore_dce_annotations: bool = false, emit_dce_annotations: bool = false, bytecode: bool = false, + output_compression: compression.OutputCompression = .none, code_coverage: bool = false, debugger: bool = false, diff --git a/src/transpiler.zig b/src/transpiler.zig index 848e389289..53fc953546 100644 --- a/src/transpiler.zig +++ b/src/transpiler.zig @@ -37,6 +37,8 @@ const TOML = @import("./toml/toml_parser.zig").TOML; const JSC = bun.JSC; const PackageManager = @import("./install/install.zig").PackageManager; const DataURL = @import("./resolver/data_url.zig").DataURL; +const compression = @import("compression.zig"); +const resolver = @import("resolver/resolver.zig"); pub const MacroJSValueType = JSC.JSValue; const default_macro_js_value = JSC.JSValue.zero; @@ -993,6 +995,13 @@ pub const Transpiler = struct { keep_json_and_toml_as_one_statement: bool = false, allow_bytecode_cache: bool = false, + + footer: bun.String = bun.String.empty, + hot_module_reloading: bool = false, + bytecode: bool = false, + output_compression: compression.OutputCompression = .none, + + entry_naming: string = "[dir]/[name].[ext]", }; pub fn parse( diff --git a/test/bundler/bundler_compression.test.ts b/test/bundler/bundler_compression.test.ts new file mode 100644 index 0000000000..6408da7e66 --- /dev/null +++ b/test/bundler/bundler_compression.test.ts @@ -0,0 +1,287 @@ +import { describe } from "bun:test"; +import { itBundled } from "./expectBundled"; +import * as zlib from "zlib"; + +describe("bundler", () => { + itBundled("compression/gz-gzip-basic", { + files: { + "/entry.ts": /* ts */ ` + import { utils } from "./utils"; + console.log(utils.greet("World")); + export const version = "1.0.0"; + `, + "/utils.ts": /* ts */ ` + export const utils = { + greet: (name: string) => \`Hello, \${name}!\` + }; + `, + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + gz: "gzip", + onAfterBundle(api) { + // Build should succeed with one output file + api.expectBundled({ + "/out/entry.js.gz": { + isGzipped: true, + contains: ["Hello, ", "World", "1.0.0"], + }, + }); + }, + }); + + itBundled("compression/gz-gzip-multiple-entry-points", { + files: { + "/entry1.ts": /* ts */ ` + export const message = "Entry 1"; + console.log(message); + `, + "/entry2.ts": /* ts */ ` + export const message = "Entry 2"; + console.log(message); + `, + }, + entryPoints: ["/entry1.ts", "/entry2.ts"], + outdir: "/out", + gz: "gzip", + onAfterBundle(api) { + api.expectBundled({ + "/out/entry1.js.gz": { + isGzipped: true, + contains: ["Entry 1"], + }, + "/out/entry2.js.gz": { + isGzipped: true, + contains: ["Entry 2"], + }, + }); + }, + }); + + itBundled("compression/gz-gzip-no-css-compression", { + files: { + "/entry.ts": /* ts */ ` + import "./styles.css"; + console.log("Hello CSS"); + `, + "/styles.css": /* css */ ` + body { color: red; } + h1 { font-size: 24px; } + `, + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + gz: "gzip", + onAfterBundle(api) { + api.expectBundled({ + "/out/entry.js.gz": { + isGzipped: true, + contains: ["Hello CSS"], + }, + "/out/entry.css": { + isFile: true, + isGzipped: false, + }, + }); + }, + }); + + itBundled("compression/gz-gzip-no-asset-compression", { + files: { + "/entry.ts": /* ts */ ` + import logo from "./logo.png"; + console.log(logo); + `, + "/logo.png": new Uint8Array([ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, // PNG header + 0x00, + 0x00, + 0x00, + 0x0d, + 0x49, + 0x48, + 0x44, + 0x52, // IHDR chunk + ]), + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + loader: { ".png": "file" }, + gz: "gzip", + onAfterBundle(api) { + api.expectBundled({ + "/out/entry.js.gz": { + isGzipped: true, + }, + "/out/logo.png": { + isFile: true, + isGzipped: false, + }, + }); + }, + }); + + itBundled("compression/gz-gzip-code-splitting", { + files: { + "/entry1.ts": /* ts */ ` + import { shared } from "./shared"; + console.log("Entry 1:", shared()); + `, + "/entry2.ts": /* ts */ ` + import { shared } from "./shared"; + console.log("Entry 2:", shared()); + `, + "/shared.ts": /* ts */ ` + export function shared() { + return "Shared code"; + } + `, + }, + entryPoints: ["/entry1.ts", "/entry2.ts"], + outdir: "/out", + splitting: true, + gz: "gzip", + onAfterBundle(api) { + // All JavaScript chunks should be compressed + const files = api.readDir("/out"); + for (const file of files) { + if (file.endsWith(".js.gz")) { + api.expectBundled({ + [`/out/${file}`]: { + isGzipped: true, + }, + }); + } + } + }, + }); + + itBundled("compression/gz-gzip-sourcemap-external", { + files: { + "/entry.ts": /* ts */ ` + const x: number = 42; + console.log(x); + `, + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + sourceMap: "external", + gz: "gzip", + onAfterBundle(api) { + api.expectBundled({ + "/out/entry.js.gz": { + isGzipped: true, + contains: ["//# sourceMappingURL=entry.js.map"], + }, + "/out/entry.js.map": { + isFile: true, + isGzipped: false, + }, + }); + }, + }); + + itBundled("compression/gz-invalid-value", { + files: { + "/entry.ts": `console.log("test");`, + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + gz: "invalid", + bundleErrors: { + "/entry.ts": ["Invalid compression type"], + }, + }); + + itBundled("compression/gz-with-compile-error", { + files: { + "/entry.ts": `console.log("test");`, + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + compile: true, + gz: "gzip", + bundleErrors: { + "/entry.ts": ["--gz cannot be used with --compile"], + }, + }); + + itBundled("compression/gz-brotli-not-implemented", { + files: { + "/entry.ts": `console.log("test");`, + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + gz: "brotli", + bundleErrors: { + "/entry.ts": ["Brotli compression is not yet implemented"], + }, + }); + + itBundled("compression/gz-gzip-with-minification", { + files: { + "/entry.ts": /* ts */ ` + function longFunctionNameThatShouldBeMinified() { + const longVariableNameThatShouldBeMinified = "Hello World"; + return longVariableNameThatShouldBeMinified; + } + console.log(longFunctionNameThatShouldBeMinified()); + `, + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + minify: true, + gz: "gzip", + onAfterBundle(api) { + api.expectBundled({ + "/out/entry.js.gz": { + isGzipped: true, + contains: ["Hello World"], + doesNotContain: ["longFunctionNameThatShouldBeMinified", "longVariableNameThatShouldBeMinified"], + }, + }); + }, + }); + + itBundled("compression/gz-gzip-with-target-node", { + files: { + "/entry.ts": /* ts */ ` + const asyncFn = async () => { + const module = await import("./dynamic"); + return module.default; + }; + asyncFn(); + `, + "/dynamic.ts": /* ts */ ` + export default "Dynamic import"; + `, + }, + entryPoints: ["/entry.ts"], + outdir: "/out", + target: "node", + gz: "gzip", + onAfterBundle(api) { + // All JS outputs should be compressed + const files = api.readDir("/out"); + for (const file of files) { + if (file.endsWith(".js")) { + throw new Error(`Found uncompressed JS file: ${file}`); + } + if (file.endsWith(".js.gz")) { + api.expectBundled({ + [`/out/${file}`]: { + isGzipped: true, + }, + }); + } + } + }, + }); +});