From 2f7ff95e5c6a84935e3adb8a988efc6aeffb02a3 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 30 Sep 2024 22:37:42 -0700 Subject: [PATCH] Introduce bytecode caching, polish `"cjs"` bundler output format (#14232) Co-authored-by: Dylan Conway <35280289+dylan-conway@users.noreply.github.com> --- Makefile | 27 +- cmake/tools/SetupWebKit.cmake | 2 +- docs/bundler/executables.md | 20 +- docs/bundler/index.md | 122 ++++-- docs/bundler/vs-esbuild.md | 2 +- packages/bun-types/bun.d.ts | 42 +- src/StandaloneModuleGraph.zig | 52 ++- src/bun.js/RuntimeTranspilerCache.zig | 3 +- src/bun.js/api/JSBundler.zig | 33 +- src/bun.js/bindings/CommonJSModuleRecord.cpp | 2 +- src/bun.js/bindings/ModuleLoader.cpp | 17 + src/bun.js/bindings/ZigGlobalObject.cpp | 23 ++ src/bun.js/bindings/ZigSourceProvider.cpp | 134 ++++++- src/bun.js/bindings/ZigSourceProvider.h | 6 +- src/bun.js/bindings/bindings.zig | 77 ++++ src/bun.js/bindings/exports.zig | 2 + src/bun.js/bindings/headers-handwritten.h | 2 + src/bun.js/javascript.zig | 2 +- src/bun.js/module_loader.zig | 17 +- src/bun.zig | 2 + src/bundler.zig | 47 ++- src/bundler/bundle_v2.zig | 393 ++++++++++++++++--- src/cli.zig | 25 +- src/cli/build_command.zig | 6 +- src/js/builtins/Module.ts | 4 +- src/js_ast.zig | 14 +- src/js_lexer.zig | 24 +- src/js_parser.zig | 42 +- src/js_printer.zig | 25 +- src/options.zig | 8 + src/renamer.zig | 24 +- src/runtime.zig | 4 + test/bundler/bun-build-api.test.ts | 28 ++ test/bundler/bundler_compile.test.ts | 127 +++++- test/bundler/esbuild/dce.test.ts | 3 + test/bundler/esbuild/default.test.ts | 51 ++- test/bundler/esbuild/extra.test.ts | 2 - test/bundler/esbuild/importstar.test.ts | 48 ++- test/bundler/expectBundled.ts | 56 ++- 39 files changed, 1284 insertions(+), 234 deletions(-) diff --git a/Makefile b/Makefile index 49a040f203..f1a8be2646 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,7 @@ BUN_RELEASE_BIN = $(PACKAGE_DIR)/bun PRETTIER ?= $(shell which prettier 2>/dev/null || echo "./node_modules/.bin/prettier") ESBUILD = "$(shell which esbuild 2>/dev/null || echo "./node_modules/.bin/esbuild")" DSYMUTIL ?= $(shell which dsymutil 2>/dev/null || which dsymutil-15 2>/dev/null) -WEBKIT_DIR ?= $(realpath src/bun.js/WebKit) +WEBKIT_DIR ?= $(realpath vendor/WebKit) WEBKIT_RELEASE_DIR ?= $(WEBKIT_DIR)/WebKitBuild/Release WEBKIT_DEBUG_DIR ?= $(WEBKIT_DIR)/WebKitBuild/Debug WEBKIT_RELEASE_DIR_LTO ?= $(WEBKIT_DIR)/WebKitBuild/ReleaseLTO @@ -138,8 +138,8 @@ endif SED = $(shell which gsed 2>/dev/null || which sed 2>/dev/null) BUN_DIR ?= $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) -BUN_DEPS_DIR ?= $(shell pwd)/src/deps -BUN_DEPS_OUT_DIR ?= $(shell pwd)/build/bun-deps +BUN_DEPS_DIR ?= $(shell pwd)/vendor +BUN_DEPS_OUT_DIR ?= $(shell pwd)/build/release CPU_COUNT = 2 ifeq ($(OS_NAME),darwin) CPU_COUNT = $(shell sysctl -n hw.logicalcpu) @@ -689,19 +689,10 @@ assert-deps: @test $(shell cargo --version | awk '{print $$2}' | cut -d. -f2) -gt 57 || (echo -e "ERROR: cargo version must be at least 1.57."; exit 1) @echo "You have the dependencies installed! Woo" -# the following allows you to run `make submodule` to update or init submodules. but we will exclude webkit -# unless you explicitly clone it yourself (a huge download) -SUBMODULE_NAMES=$(shell cat .gitmodules | grep 'path = ' | awk '{print $$3}') -ifeq ("$(wildcard src/bun.js/WebKit/.git)", "") - SUBMODULE_NAMES := $(filter-out src/bun.js/WebKit, $(SUBMODULE_NAMES)) -endif .PHONY: init-submodules init-submodules: submodule # (backwards-compatibility alias) -.PHONY: submodule -submodule: ## to init or update all submodules - git submodule update --init --recursive --progress --depth=1 --checkout $(SUBMODULE_NAMES) .PHONY: build-obj build-obj: @@ -804,7 +795,7 @@ cls: @echo -e "\n\n---\n\n" jsc-check: - @ls $(JSC_BASE_DIR) >/dev/null 2>&1 || (echo -e "Failed to access WebKit build. Please compile the WebKit submodule using the Dockerfile at $(shell pwd)/src/javascript/WebKit/Dockerfile and then copy from /output in the Docker container to $(JSC_BASE_DIR). You can override the directory via JSC_BASE_DIR. \n\n DOCKER_BUILDKIT=1 docker build -t bun-webkit $(shell pwd)/src/bun.js/WebKit -f $(shell pwd)/src/bun.js/WebKit/Dockerfile --progress=plain\n\n docker container create bun-webkit\n\n # Get the container ID\n docker container ls\n\n docker cp DOCKER_CONTAINER_ID_YOU_JUST_FOUND:/output $(JSC_BASE_DIR)" && exit 1) + @ls $(JSC_BASE_DIR) >/dev/null 2>&1 || (echo -e "Failed to access WebKit build. Please compile the WebKit submodule using the Dockerfile at $(shell pwd)/src/javascript/WebKit/Dockerfile and then copy from /output in the Docker container to $(JSC_BASE_DIR). You can override the directory via JSC_BASE_DIR. \n\n DOCKER_BUILDKIT=1 docker build -t bun-webkit $(shell pwd)/vendor/WebKit -f $(shell pwd)/vendor/WebKit/Dockerfile --progress=plain\n\n docker container create bun-webkit\n\n # Get the container ID\n docker container ls\n\n docker cp DOCKER_CONTAINER_ID_YOU_JUST_FOUND:/output $(JSC_BASE_DIR)" && exit 1) @ls $(JSC_INCLUDE_DIR) >/dev/null 2>&1 || (echo "Failed to access WebKit include directory at $(JSC_INCLUDE_DIR)." && exit 1) @ls $(JSC_LIB) >/dev/null 2>&1 || (echo "Failed to access WebKit lib directory at $(JSC_LIB)." && exit 1) @@ -945,7 +936,7 @@ jsc-bindings: headers bindings .PHONY: clone-submodules clone-submodules: - git -c submodule."src/bun.js/WebKit".update=none submodule update --init --recursive --depth=1 --progress + git -c submodule."vendor/WebKit".update=none submodule update --init --recursive --depth=1 --progress .PHONY: headers @@ -1265,7 +1256,7 @@ jsc-build-mac-compile: -DENABLE_STATIC_JSC=ON \ -DENABLE_SINGLE_THREADED_VM_ENTRY_SCOPE=ON \ -DALLOW_LINE_AND_COLUMN_NUMBER_IN_BUILTINS=ON \ - -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DUSE_THIN_ARCHIVES=OFF \ -DBUN_FAST_TLS=ON \ -DENABLE_FTL_JIT=ON \ @@ -1277,7 +1268,7 @@ jsc-build-mac-compile: $(WEBKIT_DIR) \ $(WEBKIT_RELEASE_DIR) && \ CFLAGS="$(CFLAGS) -ffat-lto-objects" CXXFLAGS="$(CXXFLAGS) -ffat-lto-objects" \ - cmake --build $(WEBKIT_RELEASE_DIR) --config Release --target jsc + cmake --build $(WEBKIT_RELEASE_DIR) --config RelWithDebInfo --target jsc .PHONY: jsc-build-mac-compile-lto jsc-build-mac-compile-lto: @@ -1379,7 +1370,7 @@ jsc-build-linux-compile-config-debug: $(WEBKIT_DEBUG_DIR) # If you get "Error: could not load cache" -# run rm -rf src/bun.js/WebKit/CMakeCache.txt +# run rm -rf vendor/WebKit/CMakeCache.txt .PHONY: jsc-build-linux-compile-build jsc-build-linux-compile-build: mkdir -p $(WEBKIT_RELEASE_DIR) && \ @@ -1414,7 +1405,7 @@ jsc-build-copy-debug: cp $(WEBKIT_DEBUG_DIR)/lib/libbmalloc.a $(BUN_DEPS_OUT_DIR)/libbmalloc.a clean-jsc: - cd src/bun.js/WebKit && rm -rf **/CMakeCache.txt **/CMakeFiles && rm -rf src/bun.js/WebKit/WebKitBuild + cd vendor/WebKit && rm -rf **/CMakeCache.txt **/CMakeFiles && rm -rf vendor/WebKit/WebKitBuild clean-bindings: rm -rf $(OBJ_DIR)/*.o $(DEBUG_OBJ_DIR)/*.o $(DEBUG_OBJ_DIR)/webcore/*.o $(DEBUG_BINDINGS_OBJ) $(OBJ_DIR)/webcore/*.o $(BINDINGS_OBJ) $(OBJ_DIR)/*.d $(DEBUG_OBJ_DIR)/*.d diff --git a/cmake/tools/SetupWebKit.cmake b/cmake/tools/SetupWebKit.cmake index 5b31574a6a..c923ce2609 100644 --- a/cmake/tools/SetupWebKit.cmake +++ b/cmake/tools/SetupWebKit.cmake @@ -2,7 +2,7 @@ option(WEBKIT_VERSION "The version of WebKit to use") option(WEBKIT_LOCAL "If a local version of WebKit should be used instead of downloading") if(NOT WEBKIT_VERSION) - set(WEBKIT_VERSION 4a2db3254a9535949a5d5380eb58cf0f77c8e15a) + set(WEBKIT_VERSION 76798f7b2fb287ee9f1ecce98bae895a2d026d93) endif() if(WEBKIT_LOCAL) diff --git a/docs/bundler/executables.md b/docs/bundler/executables.md index 7a7f05693f..6ae39a574c 100644 --- a/docs/bundler/executables.md +++ b/docs/bundler/executables.md @@ -100,12 +100,30 @@ When deploying to production, we recommend the following: bun build --compile --minify --sourcemap ./path/to/my/app.ts --outfile myapp ``` -**What do these flags do?** +### Bytecode compilation + +To improve startup time, enable bytecode compilation: + +```sh +bun build --compile --minify --sourcemap --bytecode ./path/to/my/app.ts --outfile myapp +``` + +Using bytecode compilation, `tsc` starts 2x faster: + +{% image src="https://github.com/user-attachments/assets/dc8913db-01d2-48f8-a8ef-ac4e984f9763" width="689" /%} + +Bytecode compilation moves parsing overhead for large input files from runtime to bundle time. Your app starts faster, in exchange for making the `bun build` command a little slower. It doesn't obscure source code. + +**Experimental:** Bytecode compilation is an experimental feature introduced in Bun v1.1.30. Only `cjs` format is supported (which means no top-level-await). Let us know if you run into any issues! + +### What do these flags do? The `--minify` argument optimizes the size of the transpiled output code. If you have a large application, this can save megabytes of space. For smaller applications, it might still improve start time a little. The `--sourcemap` argument embeds a sourcemap compressed with zstd, so that errors & stacktraces point to their original locations instead of the transpiled location. Bun will automatically decompress & resolve the sourcemap when an error occurs. +The `--bytecode` argument enables bytecode compilation. Every time you run JavaScript code in Bun, JavaScriptCore (the engine) will compile your source code into bytecode. We can move this parsing work from runtime to bundle time, saving you startup time. + ## Worker To use workers in a standalone executable, add the worker's entrypoint to the CLI arguments: diff --git a/docs/bundler/index.md b/docs/bundler/index.md index a3b166e8b4..aae26e4130 100644 --- a/docs/bundler/index.md +++ b/docs/bundler/index.md @@ -330,6 +330,8 @@ Depending on the target, Bun will apply different module resolution rules and op If any entrypoints contains a Bun shebang (`#!/usr/bin/env bun`) the bundler will default to `target: "bun"` instead of `"browser"`. + When using `target: "bun"` and `format: "cjs"` together, the `// @bun @bun-cjs` pragma is added and the CommonJS wrapper function is not compatible with Node.js. + --- - `node` @@ -1157,6 +1159,11 @@ Each artifact also contains the following properties: --- +- `bytecode` +- Generate bytecode for any JavaScript/TypeScript entrypoints. This can greatly improve startup times for large applications. Only supported for `"cjs"` format, only supports `"target": "bun"` and dependent on a matching version of Bun. This adds a corresponding `.jsc` file for each entrypoint + +--- + - `sourcemap` - The sourcemap file corresponding to this file, if generated. Only defined for entrypoints and chunks. @@ -1266,33 +1273,104 @@ interface Bun { build(options: BuildOptions): Promise; } -interface BuildOptions { - entrypoints: string[]; // required - outdir?: string; // default: no write (in-memory only) - format?: "esm"; // later: "cjs" | "iife" - target?: "browser" | "bun" | "node"; // "browser" - splitting?: boolean; // true - plugins?: BunPlugin[]; // [] // See https://bun.sh/docs/bundler/plugins - loader?: { [k in string]: Loader }; // See https://bun.sh/docs/bundler/loaders - manifest?: boolean; // false - external?: string[]; // [] - sourcemap?: "none" | "inline" | "linked" | "external" | "linked" | boolean; // "none" - root?: string; // computed from entrypoints +interface BuildConfig { + entrypoints: string[]; // list of file path + outdir?: string; // output directory + target?: Target; // default: "browser" + /** + * Output module format. Top-level await is only supported for `"esm"`. + * + * Can be: + * - `"esm"` + * - `"cjs"` (**experimental**) + * - `"iife"` (**experimental**) + * + * @default "esm" + */ + format?: /** + + * ECMAScript Module format + */ + | "esm" + /** + * CommonJS format + * **Experimental** + */ + | "cjs" + /** + * IIFE format + * **Experimental** + */ + | "iife"; naming?: | string | { - entry?: string; // '[dir]/[name].[ext]' - chunk?: string; // '[name]-[hash].[ext]' - asset?: string; // '[name]-[hash].[ext]' - }; - publicPath?: string; // e.g. http://mydomain.com/ + chunk?: string; + entry?: string; + asset?: string; + }; // | string; + root?: string; // project root + splitting?: boolean; // default true, enable code splitting + plugins?: BunPlugin[]; + // manifest?: boolean; // whether to return manifest + external?: string[]; + packages?: "bundle" | "external"; + publicPath?: string; + define?: Record; + // origin?: string; // e.g. http://mydomain.com + loader?: { [k in string]: Loader }; + sourcemap?: "none" | "linked" | "inline" | "external" | "linked"; // default: "none", true -> "inline" + /** + * package.json `exports` conditions used when resolving imports + * + * Equivalent to `--conditions` in `bun build` or `bun run`. + * + * https://nodejs.org/api/packages.html#exports + */ + conditions?: Array | string; minify?: - | boolean // false + | boolean | { - identifiers?: boolean; whitespace?: boolean; syntax?: boolean; + identifiers?: boolean; }; + /** + * Ignore dead code elimination/tree-shaking annotations such as @__PURE__ and package.json + * "sideEffects" fields. This should only be used as a temporary workaround for incorrect + * annotations in libraries. + */ + ignoreDCEAnnotations?: boolean; + /** + * Force emitting @__PURE__ annotations even if minify.whitespace is true. + */ + emitDCEAnnotations?: boolean; + // treeshaking?: boolean; + + // jsx?: + // | "automatic" + // | "classic" + // | /* later: "preserve" */ { + // runtime?: "automatic" | "classic"; // later: "preserve" + // /** Only works when runtime=classic */ + // factory?: string; // default: "React.createElement" + // /** Only works when runtime=classic */ + // fragment?: string; // default: "React.Fragment" + // /** Only works when runtime=automatic */ + // importSource?: string; // default: "react" + // }; + + /** + * Generate bytecode for the output. This can dramatically improve cold + * start times, but will make the final output larger and slightly increase + * memory usage. + * + * Bytecode is currently only supported for CommonJS (`format: "cjs"`). + * + * Must be `target: "bun"` + * @default false + */ + bytecode?: boolean; } interface BuildOutput { @@ -1304,9 +1382,9 @@ interface BuildOutput { interface BuildArtifact extends Blob { path: string; loader: Loader; - hash?: string; - kind: "entry-point" | "chunk" | "asset" | "sourcemap"; - sourcemap?: BuildArtifact; + hash: string | null; + kind: "entry-point" | "chunk" | "asset" | "sourcemap" | "bytecode"; + sourcemap: BuildArtifact | null; } type Loader = diff --git a/docs/bundler/vs-esbuild.md b/docs/bundler/vs-esbuild.md index c89df8bdde..a3acb93c9a 100644 --- a/docs/bundler/vs-esbuild.md +++ b/docs/bundler/vs-esbuild.md @@ -59,7 +59,7 @@ In Bun's CLI, simple boolean flags like `--minify` do not accept an argument. Ot - `--format` - `--format` -- Bun only supports `"esm"` currently but other module formats are planned. esbuild defaults to `"iife"`. +- Bun supports `"esm"` and `"cjs"` currently, but more module formats are planned. esbuild defaults to `"iife"`. --- diff --git a/packages/bun-types/bun.d.ts b/packages/bun-types/bun.d.ts index f669ca9bf6..fccf64a447 100644 --- a/packages/bun-types/bun.d.ts +++ b/packages/bun-types/bun.d.ts @@ -1497,13 +1497,35 @@ declare module "bun" { kind: ImportKind; } - type ModuleFormat = "esm"; // later: "cjs", "iife" - interface BuildConfig { entrypoints: string[]; // list of file path outdir?: string; // output directory target?: Target; // default: "browser" - format?: ModuleFormat; // later: "cjs", "iife" + /** + * Output module format. Top-level await is only supported for `"esm"`. + * + * Can be: + * - `"esm"` + * - `"cjs"` (**experimental**) + * - `"iife"` (**experimental**) + * + * @default "esm" + */ + format?: /** + + * ECMAScript Module format + */ + | "esm" + /** + * CommonJS format + * **Experimental** + */ + | "cjs" + /** + * IIFE format + * **Experimental** + */ + | "iife"; naming?: | string | { @@ -1561,6 +1583,18 @@ declare module "bun" { // /** Only works when runtime=automatic */ // importSource?: string; // default: "react" // }; + + /** + * Generate bytecode for the output. This can dramatically improve cold + * start times, but will make the final output larger and slightly increase + * memory usage. + * + * Bytecode is currently only supported for CommonJS (`format: "cjs"`). + * + * Must be `target: "bun"` + * @default false + */ + bytecode?: boolean; } namespace Password { @@ -1781,7 +1815,7 @@ declare module "bun" { path: string; loader: Loader; hash: string | null; - kind: "entry-point" | "chunk" | "asset" | "sourcemap"; + kind: "entry-point" | "chunk" | "asset" | "sourcemap" | "bytecode"; sourcemap: BuildArtifact | null; } diff --git a/src/StandaloneModuleGraph.zig b/src/StandaloneModuleGraph.zig index 8a4200f3cf..00be32eda7 100644 --- a/src/StandaloneModuleGraph.zig +++ b/src/StandaloneModuleGraph.zig @@ -75,8 +75,10 @@ pub const StandaloneModuleGraph = struct { name: Schema.StringPointer = .{}, contents: Schema.StringPointer = .{}, sourcemap: Schema.StringPointer = .{}, + bytecode: Schema.StringPointer = .{}, encoding: Encoding = .latin1, loader: bun.options.Loader = .file, + module_format: ModuleFormat = .none, }; pub const Encoding = enum(u8) { @@ -88,6 +90,12 @@ pub const StandaloneModuleGraph = struct { utf8 = 2, }; + pub const ModuleFormat = enum(u8) { + none = 0, + esm = 1, + cjs = 2, + }; + pub const File = struct { name: []const u8 = "", loader: bun.options.Loader, @@ -96,6 +104,8 @@ pub const StandaloneModuleGraph = struct { cached_blob: ?*bun.JSC.WebCore.Blob = null, encoding: Encoding = .binary, wtf_string: bun.String = bun.String.empty, + bytecode: []u8 = "", + module_format: ModuleFormat = .none, pub fn lessThanByIndex(ctx: []const File, lhs_i: u32, rhs_i: u32) bool { const lhs = ctx[lhs_i]; @@ -225,7 +235,7 @@ pub const StandaloneModuleGraph = struct { }; const modules_list_bytes = sliceTo(raw_bytes, offsets.modules_ptr); - const modules_list = std.mem.bytesAsSlice(CompiledModuleGraphFile, modules_list_bytes); + const modules_list: []align(1) const CompiledModuleGraphFile = std.mem.bytesAsSlice(CompiledModuleGraphFile, modules_list_bytes); if (offsets.entry_point_id > modules_list.len) { return error.@"Corrupted module graph: entry point ID is greater than module list count"; @@ -246,6 +256,8 @@ pub const StandaloneModuleGraph = struct { } } else .none, + .bytecode = if (module.bytecode.length > 0) @constCast(sliceTo(raw_bytes, module.bytecode)) else &.{}, + .module_format = module.module_format, }, ); } @@ -271,14 +283,14 @@ pub const StandaloneModuleGraph = struct { return bytes[ptr.offset..][0..ptr.length :0]; } - pub fn toBytes(allocator: std.mem.Allocator, prefix: []const u8, output_files: []const bun.options.OutputFile) ![]u8 { + pub fn toBytes(allocator: std.mem.Allocator, prefix: []const u8, output_files: []const bun.options.OutputFile, output_format: bun.options.Format) ![]u8 { var serialize_trace = bun.tracy.traceNamed(@src(), "StandaloneModuleGraph.serialize"); defer serialize_trace.end(); var entry_point_id: ?usize = null; var string_builder = bun.StringBuilder{}; var module_count: usize = 0; - for (output_files, 0..) |output_file, i| { + for (output_files) |output_file| { string_builder.countZ(output_file.dest_path); string_builder.countZ(prefix); if (output_file.value == .buffer) { @@ -288,10 +300,13 @@ pub const StandaloneModuleGraph = struct { // the exact amount is not possible without allocating as it // involves a JSON parser. string_builder.cap += output_file.value.buffer.bytes.len * 2; + } else if (output_file.output_kind == .bytecode) { + // Allocate up to 256 byte alignment for bytecode + string_builder.cap += (output_file.value.buffer.bytes.len + 255) / 256 * 256 + 256; } else { if (entry_point_id == null) { if (output_file.output_kind == .@"entry-point") { - entry_point_id = i; + entry_point_id = module_count; } } @@ -320,7 +335,7 @@ pub const StandaloneModuleGraph = struct { defer source_map_arena.deinit(); for (output_files) |output_file| { - if (output_file.output_kind == .sourcemap) { + if (!output_file.output_kind.isFileInStandaloneMode()) { continue; } @@ -330,6 +345,23 @@ pub const StandaloneModuleGraph = struct { const dest_path = bun.strings.removeLeadingDotSlash(output_file.dest_path); + const bytecode: StringPointer = brk: { + if (output_file.bytecode_index != std.math.maxInt(u32)) { + // Use up to 256 byte alignment for bytecode + // Not aligning it correctly will cause a runtime assertion error, or a segfault. + const bytecode = output_files[output_file.bytecode_index].value.buffer.bytes; + const aligned = std.mem.alignInSlice(string_builder.writable(), 128).?; + @memcpy(aligned[0..bytecode.len], bytecode[0..bytecode.len]); + const unaligned_space = aligned[bytecode.len..]; + const offset = @intFromPtr(aligned.ptr) - @intFromPtr(string_builder.ptr.?); + const len = bytecode.len + @min(unaligned_space.len, 128); + string_builder.len += len; + break :brk StringPointer{ .offset = @truncate(offset), .length = @truncate(len) }; + } else { + break :brk .{}; + } + }; + var module = CompiledModuleGraphFile{ .name = string_builder.fmtAppendCountZ("{s}{s}", .{ prefix, @@ -341,7 +373,14 @@ pub const StandaloneModuleGraph = struct { .js, .jsx, .ts, .tsx => .latin1, else => .binary, }, + .module_format = if (output_file.loader.isJavaScriptLike()) switch (output_format) { + .cjs => .cjs, + .esm => .esm, + else => .none, + } else .none, + .bytecode = bytecode, }; + if (output_file.source_map_index != std.math.maxInt(u32)) { defer source_map_header_list.clearRetainingCapacity(); defer source_map_string_list.clearRetainingCapacity(); @@ -619,8 +658,9 @@ pub const StandaloneModuleGraph = struct { module_prefix: []const u8, outfile: []const u8, env: *bun.DotEnv.Loader, + output_format: bun.options.Format, ) !void { - const bytes = try toBytes(allocator, module_prefix, output_files); + const bytes = try toBytes(allocator, module_prefix, output_files, output_format); if (bytes.len == 0) return; const fd = inject( diff --git a/src/bun.js/RuntimeTranspilerCache.zig b/src/bun.js/RuntimeTranspilerCache.zig index f76ee34a4c..9b9674e65f 100644 --- a/src/bun.js/RuntimeTranspilerCache.zig +++ b/src/bun.js/RuntimeTranspilerCache.zig @@ -3,7 +3,8 @@ /// Version 4: TypeScript enums are properly handled + more constant folding /// Version 5: `require.main === module` no longer marks a module as CJS /// Version 6: `use strict` is preserved in CommonJS modules when at the top of the file -const expected_version = 6; +/// Version 7: Several bundler changes that are likely to impact the runtime as well. +const expected_version = 7; const bun = @import("root").bun; const std = @import("std"); diff --git a/src/bun.js/api/JSBundler.zig b/src/bun.js/api/JSBundler.zig index 841ba7c482..3bc36542c9 100644 --- a/src/bun.js/api/JSBundler.zig +++ b/src/bun.js/api/JSBundler.zig @@ -70,6 +70,8 @@ pub const JSBundler = struct { public_path: OwnedString = OwnedString.initEmpty(bun.default_allocator), conditions: bun.StringSet = bun.StringSet.init(bun.default_allocator), packages: options.PackagesOption = .bundle, + format: options.Format = .esm, + bytecode: bool = false, pub const List = bun.StringArrayHashMapUnmanaged(Config); @@ -151,8 +153,23 @@ pub const JSBundler = struct { } } + if (try config.getOwnOptional(globalThis, "bytecode", bool)) |bytecode| { + this.bytecode = bytecode; + + if (bytecode) { + // Default to CJS for bytecode, since esm doesn't really work yet. + this.format = .cjs; + this.target = .bun; + } + } + if (try config.getOwnOptionalEnum(globalThis, "target", options.Target)) |target| { this.target = target; + + if (target != .bun and this.bytecode) { + globalThis.throwInvalidArguments("target must be 'bun' when bytecode is true", .{}); + return error.JSError; + } } var has_out_dir = false; @@ -184,12 +201,11 @@ pub const JSBundler = struct { } if (try config.getOwnOptionalEnum(globalThis, "format", options.Format)) |format| { - switch (format) { - .esm => {}, - else => { - globalThis.throwInvalidArguments("Formats besides 'esm' are not implemented", .{}); - return error.JSError; - }, + this.format = format; + + if (this.bytecode and format != .cjs) { + globalThis.throwInvalidArguments("format must be 'cjs' when bytecode is true. Eventually we'll add esm support as well.", .{}); + return error.JSError; } } @@ -1042,6 +1058,11 @@ pub const BuildArtifact = struct { @"use client", @"use server", sourcemap, + bytecode, + + pub fn isFileInStandaloneMode(this: OutputKind) bool { + return this != .sourcemap and this != .bytecode; + } }; pub fn deinit(this: *BuildArtifact) void { diff --git a/src/bun.js/bindings/CommonJSModuleRecord.cpp b/src/bun.js/bindings/CommonJSModuleRecord.cpp index de4a1320f6..b0e84311f2 100644 --- a/src/bun.js/bindings/CommonJSModuleRecord.cpp +++ b/src/bun.js/bindings/CommonJSModuleRecord.cpp @@ -763,7 +763,7 @@ void populateESMExports( bool ignoreESModuleAnnotation) { auto& vm = globalObject->vm(); - Identifier esModuleMarker = builtinNames(vm).__esModulePublicName(); + const Identifier& esModuleMarker = builtinNames(vm).__esModulePublicName(); // Bun's intepretation of the "__esModule" annotation: // diff --git a/src/bun.js/bindings/ModuleLoader.cpp b/src/bun.js/bindings/ModuleLoader.cpp index c1cd391b9b..9c5714a1d6 100644 --- a/src/bun.js/bindings/ModuleLoader.cpp +++ b/src/bun.js/bindings/ModuleLoader.cpp @@ -715,6 +715,23 @@ static JSValue fetchESMSourceCode( return reject(exception); } + // This can happen if it's a `bun build --compile`'d CommonJS file + if (res->result.value.isCommonJSModule) { + auto created = Bun::createCommonJSModule(globalObject, specifierJS, res->result.value); + + if (created.has_value()) { + return rejectOrResolve(JSSourceCode::create(vm, WTFMove(created.value()))); + } + + if constexpr (allowPromise) { + auto* exception = scope.exception(); + scope.clearException(); + return rejectedInternalPromise(globalObject, exception); + } else { + return {}; + } + } + auto moduleKey = specifier->toWTFString(BunString::ZeroCopy); auto tag = res->result.value.tag; diff --git a/src/bun.js/bindings/ZigGlobalObject.cpp b/src/bun.js/bindings/ZigGlobalObject.cpp index f5169df196..0860a1b9b2 100644 --- a/src/bun.js/bindings/ZigGlobalObject.cpp +++ b/src/bun.js/bindings/ZigGlobalObject.cpp @@ -149,6 +149,7 @@ #include "ErrorCode.h" #include "v8/shim/GlobalInternals.h" #include "EventLoopTask.h" +#include #if ENABLE(REMOTE_INSPECTOR) #include "JavaScriptCore/RemoteInspectorServer.h" @@ -191,6 +192,27 @@ static bool has_loaded_jsc = false; Structure* createMemoryFootprintStructure(JSC::VM& vm, JSC::JSGlobalObject* globalObject); extern "C" WebCore::Worker* WebWorker__getParentWorker(void*); + +#ifndef BUN_WEBKIT_VERSION +#ifndef ASSERT_ENABLED +#warning "BUN_WEBKIT_VERSION is not defined. WebKit's cmakeconfig.h is supposed to define that. If you're building a release build locally, ignore this warning. If you're seeing this warning in CI, please file an issue." +#endif + +#define WEBKIT_BYTECODE_CACHE_HASH_KEY __TIMESTAMP__ +#else +#define WEBKIT_BYTECODE_CACHE_HASH_KEY BUN_WEBKIT_VERSION +#endif +static consteval unsigned getWebKitBytecodeCacheVersion() +{ + return WTF::SuperFastHash::computeHash(WEBKIT_BYTECODE_CACHE_HASH_KEY); +} +#undef WEBKIT_BYTECODE_CACHE_HASH_KEY + +extern "C" unsigned getJSCBytecodeCacheVersion() +{ + return getWebKitBytecodeCacheVersion(); +} + extern "C" void JSCInitialize(const char* envp[], size_t envc, void (*onCrash)(const char* ptr, size_t length), bool evalMode) { // NOLINTBEGIN @@ -222,6 +244,7 @@ extern "C" void JSCInitialize(const char* envp[], size_t envc, void (*onCrash)(c JSC::Options::evalMode() = evalMode; JSC::Options::usePromiseTryMethod() = true; JSC::Options::useRegExpEscape() = true; + JSC::dangerouslyOverrideJSCBytecodeCacheVersion(getWebKitBytecodeCacheVersion()); #ifdef BUN_DEBUG JSC::Options::showPrivateScriptsInStackTraces() = true; diff --git a/src/bun.js/bindings/ZigSourceProvider.cpp b/src/bun.js/bindings/ZigSourceProvider.cpp index 3d1c137bbe..53c1236bb8 100644 --- a/src/bun.js/bindings/ZigSourceProvider.cpp +++ b/src/bun.js/bindings/ZigSourceProvider.cpp @@ -12,6 +12,9 @@ #include #include #include +#include +#include +#include extern "C" void RefString__free(void*, void*, unsigned); @@ -91,14 +94,40 @@ Ref SourceProvider::create( // https://github.com/oven-sh/bun/issues/9521 } - auto provider = adoptRef(*new SourceProvider( - globalObject->isThreadLocalDefaultGlobalObject ? globalObject : nullptr, - resolvedSource, - string.isNull() ? *StringImpl::empty() : *string.impl(), - JSC::SourceTaintedOrigin::Untainted, - toSourceOrigin(sourceURLString, isBuiltin), - sourceURLString.impl(), TextPosition(), - sourceType)); + const auto getProvider = [&]() -> Ref { + if (resolvedSource.bytecode_cache != nullptr) { + const auto destructorPtr = [](const void* ptr) { + mi_free(const_cast(ptr)); + }; + const auto destructorNoOp = [](const void* ptr) { + // no-op, for bun build --compile. + }; + const auto destructor = resolvedSource.needsDeref ? destructorPtr : destructorNoOp; + + Ref bytecode = JSC::CachedBytecode::create(std::span(resolvedSource.bytecode_cache, resolvedSource.bytecode_cache_size), destructor, {}); + auto provider = adoptRef(*new SourceProvider( + globalObject->isThreadLocalDefaultGlobalObject ? globalObject : nullptr, + resolvedSource, + string.isNull() ? *StringImpl::empty() : *string.impl(), + JSC::SourceTaintedOrigin::Untainted, + toSourceOrigin(sourceURLString, isBuiltin), + sourceURLString.impl(), TextPosition(), + sourceType)); + provider->m_cachedBytecode = WTFMove(bytecode); + return provider; + } + + return adoptRef(*new SourceProvider( + globalObject->isThreadLocalDefaultGlobalObject ? globalObject : nullptr, + resolvedSource, + string.isNull() ? *StringImpl::empty() : *string.impl(), + JSC::SourceTaintedOrigin::Untainted, + toSourceOrigin(sourceURLString, isBuiltin), + sourceURLString.impl(), TextPosition(), + sourceType)); + }; + + auto provider = getProvider(); if (shouldGenerateCodeCoverage) { ByteRangeMapping__generate(Bun::toString(provider->sourceURL()), Bun::toString(provider->source().toStringWithoutCopying()), provider->asID()); @@ -111,6 +140,11 @@ Ref SourceProvider::create( return provider; } +StringView SourceProvider::source() const +{ + return StringView(m_source.get()); +} + SourceProvider::~SourceProvider() { if (m_resolvedSource.already_bundled) { @@ -119,6 +153,90 @@ SourceProvider::~SourceProvider() } } +extern "C" void CachedBytecode__deref(JSC::CachedBytecode* cachedBytecode) +{ + cachedBytecode->deref(); +} + +static JSC::VM& getVMForBytecodeCache() +{ + static thread_local JSC::VM* vmForBytecodeCache = nullptr; + if (!vmForBytecodeCache) { + const auto heapSize = JSC::HeapType::Small; + auto& vm = JSC::VM::create(heapSize).leakRef(); + vm.ref(); + vmForBytecodeCache = &vm; + vm.heap.acquireAccess(); + } + return *vmForBytecodeCache; +} + +extern "C" bool generateCachedModuleByteCodeFromSourceCode(BunString* sourceProviderURL, const LChar* inputSourceCode, size_t inputSourceCodeSize, const uint8_t** outputByteCode, size_t* outputByteCodeSize, JSC::CachedBytecode** cachedBytecodePtr) +{ + std::span sourceCodeSpan(inputSourceCode, inputSourceCodeSize); + JSC::SourceCode sourceCode = JSC::makeSource(WTF::String(sourceCodeSpan), toSourceOrigin(sourceProviderURL->toWTFString(), false), JSC::SourceTaintedOrigin::Untainted); + + JSC::VM& vm = getVMForBytecodeCache(); + + JSC::JSLockHolder locker(vm); + LexicallyScopedFeatures lexicallyScopedFeatures = StrictModeLexicallyScopedFeature; + JSParserScriptMode scriptMode = JSParserScriptMode::Module; + EvalContextType evalContextType = EvalContextType::None; + + ParserError parserError; + UnlinkedModuleProgramCodeBlock* unlinkedCodeBlock = JSC::recursivelyGenerateUnlinkedCodeBlockForModuleProgram(vm, sourceCode, lexicallyScopedFeatures, scriptMode, {}, parserError, evalContextType); + if (parserError.isValid()) + return false; + if (!unlinkedCodeBlock) + return false; + + auto key = JSC::sourceCodeKeyForSerializedModule(vm, sourceCode); + + RefPtr cachedBytecode = JSC::encodeCodeBlock(vm, key, unlinkedCodeBlock); + if (!cachedBytecode) + return false; + + cachedBytecode->ref(); + *cachedBytecodePtr = cachedBytecode.get(); + *outputByteCode = cachedBytecode->span().data(); + *outputByteCodeSize = cachedBytecode->span().size(); + + return true; +} + +extern "C" bool generateCachedCommonJSProgramByteCodeFromSourceCode(BunString* sourceProviderURL, const LChar* inputSourceCode, size_t inputSourceCodeSize, const uint8_t** outputByteCode, size_t* outputByteCodeSize, JSC::CachedBytecode** cachedBytecodePtr) +{ + std::span sourceCodeSpan(inputSourceCode, inputSourceCodeSize); + + JSC::SourceCode sourceCode = JSC::makeSource(WTF::String(sourceCodeSpan), toSourceOrigin(sourceProviderURL->toWTFString(), false), JSC::SourceTaintedOrigin::Untainted); + JSC::VM& vm = getVMForBytecodeCache(); + + JSC::JSLockHolder locker(vm); + LexicallyScopedFeatures lexicallyScopedFeatures = NoLexicallyScopedFeatures; + JSParserScriptMode scriptMode = JSParserScriptMode::Classic; + EvalContextType evalContextType = EvalContextType::None; + + ParserError parserError; + UnlinkedProgramCodeBlock* unlinkedCodeBlock = JSC::recursivelyGenerateUnlinkedCodeBlockForProgram(vm, sourceCode, lexicallyScopedFeatures, scriptMode, {}, parserError, evalContextType); + if (parserError.isValid()) + return false; + if (!unlinkedCodeBlock) + return false; + + auto key = JSC::sourceCodeKeyForSerializedProgram(vm, sourceCode); + + RefPtr cachedBytecode = JSC::encodeCodeBlock(vm, key, unlinkedCodeBlock); + if (!cachedBytecode) + return false; + + cachedBytecode->ref(); + *cachedBytecodePtr = cachedBytecode.get(); + *outputByteCode = cachedBytecode->span().data(); + *outputByteCodeSize = cachedBytecode->span().size(); + + return true; +} + unsigned SourceProvider::hash() const { if (m_hash) { diff --git a/src/bun.js/bindings/ZigSourceProvider.h b/src/bun.js/bindings/ZigSourceProvider.h index d8ccaa69ee..eddb638665 100644 --- a/src/bun.js/bindings/ZigSourceProvider.h +++ b/src/bun.js/bindings/ZigSourceProvider.h @@ -43,11 +43,11 @@ public: bool isBuiltIn = false); ~SourceProvider(); unsigned hash() const override; - StringView source() const override { return StringView(m_source.get()); } + StringView source() const override; - RefPtr cachedBytecode() + RefPtr cachedBytecode() const final { - return nullptr; + return m_cachedBytecode.copyRef(); }; void updateCache(const UnlinkedFunctionExecutable* executable, const SourceCode&, diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index 27d04a7f71..c34ef621ea 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -105,6 +105,83 @@ pub const JSObject = extern struct { }; }; +pub const CachedBytecode = opaque { + extern fn generateCachedModuleByteCodeFromSourceCode(sourceProviderURL: *bun.String, input_code: [*]const u8, inputSourceCodeSize: usize, outputByteCode: *?[*]u8, outputByteCodeSize: *usize, cached_bytecode: *?*CachedBytecode) bool; + extern fn generateCachedCommonJSProgramByteCodeFromSourceCode(sourceProviderURL: *bun.String, input_code: [*]const u8, inputSourceCodeSize: usize, outputByteCode: *?[*]u8, outputByteCodeSize: *usize, cached_bytecode: *?*CachedBytecode) bool; + + pub fn generateForESM(sourceProviderURL: *bun.String, input: []const u8) ?struct { []const u8, *CachedBytecode } { + var this: ?*CachedBytecode = null; + + var input_code_size: usize = 0; + var input_code_ptr: ?[*]u8 = null; + if (generateCachedModuleByteCodeFromSourceCode(sourceProviderURL, input.ptr, input.len, &input_code_ptr, &input_code_size, &this)) { + return .{ input_code_ptr.?[0..input_code_size], this.? }; + } + + return null; + } + + pub fn generateForCJS(sourceProviderURL: *bun.String, input: []const u8) ?struct { []const u8, *CachedBytecode } { + var this: ?*CachedBytecode = null; + var input_code_size: usize = 0; + var input_code_ptr: ?[*]u8 = null; + if (generateCachedCommonJSProgramByteCodeFromSourceCode(sourceProviderURL, input.ptr, input.len, &input_code_ptr, &input_code_size, &this)) { + return .{ input_code_ptr.?[0..input_code_size], this.? }; + } + + return null; + } + + extern "C" fn CachedBytecode__deref(this: *CachedBytecode) void; + pub fn deref(this: *CachedBytecode) void { + return CachedBytecode__deref(this); + } + + pub fn generate(format: bun.options.Format, input: []const u8, source_provider_url: *bun.String) ?struct { []const u8, *CachedBytecode } { + return switch (format) { + .esm => generateForESM(source_provider_url, input), + .cjs => generateForCJS(source_provider_url, input), + else => null, + }; + } + + pub const VTable = &std.mem.Allocator.VTable{ + .alloc = struct { + pub fn alloc(ctx: *anyopaque, len: usize, ptr_align: u8, ret_addr: usize) ?[*]u8 { + _ = ctx; // autofix + _ = len; // autofix + _ = ptr_align; // autofix + _ = ret_addr; // autofix + @panic("Unexpectedly called CachedBytecode.alloc"); + } + }.alloc, + .resize = struct { + pub fn resize(ctx: *anyopaque, buf: []u8, buf_align: u8, new_len: usize, ret_addr: usize) bool { + _ = ctx; // autofix + _ = buf; // autofix + _ = buf_align; // autofix + _ = new_len; // autofix + _ = ret_addr; // autofix + return false; + } + }.resize, + .free = struct { + pub fn free(ctx: *anyopaque, buf: []u8, buf_align: u8, _: usize) void { + _ = buf; // autofix + _ = buf_align; // autofix + CachedBytecode__deref(@ptrCast(ctx)); + } + }.free, + }; + + pub fn allocator(this: *CachedBytecode) std.mem.Allocator { + return .{ + .ptr = this, + .vtable = VTable, + }; + } +}; + /// Prefer using bun.String instead of ZigString in new code. pub const ZigString = extern struct { /// This can be a UTF-16, Latin1, or UTF-8 string. diff --git a/src/bun.js/bindings/exports.zig b/src/bun.js/bindings/exports.zig index 27d1e91655..e836e1cb0f 100644 --- a/src/bun.js/bindings/exports.zig +++ b/src/bun.js/bindings/exports.zig @@ -215,6 +215,8 @@ pub const ResolvedSource = extern struct { /// This is for source_code source_code_needs_deref: bool = true, already_bundled: bool = false, + bytecode_cache: ?[*]u8 = null, + bytecode_cache_size: usize = 0, pub const Tag = @import("ResolvedSourceTag").ResolvedSourceTag; }; diff --git a/src/bun.js/bindings/headers-handwritten.h b/src/bun.js/bindings/headers-handwritten.h index 54efa66cfc..6e6b60a95d 100644 --- a/src/bun.js/bindings/headers-handwritten.h +++ b/src/bun.js/bindings/headers-handwritten.h @@ -101,6 +101,8 @@ typedef struct ResolvedSource { uint32_t tag; bool needsDeref; bool already_bundled; + uint8_t* bytecode_cache; + size_t bytecode_cache_size; } ResolvedSource; static const uint32_t ResolvedSourceTagPackageJSONTypeModule = 1; typedef union ErrorableResolvedSourceResult { diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 45e003084a..b1900efb55 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -2752,7 +2752,7 @@ pub const VirtualMachine = struct { JSC.JSValue.fromCell(promise).ensureStillAlive(); return promise; } else { - const promise = JSModuleLoader.loadAndEvaluateModule(this.global, &String.init(this.main)) orelse return error.JSError; + const promise = JSModuleLoader.loadAndEvaluateModule(this.global, &String.fromBytes(this.main)) orelse return error.JSError; this.pending_internal_promise = promise; JSC.JSValue.fromCell(promise).ensureStillAlive(); diff --git a/src/bun.js/module_loader.zig b/src/bun.js/module_loader.zig index e2e79283cb..1d9d176bc6 100644 --- a/src/bun.js/module_loader.zig +++ b/src/bun.js/module_loader.zig @@ -479,6 +479,7 @@ pub const RuntimeTranspilerStore = struct { setBreakPointOnFirstLine(), .runtime_transpiler_cache = if (!JSC.RuntimeTranspilerCache.is_disabled) &cache else null, .remove_cjs_module_wrapper = is_main and vm.module_loader.eval_source != null, + .allow_bytecode_cache = true, }; defer { @@ -573,8 +574,9 @@ pub const RuntimeTranspilerStore = struct { return; } - if (parse_result.already_bundled) { + if (parse_result.already_bundled != .none) { const duped = String.createUTF8(specifier); + const bytecode_slice = parse_result.already_bundled.bytecodeSlice(); this.resolved_source = ResolvedSource{ .allocator = null, .source_code = bun.String.createLatin1(parse_result.source.contents), @@ -582,6 +584,9 @@ pub const RuntimeTranspilerStore = struct { .source_url = duped.createIfDifferent(path.text), .already_bundled = true, .hash = 0, + .bytecode_cache = if (bytecode_slice.len > 0) bytecode_slice.ptr else null, + .bytecode_cache_size = bytecode_slice.len, + .is_commonjs_module = parse_result.already_bundled.isCommonJS(), }; this.resolved_source.source_code.ensureHash(); return; @@ -1615,6 +1620,7 @@ pub const ModuleLoader = struct { .allow_commonjs = true, .inject_jest_globals = jsc_vm.bundler.options.rewrite_jest_for_tests and is_main, .keep_json_and_toml_as_one_statement = true, + .allow_bytecode_cache = true, .set_breakpoint_on_first_line = is_main and jsc_vm.debugger != null and jsc_vm.debugger.?.set_breakpoint_on_first_line and @@ -1760,7 +1766,8 @@ pub const ModuleLoader = struct { }; } - if (parse_result.already_bundled) { + if (parse_result.already_bundled != .none) { + const bytecode_slice = parse_result.already_bundled.bytecodeSlice(); return ResolvedSource{ .allocator = null, .source_code = bun.String.createLatin1(parse_result.source.contents), @@ -1768,6 +1775,9 @@ pub const ModuleLoader = struct { .source_url = input_specifier.createIfDifferent(path.text), .already_bundled = true, .hash = 0, + .bytecode_cache = if (bytecode_slice.len > 0) bytecode_slice.ptr else null, + .bytecode_cache_size = if (bytecode_slice.len > 0) bytecode_slice.len else 0, + .is_commonjs_module = parse_result.already_bundled.isCommonJS(), }; } @@ -2568,6 +2578,9 @@ pub const ModuleLoader = struct { .source_url = specifier.dupeRef(), .hash = 0, .source_code_needs_deref = false, + .bytecode_cache = if (file.bytecode.len > 0) file.bytecode.ptr else null, + .bytecode_cache_size = file.bytecode.len, + .is_commonjs_module = file.module_format == .cjs, }; } } diff --git a/src/bun.zig b/src/bun.zig index db457eba7b..8f979c5196 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -3820,3 +3820,5 @@ pub fn WeakPtr(comptime T: type, comptime weakable_field: std.meta.FieldEnum(T)) } }; } + +pub const bytecode_extension = ".jsc"; diff --git a/src/bundler.zig b/src/bundler.zig index 56406a9be0..a729e3d19d 100644 --- a/src/bundler.zig +++ b/src/bundler.zig @@ -70,13 +70,36 @@ pub const ParseResult = struct { source: logger.Source, loader: options.Loader, ast: js_ast.Ast, - already_bundled: bool = false, + already_bundled: AlreadyBundled = .none, input_fd: ?StoredFileDescriptorType = null, empty: bool = false, pending_imports: _resolver.PendingResolution.List = .{}, runtime_transpiler_cache: ?*bun.JSC.RuntimeTranspilerCache = null, + pub const AlreadyBundled = union(enum) { + none: void, + source_code: void, + source_code_cjs: void, + bytecode: []u8, + bytecode_cjs: []u8, + + pub fn bytecodeSlice(this: AlreadyBundled) []u8 { + return switch (this) { + inline .bytecode, .bytecode_cjs => |slice| slice, + else => &.{}, + }; + } + + pub fn isBytecode(this: AlreadyBundled) bool { + return this == .bytecode or this == .bytecode_cjs; + } + + pub fn isCommonJS(this: AlreadyBundled) bool { + return this == .source_code_cjs or this == .bytecode_cjs; + } + }; + pub fn isPendingImport(this: *const ParseResult, id: u32) bool { const import_record_ids = this.pending_imports.items(.import_record_id); @@ -1218,6 +1241,7 @@ pub const Bundler = struct { runtime_transpiler_cache: ?*bun.JSC.RuntimeTranspilerCache = null, keep_json_and_toml_as_one_statement: bool = false, + allow_bytecode_cache: bool = false, }; pub fn parse( @@ -1399,9 +1423,26 @@ pub const Bundler = struct { .loader = loader, .input_fd = input_fd, }, - .already_bundled => ParseResult{ + .already_bundled => |already_bundled| ParseResult{ .ast = undefined, - .already_bundled = true, + .already_bundled = switch (already_bundled) { + .bun => .source_code, + .bun_cjs => .source_code_cjs, + .bytecode_cjs, .bytecode => brk: { + const default_value: ParseResult.AlreadyBundled = if (already_bundled == .bytecode_cjs) .source_code_cjs else .source_code; + if (this_parse.virtual_source == null and this_parse.allow_bytecode_cache) { + var path_buf2: bun.PathBuffer = undefined; + @memcpy(path_buf2[0..path.text.len], path.text); + path_buf2[path.text.len..][0..bun.bytecode_extension.len].* = bun.bytecode_extension.*; + const bytecode = bun.sys.File.toSourceAt(dirname_fd, path_buf2[0 .. path.text.len + bun.bytecode_extension.len], bun.default_allocator).asValue() orelse break :brk default_value; + if (bytecode.contents.len == 0) { + break :brk default_value; + } + break :brk if (already_bundled == .bytecode_cjs) .{ .bytecode_cjs = @constCast(bytecode.contents) } else .{ .bytecode = @constCast(bytecode.contents) }; + } + break :brk default_value; + }, + }, .source = source, .loader = loader, .input_fd = input_fd, diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index 9bc6814389..7723ca5ab0 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -125,7 +125,7 @@ const JSC = bun.JSC; const debugTreeShake = Output.scoped(.TreeShake, true); const BitSet = bun.bit_set.DynamicBitSetUnmanaged; const Async = bun.Async; - +const Loc = Logger.Loc; const kit = bun.kit; const logPartDependencyTree = Output.scoped(.part_dep_tree, false); @@ -834,6 +834,8 @@ pub const BundleV2 = struct { this.linker.options.output_format = bundler.options.output_format; this.linker.kit_dev_server = bundler.options.kit; + this.graph.generate_bytecode_cache = bundler.options.bytecode; + var pool = try this.graph.allocator.create(ThreadPool); if (enable_reloading) { Watcher.enableHotModuleReloading(this); @@ -1391,6 +1393,8 @@ pub const BundleV2 = struct { bundler.options.asset_naming = config.names.asset.data; bundler.options.public_path = config.public_path.list.items; + bundler.options.output_format = config.format; + bundler.options.bytecode = config.bytecode; bundler.options.output_dir = config.outdir.toOwnedSliceLeaky(); bundler.options.root_dir = config.rootdir.toOwnedSliceLeaky(); @@ -2644,6 +2648,8 @@ pub fn BundleThread(CompletionStruct: type) type { heap, ); + this.graph.generate_bytecode_cache = bundler.options.bytecode; + this.plugins = completion.plugins; this.completion = switch (CompletionStruct) { BundleV2.JSBundleCompletionTask => completion, @@ -3240,23 +3246,28 @@ pub const ParseTask = struct { else bundler.options.target; + const output_format = bundler.options.output_format; + var opts = js_parser.Parser.Options.init(task.jsx, loader); opts.bundle = true; opts.warn_about_unbundled_modules = false; opts.macro_context = &this.data.macro_context; opts.package_version = task.package_version; + opts.features.auto_polyfill_require = output_format == .esm and !target.isBun(); opts.features.allow_runtime = !source.index.isRuntime(); + opts.features.unwrap_commonjs_to_esm = output_format == .esm and FeatureFlags.unwrap_commonjs_to_esm; opts.features.use_import_meta_require = target.isBun(); - opts.features.top_level_await = true; + opts.features.top_level_await = output_format == .esm or output_format == .internal_kit_dev; opts.features.auto_import_jsx = task.jsx.parse and bundler.options.auto_import_jsx; opts.features.trim_unused_imports = loader.isTypeScript() or (bundler.options.trim_unused_imports orelse false); opts.features.inlining = bundler.options.minify_syntax; + opts.output_format = output_format; opts.features.minify_syntax = bundler.options.minify_syntax; opts.features.minify_identifiers = bundler.options.minify_identifiers; opts.features.emit_decorator_metadata = bundler.options.emit_decorator_metadata; opts.features.unwrap_commonjs_packages = bundler.options.unwrap_commonjs_packages; - opts.features.hot_module_reloading = bundler.options.output_format == .internal_kit_dev and !source.index.isRuntime(); + opts.features.hot_module_reloading = output_format == .internal_kit_dev and !source.index.isRuntime(); opts.features.react_fast_refresh = target == .browser and bundler.options.react_fast_refresh and loader.isJSX() and @@ -3748,6 +3759,8 @@ pub const JSMeta = struct { }; pub const Graph = struct { + generate_bytecode_cache: bool = false, + // TODO: consider removing references to this in favor of bundler.options.code_splitting code_splitting: bool = false, @@ -4611,6 +4624,37 @@ pub const LinkerContext = struct { this.esm_runtime_ref = runtime_named_exports.get("__esm").?.ref; this.cjs_runtime_ref = runtime_named_exports.get("__commonJS").?.ref; + + if (this.options.output_format == .cjs) { + this.unbound_module_ref = this.graph.generateNewSymbol(Index.runtime.get(), .unbound, "module"); + } + + if (this.options.output_format == .cjs or this.options.output_format == .iife) { + const exports_kind = this.graph.ast.items(.exports_kind); + const ast_flags_list = this.graph.ast.items(.flags); + const meta_flags_ist = this.graph.meta.items(.flags); + + for (entry_points) |entry_point| { + var ast_flags: js_ast.BundledAst.Flags = ast_flags_list[entry_point.get()]; + + // Loaders default to CommonJS when they are the entry point and the output + // format is not ESM-compatible since that avoids generating the ESM-to-CJS + // machinery. + if (ast_flags.has_lazy_export) { + exports_kind[entry_point.get()] = .cjs; + } + + // Entry points with ES6 exports must generate an exports object when + // targeting non-ES6 formats. Note that the IIFE format only needs this + // when the global name is present, since that's the only way the exports + // can actually be observed externally. + if (ast_flags.uses_export_keyword) { + ast_flags.uses_exports_ref = true; + ast_flags_list[entry_point.get()] = ast_flags; + meta_flags_ist[entry_point.get()].force_include_exports_for_entry_point = true; + } + } + } } pub fn computeDataForSourceMap( @@ -5637,7 +5681,7 @@ pub const LinkerContext = struct { const string_buffer_len: usize = brk: { var count: usize = 0; - if (is_entry_point and this.options.output_format == .esm) { + if (is_entry_point and output_format == .esm) { for (aliases) |alias| { count += std.fmt.count("export_{}", .{bun.fmt.fmtIdentifier(alias)}); } @@ -5652,7 +5696,7 @@ pub const LinkerContext = struct { count += "init_".len + ident_fmt_len; } - if (wrap != .cjs and export_kind != .cjs and this.options.output_format != .internal_kit_dev) { + if (wrap != .cjs and export_kind != .cjs and output_format != .internal_kit_dev) { count += "exports_".len + ident_fmt_len; count += "module_".len + ident_fmt_len; } @@ -5672,7 +5716,7 @@ pub const LinkerContext = struct { // Pre-generate symbols for re-exports CommonJS symbols in case they // are necessary later. This is done now because the symbols map cannot be // mutated later due to parallelism. - if (is_entry_point and this.options.output_format == .esm) { + if (is_entry_point and output_format == .esm) { const copies = this.allocator.alloc(Ref, aliases.len) catch unreachable; for (aliases, copies) |alias, *copy| { @@ -5700,7 +5744,7 @@ pub const LinkerContext = struct { // actual CommonJS files from being renamed. This is purely about // aesthetics and is not about correctness. This is done here because by // this point, we know the CommonJS status will not change further. - if (wrap != .cjs and export_kind != .cjs and this.options.output_format != .internal_kit_dev) { + if (wrap != .cjs and export_kind != .cjs and output_format != .internal_kit_dev) { const exports_name = builder.fmt("exports_{}", .{source.fmtIdentifier()}); const module_name = builder.fmt("module_{}", .{source.fmtIdentifier()}); @@ -5848,7 +5892,7 @@ pub const LinkerContext = struct { this.graph.meta.items(.entry_point_part_index)[id] = Index.part(entry_point_part_index); // Pull in the "__toCommonJS" symbol if we need it due to being an entry point - if (force_include_exports and this.options.output_format != .internal_kit_dev) { + if (force_include_exports and output_format != .internal_kit_dev) { this.graph.generateRuntimeSymbolImportAndUse( source_index, Index.part(entry_point_part_index), @@ -5875,7 +5919,7 @@ pub const LinkerContext = struct { // Don't follow external imports (this includes import() expressions) if (!record.source_index.isValid() or this.isExternalDynamicImport(record, source_index)) { - if (this.options.output_format == .internal_kit_dev) continue; + if (output_format == .internal_kit_dev) continue; // This is an external import. Check if it will be a "require()" call. if (kind == .require or !output_format.keepES6ImportExportSyntax() or kind == .dynamic) { @@ -5914,7 +5958,6 @@ pub const LinkerContext = struct { // - The "default" and "__esModule" exports must not be accessed // if (kind != .require and - false and // TODO: c.options.UnsupportedJSFeatures.Has(compat.DynamicImport) (kind != .stmt or record.contains_import_star or record.contains_default_alias or @@ -5948,7 +5991,7 @@ pub const LinkerContext = struct { // This is an ES6 import of a CommonJS module, so it needs the // "__toESM" wrapper as long as it's not a bare "require()" - if (kind != .require and other_export_kind == .cjs and this.options.output_format != .internal_kit_dev) { + if (kind != .require and other_export_kind == .cjs and output_format != .internal_kit_dev) { record.wrap_with_to_esm = true; to_esm_uses += 1; } @@ -6044,7 +6087,7 @@ pub const LinkerContext = struct { } } - if (this.options.output_format != .internal_kit_dev) { + if (output_format != .internal_kit_dev) { // If there's an ES6 import of a CommonJS module, then we're going to need the // "__toESM" symbol from the runtime to wrap the result of "require()" this.graph.generateRuntimeSymbolImportAndUse( @@ -6110,7 +6153,9 @@ pub const LinkerContext = struct { var ns_export_symbol_uses = js_ast.Part.SymbolUseMap{}; ns_export_symbol_uses.ensureTotalCapacity(allocator, export_aliases.len) catch bun.outOfMemory(); - const needs_exports_variable = c.graph.meta.items(.flags)[id].needs_exports_variable; + const initial_flags = c.graph.meta.items(.flags)[id]; + const needs_exports_variable = initial_flags.needs_exports_variable; + const force_include_exports_for_entry_point = c.options.output_format == .cjs and initial_flags.force_include_exports_for_entry_point; const stmts_count = // 1 statement for every export @@ -6118,7 +6163,9 @@ pub const LinkerContext = struct { // + 1 if there are non-zero exports @as(usize, @intFromBool(export_aliases.len > 0)) + // + 1 if we need to inject the exports variable - @as(usize, @intFromBool(needs_exports_variable)); + @as(usize, @intFromBool(needs_exports_variable)) + + // + 1 if we need to do module.exports = __toCommonJS(exports) + @as(usize, @intFromBool(force_include_exports_for_entry_point)); var stmts = js_ast.Stmt.Batcher.init(allocator, stmts_count) catch bun.outOfMemory(); defer stmts.done(); @@ -6210,7 +6257,9 @@ pub const LinkerContext = struct { var declared_symbols = js_ast.DeclaredSymbol.List{}; const exports_ref = c.graph.ast.items(.exports_ref)[id]; - const all_export_stmts: []js_ast.Stmt = stmts.head[0 .. @as(usize, @intFromBool(needs_exports_variable)) + @as(usize, @intFromBool(properties.items.len > 0))]; + const all_export_stmts: []js_ast.Stmt = stmts.head[0 .. @as(usize, @intFromBool(needs_exports_variable)) + + @as(usize, @intFromBool(properties.items.len > 0) + + @as(usize, @intFromBool(force_include_exports_for_entry_point)))]; stmts.head = stmts.head[all_export_stmts.len..]; var remaining_stmts = all_export_stmts; defer bun.assert(remaining_stmts.len == 0); // all must be used @@ -6243,7 +6292,7 @@ pub const LinkerContext = struct { // "__export(exports, { foo: () => foo })" var export_ref = Ref.None; if (properties.items.len > 0) { - export_ref = c.graph.ast.items(.module_scope)[Index.runtime.get()].members.get("__export").?.ref; + export_ref = c.runtimeFunction("__export"); var args = allocator.alloc(js_ast.Expr, 2) catch unreachable; args[0..2].* = [_]js_ast.Expr{ js_ast.Expr.initIdentifier(exports_ref, loc), @@ -6279,6 +6328,40 @@ pub const LinkerContext = struct { c.graph.ast.items(.flags)[id].uses_exports_ref = true; } + // Decorate "module.exports" with the "__esModule" flag to indicate that + // we used to be an ES module. This is done by wrapping the exports object + // instead of by mutating the exports object because other modules in the + // bundle (including the entry point module) may do "import * as" to get + // access to the exports object and should NOT see the "__esModule" flag. + if (force_include_exports_for_entry_point) { + const toCommonJSRef = c.runtimeFunction("__toCommonJS"); + + var call_args = allocator.alloc(js_ast.Expr, 1) catch unreachable; + call_args[0] = Expr.initIdentifier(exports_ref, Loc.Empty); + remaining_stmts[0] = js_ast.Stmt.assign( + Expr.allocate( + allocator, + E.Dot, + E.Dot{ + .name = "exports", + .name_loc = Loc.Empty, + .target = Expr.initIdentifier(c.unbound_module_ref, Loc.Empty), + }, + Loc.Empty, + ), + Expr.allocate( + allocator, + E.Call, + E.Call{ + .target = Expr.initIdentifier(toCommonJSRef, Loc.Empty), + .args = js_ast.ExprNodeList.init(call_args), + }, + Loc.Empty, + ), + ); + remaining_stmts = remaining_stmts[1..]; + } + // No need to generate a part if it'll be empty if (all_export_stmts.len > 0) { // - we must already have preallocated the parts array @@ -7130,7 +7213,7 @@ pub const LinkerContext = struct { const all_wrapper_refs: []const Ref = c.graph.ast.items(.wrapper_ref); const all_import_records: []const ImportRecord.List = c.graph.ast.items(.import_records); - var reserved_names = try renamer.computeInitialReservedNames(allocator); + var reserved_names = try renamer.computeInitialReservedNames(allocator, c.options.output_format); for (files_in_order) |source_index| { renamer.computeReservedNamesForScope(&all_module_scopes[source_index], &c.graph.symbols, &reserved_names, allocator); } @@ -7553,6 +7636,7 @@ pub const LinkerContext = struct { .input = chunk.unique_key, }, }; + const output_format = c.options.output_format; var line_offset: bun.sourcemap.LineColumnOffset.Optional = if (c.options.source_maps != .none) .{ .value = .{} } else .{ .null = {} }; @@ -7577,14 +7661,37 @@ pub const LinkerContext = struct { } if (is_bun) { - j.pushStatic("// @bun\n"); - line_offset.advance("// @bun\n"); + const cjs_entry_chunk = "(function(exports, require, module, __filename, __dirname) {"; + if (ctx.c.parse_graph.generate_bytecode_cache and output_format == .cjs) { + const input = "// @bun @bytecode @bun-cjs\n" ++ cjs_entry_chunk; + j.pushStatic(input); + line_offset.advance(input); + } else if (ctx.c.parse_graph.generate_bytecode_cache) { + j.pushStatic("// @bun @bytecode\n"); + line_offset.advance("// @bun @bytecode\n"); + } else if (output_format == .cjs) { + j.pushStatic("// @bun @bun-cjs\n" ++ cjs_entry_chunk); + line_offset.advance("// @bun @bun-cjs\n" ++ cjs_entry_chunk); + } else { + j.pushStatic("// @bun\n"); + line_offset.advance("// @bun\n"); + } } } // TODO: banner - // TODO: directive + // Add the top-level directive if present (but omit "use strict" in ES + // modules because all ES modules are automatically in strict mode) + if (chunk.isEntryPoint() and !output_format.isAlwaysStrictMode()) { + const flags: JSAst.Flags = c.graph.ast.items(.flags)[chunk.entry_point.source_index]; + + if (flags.has_explicit_use_strict_directive) { + j.pushStatic("\"use strict\";\n"); + line_offset.advance("\"use strict\";\n"); + newline_before_comment = true; + } + } // For Kit, hoist runtime.js outside of the IIFE const compile_results = chunk.compile_results_for_chunk; @@ -7658,7 +7765,7 @@ pub const LinkerContext = struct { CommentType.single; if (!c.options.minify_whitespace and - (c.options.output_format == .iife or c.options.output_format == .internal_kit_dev)) + (output_format == .iife or output_format == .internal_kit_dev)) { j.pushStatic(" "); line_offset.advance(" "); @@ -7734,7 +7841,7 @@ pub const LinkerContext = struct { j.push(cross_chunk_suffix, bun.default_allocator); } - switch (c.options.output_format) { + switch (output_format) { .iife => { const without_newline = "})();"; @@ -7770,6 +7877,15 @@ pub const LinkerContext = struct { line_offset.advance(str); } }, + .cjs => { + if (chunk.isEntryPoint()) { + const is_bun = ctx.c.graph.ast.items(.target)[chunk.entry_point.source_index].isBun(); + if (is_bun) { + j.pushStatic("})\n"); + line_offset.advance("})\n"); + } + } + }, else => {}, } @@ -8831,6 +8947,7 @@ pub const LinkerContext = struct { const shouldStripExports = c.options.mode != .passthrough or c.graph.files.items(.entry_point_kind)[source_index] != .none; const flags = c.graph.meta.items(.flags); + const output_format = c.options.output_format; // If this file is a CommonJS entry point, double-write re-exports to the // external CommonJS "module.exports" object in addition to our internal ESM @@ -8840,11 +8957,13 @@ pub const LinkerContext = struct { // importing itself should not see the "__esModule" marker but a CommonJS module // importing us should see the "__esModule" marker. var module_exports_for_export: ?Expr = null; - if (c.options.output_format == .cjs and chunk.isEntryPoint()) { - module_exports_for_export = Expr.init( + if (output_format == .cjs and chunk.isEntryPoint()) { + module_exports_for_export = Expr.allocate( + allocator, E.Dot, E.Dot{ - .target = Expr.init( + .target = Expr.allocate( + allocator, E.Identifier, E.Identifier{ .ref = c.unbound_module_ref, @@ -8967,10 +9086,12 @@ pub const LinkerContext = struct { Stmt.alloc( S.SExpr, S.SExpr{ - .value = Expr.init( + .value = Expr.allocate( + allocator, E.Call, E.Call{ - .target = Expr.init( + .target = Expr.allocate( + allocator, E.Identifier, E.Identifier{ .ref = export_star_ref, @@ -9014,11 +9135,10 @@ pub const LinkerContext = struct { } if (record.calls_runtime_re_export_fn) { - const other_source_index = record.source_index.get(); const target: Expr = brk: { - if (c.graph.ast.items(.exports_kind)[other_source_index].isESMWithDynamicFallback()) { + if (record.source_index.isValid() and c.graph.ast.items(.exports_kind)[record.source_index.get()].isESMWithDynamicFallback()) { // Prefix this module with "__reExport(exports, otherExports, module.exports)" - break :brk Expr.initIdentifier(c.graph.ast.items(.exports_ref)[other_source_index], stmt.loc); + break :brk Expr.initIdentifier(c.graph.ast.items(.exports_ref)[record.source_index.get()], stmt.loc); } break :brk Expr.init( @@ -9045,7 +9165,7 @@ pub const LinkerContext = struct { }; if (module_exports_for_export) |mod| { - args[3] = mod; + args[2] = mod; } try stmts.inside_wrapper_prefix.append( @@ -9461,6 +9581,16 @@ pub const LinkerContext = struct { break :brk std.math.maxInt(u32); }; + const output_format = c.options.output_format; + + // The top-level directive must come first (the non-wrapped case is handled + // by the chunk generation code, although only for the entry point) + if (flags.wrap != .none and ast.flags.has_explicit_use_strict_directive and !chunk.isEntryPoint() and !output_format.isAlwaysStrictMode()) { + stmts.inside_wrapper_prefix.append(Stmt.alloc(S.Directive, .{ + .value = "use strict", + }, Logger.Loc.Empty)) catch unreachable; + } + // TODO: handle directive if (namespace_export_part_index >= part_range.part_index_begin and namespace_export_part_index < part_range.part_index_end and @@ -9633,7 +9763,7 @@ pub const LinkerContext = struct { // Turn each module into a function if this is Kit var stmt_storage: Stmt = undefined; - if (c.options.output_format == .internal_kit_dev and !part_range.source_index.isRuntime()) { + if (output_format == .internal_kit_dev and !part_range.source_index.isRuntime()) { if (stmts.all_stmts.items.len == 0) { // TODO: these chunks should just not exist in the first place // they seem to happen on the entry point? or JSX? not clear @@ -9990,7 +10120,7 @@ pub const LinkerContext = struct { .indent = .{}, .commonjs_named_exports = ast.commonjs_named_exports, .commonjs_named_exports_ref = ast.exports_ref, - .commonjs_module_ref = if (ast.flags.uses_module_ref or c.options.output_format == .internal_kit_dev) + .commonjs_module_ref = if (ast.flags.uses_module_ref or output_format == .internal_kit_dev) ast.module_ref else Ref.None, @@ -10001,7 +10131,7 @@ pub const LinkerContext = struct { .minify_whitespace = c.options.minify_whitespace, .minify_syntax = c.options.minify_syntax, - .module_type = switch (c.options.output_format) { + .module_type = switch (output_format) { else => |format| format, .internal_kit_dev => if (part_range.source_index.isRuntime()) .esm else .internal_kit_dev, }, @@ -10011,7 +10141,11 @@ pub const LinkerContext = struct { .allocator = allocator, .to_esm_ref = toESMRef, .to_commonjs_ref = toCommonJSRef, - .require_ref = if (c.options.output_format == .internal_kit_dev) ast.require_ref else runtimeRequireRef, + .require_ref = switch (c.options.output_format) { + .internal_kit_dev => ast.require_ref, + .cjs => null, + else => runtimeRequireRef, + }, .require_or_import_meta_for_source_callback = js_printer.RequireOrImportMeta.Callback.init( LinkerContext, requireOrImportMetaForSource, @@ -10020,7 +10154,7 @@ pub const LinkerContext = struct { .line_offset_tables = c.graph.files.items(.line_offset_table)[part_range.source_index.get()], .target = c.options.target, - .input_files_for_kit = if (c.options.output_format == .internal_kit_dev and !part_range.source_index.isRuntime()) + .input_files_for_kit = if (output_format == .internal_kit_dev and !part_range.source_index.isRuntime()) c.parse_graph.input_files.items(.source) else null, @@ -10306,7 +10440,7 @@ pub const LinkerContext = struct { const root_path = c.resolver.opts.output_dir; - if (root_path.len == 0 and c.parse_graph.additional_output_files.items.len > 0 and !c.resolver.opts.compile) { + if (root_path.len == 0 and (c.parse_graph.additional_output_files.items.len > 0 or c.parse_graph.generate_bytecode_cache) and !c.resolver.opts.compile) { try c.log.addError(null, Logger.Loc.Empty, "cannot write multiple output files without an output directory"); return error.MultipleOutputFilesWithoutOutputDir; } @@ -10328,7 +10462,6 @@ pub const LinkerContext = struct { &display_size, c.options.source_maps != .none, ); - var code_result = _code_result catch @panic("Failed to allocate memory for output file"); var sourcemap_output_file: ?options.OutputFile = null; @@ -10404,7 +10537,71 @@ pub const LinkerContext = struct { .none => {}, } - output_files.appendAssumeCapacity( + const bytecode_output_file: ?options.OutputFile = brk: { + if (c.parse_graph.generate_bytecode_cache) { + const loader: Loader = if (chunk.entry_point.is_entry_point) + c.parse_graph.input_files.items(.loader)[ + chunk.entry_point.source_index + ] + else + .js; + + if (loader.isJavaScriptLike()) { + JSC.initialize(false); + var fdpath: bun.PathBuffer = undefined; + var source_provider_url = try bun.String.createFormat("{s}" ++ bun.bytecode_extension, .{chunk.final_rel_path}); + source_provider_url.ref(); + + defer source_provider_url.deref(); + + if (JSC.CachedBytecode.generate(c.options.output_format, code_result.buffer, &source_provider_url)) |result| { + const bytecode, const cached_bytecode = result; + const source_provider_url_str = source_provider_url.toSlice(bun.default_allocator); + defer source_provider_url_str.deinit(); + debug("Bytecode cache generated {s}: {}", .{ source_provider_url_str.slice(), bun.fmt.size(bytecode.len, .{ .space_between_number_and_unit = true }) }); + @memcpy(fdpath[0..chunk.final_rel_path.len], chunk.final_rel_path); + fdpath[chunk.final_rel_path.len..][0..bun.bytecode_extension.len].* = bun.bytecode_extension.*; + + break :brk options.OutputFile.init( + options.OutputFile.Options{ + .output_path = bun.default_allocator.dupe(u8, source_provider_url_str.slice()) catch unreachable, + .input_path = std.fmt.allocPrint(bun.default_allocator, "{s}" ++ bun.bytecode_extension, .{chunk.final_rel_path}) catch unreachable, + .input_loader = .js, + .hash = if (chunk.template.placeholder.hash != null) bun.hash(bytecode) else null, + .output_kind = .bytecode, + .loader = .file, + .size = @as(u32, @truncate(bytecode.len)), + .display_size = @as(u32, @truncate(bytecode.len)), + .data = .{ + .buffer = .{ .data = bytecode, .allocator = cached_bytecode.allocator() }, + }, + }, + ); + } else { + // an error + c.log.addErrorFmt(null, Logger.Loc.Empty, bun.default_allocator, "Failed to generate bytecode for {s}", .{ + chunk.final_rel_path, + }) catch unreachable; + } + } + } + + break :brk null; + }; + + const source_map_index: ?u32 = if (sourcemap_output_file != null) + @as(u32, @truncate(output_files.items.len + 1)) + else + null; + + const bytecode_index: ?u32 = if (bytecode_output_file != null and source_map_index != null) + @as(u32, @truncate(output_files.items.len + 2)) + else if (bytecode_output_file != null) + @as(u32, @truncate(output_files.items.len + 1)) + else + null; + + try output_files.append( options.OutputFile.init( options.OutputFile.Options{ .data = .{ @@ -10424,19 +10621,20 @@ pub const LinkerContext = struct { .input_loader = if (chunk.entry_point.is_entry_point) c.parse_graph.input_files.items(.loader)[chunk.entry_point.source_index] else .js, .output_path = try bun.default_allocator.dupe(u8, chunk.final_rel_path), .is_executable = chunk.is_executable, - .source_map_index = if (sourcemap_output_file != null) - @as(u32, @truncate(output_files.items.len + 1)) - else - null, + .source_map_index = source_map_index, + .bytecode_index = bytecode_index, }, ), ); if (sourcemap_output_file) |sourcemap_file| { - output_files.appendAssumeCapacity(sourcemap_file); + try output_files.append(sourcemap_file); + } + if (bytecode_output_file) |bytecode_file| { + try output_files.append(bytecode_file); } } - output_files.appendSliceAssumeCapacity(c.parse_graph.additional_output_files.items); + try output_files.appendSlice(c.parse_graph.additional_output_files.items); } return output_files; @@ -10668,6 +10866,85 @@ pub const LinkerContext = struct { }, .none => {}, } + const bytecode_output_file: ?options.OutputFile = brk: { + if (c.parse_graph.generate_bytecode_cache) { + const loader: Loader = if (chunk.entry_point.is_entry_point) + c.parse_graph.input_files.items(.loader)[ + chunk.entry_point.source_index + ] + else + .js; + + if (loader.isJavaScriptLike()) { + JSC.initialize(false); + var fdpath: bun.PathBuffer = undefined; + var source_provider_url = try bun.String.createFormat("{s}" ++ bun.bytecode_extension, .{chunk.final_rel_path}); + source_provider_url.ref(); + + defer source_provider_url.deref(); + + if (JSC.CachedBytecode.generate(c.options.output_format, code_result.buffer, &source_provider_url)) |result| { + const source_provider_url_str = source_provider_url.toSlice(bun.default_allocator); + defer source_provider_url_str.deinit(); + const bytecode, const cached_bytecode = result; + debug("Bytecode cache generated {s}: {}", .{ source_provider_url_str.slice(), bun.fmt.size(bytecode.len, .{ .space_between_number_and_unit = true }) }); + @memcpy(fdpath[0..chunk.final_rel_path.len], chunk.final_rel_path); + fdpath[chunk.final_rel_path.len..][0..bun.bytecode_extension.len].* = bun.bytecode_extension.*; + defer cached_bytecode.deref(); + switch (JSC.Node.NodeFS.writeFileWithPathBuffer( + &pathbuf, + JSC.Node.Arguments.WriteFile{ + .data = JSC.Node.StringOrBuffer{ + .buffer = JSC.Buffer{ + .buffer = .{ + .ptr = @constCast(bytecode.ptr), + .len = @as(u32, @truncate(bytecode.len)), + .byte_len = @as(u32, @truncate(bytecode.len)), + }, + }, + }, + .encoding = .buffer, + .mode = if (chunk.is_executable) 0o755 else 0o644, + + .dirfd = bun.toFD(root_dir.fd), + .file = .{ + .path = JSC.Node.PathLike{ + .string = JSC.PathString.init(fdpath[0 .. chunk.final_rel_path.len + bun.bytecode_extension.len]), + }, + }, + }, + )) { + .result => {}, + .err => |err| { + c.log.addErrorFmt(null, Logger.Loc.Empty, bun.default_allocator, "{} writing bytecode for chunk {}", .{ + err, + bun.fmt.quote(chunk.final_rel_path), + }) catch unreachable; + return error.WriteFailed; + }, + } + + break :brk options.OutputFile.init( + options.OutputFile.Options{ + .output_path = bun.default_allocator.dupe(u8, source_provider_url_str.slice()) catch unreachable, + .input_path = std.fmt.allocPrint(bun.default_allocator, "{s}" ++ bun.bytecode_extension, .{chunk.final_rel_path}) catch unreachable, + .input_loader = .file, + .hash = if (chunk.template.placeholder.hash != null) bun.hash(bytecode) else null, + .output_kind = .bytecode, + .loader = .file, + .size = @as(u32, @truncate(bytecode.len)), + .display_size = @as(u32, @truncate(bytecode.len)), + .data = .{ + .saved = 0, + }, + }, + ); + } + } + } + + break :brk null; + }; switch (JSC.Node.NodeFS.writeFileWithPathBuffer( &pathbuf, @@ -10705,7 +10982,19 @@ pub const LinkerContext = struct { .result => {}, } - output_files.appendAssumeCapacity( + const source_map_index: ?u32 = if (source_map_output_file != null) + @as(u32, @truncate(output_files.items.len + 1)) + else + null; + + const bytecode_index: ?u32 = if (bytecode_output_file != null and source_map_index != null) + @as(u32, @truncate(output_files.items.len + 2)) + else if (bytecode_output_file != null) + @as(u32, @truncate(output_files.items.len + 1)) + else + null; + + try output_files.append( options.OutputFile.init( options.OutputFile.Options{ .output_path = bun.default_allocator.dupe(u8, chunk.final_rel_path) catch unreachable, @@ -10720,10 +11009,8 @@ pub const LinkerContext = struct { else .chunk, .loader = .js, - .source_map_index = if (source_map_output_file != null) - @as(u32, @truncate(output_files.items.len + 1)) - else - null, + .source_map_index = source_map_index, + .bytecode_index = bytecode_index, .size = @as(u32, @truncate(code_result.buffer.len)), .display_size = @as(u32, @truncate(display_size)), .is_executable = chunk.is_executable, @@ -10735,7 +11022,11 @@ pub const LinkerContext = struct { ); if (source_map_output_file) |sourcemap_file| { - output_files.appendAssumeCapacity(sourcemap_file); + try output_files.append(sourcemap_file); + } + + if (bytecode_output_file) |bytecode_file| { + try output_files.append(bytecode_file); } } diff --git a/src/cli.zig b/src/cli.zig index 67b44bdfbb..8a14b44e24 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -237,6 +237,7 @@ pub const Arguments = struct { const build_only_params = [_]ParamType{ clap.parseParam("--compile Generate a standalone Bun executable containing your bundled code") catch unreachable, + clap.parseParam("--bytecode Use a bytecode cache") catch unreachable, clap.parseParam("--watch Automatically restart the process on file change") catch unreachable, clap.parseParam("--no-clear-screen Disable clearing the terminal screen on reload when --watch is enabled") catch unreachable, clap.parseParam("--target The intended execution environment for the bundle. \"browser\", \"bun\" or \"node\"") catch unreachable, @@ -725,6 +726,13 @@ pub const Arguments = struct { if (cmd == .BuildCommand) { ctx.bundler_options.transform_only = args.flag("--no-bundle"); + ctx.bundler_options.bytecode = args.flag("--bytecode"); + + // TODO: support --format=esm + if (ctx.bundler_options.bytecode) { + ctx.bundler_options.output_format = .cjs; + ctx.args.target = .bun; + } if (args.option("--public-path")) |public_path| { ctx.bundler_options.public_path = public_path; @@ -783,6 +791,11 @@ pub const Arguments = struct { if (opts.target.? == .bun) ctx.debug.run_in_bun = opts.target.? == .bun; + + if (opts.target.? != .bun and ctx.bundler_options.bytecode) { + Output.errGeneric("target must be 'bun' when bytecode is true. Received: {s}", .{@tagName(opts.target.?)}); + Global.exit(1); + } } if (args.flag("--watch")) { @@ -827,17 +840,18 @@ pub const Arguments = struct { bun.Output.flush(); }, .cjs => { - // Make this a soft error in debug to allow experimenting with these flags. - const function = if (Environment.isDebug) Output.debugWarn else Output.errGeneric; - function("Format '{s}' are not implemented", .{@tagName(format)}); - if (!Environment.isDebug) { - Global.crash(); + if (ctx.args.target == null) { + ctx.args.target = .node; } }, else => {}, } ctx.bundler_options.output_format = format; + if (format != .cjs and ctx.bundler_options.bytecode) { + Output.errGeneric("format must be 'cjs' when bytecode is true. Eventually we'll add esm support as well.", .{}); + Global.exit(1); + } } if (args.flag("--splitting")) { @@ -1344,6 +1358,7 @@ pub const Command = struct { ignore_dce_annotations: bool = false, emit_dce_annotations: bool = true, output_format: options.Format = .esm, + bytecode: bool = false, }; pub fn create(allocator: std.mem.Allocator, log: *logger.Log, comptime command: Command.Tag) anyerror!Context { diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index 5630ee9b14..b1a1c64f51 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -42,7 +42,7 @@ pub const BuildCommand = struct { Global.configureAllocator(.{ .long_running = true }); const allocator = ctx.allocator; var log = ctx.log; - if (ctx.bundler_options.compile) { + if (ctx.bundler_options.compile or ctx.bundler_options.bytecode) { // set this early so that externals are set up correctly and define is right ctx.args.target = .bun; } @@ -96,6 +96,7 @@ pub const BuildCommand = struct { this_bundler.options.minify_identifiers = ctx.bundler_options.minify_identifiers; this_bundler.options.emit_dce_annotations = ctx.bundler_options.emit_dce_annotations; this_bundler.options.ignore_dce_annotations = ctx.bundler_options.ignore_dce_annotations; + this_bundler.options.output_dir = ctx.bundler_options.outdir; this_bundler.options.output_format = ctx.bundler_options.output_format; @@ -103,6 +104,8 @@ pub const BuildCommand = struct { this_bundler.options.tree_shaking = false; } + this_bundler.options.bytecode = ctx.bundler_options.bytecode; + if (ctx.bundler_options.compile) { if (ctx.bundler_options.code_splitting) { Output.prettyErrorln("error: cannot use --compile with --splitting", .{}); @@ -389,6 +392,7 @@ pub const BuildCommand = struct { this_bundler.options.public_path, outfile, this_bundler.env, + this_bundler.options.output_format, ); const compiled_elapsed = @divTrunc(@as(i64, @truncate(std.time.nanoTimestamp() - bundled_end)), @as(i64, std.time.ns_per_ms)); const compiled_elapsed_digit_count: isize = switch (compiled_elapsed) { diff --git a/src/js/builtins/Module.ts b/src/js/builtins/Module.ts index 0b528b6439..c48de2488d 100644 --- a/src/js/builtins/Module.ts +++ b/src/js/builtins/Module.ts @@ -76,9 +76,7 @@ export function overridableRequire(this: CommonJSModuleRecord, id: string) { // If we can pull out a ModuleNamespaceObject, let's do it. if (esm?.evaluated && (esm.state ?? 0) >= $ModuleReady) { const namespace = Loader.getModuleNamespaceObject(esm!.module); - return (mod.exports = - // if they choose a module - namespace.__esModule ? namespace : Object.create(namespace, { __esModule: { value: true } })); + return (mod.exports = namespace); } } diff --git a/src/js_ast.zig b/src/js_ast.zig index f36bf9aa7c..5c0177e95e 100644 --- a/src/js_ast.zig +++ b/src/js_ast.zig @@ -6985,8 +6985,7 @@ pub const BundledAst = struct { force_cjs_to_esm: bool = false, has_lazy_export: bool = false, commonjs_module_exports_assigned_deoptimized: bool = false, - - _: u1 = 0, + has_explicit_use_strict_directive: bool = false, }; pub const empty = BundledAst.init(Ast.empty); @@ -7037,6 +7036,7 @@ pub const BundledAst = struct { .force_cjs_to_esm = this.flags.force_cjs_to_esm, .has_lazy_export = this.flags.has_lazy_export, .commonjs_module_exports_assigned_deoptimized = this.flags.commonjs_module_exports_assigned_deoptimized, + .directive = if (this.flags.has_explicit_use_strict_directive) "use strict" else null, }; } @@ -7089,6 +7089,7 @@ pub const BundledAst = struct { .force_cjs_to_esm = ast.force_cjs_to_esm, .has_lazy_export = ast.has_lazy_export, .commonjs_module_exports_assigned_deoptimized = ast.commonjs_module_exports_assigned_deoptimized, + .has_explicit_use_strict_directive = strings.eqlComptime(ast.directive orelse "", "use strict"), }, }; } @@ -7505,9 +7506,16 @@ pub const Part = struct { }; pub const Result = union(enum) { - already_bundled: void, + already_bundled: AlreadyBundled, cached: void, ast: Ast, + + pub const AlreadyBundled = enum { + bun, + bun_cjs, + bytecode, + bytecode_cjs, + }; }; pub const StmtOrExpr = union(enum) { diff --git a/src/js_lexer.zig b/src/js_lexer.zig index 7e7a41298e..ff310c3156 100644 --- a/src/js_lexer.zig +++ b/src/js_lexer.zig @@ -172,7 +172,13 @@ fn NewLexer_( code_point: CodePoint = -1, identifier: []const u8 = "", jsx_pragma: JSXPragma = .{}, - bun_pragma: bool = false, + bun_pragma: enum { + none, + bun, + bun_cjs, + bytecode, + bytecode_cjs, + } = .none, source_mapping_url: ?js_ast.Span = null, number: f64 = 0.0, rescan_close_brace_as_template_token: bool = false, @@ -2023,8 +2029,8 @@ fn NewLexer_( // } } - if (strings.hasPrefixWithWordBoundary(chunk, "bun")) { - lexer.bun_pragma = true; + if (lexer.bun_pragma == .none and strings.hasPrefixWithWordBoundary(chunk, "bun")) { + lexer.bun_pragma = .bun; } else if (strings.hasPrefixWithWordBoundary(chunk, "jsx")) { if (PragmaArg.scan(.skip_space_first, lexer.start + i + 1, "jsx", chunk)) |span| { lexer.jsx_pragma._jsx = span; @@ -2045,6 +2051,10 @@ fn NewLexer_( if (PragmaArg.scan(.no_space_first, lexer.start + i + 1, " sourceMappingURL=", chunk)) |span| { lexer.source_mapping_url = span; } + } else if ((lexer.bun_pragma == .bun or lexer.bun_pragma == .bun_cjs) and strings.hasPrefixWithWordBoundary(chunk, "bytecode")) { + lexer.bun_pragma = if (lexer.bun_pragma == .bun) .bytecode else .bytecode_cjs; + } else if ((lexer.bun_pragma == .bytecode or lexer.bun_pragma == .bun) and strings.hasPrefixWithWordBoundary(chunk, "bun-cjs")) { + lexer.bun_pragma = if (lexer.bun_pragma == .bytecode) .bytecode_cjs else .bun_cjs; } }, else => {}, @@ -2074,8 +2084,8 @@ fn NewLexer_( } } - if (strings.hasPrefixWithWordBoundary(chunk, "bun")) { - lexer.bun_pragma = true; + if (lexer.bun_pragma == .none and strings.hasPrefixWithWordBoundary(chunk, "bun")) { + lexer.bun_pragma = .bun; } else if (strings.hasPrefixWithWordBoundary(chunk, "jsx")) { if (PragmaArg.scan(.skip_space_first, lexer.start + i + 1, "jsx", chunk)) |span| { lexer.jsx_pragma._jsx = span; @@ -2096,6 +2106,10 @@ fn NewLexer_( if (PragmaArg.scan(.no_space_first, lexer.start + i + 1, " sourceMappingURL=", chunk)) |span| { lexer.source_mapping_url = span; } + } else if ((lexer.bun_pragma == .bun or lexer.bun_pragma == .bun_cjs) and strings.hasPrefixWithWordBoundary(chunk, "bytecode")) { + lexer.bun_pragma = if (lexer.bun_pragma == .bun) .bytecode else .bytecode_cjs; + } else if ((lexer.bun_pragma == .bytecode or lexer.bun_pragma == .bun) and strings.hasPrefixWithWordBoundary(chunk, "bun-cjs")) { + lexer.bun_pragma = if (lexer.bun_pragma == .bytecode) .bytecode_cjs else .bun_cjs; } }, else => {}, diff --git a/src/js_parser.zig b/src/js_parser.zig index 98cbde5f04..c4fa817b6f 100644 --- a/src/js_parser.zig +++ b/src/js_parser.zig @@ -1320,6 +1320,7 @@ pub const ImportScanner = struct { ) catch bun.outOfMemory(); if (st.star_name_loc) |loc| { + record.contains_import_star = true; p.named_imports.putAssumeCapacity( namespace_ref, js_ast.NamedImport{ @@ -1333,6 +1334,7 @@ pub const ImportScanner = struct { } if (st.default_name) |default| { + record.contains_default_alias = true; p.named_imports.putAssumeCapacity( default.ref.?, .{ @@ -1366,8 +1368,7 @@ pub const ImportScanner = struct { // We do not know at this stage whether or not the import statement is bundled // This keeps track of the `namespace_alias` incase, at printing time, we determine that we should print it with the namespace for (st.items) |item| { - const is_default = strings.eqlComptime(item.alias, "default"); - record.contains_default_alias = record.contains_default_alias or is_default; + record.contains_default_alias = record.contains_default_alias or strings.eqlComptime(item.alias, "default"); const name: LocRef = item.name; const name_ref = name.ref.?; @@ -2895,6 +2896,7 @@ pub const Parser = struct { warn_about_unbundled_modules: bool = true, module_type: options.ModuleType = .unknown, + output_format: options.Format = .esm, transform_only: bool = false, @@ -3226,9 +3228,15 @@ pub const Parser = struct { } // Detect a leading "// @bun" pragma - if (p.lexer.bun_pragma and p.options.features.dont_bundle_twice) { + if (p.lexer.bun_pragma != .none and p.options.features.dont_bundle_twice) { return js_ast.Result{ - .already_bundled = {}, + .already_bundled = switch (p.lexer.bun_pragma) { + .bun => .bun, + .bytecode => .bytecode, + .bytecode_cjs => .bytecode_cjs, + .bun_cjs => .bun_cjs, + else => unreachable, + }, }; } @@ -3655,7 +3663,7 @@ pub const Parser = struct { } } - if (p.options.bundle and parts.items.len < 4 and parts.items.len > 0) { + if (parts.items.len < 4 and parts.items.len > 0 and p.options.features.unwrap_commonjs_to_esm) { // Specially handle modules shaped like this: // @@ -3731,6 +3739,7 @@ pub const Parser = struct { } if (p.commonjs_named_exports_deoptimized and + p.options.features.unwrap_commonjs_to_esm and p.unwrap_all_requires and p.imports_to_convert_from_require.items.len == 1 and p.import_records.items.len == 1 and @@ -5273,7 +5282,7 @@ fn NewParser_( // For unwrapping CommonJS into ESM to fully work // we must also unwrap requires into imports. - const should_unwrap_require = !p.options.features.hot_module_reloading and + const should_unwrap_require = p.options.features.unwrap_commonjs_to_esm and (p.unwrap_all_requires or if (path.packageName()) |pkg| p.options.features.shouldUnwrapRequire(pkg) else false) and // We cannot unwrap a require wrapped in a try/catch because @@ -5336,9 +5345,8 @@ fn NewParser_( } } - pub fn shouldUnwrapCommonJSToESM(p: *P) bool { - // hot module loading opts out of this because we want to produce a cjs bundle at the end - return FeatureFlags.unwrap_commonjs_to_esm and !p.options.features.hot_module_reloading; + pub inline fn shouldUnwrapCommonJSToESM(p: *const P) bool { + return p.options.features.unwrap_commonjs_to_esm; } fn isBindingUsed(p: *P, binding: Binding, default_export_ref: Ref) bool { @@ -12457,8 +12465,7 @@ fn NewParser_( } pub inline fn isStrictModeOutputFormat(p: *P) bool { - // TODO: once CJS or IIFE is supported, this will need to be updated - return p.options.bundle; + return p.options.bundle and p.options.output_format.isESM(); } pub fn declareCommonJSSymbol(p: *P, comptime kind: Symbol.Kind, comptime name: string) !Ref { @@ -17637,7 +17644,7 @@ fn NewParser_( /// If --target=bun, this does nothing. fn recordUsageOfRuntimeRequire(p: *P) void { // target bun does not have __require - if (!p.options.features.use_import_meta_require) { + if (p.options.features.auto_polyfill_require) { bun.assert(p.options.features.allow_runtime); p.ensureRequireSymbol(); @@ -17646,9 +17653,7 @@ fn NewParser_( } fn ignoreUsageOfRuntimeRequire(p: *P) void { - if (!p.options.features.use_import_meta_require and - p.options.features.allow_runtime) - { + if (p.options.features.auto_polyfill_require) { bun.assert(p.runtime_imports.__require != null); p.ignoreUsage(p.runtimeIdentifierRef(logger.Loc.Empty, "__require")); p.symbols.items[p.require_ref.innerIndex()].use_count_estimate -|= 1; @@ -23511,6 +23516,7 @@ fn NewParser_( .export_keyword = p.esm_export_keyword, .top_level_symbols_to_parts = top_level_symbols_to_parts, .char_freq = p.computeCharacterFrequency(), + .directive = if (p.module_scope.strict_mode == .explicit_strict_mode) "use strict" else null, // Assign slots to symbols in nested scopes. This is some precomputation for // the symbol renaming pass that will happen later in the linker. It's done @@ -23645,13 +23651,13 @@ fn NewParser_( .needs_jsx_import = if (comptime only_scan_imports_and_do_not_visit) false else NeedsJSXType{}, .lexer = lexer, - // Only enable during bundling - .commonjs_named_exports_deoptimized = !opts.bundle, + // Only enable during bundling, when not bundling CJS + .commonjs_named_exports_deoptimized = if (opts.bundle) opts.output_format == .cjs else true, }; this.lexer.track_comments = opts.features.minify_identifiers; this.unwrap_all_requires = brk: { - if (opts.bundle) { + if (opts.bundle and opts.output_format != .cjs) { if (source.path.packageName()) |pkg| { if (opts.features.shouldUnwrapRequire(pkg)) { if (strings.eqlComptime(pkg, "react") or strings.eqlComptime(pkg, "react-dom")) { diff --git a/src/js_printer.zig b/src/js_printer.zig index 6656faacb2..026e8b30e9 100644 --- a/src/js_printer.zig +++ b/src/js_printer.zig @@ -1939,6 +1939,7 @@ fn NewPrinter( assert(p.import_records.len > import_record_index); const record = p.importRecord(import_record_index); + const module_type = p.options.module_type; if (comptime is_bun_platform) { // "bun" is not a real module. It's just globalThis.Bun. @@ -1962,13 +1963,13 @@ fn NewPrinter( }, .bun_test => { if (record.kind == .dynamic) { - if (p.options.module_type == .cjs) { + if (module_type == .cjs) { p.print("Promise.resolve(globalThis.Bun.jest(__filename))"); } else { p.print("Promise.resolve(globalThis.Bun.jest(import.meta.path))"); } } else if (record.kind == .require) { - if (p.options.module_type == .cjs) { + if (module_type == .cjs) { p.print("globalThis.Bun.jest(__filename)"); } else { p.print("globalThis.Bun.jest(import.meta.path)"); @@ -2027,7 +2028,7 @@ fn NewPrinter( } if (p.options.input_files_for_kit) |input_files| { - bun.assert(p.options.module_type == .internal_kit_dev); + bun.assert(module_type == .internal_kit_dev); p.printSpaceBeforeIdentifier(); p.printSymbol(p.options.commonjs_module_ref); p.print(".require("); @@ -2070,7 +2071,7 @@ fn NewPrinter( } if (wrap_with_to_esm) { - if (p.options.module_type.isESM()) { + if (module_type.isESM()) { p.print(","); p.printSpace(); p.print("1"); @@ -2097,7 +2098,9 @@ fn NewPrinter( } } - if (p.options.module_type == .internal_kit_dev) { + const wrap_with_to_esm = record.wrap_with_to_esm; + + if (module_type == .internal_kit_dev) { p.printSpaceBeforeIdentifier(); p.printSymbol(p.options.commonjs_module_ref); if (record.tag == .builtin) @@ -2112,9 +2115,13 @@ fn NewPrinter( } p.print(")"); return; + } else if (wrap_with_to_esm) { + p.printSpaceBeforeIdentifier(); + p.printSymbol(p.options.to_esm_ref); + p.print("("); } - if (p.options.module_type == .esm and is_bun_platform) { + if (module_type == .esm and is_bun_platform) { p.print("import.meta.require"); } else if (p.options.require_ref) |ref| { p.printSymbol(ref); @@ -2125,6 +2132,10 @@ fn NewPrinter( p.print("("); p.printImportRecordPath(record); p.print(")"); + + if (wrap_with_to_esm) { + p.print(")"); + } return; } @@ -6059,7 +6070,7 @@ pub fn printAst( var module_scope = tree.module_scope; if (opts.minify_identifiers) { const allocator = opts.allocator; - var reserved_names = try rename.computeInitialReservedNames(allocator); + var reserved_names = try rename.computeInitialReservedNames(allocator, opts.module_type); for (module_scope.children.slice()) |child| { child.parent = &module_scope; } diff --git a/src/options.zig b/src/options.zig index fa402cfa95..67f4ec2bba 100644 --- a/src/options.zig +++ b/src/options.zig @@ -599,6 +599,10 @@ pub const Format = enum { return this == .esm; } + pub inline fn isAlwaysStrictMode(this: Format) bool { + return this == .esm; + } + pub const Map = bun.ComptimeStringMap(Format, .{ .{ "esm", .esm }, .{ "cjs", .cjs }, @@ -1484,6 +1488,7 @@ pub const BundleOptions = struct { ignore_dce_annotations: bool = false, emit_dce_annotations: bool = false, + bytecode: bool = false, code_coverage: bool = false, debugger: bool = false, @@ -1825,6 +1830,7 @@ pub const OutputFile = struct { hash: u64 = 0, is_executable: bool = false, source_map_index: u32 = std.math.maxInt(u32), + bytecode_index: u32 = std.math.maxInt(u32), output_kind: JSC.API.BuildArtifact.OutputKind = .chunk, dest_path: []const u8 = "", @@ -1929,6 +1935,7 @@ pub const OutputFile = struct { input_loader: Loader, hash: ?u64 = null, source_map_index: ?u32 = null, + bytecode_index: ?u32 = null, output_path: string, size: ?usize = null, input_path: []const u8 = "", @@ -1963,6 +1970,7 @@ pub const OutputFile = struct { .size_without_sourcemap = options.display_size, .hash = options.hash orelse 0, .output_kind = options.output_kind, + .bytecode_index = options.bytecode_index orelse std.math.maxInt(u32), .source_map_index = options.source_map_index orelse std.math.maxInt(u32), .is_executable = options.is_executable, .value = switch (options.data) { diff --git a/src/renamer.zig b/src/renamer.zig index 30c6a21ecf..64e9e93e56 100644 --- a/src/renamer.zig +++ b/src/renamer.zig @@ -875,6 +875,7 @@ pub const ExportRenamer = struct { pub fn computeInitialReservedNames( allocator: std.mem.Allocator, + output_format: bun.options.Format, ) !bun.StringHashMapUnmanaged(u32) { if (comptime bun.Environment.isWasm) { unreachable; @@ -887,9 +888,17 @@ pub fn computeInitialReservedNames( "Require", }; + const cjs_names = .{ + "exports", + "module", + }; + + const cjs_names_len: u32 = if (output_format == .cjs) cjs_names.len else 0; + try names.ensureTotalCapacityContext( allocator, - @as(u32, @truncate(JSLexer.Keywords.keys().len + JSLexer.StrictModeReservedWords.keys().len + 1 + extras.len)), + cjs_names_len + + @as(u32, @truncate(JSLexer.Keywords.keys().len + JSLexer.StrictModeReservedWords.keys().len + 1 + extras.len)), bun.StringHashMapContext{}, ); @@ -901,6 +910,19 @@ pub fn computeInitialReservedNames( names.putAssumeCapacity(keyword, 1); } + // Node contains code that scans CommonJS modules in an attempt to statically + // detect the set of export names that a module will use. However, it doesn't + // do any scope analysis so it can be fooled by local variables with the same + // name as the CommonJS module-scope variables "exports" and "module". Avoid + // using these names in this case even if there is not a risk of a name + // collision because there is still a risk of node incorrectly detecting + // something in a nested scope as an top-level export. + if (output_format == .cjs) { + inline for (cjs_names) |name| { + names.putAssumeCapacity(name, 1); + } + } + inline for (comptime extras) |extra| { names.putAssumeCapacity(extra, 1); } diff --git a/src/runtime.zig b/src/runtime.zig index d42d657c7d..932ecfd505 100644 --- a/src/runtime.zig +++ b/src/runtime.zig @@ -241,6 +241,9 @@ pub const Runtime = struct { /// This is only supported with --target=bun use_import_meta_require: bool = false, + /// Allow runtime usage of require(), converting `require` into `__require` + auto_polyfill_require: bool = false, + replace_exports: ReplaceableExport.Map = .{}, /// Scan for '// @bun' at the top of this file, halting a parse if it is @@ -259,6 +262,7 @@ pub const Runtime = struct { unwrap_commonjs_packages: []const string = &.{}, commonjs_at_runtime: bool = false, + unwrap_commonjs_to_esm: bool = false, emit_decorator_metadata: bool = false, diff --git a/test/bundler/bun-build-api.test.ts b/test/bundler/bun-build-api.test.ts index 3ad22e8dad..10248b6a29 100644 --- a/test/bundler/bun-build-api.test.ts +++ b/test/bundler/bun-build-api.test.ts @@ -4,6 +4,34 @@ import { bunEnv, bunExe, tempDirWithFiles } from "harness"; import path, { join } from "path"; describe("Bun.build", () => { + test("bytecode works", async () => { + const dir = tempDirWithFiles("bun-build-api-bytecode", { + "package.json": `{}`, + "index.ts": ` + export function hello() { + return "world"; + } + + console.log(hello()); + `, + out: { + "hmm.js": "hmm", + }, + }); + + const build = await Bun.build({ + entrypoints: [join(dir, "index.ts")], + outdir: join(dir, "out"), + target: "bun", + bytecode: true, + }); + + expect(build.outputs).toHaveLength(2); + expect(build.outputs[0].kind).toBe("entry-point"); + expect(build.outputs[1].kind).toBe("bytecode"); + expect([build.outputs[0].path]).toRun("world\n"); + }); + test("passing undefined doesnt segfault", () => { try { // @ts-ignore diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index 078f9bb3c2..60a811bdc9 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -13,6 +13,27 @@ describe("bundler", () => { }, run: { stdout: "Hello, world!" }, }); + itBundled("compile/HelloWorldBytecode", { + compile: true, + bytecode: true, + files: { + "/entry.ts": /* js */ ` + console.log("Hello, world!"); + `, + }, + run: { + stdout: "Hello, world!", + stderr: [ + "[Disk Cache] Cache hit for sourceCode", + + // TODO: remove this line once bun:main is removed. + "[Disk Cache] Cache miss for sourceCode", + ].join("\n"), + env: { + BUN_JSC_verboseDiskCache: "1", + }, + }, + }); // https://github.com/oven-sh/bun/issues/8697 itBundled("compile/EmbeddedFileOutfile", { compile: true, @@ -66,6 +87,43 @@ describe("bundler", () => { outfile: "dist/out", run: { stdout: "Hello, world!\nWorker loaded!\n", file: "dist/out", setCwd: true }, }); + itBundled("compile/WorkerRelativePathTSExtensionBytecode", { + compile: true, + bytecode: true, + files: { + "/entry.ts": /* js */ ` + import {rmSync} from 'fs'; + // Verify we're not just importing from the filesystem + rmSync("./worker.ts", {force: true}); + console.log("Hello, world!"); + new Worker("./worker.ts"); + `, + "/worker.ts": /* js */ ` + console.log("Worker loaded!"); + `.trim(), + }, + entryPointsRaw: ["./entry.ts", "./worker.ts"], + outfile: "dist/out", + run: { + stdout: "Hello, world!\nWorker loaded!\n", + file: "dist/out", + setCwd: true, + stderr: [ + "[Disk Cache] Cache hit for sourceCode", + + // TODO: remove this line once bun:main is removed. + "[Disk Cache] Cache miss for sourceCode", + + "[Disk Cache] Cache hit for sourceCode", + + // TODO: remove this line once bun:main is removed. + "[Disk Cache] Cache miss for sourceCode", + ].join("\n"), + env: { + BUN_JSC_verboseDiskCache: "1", + }, + }, + }); itBundled("compile/Bun.embeddedFiles", { compile: true, // TODO: this shouldn't be necessary, or we should add a map aliasing files. @@ -188,10 +246,25 @@ describe("bundler", () => { }, run: { stdout: "ok" }, }); - itBundled("compile/ReactSSR", { - install: ["react@next", "react-dom@next"], - files: { - "/entry.tsx": /* tsx */ ` + + for (const additionalOptions of [ + { bytecode: true, minify: true, format: "cjs" }, + { format: "cjs" }, + { format: "cjs", minify: true }, + { format: "esm" }, + { format: "esm", minify: true }, + ]) { + const { bytecode = false, format, minify = false } = additionalOptions; + const NODE_ENV = minify ? "'production'" : undefined; + itBundled("compile/ReactSSR" + (bytecode ? "+bytecode" : "") + "+" + format + (minify ? "+minify" : ""), { + install: ["react@next", "react-dom@next"], + format, + minifySyntax: minify, + minifyIdentifiers: minify, + minifyWhitespace: minify, + define: NODE_ENV ? { "process.env.NODE_ENV": NODE_ENV } : undefined, + files: { + "/entry.tsx": /* tsx */ ` import React from "react"; import { renderToReadableStream } from "react-dom/server"; @@ -210,23 +283,37 @@ describe("bundler", () => { ); - const port = 0; - using server = Bun.serve({ - port, - async fetch(req) { - return new Response(await renderToReadableStream(), headers); - }, - }); - const res = await fetch(server.url); - if (res.status !== 200) throw "status error"; - console.log(await res.text()); + async function main() { + const port = 0; + using server = Bun.serve({ + port, + async fetch(req) { + return new Response(await renderToReadableStream(), headers); + }, + }); + const res = await fetch(server.url); + if (res.status !== 200) throw "status error"; + console.log(await res.text()); + } + + main(); `, - }, - run: { - stdout: "

Hello World

This is an example.

", - }, - compile: true, - }); + }, + run: { + stdout: "

Hello World

This is an example.

", + stderr: bytecode + ? "[Disk Cache] Cache hit for sourceCode\n[Disk Cache] Cache miss for sourceCode\n" + : undefined, + env: bytecode + ? { + BUN_JSC_verboseDiskCache: "1", + } + : undefined, + }, + compile: true, + bytecode, + }); + } itBundled("compile/DynamicRequire", { files: { "/entry.tsx": /* tsx */ ` diff --git a/test/bundler/esbuild/dce.test.ts b/test/bundler/esbuild/dce.test.ts index f718bd9670..75ccbbc22c 100644 --- a/test/bundler/esbuild/dce.test.ts +++ b/test/bundler/esbuild/dce.test.ts @@ -1446,6 +1446,7 @@ describe("bundler", () => { format: "cjs", treeShaking: true, bundling: false, + todo: true, }); itBundled("dce/TreeShakingNoBundleIIFE", { files: { @@ -1459,6 +1460,7 @@ describe("bundler", () => { format: "iife", treeShaking: true, bundling: false, + todo: true, }); itBundled("dce/TreeShakingInESMWrapper", { files: { @@ -1687,6 +1689,7 @@ describe("bundler", () => { `, }, format: "iife", + todo: true, dce: true, }); itBundled("dce/RemoveUnusedImports", { diff --git a/test/bundler/esbuild/default.test.ts b/test/bundler/esbuild/default.test.ts index bc8b9ba89c..39847ec22a 100644 --- a/test/bundler/esbuild/default.test.ts +++ b/test/bundler/esbuild/default.test.ts @@ -193,6 +193,7 @@ describe("bundler", () => { format: "iife", globalName: "globalName", run: true, + todo: true, onAfterBundle(api) { api.appendFile( "/out.js", @@ -1318,7 +1319,7 @@ describe("bundler", () => { console.log('writeFileSync' in fs, readFileSync, 'writeFileSync' in defaultValue) `, }, - target: "node", + target: "bun", format: "cjs", run: { stdout: "true [Function: readFileSync] true", @@ -2580,6 +2581,7 @@ describe("bundler", () => { minifySyntax: true, minifyWhitespace: true, bundling: false, + todo: true, onAfterBundle(api) { assert(api.readFile("/out.js").includes('"use strict";'), '"use strict"; was emitted'); }, @@ -2797,14 +2799,16 @@ describe("bundler", () => { }); itBundled("default/ImportMetaCommonJS", { files: { - "/entry.js": `console.log(import.meta.url, import.meta.path)`, + "/entry.js": ` + import fs from "fs"; + import { fileURLToPath } from "url"; + console.log(fs.existsSync(fileURLToPath(import.meta.url)), fs.existsSync(import.meta.path)); + `, }, format: "cjs", - bundleWarnings: { - "/entry.js": [`"import.meta" is not available with the "cjs" output format and will be empty`], - }, + target: "node", run: { - stdout: "undefined undefined", + stdout: "true true", }, }); itBundled("default/ImportMetaES6", { @@ -3431,6 +3435,7 @@ describe("bundler", () => { `, }, format: "iife", + todo: true, bundleErrors: { "/entry.js": ['Top-level await is currently not supported with the "iife" output format'], }, @@ -3453,6 +3458,7 @@ describe("bundler", () => { `, }, format: "cjs", + todo: true, bundleErrors: { "/entry.js": ['Top-level await is currently not supported with the "cjs" output format'], }, @@ -3908,6 +3914,13 @@ describe("bundler", () => { }, target: "node", format: "cjs", + bundleErrors: { + "/entry.js": [ + 'Could not resolve: "./missing-file"', + 'Could not resolve: "missing-pkg"', + 'Could not resolve: "@scope/missing-pkg"', + ], + }, external: ["external-pkg", "@scope/external-pkg", "{{root}}/external-file"], }); itBundled("default/InjectMissing", { @@ -4754,20 +4767,18 @@ describe("bundler", () => { assert([...code.matchAll(/let/g)].length === 3, "should have 3 let statements"); }, }); - // TODO: this is hard to test since bun runtime doesn't support require.main and require.cache - // i'm not even sure what we want our behavior to be for this case. - // itBundled("default/RequireMainCacheCommonJS", { - // files: { - // "/entry.js": /* js */ ` - // console.log('is main:', require.main === module) - // console.log(require('./is-main')) - // console.log('cache:', require.cache); - // `, - // "/is-main.js": `module.exports = require.main === module`, - // }, - // format: "cjs", - // platform: "node", - // }); + itBundled("default/RequireMainCacheCommonJS", { + files: { + "/entry.js": /* js */ ` + console.log('is main:', require.main === module) + console.log(require('./is-main')) + console.log('cache:', require.cache); + `, + "/is-main.js": `module.exports = require.main === module`, + }, + format: "cjs", + platform: "node", + }); itBundled("default/ExternalES6ConvertedToCommonJS", { todo: true, files: { diff --git a/test/bundler/esbuild/extra.test.ts b/test/bundler/esbuild/extra.test.ts index fae84dd219..da25b13476 100644 --- a/test/bundler/esbuild/extra.test.ts +++ b/test/bundler/esbuild/extra.test.ts @@ -432,7 +432,6 @@ describe("bundler", () => { // Use "eval" to access CommonJS variables itBundled("extra/CJSEval1", { - todo: true, files: { "in.js": `if (require('./eval').foo !== 123) throw 'fail'`, "eval.js": `exports.foo=234;eval('exports.foo = 123')`, @@ -440,7 +439,6 @@ describe("bundler", () => { run: true, }); itBundled("extra/CJSEval2", { - todo: true, files: { "in.js": `if (require('./eval').foo !== 123) throw 'fail'`, "eval.js": `module.exports={foo:234};eval('module.exports = {foo: 123}')`, diff --git a/test/bundler/esbuild/importstar.test.ts b/test/bundler/esbuild/importstar.test.ts index 3cec66e2b8..8743b1df9a 100644 --- a/test/bundler/esbuild/importstar.test.ts +++ b/test/bundler/esbuild/importstar.test.ts @@ -561,6 +561,7 @@ describe("bundler", () => { format: "iife", }); itBundled("importstar/ExportSelfIIFEWithName", { + todo: true, files: { "/entry.js": /* js */ ` export const foo = 123 @@ -603,6 +604,7 @@ describe("bundler", () => { `, }, format: "cjs", + runtimeFiles: { "/test.js": /* js */ ` console.log(JSON.stringify(require("./out.js"))); @@ -622,6 +624,7 @@ describe("bundler", () => { }, minifyIdentifiers: true, format: "cjs", + run: { stdout: '{"foo":123}', }, @@ -635,6 +638,7 @@ describe("bundler", () => { `, }, format: "cjs", + runtimeFiles: { "/test.js": /* js */ ` console.log('2', JSON.stringify(require("./out.js"))); @@ -776,6 +780,7 @@ describe("bundler", () => { `, }, format: "cjs", + runtimeFiles: { "/test.js": /* js */ ` const foo = require('./out.js') @@ -795,6 +800,7 @@ describe("bundler", () => { `, }, format: "cjs", + run: { stdout: '{"foo":123}', }, @@ -808,6 +814,7 @@ describe("bundler", () => { `, }, format: "cjs", + run: { stdout: '{"foo":123}', }, @@ -818,6 +825,7 @@ describe("bundler", () => { "/foo.js": `exports.foo = 123`, }, format: "cjs", + runtimeFiles: { "/test.js": /* js */ ` const foo = require('./out.js') @@ -860,6 +868,7 @@ describe("bundler", () => { "/foo.js": `exports.foo = 123`, }, format: "cjs", + runtimeFiles: { "/test.js": /* js */ ` const foo = require('./out.js') @@ -879,6 +888,7 @@ describe("bundler", () => { "/foo.js": `exports.foo = 123`, }, format: "cjs", + runtimeFiles: { "/test.js": /* js */ ` const foo = require('./out.js') @@ -1025,6 +1035,7 @@ describe("bundler", () => { `, }, format: "cjs", + dce: true, runtimeFiles: { "/test.js": /* js */ ` @@ -1052,6 +1063,7 @@ describe("bundler", () => { }, }); itBundled("importstar/ReExportStarExternalIIFE", { + todo: true, files: { "/entry.js": `export * from "foo"`, }, @@ -1098,6 +1110,7 @@ describe("bundler", () => { }, external: ["foo"], format: "cjs", + runtimeFiles: { "/node_modules/foo/index.js": /* js */ ` module.exports = { bar: 'bar', foo: 'foo' } @@ -1361,7 +1374,7 @@ describe("bundler", () => { ], }, }); - itBundled("importstar/ReExportStarEntryPointAndInnerFile", { + itBundled("importstar/ReExportStarEntryPointAndInnerFileExternal", { files: { "/entry.js": /* js */ ` export * from 'a' @@ -1373,13 +1386,38 @@ describe("bundler", () => { format: "cjs", external: ["a", "b"], runtimeFiles: { + "/test.js": /* js */ ` + console.log(JSON.stringify(require('./out.js'))) + `, "/node_modules/a/index.js": /* js */ ` - export const a = 123; - `, + export const a = 123; + `, "/node_modules/b/index.js": /* js */ ` - export const b = 456; + export const b = 456; + `, + }, + run: { + file: "/test.js", + stdout: '{"inner":{"b":456},"a":123,"b":456}', + }, + }); + itBundled("importstar/ReExportStarEntryPointAndInnerFile", { + files: { + "/entry.js": /* js */ ` + export * from 'a' + import * as inner from './inner.js' + export { inner } `, - + "/inner.js": `export * from 'b'`, + "/node_modules/a/index.js": /* js */ ` + export const a = 123; + `, + "/node_modules/b/index.js": /* js */ ` + export const b = 456; + `, + }, + format: "cjs", + runtimeFiles: { "/test.js": /* js */ ` console.log(JSON.stringify(require('./out.js'))) `, diff --git a/test/bundler/expectBundled.ts b/test/bundler/expectBundled.ts index d1177a8102..94e0d38095 100644 --- a/test/bundler/expectBundled.ts +++ b/test/bundler/expectBundled.ts @@ -326,6 +326,7 @@ export interface BundlerTestRunOptions { bunArgs?: string[]; /** match exact stdout */ stdout?: string | RegExp; + stderr?: string; /** partial match stdout (toContain()) */ partialStdout?: string; /** match exact error message, example "ReferenceError: Can't find variable: bar" */ @@ -444,6 +445,7 @@ function expectBundled( unsupportedJSFeatures, useDefineForClassFields, ignoreDCEAnnotations, + bytecode = false, emitDCEAnnotations, // @ts-expect-error _referenceFn, @@ -467,8 +469,15 @@ function expectBundled( // Resolve defaults for options and some related things bundling ??= true; - target ??= "browser"; + + if (bytecode) { + format ??= "cjs"; + target ??= "bun"; + } + format ??= "esm"; + target ??= "browser"; + entryPoints ??= entryPointsRaw ? [] : [Object.keys(files)[0]]; if (run === true) run = {}; if (metafile === true) metafile = "/metafile.json"; @@ -479,9 +488,7 @@ function expectBundled( if (bundling === false && entryPoints.length > 1) { throw new Error("bundling:false only supports a single entry point"); } - if (!ESBUILD && (format === "cjs" || format === "iife")) { - throw new Error(`format ${format} not implemented in bun build`); - } + if (!ESBUILD && metafile) { throw new Error("metafile not implemented in bun build"); } @@ -514,6 +521,9 @@ function expectBundled( throw new Error(`loader '${unsupportedLoaderTypes.join("', '")}' not implemented in bun build`); } } + if (ESBUILD && bytecode) { + throw new Error("bytecode not implemented in esbuild"); + } if (ESBUILD && skipOnEsbuild) { return testRef(id, opts); } @@ -660,6 +670,7 @@ function expectBundled( // mainFields && `--main-fields=${mainFields}`, loader && Object.entries(loader).map(([k, v]) => ["--loader", `${k}:${v}`]), publicPath && `--public-path=${publicPath}`, + bytecode && "--bytecode", ] : [ ESBUILD_PATH, @@ -963,6 +974,7 @@ function expectBundled( sourcemap: sourceMap, splitting, target, + bytecode, publicPath, emitDCEAnnotations, ignoreDCEAnnotations, @@ -1419,7 +1431,6 @@ for (const [key, blob] of build.outputs) { } else { throw new Error(prefix + "run.file is required when there is more than one entrypoint."); } - const args = [ ...(compile ? [] : [(run.runtime ?? "bun") === "bun" ? bunExe() : "node"]), ...(run.bunArgs ?? []), @@ -1431,6 +1442,7 @@ for (const [key, blob] of build.outputs) { cmd: args, env: { ...bunEnv, + ...(run.env || {}), FORCE_COLOR: "0", IS_TEST_RUNNER: "1", }, @@ -1504,28 +1516,31 @@ for (const [key, blob] of build.outputs) { run.validate({ stderr: stderr.toUnixString(), stdout: stdout.toUnixString() }); } - if (run.stdout !== undefined) { - const result = stdout!.toUnixString().trim(); - if (typeof run.stdout === "string") { - const expected = dedent(run.stdout).trim(); + for (let [name, expected, out] of [ + ["stdout", run.stdout, stdout], + ["stderr", run.stderr, stderr], + ].filter(([, v]) => v !== undefined)) { + const result = out!.toUnixString().trim(); + if (typeof expected === "string") { + expected = dedent(expected).trim(); if (expected !== result) { console.log(`runtime failed file: ${file}`); - console.log(`reference stdout:`); + console.log(`${name} output:`); console.log(result); console.log(`---`); - console.log(`expected stdout:`); + console.log(`expected ${name}:`); console.log(expected); console.log(`---`); } expect(result).toBe(expected); } else { - if (!run.stdout.test(result)) { + if (!expected.test(result)) { console.log(`runtime failed file: ${file}`); - console.log(`reference stdout:`); + console.log(`${name} output:`); console.log(result); console.log(`---`); } - expect(result).toMatch(run.stdout); + expect(result).toMatch(expected); } } @@ -1546,7 +1561,7 @@ for (const [key, blob] of build.outputs) { return testRef(id, opts); })(); } - +let anyOnly = false; /** Shorthand for test and expectBundled. See `expectBundled` for what this does. */ export function itBundled( @@ -1584,6 +1599,17 @@ export function itBundled( } return ref; } +itBundled.only = (id: string, opts: BundlerTestInput) => { + const { it } = testForFile(currentFile ?? callerSourceOrigin()); + + it.only( + id, + () => expectBundled(id, opts as any), + // sourcemap code is slow + isDebug ? Infinity : opts.snapshotSourceMap ? 30_000 : undefined, + ); +}; + itBundled.skip = (id: string, opts: BundlerTestInput) => { if (FILTER && !filterMatches(id)) { return testRef(id, opts);