mirror of
https://github.com/oven-sh/bun
synced 2026-02-05 16:38:55 +00:00
Compare commits
14 Commits
dylan/pyth
...
bun-build-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
945e257600 | ||
|
|
e934b1f3e0 | ||
|
|
b4ef485ac8 | ||
|
|
116e2b3669 | ||
|
|
8d8345a114 | ||
|
|
7c43902646 | ||
|
|
4ea6247c66 | ||
|
|
9e391275d3 | ||
|
|
58c008d51f | ||
|
|
7ebbddacaa | ||
|
|
d984e618bd | ||
|
|
c1103ef0e3 | ||
|
|
cded1d040c | ||
|
|
cd84b52714 |
@@ -1273,9 +1273,15 @@ if(LINUX)
|
||||
target_link_libraries(${bun} PUBLIC libatomic.so)
|
||||
endif()
|
||||
|
||||
target_link_libraries(${bun} PRIVATE ${WEBKIT_LIB_PATH}/libicudata.a)
|
||||
target_link_libraries(${bun} PRIVATE ${WEBKIT_LIB_PATH}/libicui18n.a)
|
||||
target_link_libraries(${bun} PRIVATE ${WEBKIT_LIB_PATH}/libicuuc.a)
|
||||
if(USE_WEBKIT_ICU)
|
||||
target_link_libraries(${bun} PRIVATE ${WEBKIT_LIB_PATH}/libicudata.a)
|
||||
target_link_libraries(${bun} PRIVATE ${WEBKIT_LIB_PATH}/libicui18n.a)
|
||||
target_link_libraries(${bun} PRIVATE ${WEBKIT_LIB_PATH}/libicuuc.a)
|
||||
else()
|
||||
# Use system ICU libraries
|
||||
find_package(ICU REQUIRED COMPONENTS data i18n uc)
|
||||
target_link_libraries(${bun} PRIVATE ICU::data ICU::i18n ICU::uc)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(WIN32)
|
||||
|
||||
@@ -28,12 +28,13 @@ if(WEBKIT_LOCAL)
|
||||
# make jsc-compile-debug jsc-copy-headers
|
||||
include_directories(
|
||||
${WEBKIT_PATH}
|
||||
${WEBKIT_PATH}/JavaScriptCore/Headers
|
||||
${WEBKIT_PATH}/JavaScriptCore/Headers/JavaScriptCore
|
||||
${WEBKIT_PATH}/JavaScriptCore/PrivateHeaders
|
||||
${WEBKIT_PATH}/JavaScriptCore/PrivateHeaders/JavaScriptCore
|
||||
${WEBKIT_PATH}/bmalloc/Headers
|
||||
${WEBKIT_PATH}/WTF/Headers
|
||||
${WEBKIT_PATH}/JavaScriptCore/DerivedSources/inspector
|
||||
${WEBKIT_PATH}/JavaScriptCore/PrivateHeaders/JavaScriptCore
|
||||
)
|
||||
endif()
|
||||
|
||||
|
||||
220
docs/esm-bytecode-cache.md
Normal file
220
docs/esm-bytecode-cache.md
Normal file
@@ -0,0 +1,220 @@
|
||||
# ESM Bytecode Cache Architecture
|
||||
|
||||
## Overview
|
||||
|
||||
ESM Bytecode Cache is an experimental feature that enables bytecode caching for ES Modules in Bun's bundler. This allows compiled executables to skip the JavaScript parsing phase during module loading, improving startup performance for large applications.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Binary Format: BMES v3
|
||||
|
||||
The ESM bytecode cache uses a custom binary format called BMES (Bun Module ES) version 3. This format stores both the bytecode and module metadata required to reconstruct a `JSModuleRecord` without parsing.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
│ BMES Header │
|
||||
├─────────────────────────────────────────┤
|
||||
│ Magic Number: "SEMB" (4 bytes) │
|
||||
│ Version: 3 (4 bytes) │
|
||||
│ Bytecode Offset (4 bytes) │
|
||||
│ Bytecode Size (4 bytes) │
|
||||
├─────────────────────────────────────────┤
|
||||
│ Module Metadata │
|
||||
├─────────────────────────────────────────┤
|
||||
│ Requested Modules (dependencies) │
|
||||
│ Import Entries │
|
||||
│ Export Entries │
|
||||
│ Star Export Entries │
|
||||
│ Declared Variables (var) │
|
||||
│ Lexical Variables (let/const) │
|
||||
│ Code Features flags │
|
||||
├─────────────────────────────────────────┤
|
||||
│ JSC Bytecode │
|
||||
│ (WebKit JavaScriptCore format) │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Module Loading Flow
|
||||
|
||||
#### Without Bytecode Cache (Traditional)
|
||||
|
||||
```
|
||||
Source Code → Parse → ModuleAnalyzer → JSModuleRecord → Link → Evaluate
|
||||
↓
|
||||
AST Generation
|
||||
Import/Export Analysis
|
||||
Variable Declaration Extraction
|
||||
```
|
||||
|
||||
#### With Bytecode Cache
|
||||
|
||||
```
|
||||
BMES File → Deserialize Metadata → JSModuleRecord::create() → Link → Evaluate
|
||||
↓
|
||||
Skip Parsing!
|
||||
Skip AST Generation!
|
||||
Skip Module Analysis!
|
||||
```
|
||||
|
||||
### Implementation Components
|
||||
|
||||
#### 1. Metadata Serialization (`src/bun.js/bindings/ZigSourceProvider.cpp`)
|
||||
|
||||
During build time, module metadata is serialized into the BMES format:
|
||||
|
||||
```cpp
|
||||
struct CachedModuleMetadata {
|
||||
Vector<ModuleRequest> requestedModules; // import specifiers
|
||||
Vector<ImportEntry> importEntries; // import bindings
|
||||
Vector<ExportEntry> exportEntries; // export bindings
|
||||
Vector<WTF::String> starExportEntries; // export * from
|
||||
Vector<VariableEntry> declaredVariables; // var declarations
|
||||
Vector<VariableEntry> lexicalVariables; // let/const declarations
|
||||
uint32_t codeFeatures; // feature flags
|
||||
};
|
||||
```
|
||||
|
||||
#### 2. JSC Integration (`vendor/WebKit/Source/JavaScriptCore/`)
|
||||
|
||||
New virtual methods added to `SourceProvider`:
|
||||
|
||||
```cpp
|
||||
// SourceProvider.h
|
||||
virtual bool hasCachedModuleMetadata() const { return false; }
|
||||
virtual JSModuleRecord* createModuleRecordFromCache(
|
||||
JSGlobalObject*, const Identifier&) { return nullptr; }
|
||||
```
|
||||
|
||||
Cache check in module loader (`JSModuleLoader.cpp`):
|
||||
|
||||
```cpp
|
||||
// Check if we can skip parsing by using cached module metadata
|
||||
if (sourceCode.provider()->hasCachedModuleMetadata()) {
|
||||
JSModuleRecord* moduleRecord =
|
||||
sourceCode.provider()->createModuleRecordFromCache(globalObject, moduleKey);
|
||||
if (moduleRecord) {
|
||||
// Skip parsing entirely!
|
||||
promise->fulfillWithNonPromise(globalObject, moduleRecord);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Fall through to normal parsing...
|
||||
```
|
||||
|
||||
#### 3. Module Record Reconstruction (`ZigSourceProvider.cpp`)
|
||||
|
||||
The `createModuleRecordFromCache` method reconstructs a complete `JSModuleRecord`:
|
||||
|
||||
1. Deserialize variable environments (declared + lexical)
|
||||
2. Create `JSModuleRecord` with `JSModuleRecord::create()`
|
||||
3. Add requested modules (dependencies)
|
||||
4. Add import entries
|
||||
5. Add export entries (local, indirect, star exports)
|
||||
|
||||
## Usage
|
||||
|
||||
### Bun.build API
|
||||
|
||||
```typescript
|
||||
await Bun.build({
|
||||
entrypoints: ["./src/index.ts"],
|
||||
outdir: "./dist",
|
||||
target: "bun",
|
||||
compile: true,
|
||||
experimentalEsmBytecode: true, // Enable ESM bytecode cache
|
||||
});
|
||||
```
|
||||
|
||||
### Output Files
|
||||
|
||||
When `experimentalEsmBytecode` is enabled with `compile: true`, the bytecode is embedded directly into the single-file executable.
|
||||
|
||||
## Performance
|
||||
|
||||
### Benchmark Results
|
||||
|
||||
Tested with synthetic modules containing realistic JavaScript patterns (classes, async functions, generators, destructuring, spread operators, etc.)
|
||||
|
||||
#### ESM Bytecode Cache Performance
|
||||
|
||||
| Size | Source | Binary Increase | No Cache | With Cache | Improvement | Time Saved |
|
||||
|------|--------|-----------------|----------|------------|-------------|------------|
|
||||
| tiny | 12 KB | +16 KB | 9.26 ms | 10.13 ms | -9.4% | -0.87 ms |
|
||||
| small | 28 KB | +54 KB | 9.78 ms | 10.43 ms | -6.7% | -0.66 ms |
|
||||
| medium | 212 KB | +483 KB | 13.92 ms | 13.56 ms | +2.6% | +0.36 ms |
|
||||
| large | 2 MB | +4.9 MB | 48.27 ms | 41.83 ms | **+13.4%** | +6.44 ms |
|
||||
| xlarge | 10 MB | +24 MB | 214.80 ms | 192.13 ms | **+10.6%** | +22.67 ms |
|
||||
| huge | 20 MB | +49 MB | 430.52 ms | 364.97 ms | **+15.2%** | **+65.56 ms** |
|
||||
|
||||
#### Comparison with CJS Bytecode Cache
|
||||
|
||||
| Size | ESM Improvement | CJS Improvement | ESM Time Saved | CJS Time Saved |
|
||||
|------|-----------------|-----------------|----------------|----------------|
|
||||
| tiny | -9.4% | -4.6% | -0.87 ms | -0.43 ms |
|
||||
| small | -6.7% | -3.2% | -0.66 ms | -0.31 ms |
|
||||
| medium | +2.6% | +3.6% | +0.36 ms | +0.53 ms |
|
||||
| large | +13.4% | +16.6% | +6.44 ms | +10.40 ms |
|
||||
| xlarge | +10.6% | +14.4% | +22.67 ms | +39.99 ms |
|
||||
| huge | +15.2% | +19.4% | +65.56 ms | +109.84 ms |
|
||||
|
||||
### Key Findings
|
||||
|
||||
1. **Break-even point**: ~200KB of source code
|
||||
- Below this threshold, bytecode cache overhead exceeds parsing time
|
||||
- Above this threshold, performance improvements are significant
|
||||
|
||||
2. **Large applications benefit most**:
|
||||
- 10-15% faster startup for ESM
|
||||
- 14-19% faster startup for CJS
|
||||
- Up to 65ms saved for ~20MB bundles (ESM)
|
||||
- Up to 110ms saved for ~20MB bundles (CJS)
|
||||
|
||||
3. **Binary size trade-off**:
|
||||
- ESM bytecode adds ~2.5x the source size
|
||||
- CJS bytecode adds ~4.5x the source size
|
||||
|
||||
4. **CJS vs ESM performance difference**:
|
||||
- CJS shows higher improvement percentages because:
|
||||
- CJS has additional `require()` resolution overhead
|
||||
- CJS bytecode skips both parsing AND bytecode generation
|
||||
- ESM bytecode currently skips parsing but bytecode is regenerated from cache
|
||||
|
||||
## Limitations
|
||||
|
||||
1. **Small files**: Not recommended for applications under ~200KB as the bytecode loading overhead exceeds parsing time savings.
|
||||
|
||||
2. **Binary size**: Bytecode significantly increases the compiled binary size. Consider this trade-off for deployment scenarios with size constraints.
|
||||
|
||||
3. **Experimental status**: This feature is experimental and the binary format may change in future versions.
|
||||
|
||||
## Future Improvements
|
||||
|
||||
1. **Bytecode execution from cache**: Currently, the bytecode is stored but JSC still regenerates it from the cached data. Direct bytecode execution would further improve performance.
|
||||
|
||||
2. **Lazy bytecode loading**: Load bytecode on-demand for modules that may not be executed.
|
||||
|
||||
3. **Incremental updates**: Support for updating individual module bytecode without rebuilding the entire cache.
|
||||
|
||||
## Technical Details
|
||||
|
||||
### What Gets Skipped
|
||||
|
||||
With ESM bytecode cache enabled, the following operations are skipped during module loading:
|
||||
|
||||
- **Lexical analysis**: Tokenizing the source code
|
||||
- **Parsing**: Building the Abstract Syntax Tree (AST)
|
||||
- **Module analysis**: Extracting import/export declarations
|
||||
- **Scope analysis**: Determining variable bindings
|
||||
|
||||
### What Still Happens
|
||||
|
||||
- **Bytecode validation**: JSC validates the cached bytecode
|
||||
- **Module linking**: Resolving import/export bindings between modules
|
||||
- **Module evaluation**: Executing the module code
|
||||
|
||||
### Memory Considerations
|
||||
|
||||
The module metadata is kept in memory after loading to support:
|
||||
- Module namespace object creation
|
||||
- Dynamic import resolution
|
||||
- Hot module replacement (future)
|
||||
@@ -42,8 +42,8 @@
|
||||
"build:logs": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DENABLE_LOGS=ON -B build/release-logs",
|
||||
"build:safe": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DZIG_OPTIMIZE=ReleaseSafe -B build/release-safe",
|
||||
"build:smol": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=MinSizeRel -B build/release-smol",
|
||||
"build:local": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Debug -DWEBKIT_LOCAL=ON -B build/debug-local",
|
||||
"build:release:local": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DWEBKIT_LOCAL=ON -B build/release-local",
|
||||
"build:local": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Debug -DWEBKIT_LOCAL=ON -DUSE_WEBKIT_ICU=OFF -B build/debug-local",
|
||||
"build:release:local": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DWEBKIT_LOCAL=ON -DUSE_WEBKIT_ICU=OFF -B build/release-local",
|
||||
"build:release:with_logs": "cmake . -DCMAKE_BUILD_TYPE=Release -DENABLE_LOGS=true -GNinja -Bbuild-release && ninja -Cbuild-release",
|
||||
"build:debug-zig-release": "cmake . -DCMAKE_BUILD_TYPE=Release -DZIG_OPTIMIZE=Debug -GNinja -Bbuild-debug-zig-release && ninja -Cbuild-debug-zig-release",
|
||||
"run:linux": "docker run --rm -v \"$PWD:/root/bun/\" -w /root/bun ghcr.io/oven-sh/bun-development-docker-image",
|
||||
|
||||
@@ -134,16 +134,20 @@ const getBuildFlags = (config: BuildConfig) => {
|
||||
const getBuildEnv = () => {
|
||||
const env = { ...process.env };
|
||||
|
||||
const cflags = ["-ffat-lto-objects"];
|
||||
const cxxflags = ["-ffat-lto-objects"];
|
||||
const cflags: string[] = [];
|
||||
const cxxflags: string[] = [];
|
||||
|
||||
if (IS_LINUX && buildConfig !== "lto") {
|
||||
cflags.push("-Wl,--whole-archive");
|
||||
cxxflags.push("-Wl,--whole-archive", "-DUSE_BUN_JSC_ADDITIONS=ON", "-DUSE_BUN_EVENT_LOOP=ON");
|
||||
// Note: -ffat-lto-objects and --whole-archive removed due to libgcc multiple definition errors
|
||||
cxxflags.push("-DUSE_BUN_JSC_ADDITIONS=ON", "-DUSE_BUN_EVENT_LOOP=ON");
|
||||
}
|
||||
|
||||
env.CFLAGS = (env.CFLAGS || "") + " " + cflags.join(" ");
|
||||
env.CXXFLAGS = (env.CXXFLAGS || "") + " " + cxxflags.join(" ");
|
||||
if (cflags.length > 0) {
|
||||
env.CFLAGS = (env.CFLAGS || "") + " " + cflags.join(" ");
|
||||
}
|
||||
if (cxxflags.length > 0) {
|
||||
env.CXXFLAGS = (env.CXXFLAGS || "") + " " + cxxflags.join(" ");
|
||||
}
|
||||
|
||||
if (IS_MAC) {
|
||||
env.ICU_INCLUDE_DIRS = `${HOMEBREW_PREFIX}opt/icu4c/include`;
|
||||
|
||||
@@ -34,6 +34,7 @@ pub const JSBundler = struct {
|
||||
packages: options.PackagesOption = .bundle,
|
||||
format: options.Format = .esm,
|
||||
bytecode: bool = false,
|
||||
experimental_esm_bytecode: bool = false,
|
||||
banner: OwnedString = OwnedString.initEmpty(bun.default_allocator),
|
||||
footer: OwnedString = OwnedString.initEmpty(bun.default_allocator),
|
||||
css_chunking: bool = false,
|
||||
@@ -330,6 +331,21 @@ pub const JSBundler = struct {
|
||||
}
|
||||
}
|
||||
|
||||
if (try config.getBooleanLoose(globalThis, "experimentalEsmBytecode")) |experimental_esm_bytecode| {
|
||||
this.experimental_esm_bytecode = experimental_esm_bytecode;
|
||||
|
||||
if (experimental_esm_bytecode) {
|
||||
// Default to ESM format for ESM bytecode cache
|
||||
this.format = .esm;
|
||||
|
||||
// Must target bun
|
||||
if (did_set_target and this.target != .bun and this.experimental_esm_bytecode) {
|
||||
return globalThis.throwInvalidArguments("target must be 'bun' when experimentalEsmBytecode is true", .{});
|
||||
}
|
||||
this.target = .bun;
|
||||
}
|
||||
}
|
||||
|
||||
var has_out_dir = false;
|
||||
if (try config.getOptional(globalThis, "outdir", ZigString.Slice)) |slice| {
|
||||
defer slice.deinit();
|
||||
@@ -448,6 +464,10 @@ pub const JSBundler = struct {
|
||||
if (this.bytecode and format != .cjs) {
|
||||
return globalThis.throwInvalidArguments("format must be 'cjs' when bytecode is true. Eventually we'll add esm support as well.", .{});
|
||||
}
|
||||
|
||||
if (this.experimental_esm_bytecode and format != .esm) {
|
||||
return globalThis.throwInvalidArguments("format must be 'esm' when experimentalEsmBytecode is true", .{});
|
||||
}
|
||||
}
|
||||
|
||||
if (try config.getBooleanLoose(globalThis, "splitting")) |hot| {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
pub const CachedBytecode = opaque {
|
||||
extern fn generateCachedModuleByteCodeFromSourceCode(sourceProviderURL: *bun.String, input_code: [*]const u8, inputSourceCodeSize: usize, outputByteCode: *?[*]u8, outputByteCodeSize: *usize, cached_bytecode: *?*CachedBytecode) bool;
|
||||
extern fn generateCachedCommonJSProgramByteCodeFromSourceCode(sourceProviderURL: *bun.String, input_code: [*]const u8, inputSourceCodeSize: usize, outputByteCode: *?[*]u8, outputByteCodeSize: *usize, cached_bytecode: *?*CachedBytecode) bool;
|
||||
extern fn generateCachedModuleByteCodeWithMetadata(sourceProviderURL: *bun.String, input_code: [*]const u8, inputSourceCodeSize: usize, outputByteCode: *?[*]u8, outputByteCodeSize: *usize, cached_bytecode: *?*CachedBytecode) bool;
|
||||
extern fn validateCachedModuleMetadata(cacheData: [*]const u8, cacheSize: usize) bool;
|
||||
|
||||
pub fn generateForESM(sourceProviderURL: *bun.String, input: []const u8) ?struct { []const u8, *CachedBytecode } {
|
||||
var this: ?*CachedBytecode = null;
|
||||
@@ -14,6 +16,22 @@ pub const CachedBytecode = opaque {
|
||||
return null;
|
||||
}
|
||||
|
||||
pub fn generateForESMWithMetadata(sourceProviderURL: *bun.String, input: []const u8) ?struct { []const u8, *CachedBytecode } {
|
||||
var this: ?*CachedBytecode = null;
|
||||
|
||||
var input_code_size: usize = 0;
|
||||
var input_code_ptr: ?[*]u8 = null;
|
||||
if (generateCachedModuleByteCodeWithMetadata(sourceProviderURL, input.ptr, input.len, &input_code_ptr, &input_code_size, &this)) {
|
||||
return .{ input_code_ptr.?[0..input_code_size], this.? };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
pub fn validateMetadata(cache: []const u8) bool {
|
||||
return validateCachedModuleMetadata(cache.ptr, cache.len);
|
||||
}
|
||||
|
||||
pub fn generateForCJS(sourceProviderURL: *bun.String, input: []const u8) ?struct { []const u8, *CachedBytecode } {
|
||||
var this: ?*CachedBytecode = null;
|
||||
var input_code_size: usize = 0;
|
||||
@@ -74,3 +92,48 @@ pub const CachedBytecode = opaque {
|
||||
|
||||
const bun = @import("bun");
|
||||
const std = @import("std");
|
||||
const jsc = bun.jsc;
|
||||
|
||||
pub const TestingAPIs = struct {
|
||||
pub fn generateForESMWithMetadata(global: *jsc.JSGlobalObject, call_frame: *jsc.CallFrame) bun.JSError!jsc.JSValue {
|
||||
const source_url_val = call_frame.argument(0);
|
||||
const input_val = call_frame.argument(1);
|
||||
|
||||
if (!source_url_val.isString()) {
|
||||
return global.throw("Expected source URL string as first argument", .{});
|
||||
}
|
||||
|
||||
if (!input_val.isString()) {
|
||||
return global.throw("Expected source code string as second argument", .{});
|
||||
}
|
||||
|
||||
var source_url_str = try source_url_val.toSlice(global, bun.default_allocator);
|
||||
defer source_url_str.deinit();
|
||||
|
||||
var input_str = try input_val.toSlice(global, bun.default_allocator);
|
||||
defer input_str.deinit();
|
||||
|
||||
var source_url_bun = bun.String.init(source_url_str.slice());
|
||||
defer source_url_bun.deref();
|
||||
|
||||
const result = CachedBytecode.generateForESMWithMetadata(&source_url_bun, input_str.slice()) orelse {
|
||||
return jsc.JSValue.jsNull();
|
||||
};
|
||||
|
||||
const cache_data, _ = result;
|
||||
|
||||
// Create a Uint8Array for the cache data (without copying)
|
||||
return try jsc.ArrayBuffer.createUint8Array(global, cache_data);
|
||||
}
|
||||
|
||||
pub fn validateMetadata(global: *jsc.JSGlobalObject, call_frame: *jsc.CallFrame) bun.JSError!jsc.JSValue {
|
||||
const cache_val = call_frame.argument(0);
|
||||
|
||||
const array_buffer = cache_val.asArrayBuffer(global) orelse {
|
||||
return global.throw("Expected Uint8Array or ArrayBuffer as first argument", .{});
|
||||
};
|
||||
|
||||
const cache_slice = array_buffer.byteSlice();
|
||||
return jsc.JSValue.jsBoolean(CachedBytecode.validateMetadata(cache_slice));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -15,6 +15,11 @@
|
||||
#include <JavaScriptCore/SourceCodeKey.h>
|
||||
#include <mimalloc.h>
|
||||
#include <JavaScriptCore/CodeCache.h>
|
||||
#include <JavaScriptCore/ModuleAnalyzer.h>
|
||||
#include <JavaScriptCore/JSModuleRecord.h>
|
||||
#include <JavaScriptCore/Parser.h>
|
||||
#include <JavaScriptCore/Nodes.h>
|
||||
#include <JavaScriptCore/VariableEnvironment.h>
|
||||
|
||||
namespace Zig {
|
||||
|
||||
@@ -69,6 +74,283 @@ extern "C" bool BunTest__shouldGenerateCodeCoverage(BunString sourceURL);
|
||||
extern "C" void Bun__addSourceProviderSourceMap(void* bun_vm, SourceProvider* opaque_source_provider, BunString* specifier);
|
||||
extern "C" void Bun__removeSourceProviderSourceMap(void* bun_vm, SourceProvider* opaque_source_provider, BunString* specifier);
|
||||
|
||||
// BMES format constants and helper functions
|
||||
static constexpr uint32_t MODULE_CACHE_MAGIC = 0x424D4553; // "BMES"
|
||||
static constexpr uint32_t MODULE_CACHE_VERSION = 3; // Version 3: includes VariableEnvironment and CodeFeatures
|
||||
|
||||
// BMES v3 Header layout (16 bytes total):
|
||||
// [4 bytes: MAGIC] "BMES"
|
||||
// [4 bytes: VERSION] 3
|
||||
// [4 bytes: BYTECODE_OFFSET] offset from start of buffer to bytecode data
|
||||
// [4 bytes: BYTECODE_SIZE] size of bytecode data
|
||||
// ... metadata (includes VariableEnvironment and CodeFeatures) ...
|
||||
// [BYTECODE_SIZE bytes: BYTECODE_DATA]
|
||||
|
||||
static constexpr size_t BMES_HEADER_SIZE = 16; // magic + version + offset + size
|
||||
|
||||
// Quick bytecode extraction from BMES format - O(1) for v2/v3
|
||||
// Returns true if extraction was successful, and sets bytecodeStart/bytecodeSize
|
||||
static bool extractBytecodeFromBMES(
|
||||
const uint8_t* cacheData,
|
||||
size_t cacheSize,
|
||||
const uint8_t*& bytecodeStart,
|
||||
size_t& bytecodeSize)
|
||||
{
|
||||
if (cacheSize < 8) return false;
|
||||
|
||||
// Read and validate magic
|
||||
const uint8_t* ptr = cacheData;
|
||||
uint32_t magic = static_cast<uint32_t>(ptr[0]) |
|
||||
(static_cast<uint32_t>(ptr[1]) << 8) |
|
||||
(static_cast<uint32_t>(ptr[2]) << 16) |
|
||||
(static_cast<uint32_t>(ptr[3]) << 24);
|
||||
if (magic != MODULE_CACHE_MAGIC) return false;
|
||||
ptr += 4;
|
||||
|
||||
// Read version
|
||||
uint32_t version = static_cast<uint32_t>(ptr[0]) |
|
||||
(static_cast<uint32_t>(ptr[1]) << 8) |
|
||||
(static_cast<uint32_t>(ptr[2]) << 16) |
|
||||
(static_cast<uint32_t>(ptr[3]) << 24);
|
||||
ptr += 4;
|
||||
|
||||
// Version 2 and 3: O(1) bytecode extraction using header offset
|
||||
// Version 2 and 3 share the same header layout
|
||||
if (version == 2 || version == 3) {
|
||||
if (cacheSize < BMES_HEADER_SIZE) return false;
|
||||
|
||||
// Read bytecode offset
|
||||
uint32_t bytecodeOffset = static_cast<uint32_t>(ptr[0]) |
|
||||
(static_cast<uint32_t>(ptr[1]) << 8) |
|
||||
(static_cast<uint32_t>(ptr[2]) << 16) |
|
||||
(static_cast<uint32_t>(ptr[3]) << 24);
|
||||
ptr += 4;
|
||||
|
||||
// Read bytecode size
|
||||
bytecodeSize = static_cast<uint32_t>(ptr[0]) |
|
||||
(static_cast<uint32_t>(ptr[1]) << 8) |
|
||||
(static_cast<uint32_t>(ptr[2]) << 16) |
|
||||
(static_cast<uint32_t>(ptr[3]) << 24);
|
||||
|
||||
// Validate offset and size
|
||||
if (bytecodeOffset + bytecodeSize > cacheSize) return false;
|
||||
|
||||
bytecodeStart = cacheData + bytecodeOffset;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Unknown version
|
||||
return false;
|
||||
}
|
||||
|
||||
// Helper functions for reading serialized data
|
||||
static uint32_t readUint32(const uint8_t*& ptr)
|
||||
{
|
||||
uint32_t value = static_cast<uint32_t>(ptr[0]) |
|
||||
(static_cast<uint32_t>(ptr[1]) << 8) |
|
||||
(static_cast<uint32_t>(ptr[2]) << 16) |
|
||||
(static_cast<uint32_t>(ptr[3]) << 24);
|
||||
ptr += 4;
|
||||
return value;
|
||||
}
|
||||
|
||||
static WTF::String readString(JSC::VM& vm, const uint8_t*& ptr)
|
||||
{
|
||||
uint32_t length = readUint32(ptr);
|
||||
if (length == 0)
|
||||
return WTF::String();
|
||||
WTF::String result = WTF::String::fromUTF8(std::span(ptr, length));
|
||||
ptr += length;
|
||||
return result;
|
||||
}
|
||||
|
||||
// Structure to hold deserialized module metadata
|
||||
struct DeserializedModuleMetadata {
|
||||
struct ModuleRequest {
|
||||
WTF::String specifier;
|
||||
// Attributes omitted for now
|
||||
};
|
||||
|
||||
struct ImportEntry {
|
||||
uint32_t type; // 0=Single, 1=SingleTypeScript, 2=Namespace
|
||||
WTF::String moduleRequest;
|
||||
WTF::String importName;
|
||||
WTF::String localName;
|
||||
};
|
||||
|
||||
struct ExportEntry {
|
||||
uint32_t type; // 0=Local, 1=Indirect, 2=Namespace
|
||||
WTF::String exportName;
|
||||
WTF::String moduleName;
|
||||
WTF::String importName;
|
||||
WTF::String localName;
|
||||
};
|
||||
|
||||
struct VariableEntry {
|
||||
WTF::String name;
|
||||
uint32_t bits;
|
||||
};
|
||||
|
||||
Vector<ModuleRequest> requestedModules;
|
||||
Vector<ImportEntry> importEntries;
|
||||
Vector<ExportEntry> exportEntries;
|
||||
Vector<WTF::String> starExports;
|
||||
Vector<VariableEntry> declaredVariables;
|
||||
Vector<VariableEntry> lexicalVariables;
|
||||
uint32_t codeFeatures = 0;
|
||||
const uint8_t* bytecodeStart = nullptr;
|
||||
size_t bytecodeSize = 0;
|
||||
};
|
||||
|
||||
// Validate and deserialize cached module metadata
|
||||
// Returns std::nullopt if cache is invalid
|
||||
static std::optional<DeserializedModuleMetadata> deserializeCachedModuleMetadata(
|
||||
JSC::VM& vm,
|
||||
const uint8_t* cacheData,
|
||||
size_t cacheSize)
|
||||
{
|
||||
if (cacheSize < 16) // At least magic + version + bytecode_offset + bytecode_size
|
||||
return std::nullopt;
|
||||
|
||||
const uint8_t* ptr = cacheData;
|
||||
const uint8_t* end = cacheData + cacheSize;
|
||||
|
||||
// Check magic number
|
||||
uint32_t magic = readUint32(ptr);
|
||||
if (magic != MODULE_CACHE_MAGIC)
|
||||
return std::nullopt;
|
||||
|
||||
// Check version
|
||||
uint32_t version = readUint32(ptr);
|
||||
if (version != MODULE_CACHE_VERSION)
|
||||
return std::nullopt;
|
||||
|
||||
// Read bytecode offset and size from header (v3 format)
|
||||
uint32_t bytecodeOffset = readUint32(ptr);
|
||||
uint32_t bytecodeSize = readUint32(ptr);
|
||||
|
||||
DeserializedModuleMetadata metadata;
|
||||
metadata.bytecodeSize = bytecodeSize;
|
||||
metadata.bytecodeStart = cacheData + bytecodeOffset;
|
||||
|
||||
// Read requested modules
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t moduleCount = readUint32(ptr);
|
||||
metadata.requestedModules.reserveInitialCapacity(moduleCount);
|
||||
|
||||
for (uint32_t i = 0; i < moduleCount; ++i) {
|
||||
if (ptr >= end) return std::nullopt;
|
||||
WTF::String specifier = readString(vm, ptr);
|
||||
|
||||
// Read has_attributes flag
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t hasAttributes = readUint32(ptr);
|
||||
|
||||
if (hasAttributes) {
|
||||
// Skip attributes for now
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t attrCount = readUint32(ptr);
|
||||
for (uint32_t j = 0; j < attrCount; ++j) {
|
||||
if (ptr >= end) return std::nullopt;
|
||||
readString(vm, ptr); // key
|
||||
readString(vm, ptr); // value
|
||||
}
|
||||
}
|
||||
|
||||
metadata.requestedModules.append({ WTFMove(specifier) });
|
||||
}
|
||||
|
||||
// Read import entries
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t importCount = readUint32(ptr);
|
||||
metadata.importEntries.reserveInitialCapacity(importCount);
|
||||
|
||||
for (uint32_t i = 0; i < importCount; ++i) {
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t type = readUint32(ptr);
|
||||
|
||||
if (ptr >= end) return std::nullopt;
|
||||
WTF::String moduleRequest = readString(vm, ptr);
|
||||
WTF::String importName = readString(vm, ptr);
|
||||
WTF::String localName = readString(vm, ptr);
|
||||
|
||||
metadata.importEntries.append({
|
||||
type,
|
||||
WTFMove(moduleRequest),
|
||||
WTFMove(importName),
|
||||
WTFMove(localName)
|
||||
});
|
||||
}
|
||||
|
||||
// Read export entries
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t exportCount = readUint32(ptr);
|
||||
metadata.exportEntries.reserveInitialCapacity(exportCount);
|
||||
|
||||
for (uint32_t i = 0; i < exportCount; ++i) {
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t type = readUint32(ptr);
|
||||
|
||||
if (ptr >= end) return std::nullopt;
|
||||
WTF::String exportName = readString(vm, ptr);
|
||||
WTF::String moduleName = readString(vm, ptr);
|
||||
WTF::String importName = readString(vm, ptr);
|
||||
WTF::String localName = readString(vm, ptr);
|
||||
|
||||
metadata.exportEntries.append({
|
||||
type,
|
||||
WTFMove(exportName),
|
||||
WTFMove(moduleName),
|
||||
WTFMove(importName),
|
||||
WTFMove(localName)
|
||||
});
|
||||
}
|
||||
|
||||
// Read star exports
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t starExportCount = readUint32(ptr);
|
||||
metadata.starExports.reserveInitialCapacity(starExportCount);
|
||||
|
||||
for (uint32_t i = 0; i < starExportCount; ++i) {
|
||||
if (ptr >= end) return std::nullopt;
|
||||
metadata.starExports.append(readString(vm, ptr));
|
||||
}
|
||||
|
||||
// Read declared variables (v3+)
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t declaredVarCount = readUint32(ptr);
|
||||
metadata.declaredVariables.reserveInitialCapacity(declaredVarCount);
|
||||
|
||||
for (uint32_t i = 0; i < declaredVarCount; ++i) {
|
||||
if (ptr >= end) return std::nullopt;
|
||||
WTF::String name = readString(vm, ptr);
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t bits = readUint32(ptr);
|
||||
metadata.declaredVariables.append({ WTFMove(name), bits });
|
||||
}
|
||||
|
||||
// Read lexical variables (v3+)
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t lexicalVarCount = readUint32(ptr);
|
||||
metadata.lexicalVariables.reserveInitialCapacity(lexicalVarCount);
|
||||
|
||||
for (uint32_t i = 0; i < lexicalVarCount; ++i) {
|
||||
if (ptr >= end) return std::nullopt;
|
||||
WTF::String name = readString(vm, ptr);
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
uint32_t bits = readUint32(ptr);
|
||||
metadata.lexicalVariables.append({ WTFMove(name), bits });
|
||||
}
|
||||
|
||||
// Read code features (v3+)
|
||||
if (ptr + 4 > end) return std::nullopt;
|
||||
metadata.codeFeatures = readUint32(ptr);
|
||||
|
||||
// bytecodeStart and bytecodeSize are already set from header
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
Ref<SourceProvider> SourceProvider::create(
|
||||
Zig::GlobalObject* globalObject,
|
||||
ResolvedSource& resolvedSource,
|
||||
@@ -101,6 +383,97 @@ Ref<SourceProvider> SourceProvider::create(
|
||||
};
|
||||
const auto destructor = resolvedSource.needsDeref ? destructorPtr : destructorNoOp;
|
||||
|
||||
uint8_t* bytecodeData = resolvedSource.bytecode_cache;
|
||||
size_t bytecodeSize = resolvedSource.bytecode_cache_size;
|
||||
|
||||
// Check if this is BMES format (ESM bytecode with metadata)
|
||||
// BMES format starts with magic number 0x424D4553 ("BMES")
|
||||
const uint8_t* extractedBytecodeStart = nullptr;
|
||||
size_t extractedBytecodeSize = 0;
|
||||
if (extractBytecodeFromBMES(bytecodeData, bytecodeSize, extractedBytecodeStart, extractedBytecodeSize)) {
|
||||
// This is BMES format - use zero-copy approach
|
||||
// Keep the original buffer alive, use span pointing to bytecode section
|
||||
// The destructor will free the original BMES buffer when the bytecode is no longer needed
|
||||
|
||||
// Store the original buffer pointer for the destructor
|
||||
uint8_t* originalBuffer = bytecodeData;
|
||||
bool needsDeref = resolvedSource.needsDeref;
|
||||
|
||||
// Create a destructor that frees the original BMES buffer (not the span pointer)
|
||||
WTF::Function<void(const void*)> bmesDestructor = needsDeref
|
||||
? WTF::Function<void(const void*)>([originalBuffer](const void*) {
|
||||
mi_free(originalBuffer);
|
||||
})
|
||||
: WTF::Function<void(const void*)>([](const void*) {
|
||||
// no-op for bun build --compile
|
||||
});
|
||||
|
||||
// Create CachedBytecode with span pointing directly into BMES buffer
|
||||
Ref<JSC::CachedBytecode> bytecode = JSC::CachedBytecode::create(
|
||||
std::span<uint8_t>(const_cast<uint8_t*>(extractedBytecodeStart), extractedBytecodeSize),
|
||||
WTFMove(bmesDestructor), {});
|
||||
|
||||
auto provider = adoptRef(*new SourceProvider(
|
||||
globalObject->isThreadLocalDefaultGlobalObject ? globalObject : nullptr,
|
||||
resolvedSource,
|
||||
string.isNull() ? *StringImpl::empty() : *string.impl(),
|
||||
JSC::SourceTaintedOrigin::Untainted,
|
||||
toSourceOrigin(sourceURLString, isBuiltin),
|
||||
sourceURLString.impl(), TextPosition(),
|
||||
sourceType));
|
||||
provider->m_cachedBytecode = WTFMove(bytecode);
|
||||
|
||||
// Also deserialize module metadata from BMES v3+
|
||||
auto deserializedMetadata = deserializeCachedModuleMetadata(globalObject->vm(), bytecodeData, bytecodeSize);
|
||||
if (deserializedMetadata) {
|
||||
// Convert DeserializedModuleMetadata to CachedModuleMetadata
|
||||
CachedModuleMetadata cachedMetadata;
|
||||
cachedMetadata.requestedModules.reserveInitialCapacity(deserializedMetadata->requestedModules.size());
|
||||
for (const auto& req : deserializedMetadata->requestedModules) {
|
||||
cachedMetadata.requestedModules.append({ req.specifier });
|
||||
}
|
||||
|
||||
cachedMetadata.importEntries.reserveInitialCapacity(deserializedMetadata->importEntries.size());
|
||||
for (const auto& entry : deserializedMetadata->importEntries) {
|
||||
cachedMetadata.importEntries.append({
|
||||
entry.type,
|
||||
entry.moduleRequest,
|
||||
entry.importName,
|
||||
entry.localName
|
||||
});
|
||||
}
|
||||
|
||||
cachedMetadata.exportEntries.reserveInitialCapacity(deserializedMetadata->exportEntries.size());
|
||||
for (const auto& entry : deserializedMetadata->exportEntries) {
|
||||
cachedMetadata.exportEntries.append({
|
||||
entry.type,
|
||||
entry.exportName,
|
||||
entry.moduleName,
|
||||
entry.importName,
|
||||
entry.localName
|
||||
});
|
||||
}
|
||||
|
||||
cachedMetadata.starExports = WTFMove(deserializedMetadata->starExports);
|
||||
|
||||
cachedMetadata.declaredVariables.reserveInitialCapacity(deserializedMetadata->declaredVariables.size());
|
||||
for (const auto& entry : deserializedMetadata->declaredVariables) {
|
||||
cachedMetadata.declaredVariables.append({ entry.name, entry.bits });
|
||||
}
|
||||
|
||||
cachedMetadata.lexicalVariables.reserveInitialCapacity(deserializedMetadata->lexicalVariables.size());
|
||||
for (const auto& entry : deserializedMetadata->lexicalVariables) {
|
||||
cachedMetadata.lexicalVariables.append({ entry.name, entry.bits });
|
||||
}
|
||||
|
||||
cachedMetadata.codeFeatures = deserializedMetadata->codeFeatures;
|
||||
|
||||
provider->m_cachedModuleMetadata = WTFMove(cachedMetadata);
|
||||
}
|
||||
|
||||
return provider;
|
||||
}
|
||||
|
||||
Ref<JSC::CachedBytecode> bytecode = JSC::CachedBytecode::create(std::span<uint8_t>(resolvedSource.bytecode_cache, resolvedSource.bytecode_cache_size), destructor, {});
|
||||
auto provider = adoptRef(*new SourceProvider(
|
||||
globalObject->isThreadLocalDefaultGlobalObject ? globalObject : nullptr,
|
||||
@@ -168,6 +541,290 @@ static JSC::VM& getVMForBytecodeCache()
|
||||
return *vmForBytecodeCache;
|
||||
}
|
||||
|
||||
// Module metadata serialization format:
|
||||
// [4 bytes: MAGIC] "BMES" (Bun Module ESM Serialization)
|
||||
// [4 bytes: VERSION] Current version
|
||||
// [4 bytes: MODULE_REQUEST_COUNT]
|
||||
// For each module request:
|
||||
// [4 bytes: SPECIFIER_LENGTH]
|
||||
// [SPECIFIER_LENGTH bytes: SPECIFIER_UTF8]
|
||||
// [4 bytes: HAS_ATTRIBUTES] (0 or 1)
|
||||
// If HAS_ATTRIBUTES:
|
||||
// [4 bytes: ATTRIBUTE_COUNT]
|
||||
// For each attribute:
|
||||
// [4 bytes: KEY_LENGTH]
|
||||
// [KEY_LENGTH bytes: KEY_UTF8]
|
||||
// [4 bytes: VALUE_LENGTH]
|
||||
// [VALUE_LENGTH bytes: VALUE_UTF8]
|
||||
// [4 bytes: IMPORT_ENTRY_COUNT]
|
||||
// For each import entry:
|
||||
// [4 bytes: TYPE] (0=Single, 1=SingleTypeScript, 2=Namespace)
|
||||
// [4 bytes: MODULE_REQUEST_LENGTH]
|
||||
// [MODULE_REQUEST_LENGTH bytes: MODULE_REQUEST_UTF8]
|
||||
// [4 bytes: IMPORT_NAME_LENGTH]
|
||||
// [IMPORT_NAME_LENGTH bytes: IMPORT_NAME_UTF8]
|
||||
// [4 bytes: LOCAL_NAME_LENGTH]
|
||||
// [LOCAL_NAME_LENGTH bytes: LOCAL_NAME_UTF8]
|
||||
// [4 bytes: EXPORT_ENTRY_COUNT]
|
||||
// For each export entry:
|
||||
// [4 bytes: TYPE] (0=Local, 1=Indirect, 2=Namespace)
|
||||
// [4 bytes: EXPORT_NAME_LENGTH]
|
||||
// [EXPORT_NAME_LENGTH bytes: EXPORT_NAME_UTF8]
|
||||
// [4 bytes: MODULE_NAME_LENGTH]
|
||||
// [MODULE_NAME_LENGTH bytes: MODULE_NAME_UTF8]
|
||||
// [4 bytes: IMPORT_NAME_LENGTH]
|
||||
// [IMPORT_NAME_LENGTH bytes: IMPORT_NAME_UTF8]
|
||||
// [4 bytes: LOCAL_NAME_LENGTH]
|
||||
// [LOCAL_NAME_LENGTH bytes: LOCAL_NAME_UTF8]
|
||||
// [4 bytes: STAR_EXPORT_COUNT]
|
||||
// For each star export:
|
||||
// [4 bytes: MODULE_NAME_LENGTH]
|
||||
// [MODULE_NAME_LENGTH bytes: MODULE_NAME_UTF8]
|
||||
// [4 bytes: BYTECODE_SIZE]
|
||||
// [BYTECODE_SIZE bytes: BYTECODE_DATA]
|
||||
|
||||
// MODULE_CACHE_MAGIC and MODULE_CACHE_VERSION are defined at the top of this file
|
||||
|
||||
static void writeUint32(Vector<uint8_t>& buffer, uint32_t value)
|
||||
{
|
||||
buffer.append(static_cast<uint8_t>(value & 0xFF));
|
||||
buffer.append(static_cast<uint8_t>((value >> 8) & 0xFF));
|
||||
buffer.append(static_cast<uint8_t>((value >> 16) & 0xFF));
|
||||
buffer.append(static_cast<uint8_t>((value >> 24) & 0xFF));
|
||||
}
|
||||
|
||||
static void writeString(Vector<uint8_t>& buffer, const WTF::String& str)
|
||||
{
|
||||
if (str.isNull() || str.isEmpty()) {
|
||||
writeUint32(buffer, 0);
|
||||
return;
|
||||
}
|
||||
CString utf8 = str.utf8();
|
||||
writeUint32(buffer, utf8.length());
|
||||
buffer.appendVector(Vector<uint8_t>(std::span(reinterpret_cast<const uint8_t*>(utf8.data()), utf8.length())));
|
||||
}
|
||||
|
||||
// Check if cached metadata is valid for given source
|
||||
// Only accepts v3 format (with VariableEnvironment and CodeFeatures)
|
||||
extern "C" bool validateCachedModuleMetadata(
|
||||
const uint8_t* cacheData,
|
||||
size_t cacheSize)
|
||||
{
|
||||
if (cacheSize < 8)
|
||||
return false;
|
||||
|
||||
const uint8_t* ptr = cacheData;
|
||||
|
||||
// Check magic
|
||||
uint32_t magic = readUint32(ptr);
|
||||
if (magic != MODULE_CACHE_MAGIC)
|
||||
return false;
|
||||
|
||||
// Check version - only accept v3
|
||||
uint32_t version = readUint32(ptr);
|
||||
if (version != 3)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Generate cached bytecode WITH module metadata
|
||||
// Uses BMES v3 format with bytecode offset in header for O(1) extraction
|
||||
extern "C" bool generateCachedModuleByteCodeWithMetadata(
|
||||
BunString* sourceProviderURL,
|
||||
const Latin1Character* inputSourceCode,
|
||||
size_t inputSourceCodeSize,
|
||||
const uint8_t** outputByteCode,
|
||||
size_t* outputByteCodeSize,
|
||||
JSC::CachedBytecode** cachedBytecodePtr)
|
||||
{
|
||||
using namespace JSC;
|
||||
|
||||
std::span<const Latin1Character> sourceCodeSpan(inputSourceCode, inputSourceCodeSize);
|
||||
SourceCode sourceCode = makeSource(WTF::String(sourceCodeSpan), toSourceOrigin(sourceProviderURL->toWTFString(), false), SourceTaintedOrigin::Untainted);
|
||||
|
||||
VM& vm = getVMForBytecodeCache();
|
||||
JSLockHolder locker(vm);
|
||||
|
||||
// Parse the module to extract metadata
|
||||
ParserError parserError;
|
||||
std::unique_ptr<ModuleProgramNode> moduleProgramNode = parseRootNode<ModuleProgramNode>(
|
||||
vm, sourceCode,
|
||||
ImplementationVisibility::Public,
|
||||
JSParserBuiltinMode::NotBuiltin,
|
||||
StrictModeLexicallyScopedFeature,
|
||||
JSParserScriptMode::Module,
|
||||
SourceParseMode::ModuleAnalyzeMode,
|
||||
parserError
|
||||
);
|
||||
|
||||
if (parserError.isValid() || !moduleProgramNode)
|
||||
return false;
|
||||
|
||||
// Create a temporary global object for analysis
|
||||
Structure* structure = JSGlobalObject::createStructure(vm, jsNull());
|
||||
JSGlobalObject* globalObject = JSGlobalObject::create(vm, structure);
|
||||
|
||||
// Analyze the module
|
||||
ModuleAnalyzer analyzer(globalObject, Identifier::fromString(vm, sourceProviderURL->toWTFString()),
|
||||
sourceCode, moduleProgramNode->varDeclarations(),
|
||||
moduleProgramNode->lexicalVariables(), AllFeatures);
|
||||
|
||||
auto result = analyzer.analyze(*moduleProgramNode);
|
||||
if (!result)
|
||||
return false;
|
||||
|
||||
JSModuleRecord* moduleRecord = *result;
|
||||
|
||||
// Generate bytecode first to know its size
|
||||
UnlinkedModuleProgramCodeBlock* unlinkedCodeBlock = recursivelyGenerateUnlinkedCodeBlockForModuleProgram(
|
||||
vm, sourceCode, StrictModeLexicallyScopedFeature, JSParserScriptMode::Module,
|
||||
{}, parserError, EvalContextType::None
|
||||
);
|
||||
|
||||
if (parserError.isValid() || !unlinkedCodeBlock)
|
||||
return false;
|
||||
|
||||
auto key = sourceCodeKeyForSerializedModule(vm, sourceCode);
|
||||
RefPtr<CachedBytecode> bytecodeCache = encodeCodeBlock(vm, key, unlinkedCodeBlock);
|
||||
|
||||
if (!bytecodeCache)
|
||||
return false;
|
||||
|
||||
// BMES v3 Format:
|
||||
// [4 bytes: MAGIC] [4 bytes: VERSION] [4 bytes: BYTECODE_OFFSET] [4 bytes: BYTECODE_SIZE]
|
||||
// [... metadata (includes VariableEnvironment and CodeFeatures) ...]
|
||||
// [BYTECODE_SIZE bytes: BYTECODE_DATA]
|
||||
|
||||
Vector<uint8_t> metadataBuffer;
|
||||
metadataBuffer.reserveInitialCapacity(4096);
|
||||
|
||||
// Write header - offset and size will be filled in later
|
||||
writeUint32(metadataBuffer, MODULE_CACHE_MAGIC);
|
||||
writeUint32(metadataBuffer, MODULE_CACHE_VERSION); // Version 3
|
||||
size_t offsetPosition = metadataBuffer.size();
|
||||
writeUint32(metadataBuffer, 0); // Placeholder for bytecode offset
|
||||
writeUint32(metadataBuffer, static_cast<uint32_t>(bytecodeCache->span().size())); // Bytecode size
|
||||
|
||||
// Serialize requested modules
|
||||
const auto& requestedModules = moduleRecord->requestedModules();
|
||||
writeUint32(metadataBuffer, requestedModules.size());
|
||||
|
||||
for (const auto& request : requestedModules) {
|
||||
writeString(metadataBuffer, *request.m_specifier);
|
||||
|
||||
// Serialize attributes
|
||||
if (request.m_attributes) {
|
||||
writeUint32(metadataBuffer, 1); // has attributes
|
||||
// For now, we'll skip detailed attribute serialization
|
||||
// This can be extended later
|
||||
writeUint32(metadataBuffer, 0); // attribute count
|
||||
} else {
|
||||
writeUint32(metadataBuffer, 0); // no attributes
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize import entries
|
||||
const auto& importEntries = moduleRecord->importEntries();
|
||||
writeUint32(metadataBuffer, importEntries.size());
|
||||
|
||||
for (const auto& entry : importEntries) {
|
||||
writeUint32(metadataBuffer, static_cast<uint32_t>(entry.value.type));
|
||||
writeString(metadataBuffer, entry.value.moduleRequest.string());
|
||||
writeString(metadataBuffer, entry.value.importName.string());
|
||||
writeString(metadataBuffer, entry.value.localName.string());
|
||||
}
|
||||
|
||||
// Serialize export entries
|
||||
const auto& exportEntries = moduleRecord->exportEntries();
|
||||
writeUint32(metadataBuffer, exportEntries.size());
|
||||
|
||||
for (const auto& entry : exportEntries) {
|
||||
writeUint32(metadataBuffer, static_cast<uint32_t>(entry.value.type));
|
||||
writeString(metadataBuffer, entry.value.exportName.string());
|
||||
writeString(metadataBuffer, entry.value.moduleName.string());
|
||||
writeString(metadataBuffer, entry.value.importName.string());
|
||||
writeString(metadataBuffer, entry.value.localName.string());
|
||||
}
|
||||
|
||||
// Serialize star exports
|
||||
const auto& starExports = moduleRecord->starExportEntries();
|
||||
writeUint32(metadataBuffer, starExports.size());
|
||||
|
||||
for (const auto& moduleName : starExports) {
|
||||
writeString(metadataBuffer, *moduleName);
|
||||
}
|
||||
|
||||
// Serialize declared variables (from ModuleProgramNode)
|
||||
const auto& declaredVars = moduleProgramNode->varDeclarations();
|
||||
writeUint32(metadataBuffer, declaredVars.size());
|
||||
for (const auto& entry : declaredVars) {
|
||||
writeString(metadataBuffer, String(entry.key.get()));
|
||||
writeUint32(metadataBuffer, entry.value.bits());
|
||||
}
|
||||
|
||||
// Serialize lexical variables (from ModuleProgramNode)
|
||||
const auto& lexicalVars = moduleProgramNode->lexicalVariables();
|
||||
writeUint32(metadataBuffer, lexicalVars.size());
|
||||
for (const auto& entry : lexicalVars) {
|
||||
writeString(metadataBuffer, String(entry.key.get()));
|
||||
writeUint32(metadataBuffer, entry.value.bits());
|
||||
}
|
||||
|
||||
// Serialize code features
|
||||
writeUint32(metadataBuffer, static_cast<uint32_t>(moduleProgramNode->features()));
|
||||
|
||||
// Align bytecode offset to 8-byte boundary for JSC's CachedBytecode requirements
|
||||
size_t currentSize = metadataBuffer.size();
|
||||
size_t alignedOffset = (currentSize + 7) & ~static_cast<size_t>(7);
|
||||
size_t paddingNeeded = alignedOffset - currentSize;
|
||||
|
||||
// Add padding bytes
|
||||
for (size_t i = 0; i < paddingNeeded; ++i) {
|
||||
metadataBuffer.append(0);
|
||||
}
|
||||
|
||||
// Record bytecode offset (now aligned to 8-byte boundary)
|
||||
uint32_t bytecodeOffset = static_cast<uint32_t>(metadataBuffer.size());
|
||||
|
||||
// Write bytecode offset back into header
|
||||
metadataBuffer[offsetPosition] = static_cast<uint8_t>(bytecodeOffset & 0xFF);
|
||||
metadataBuffer[offsetPosition + 1] = static_cast<uint8_t>((bytecodeOffset >> 8) & 0xFF);
|
||||
metadataBuffer[offsetPosition + 2] = static_cast<uint8_t>((bytecodeOffset >> 16) & 0xFF);
|
||||
metadataBuffer[offsetPosition + 3] = static_cast<uint8_t>((bytecodeOffset >> 24) & 0xFF);
|
||||
|
||||
// Append bytecode data
|
||||
metadataBuffer.appendVector(Vector<uint8_t>(bytecodeCache->span()));
|
||||
|
||||
// Create final cached bytecode
|
||||
WTF::Function<void(const void*)> finalDestructor = [](const void* ptr) {
|
||||
mi_free(const_cast<void*>(ptr));
|
||||
};
|
||||
|
||||
// Use mi_malloc instead of new[] for consistency
|
||||
uint8_t* finalBuffer = static_cast<uint8_t*>(mi_malloc(metadataBuffer.size()));
|
||||
if (!finalBuffer)
|
||||
return false;
|
||||
|
||||
// Copy using range-based iteration to avoid accessing private data()
|
||||
for (size_t i = 0; i < metadataBuffer.size(); ++i) {
|
||||
finalBuffer[i] = metadataBuffer[i];
|
||||
}
|
||||
|
||||
RefPtr<CachedBytecode> finalCache = CachedBytecode::create(
|
||||
std::span<uint8_t>(finalBuffer, metadataBuffer.size()),
|
||||
WTFMove(finalDestructor),
|
||||
{}
|
||||
);
|
||||
|
||||
finalCache->ref();
|
||||
*cachedBytecodePtr = finalCache.get();
|
||||
*outputByteCode = finalBuffer;
|
||||
*outputByteCodeSize = metadataBuffer.size();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
extern "C" bool generateCachedModuleByteCodeFromSourceCode(BunString* sourceProviderURL, const Latin1Character* inputSourceCode, size_t inputSourceCodeSize, const uint8_t** outputByteCode, size_t* outputByteCodeSize, JSC::CachedBytecode** cachedBytecodePtr)
|
||||
{
|
||||
std::span<const Latin1Character> sourceCodeSpan(inputSourceCode, inputSourceCodeSize);
|
||||
@@ -365,4 +1022,145 @@ extern "C" BunString ZigSourceProvider__getSourceSlice(SourceProvider* provider)
|
||||
return Bun::toStringView(provider->source());
|
||||
}
|
||||
|
||||
JSC::JSModuleRecord* SourceProvider::createModuleRecordFromCache(
|
||||
JSC::JSGlobalObject* globalObject,
|
||||
const JSC::Identifier& moduleKey)
|
||||
{
|
||||
if (!m_cachedModuleMetadata.has_value()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
JSC::VM& vm = globalObject->vm();
|
||||
auto scope = DECLARE_THROW_SCOPE(vm);
|
||||
|
||||
const auto& metadata = m_cachedModuleMetadata.value();
|
||||
|
||||
// Helper lambda to set VariableEnvironmentEntry bits
|
||||
// Set flags individually to be compatible with both prebuilt and local JSC
|
||||
auto setEntryBits = [](JSC::VariableEnvironmentEntry& varEntry, uint32_t bits) {
|
||||
// These bit positions match VariableEnvironmentEntry::Traits enum
|
||||
if (bits & (1 << 0)) varEntry.setIsCaptured();
|
||||
if (bits & (1 << 1)) varEntry.setIsConst();
|
||||
if (bits & (1 << 2)) varEntry.setIsVar();
|
||||
if (bits & (1 << 3)) varEntry.setIsLet();
|
||||
if (bits & (1 << 4)) varEntry.setIsExported();
|
||||
if (bits & (1 << 5)) varEntry.setIsImported();
|
||||
if (bits & (1 << 6)) varEntry.setIsImportedNamespace();
|
||||
if (bits & (1 << 7)) varEntry.setIsFunction();
|
||||
if (bits & (1 << 8)) varEntry.setIsParameter();
|
||||
if (bits & (1 << 9)) varEntry.setIsSloppyModeHoistedFunction();
|
||||
if (bits & (1 << 10)) varEntry.setIsPrivateField();
|
||||
if (bits & (1 << 11)) varEntry.setIsPrivateMethod();
|
||||
if (bits & (1 << 12)) varEntry.setIsPrivateGetter();
|
||||
if (bits & (1 << 13)) varEntry.setIsPrivateSetter();
|
||||
};
|
||||
|
||||
// Build VariableEnvironment for declared variables
|
||||
JSC::VariableEnvironment declaredVariables;
|
||||
for (const auto& entry : metadata.declaredVariables) {
|
||||
auto identifier = JSC::Identifier::fromString(vm, entry.name);
|
||||
auto result = declaredVariables.add(identifier);
|
||||
setEntryBits(result.iterator->value, entry.bits);
|
||||
}
|
||||
|
||||
// Build VariableEnvironment for lexical variables
|
||||
JSC::VariableEnvironment lexicalVariables;
|
||||
for (const auto& entry : metadata.lexicalVariables) {
|
||||
auto identifier = JSC::Identifier::fromString(vm, entry.name);
|
||||
auto result = lexicalVariables.add(identifier);
|
||||
setEntryBits(result.iterator->value, entry.bits);
|
||||
}
|
||||
|
||||
// Create SourceCode
|
||||
JSC::SourceCode sourceCode(this, 0, source().length(), 0, 0);
|
||||
|
||||
// Get the structure for JSModuleRecord
|
||||
JSC::Structure* moduleRecordStructure = globalObject->moduleRecordStructure();
|
||||
if (!moduleRecordStructure) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Create JSModuleRecord
|
||||
JSC::JSModuleRecord* moduleRecord = JSC::JSModuleRecord::create(
|
||||
globalObject,
|
||||
vm,
|
||||
moduleRecordStructure,
|
||||
moduleKey,
|
||||
sourceCode,
|
||||
declaredVariables,
|
||||
lexicalVariables,
|
||||
static_cast<JSC::CodeFeatures>(metadata.codeFeatures)
|
||||
);
|
||||
RETURN_IF_EXCEPTION(scope, nullptr);
|
||||
|
||||
if (!moduleRecord) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Add requested modules
|
||||
for (const auto& request : metadata.requestedModules) {
|
||||
auto specifier = JSC::Identifier::fromString(vm, request.specifier);
|
||||
moduleRecord->appendRequestedModule(specifier, nullptr);
|
||||
}
|
||||
|
||||
// Add import entries
|
||||
for (const auto& entry : metadata.importEntries) {
|
||||
JSC::AbstractModuleRecord::ImportEntry importEntry;
|
||||
switch (entry.type) {
|
||||
case 0:
|
||||
importEntry.type = JSC::AbstractModuleRecord::ImportEntryType::Single;
|
||||
break;
|
||||
case 1:
|
||||
#if USE(BUN_JSC_ADDITIONS)
|
||||
importEntry.type = JSC::AbstractModuleRecord::ImportEntryType::SingleTypeScript;
|
||||
break;
|
||||
#else
|
||||
importEntry.type = JSC::AbstractModuleRecord::ImportEntryType::Single;
|
||||
break;
|
||||
#endif
|
||||
case 2:
|
||||
default:
|
||||
importEntry.type = JSC::AbstractModuleRecord::ImportEntryType::Namespace;
|
||||
break;
|
||||
}
|
||||
// Use empty identifier for null strings to avoid crash
|
||||
importEntry.moduleRequest = entry.moduleRequest.isNull() ? JSC::Identifier() : JSC::Identifier::fromString(vm, entry.moduleRequest);
|
||||
importEntry.importName = entry.importName.isNull() ? JSC::Identifier() : JSC::Identifier::fromString(vm, entry.importName);
|
||||
importEntry.localName = entry.localName.isNull() ? JSC::Identifier() : JSC::Identifier::fromString(vm, entry.localName);
|
||||
moduleRecord->addImportEntry(importEntry);
|
||||
}
|
||||
|
||||
// Add export entries
|
||||
for (const auto& entry : metadata.exportEntries) {
|
||||
JSC::AbstractModuleRecord::ExportEntry exportEntry;
|
||||
// Use empty identifier for null strings to avoid crash
|
||||
auto exportName = entry.exportName.isNull() ? JSC::Identifier() : JSC::Identifier::fromString(vm, entry.exportName);
|
||||
auto moduleName = entry.moduleName.isNull() ? JSC::Identifier() : JSC::Identifier::fromString(vm, entry.moduleName);
|
||||
auto importName = entry.importName.isNull() ? JSC::Identifier() : JSC::Identifier::fromString(vm, entry.importName);
|
||||
auto localName = entry.localName.isNull() ? JSC::Identifier() : JSC::Identifier::fromString(vm, entry.localName);
|
||||
|
||||
switch (entry.type) {
|
||||
case 0:
|
||||
exportEntry = JSC::AbstractModuleRecord::ExportEntry::createLocal(exportName, localName);
|
||||
break;
|
||||
case 1:
|
||||
exportEntry = JSC::AbstractModuleRecord::ExportEntry::createIndirect(exportName, importName, moduleName);
|
||||
break;
|
||||
case 2:
|
||||
default:
|
||||
exportEntry = JSC::AbstractModuleRecord::ExportEntry::createNamespace(exportName, moduleName);
|
||||
break;
|
||||
}
|
||||
moduleRecord->addExportEntry(exportEntry);
|
||||
}
|
||||
|
||||
// Add star exports
|
||||
for (const auto& specifier : metadata.starExports) {
|
||||
auto identifier = JSC::Identifier::fromString(vm, specifier);
|
||||
moduleRecord->addStarExportEntry(identifier);
|
||||
}
|
||||
|
||||
return moduleRecord;
|
||||
}
|
||||
|
||||
}; // namespace Zig
|
||||
|
||||
@@ -8,6 +8,8 @@ class Structure;
|
||||
class Identifier;
|
||||
class SourceCodeKey;
|
||||
class SourceProvider;
|
||||
class JSModuleRecord;
|
||||
class VariableEnvironment;
|
||||
} // namespace JSC
|
||||
|
||||
#include <JavaScriptCore/CachedBytecode.h>
|
||||
@@ -15,11 +17,49 @@ class SourceProvider;
|
||||
#include <JavaScriptCore/JSTypeInfo.h>
|
||||
#include <JavaScriptCore/SourceProvider.h>
|
||||
#include <JavaScriptCore/Structure.h>
|
||||
#include <optional>
|
||||
|
||||
namespace Zig {
|
||||
|
||||
class GlobalObject;
|
||||
|
||||
// Cached module metadata for ESM bytecode cache
|
||||
// This structure holds the deserialized import/export information
|
||||
// that allows skipping the ModuleAnalyzeMode parsing phase
|
||||
struct CachedModuleMetadata {
|
||||
struct ModuleRequest {
|
||||
WTF::String specifier;
|
||||
};
|
||||
|
||||
struct ImportEntry {
|
||||
uint32_t type; // 0=Single, 1=SingleTypeScript, 2=Namespace
|
||||
WTF::String moduleRequest;
|
||||
WTF::String importName;
|
||||
WTF::String localName;
|
||||
};
|
||||
|
||||
struct ExportEntry {
|
||||
uint32_t type; // 0=Local, 1=Indirect, 2=Namespace
|
||||
WTF::String exportName;
|
||||
WTF::String moduleName;
|
||||
WTF::String importName;
|
||||
WTF::String localName;
|
||||
};
|
||||
|
||||
struct VariableEntry {
|
||||
WTF::String name;
|
||||
uint32_t bits; // VariableEnvironmentEntry bits
|
||||
};
|
||||
|
||||
Vector<ModuleRequest> requestedModules;
|
||||
Vector<ImportEntry> importEntries;
|
||||
Vector<ExportEntry> exportEntries;
|
||||
Vector<WTF::String> starExports;
|
||||
Vector<VariableEntry> declaredVariables;
|
||||
Vector<VariableEntry> lexicalVariables;
|
||||
uint32_t codeFeatures;
|
||||
};
|
||||
|
||||
void forEachSourceProvider(WTF::Function<void(JSC::SourceID)>);
|
||||
JSC::SourceID sourceIDForSourceURL(const WTF::String& sourceURL);
|
||||
void* sourceMappingForSourceURL(const WTF::String& sourceURL);
|
||||
@@ -50,6 +90,16 @@ public:
|
||||
return m_cachedBytecode.copyRef();
|
||||
};
|
||||
|
||||
// ESM bytecode cache support - virtual overrides from JSC::SourceProvider
|
||||
bool hasCachedModuleMetadata() const override
|
||||
{
|
||||
return m_cachedModuleMetadata.has_value();
|
||||
}
|
||||
|
||||
JSC::JSModuleRecord* createModuleRecordFromCache(
|
||||
JSC::JSGlobalObject* globalObject,
|
||||
const JSC::Identifier& moduleKey) override;
|
||||
|
||||
void updateCache(const UnlinkedFunctionExecutable* executable, const SourceCode&, CodeSpecializationKind kind, const UnlinkedFunctionCodeBlock* codeBlock);
|
||||
void cacheBytecode(const BytecodeCacheGenerator& generator);
|
||||
void commitCachedBytecode();
|
||||
@@ -73,6 +123,7 @@ private:
|
||||
|
||||
Zig::GlobalObject* m_globalObject;
|
||||
RefPtr<JSC::CachedBytecode> m_cachedBytecode;
|
||||
std::optional<CachedModuleMetadata> m_cachedModuleMetadata;
|
||||
Ref<WTF::StringImpl> m_source;
|
||||
unsigned m_hash = 0;
|
||||
};
|
||||
|
||||
@@ -57,6 +57,7 @@ pub const LinkerContext = struct {
|
||||
|
||||
pub const LinkerOptions = struct {
|
||||
generate_bytecode_cache: bool = false,
|
||||
experimental_esm_bytecode_cache: bool = false,
|
||||
output_format: options.Format = .esm,
|
||||
ignore_dce_annotations: bool = false,
|
||||
emit_dce_annotations: bool = true,
|
||||
|
||||
@@ -929,6 +929,7 @@ pub const BundleV2 = struct {
|
||||
this.linker.options.target = transpiler.options.target;
|
||||
this.linker.options.output_format = transpiler.options.output_format;
|
||||
this.linker.options.generate_bytecode_cache = transpiler.options.bytecode;
|
||||
this.linker.options.experimental_esm_bytecode_cache = transpiler.options.experimental_esm_bytecode;
|
||||
|
||||
this.linker.dev_server = transpiler.options.dev_server;
|
||||
|
||||
@@ -1876,6 +1877,7 @@ pub const BundleV2 = struct {
|
||||
|
||||
transpiler.options.output_format = config.format;
|
||||
transpiler.options.bytecode = config.bytecode;
|
||||
transpiler.options.experimental_esm_bytecode = config.experimental_esm_bytecode;
|
||||
transpiler.options.compile = config.compile != null;
|
||||
|
||||
// For compile mode, set the public_path to the target-specific base path
|
||||
|
||||
@@ -77,7 +77,7 @@ pub fn calculateOutputFileListCapacity(c: *const bun.bundle_v2.LinkerContext, ch
|
||||
}
|
||||
break :brk count;
|
||||
} else 0;
|
||||
const bytecode_count = if (c.options.generate_bytecode_cache) bytecode_count: {
|
||||
const bytecode_count = if (c.options.generate_bytecode_cache or c.options.experimental_esm_bytecode_cache) bytecode_count: {
|
||||
var bytecode_count: usize = 0;
|
||||
for (chunks) |*chunk| {
|
||||
const loader: bun.options.Loader = if (chunk.entry_point.is_entry_point)
|
||||
|
||||
@@ -307,7 +307,7 @@ pub fn generateChunksInParallel(
|
||||
var output_files = try OutputFileListBuilder.init(bun.default_allocator, c, chunks, c.parse_graph.additional_output_files.items.len);
|
||||
|
||||
const root_path = c.resolver.opts.output_dir;
|
||||
const more_than_one_output = c.parse_graph.additional_output_files.items.len > 0 or c.options.generate_bytecode_cache or (has_css_chunk and has_js_chunk) or (has_html_chunk and (has_js_chunk or has_css_chunk));
|
||||
const more_than_one_output = c.parse_graph.additional_output_files.items.len > 0 or c.options.generate_bytecode_cache or c.options.experimental_esm_bytecode_cache or (has_css_chunk and has_js_chunk) or (has_html_chunk and (has_js_chunk or has_css_chunk));
|
||||
|
||||
if (!c.resolver.opts.compile and more_than_one_output and !c.resolver.opts.supports_multiple_outputs) {
|
||||
try c.log.addError(null, Logger.Loc.Empty, "cannot write multiple output files without an output directory");
|
||||
@@ -420,7 +420,7 @@ pub fn generateChunksInParallel(
|
||||
}
|
||||
|
||||
const bytecode_output_file: ?options.OutputFile = brk: {
|
||||
if (c.options.generate_bytecode_cache) {
|
||||
if (c.options.generate_bytecode_cache or c.options.experimental_esm_bytecode_cache) {
|
||||
const loader: Loader = if (chunk.entry_point.is_entry_point)
|
||||
c.parse_graph.input_files.items(.loader)[
|
||||
chunk.entry_point.source_index
|
||||
@@ -437,7 +437,12 @@ pub fn generateChunksInParallel(
|
||||
|
||||
defer source_provider_url.deref();
|
||||
|
||||
if (jsc.CachedBytecode.generate(c.options.output_format, code_result.buffer, &source_provider_url)) |result| {
|
||||
const result_opt = if (c.options.experimental_esm_bytecode_cache and c.options.output_format == .esm)
|
||||
jsc.CachedBytecode.generateForESMWithMetadata(&source_provider_url, code_result.buffer)
|
||||
else
|
||||
jsc.CachedBytecode.generate(c.options.output_format, code_result.buffer, &source_provider_url);
|
||||
|
||||
if (result_opt) |result| {
|
||||
const bytecode, const cached_bytecode = result;
|
||||
const source_provider_url_str = source_provider_url.toSlice(bun.default_allocator);
|
||||
defer source_provider_url_str.deinit();
|
||||
|
||||
@@ -157,7 +157,7 @@ pub fn postProcessJSChunk(ctx: GenerateChunkCtx, worker: *ThreadPool.Worker, chu
|
||||
const input = "// @bun @bytecode @bun-cjs\n" ++ cjs_entry_chunk;
|
||||
j.pushStatic(input);
|
||||
line_offset.advance(input);
|
||||
} else if (ctx.c.options.generate_bytecode_cache) {
|
||||
} else if (ctx.c.options.generate_bytecode_cache or (ctx.c.options.experimental_esm_bytecode_cache and output_format == .esm)) {
|
||||
j.pushStatic("// @bun @bytecode\n");
|
||||
line_offset.advance("// @bun @bytecode\n");
|
||||
} else if (output_format == .cjs) {
|
||||
|
||||
@@ -178,7 +178,7 @@ pub fn writeOutputFilesToDisk(
|
||||
.none => {},
|
||||
}
|
||||
const bytecode_output_file: ?options.OutputFile = brk: {
|
||||
if (c.options.generate_bytecode_cache) {
|
||||
if (c.options.generate_bytecode_cache or c.options.experimental_esm_bytecode_cache) {
|
||||
const loader: Loader = if (chunk.entry_point.is_entry_point)
|
||||
c.parse_graph.input_files.items(.loader)[
|
||||
chunk.entry_point.source_index
|
||||
@@ -195,7 +195,12 @@ pub fn writeOutputFilesToDisk(
|
||||
|
||||
defer source_provider_url.deref();
|
||||
|
||||
if (jsc.CachedBytecode.generate(c.options.output_format, code_result.buffer, &source_provider_url)) |result| {
|
||||
const result_opt = if (c.options.experimental_esm_bytecode_cache and c.options.output_format == .esm)
|
||||
jsc.CachedBytecode.generateForESMWithMetadata(&source_provider_url, code_result.buffer)
|
||||
else
|
||||
jsc.CachedBytecode.generate(c.options.output_format, code_result.buffer, &source_provider_url);
|
||||
|
||||
if (result_opt) |result| {
|
||||
const source_provider_url_str = source_provider_url.toSlice(bun.default_allocator);
|
||||
defer source_provider_url_str.deinit();
|
||||
const bytecode, const cached_bytecode = result;
|
||||
|
||||
@@ -218,3 +218,8 @@ export const hostedGitInfo = {
|
||||
parseUrl: $newZigFunction("hosted_git_info.zig", "TestingAPIs.jsParseUrl", 1),
|
||||
fromUrl: $newZigFunction("hosted_git_info.zig", "TestingAPIs.jsFromUrl", 1),
|
||||
};
|
||||
|
||||
export const CachedBytecode = {
|
||||
generateForESMWithMetadata: $newZigFunction("CachedBytecode.zig", "TestingAPIs.generateForESMWithMetadata", 2),
|
||||
validateMetadata: $newZigFunction("CachedBytecode.zig", "TestingAPIs.validateMetadata", 1),
|
||||
};
|
||||
|
||||
@@ -1803,6 +1803,7 @@ pub const BundleOptions = struct {
|
||||
ignore_dce_annotations: bool = false,
|
||||
emit_dce_annotations: bool = false,
|
||||
bytecode: bool = false,
|
||||
experimental_esm_bytecode: bool = false,
|
||||
|
||||
code_coverage: bool = false,
|
||||
debugger: bool = false,
|
||||
|
||||
@@ -1162,7 +1162,44 @@ pub const Transpiler = struct {
|
||||
var path_buf2: bun.PathBuffer = undefined;
|
||||
@memcpy(path_buf2[0..path.text.len], path.text);
|
||||
path_buf2[path.text.len..][0..bun.bytecode_extension.len].* = bun.bytecode_extension.*;
|
||||
const bytecode = bun.sys.File.toSourceAt(dirname_fd.unwrapValid() orelse bun.FD.cwd(), path_buf2[0 .. path.text.len + bun.bytecode_extension.len], bun.default_allocator, .{}).asValue() orelse break :brk default_value;
|
||||
|
||||
const dir_fd = dirname_fd.unwrapValid() orelse bun.FD.cwd();
|
||||
// Add null terminator for bytecode path
|
||||
path_buf2[path.text.len + bun.bytecode_extension.len] = 0;
|
||||
const bytecode_path: [:0]const u8 = path_buf2[0 .. path.text.len + bun.bytecode_extension.len :0];
|
||||
|
||||
// Check timestamps: if source file is newer than bytecode cache, invalidate cache
|
||||
// This ensures the cache is regenerated when source changes
|
||||
// Create null-terminated source path
|
||||
var source_path_buf: bun.PathBuffer = undefined;
|
||||
@memcpy(source_path_buf[0..path.text.len], path.text);
|
||||
source_path_buf[path.text.len] = 0;
|
||||
const source_path: [:0]const u8 = source_path_buf[0..path.text.len :0];
|
||||
|
||||
const source_stat = bun.sys.fstatat(dir_fd, source_path);
|
||||
const bytecode_stat = bun.sys.fstatat(dir_fd, bytecode_path);
|
||||
|
||||
if (source_stat == .result and bytecode_stat == .result) {
|
||||
const source_mtime = source_stat.result.mtime();
|
||||
const bytecode_mtime = bytecode_stat.result.mtime();
|
||||
|
||||
// Compare timestamps: if source is newer than bytecode by more than 1 second, skip cache
|
||||
// We use 1 second tolerance because bytecode is written just before the source file
|
||||
// during Bun.build, so there can be sub-second timing differences
|
||||
// (source_mtime > bytecode_mtime + 1 second)
|
||||
if (source_mtime.sec > bytecode_mtime.sec + 1 or
|
||||
(source_mtime.sec == bytecode_mtime.sec + 1 and source_mtime.nsec > bytecode_mtime.nsec))
|
||||
{
|
||||
break :brk default_value;
|
||||
}
|
||||
} else {
|
||||
// If we can't stat one of the files, skip cache
|
||||
if (bytecode_stat == .err) {
|
||||
break :brk default_value;
|
||||
}
|
||||
}
|
||||
|
||||
const bytecode = bun.sys.File.toSourceAt(dir_fd, bytecode_path, bun.default_allocator, .{}).asValue() orelse break :brk default_value;
|
||||
if (bytecode.contents.len == 0) {
|
||||
break :brk default_value;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user