Compare commits

...

12 Commits

Author SHA1 Message Date
Dylan Conway
46a7cd50ad build: sync cmake from main, pass CMAKE_MSVC_RUNTIME_LIBRARY to dep configures on Windows 2026-03-12 00:23:08 +00:00
Dylan Conway
a18090fa23 Merge remote-tracking branch 'origin/main' into claude/build-system-js 2026-03-12 00:22:03 +00:00
Dylan Conway
d03b6aa24d build: sync WebKit/Zig versions, -baseline WebKit suffix, strip eh_frame unconditionally 2026-03-11 20:17:48 +00:00
Dylan Conway
f977382424 Merge remote-tracking branch 'origin/main' into claude/build-system-js 2026-03-11 20:16:24 +00:00
Dylan Conway
d12b60c106 build: add -std=gnu17 for C files (matches cmake C_STANDARD 17 with default C_EXTENSIONS) 2026-03-11 00:56:10 +00:00
Dylan Conway
a054f776ee build: default assertions=true when asan=true — ABI-coupled via ASSERT_ENABLED, matches cmake build:asan 2026-03-11 00:41:31 +00:00
Dylan Conway
48e00fa1fa build: fix release-assertions profile, track symbol files as link implicit inputs, use writeSync for stream.ts final writes 2026-03-11 00:00:09 +00:00
Dylan Conway
908bdf5fe5 build: revert setarch wrapper for features.mjs — cmake doesn't use one, containerized CI can't setarch 2026-03-10 23:51:02 +00:00
Dylan Conway
c88b119bac build: fix MSVC toolset version comparison, add setarch wrapper for features.mjs on ASAN, skip perl check for zig-only/link-only 2026-03-10 23:21:36 +00:00
Dylan Conway
6dbb63e070 Merge branch 'main' into claude/build-system-js 2026-03-10 16:14:55 -07:00
Dylan Conway
8993389bc2 Revert "fix(windows): use TerminateProcess to prevent NAPI module segfault on exit (#27829)" 2026-03-10 22:23:22 +00:00
Dylan Conway
d61e306d2c build: replace CMake with TypeScript-based ninja generator 2026-03-10 22:23:22 +00:00
51 changed files with 10777 additions and 106 deletions

View File

@@ -444,39 +444,54 @@ function getTestAgent(platform, options) {
*/
/**
* Build the scripts/build.ts argument list from a target's properties.
* Replaces the old getBuildEnv (cmake -D env vars) + getBuildCommand
* (--target passthrough) with direct build.ts flags.
*
* @param {Target} target
* @param {PipelineOptions} options
* @returns {Record<string, string | undefined>}
* @param {"cpp-only" | "zig-only" | "link-only"} mode
* @returns {string}
*/
function getBuildEnv(target, options) {
const { baseline, abi } = target;
function getBuildArgs(target, options, mode) {
const { os, arch, abi, baseline, profile } = target;
const { canary } = options;
const revision = typeof canary === "number" ? canary : 1;
return {
ENABLE_BASELINE: baseline ? "ON" : "OFF",
ENABLE_CANARY: revision > 0 ? "ON" : "OFF",
CANARY_REVISION: revision,
ABI: abi === "musl" ? "musl" : undefined,
CMAKE_VERBOSE_MAKEFILE: "ON",
CMAKE_TLS_VERIFY: "0",
};
const args = [`--profile=ci-${mode}`];
// zig-only cross-compiles (linux host → all targets); os/arch/abi must
// all be explicit — host detection (detectLinuxAbi checks /etc/alpine-release)
// would report the build box's abi (Alpine→musl), not the target's.
// cpp-only/link-only: native build, host detection is correct.
if (mode === "zig-only") {
args.push(`--os=${os}`, `--arch=${arch}`);
if (os === "linux") args.push(`--abi=${abi ?? "gnu"}`);
} else if (abi === "musl") {
args.push("--abi=musl");
}
if (baseline) args.push("--baseline=on");
if (profile === "asan") args.push("--asan=on");
// canary: options.canary can be number (revision count) or undefined
// (default on). Old system used CANARY_REVISION as a counter; build.ts
// has only on/off — disabled only when explicitly 0.
const canaryRev = typeof canary === "number" ? canary : 1;
if (canaryRev === 0) args.push("--canary=off");
return args.join(" ");
}
/**
* @param {Target} target
* @param {PipelineOptions} options
* @param {"cpp-only" | "zig-only" | "link-only"} mode
* @returns {string}
*/
function getBuildCommand(target, options, label) {
const { profile } = target;
const buildProfile = profile || "release";
function getBuildCommand(target, options, mode) {
// Windows code signing is handled by a dedicated 'windows-sign' step after
// all Windows builds complete — see getWindowsSignStep(). smctl is x64-only,
// so signing on the build agent wouldn't work for ARM64 anyway.
return `bun run build:${buildProfile}`;
return `bun scripts/build.ts ${getBuildArgs(target, options, mode)}`;
}
/**
@@ -485,59 +500,34 @@ function getBuildCommand(target, options, label) {
* @returns {Step}
*/
function getBuildCppStep(platform, options) {
const command = getBuildCommand(platform, options);
return {
key: `${getTargetKey(platform)}-build-cpp`,
label: `${getTargetLabel(platform)} - build-cpp`,
agents: getCppAgent(platform, options),
retry: getRetry(),
cancel_on_build_failing: isMergeQueue(),
env: {
BUN_CPP_ONLY: "ON",
...getBuildEnv(platform, options),
},
// We used to build the C++ dependencies and bun in separate steps.
// However, as long as the zig build takes longer than both sequentially,
// it's cheaper to run them in the same step. Can be revisited in the future.
command: [`${command} --target bun`, `${command} --target dependencies`],
// cpp-only builds deps + bun's C++ in one ninja graph (ninja pulls
// everything the archive transitively needs). The old two-command
// split (--target bun, --target dependencies) was a cmake artifact.
command: getBuildCommand(platform, options, "cpp-only"),
};
}
/**
* @param {Target} target
* @returns {string}
*/
function getBuildToolchain(target) {
const { os, arch, abi, baseline } = target;
let key = `${os}-${arch}`;
if (abi) {
key += `-${abi}`;
}
if (baseline) {
key += "-baseline";
}
return key;
}
/**
* @param {Platform} platform
* @param {PipelineOptions} options
* @returns {Step}
*/
function getBuildZigStep(platform, options) {
const { os, arch } = platform;
const toolchain = getBuildToolchain(platform);
// Native Windows builds don't need a cross-compilation toolchain
const toolchainArg = os === "windows" ? "" : ` --toolchain ${toolchain}`;
return {
key: `${getTargetKey(platform)}-build-zig`,
retry: getRetry(),
label: `${getTargetLabel(platform)} - build-zig`,
agents: getZigAgent(platform, options),
cancel_on_build_failing: isMergeQueue(),
env: getBuildEnv(platform, options),
command: `${getBuildCommand(platform, options)} --target bun-zig${toolchainArg}`,
// zig cross-compiles via --os/--arch in build args. No separate
// toolchain file — zig handles cross-compilation natively.
command: getBuildCommand(platform, options, "zig-only"),
timeout_in_minutes: 35,
};
}
@@ -556,11 +546,13 @@ function getLinkBunStep(platform, options) {
retry: getRetry(),
cancel_on_build_failing: isMergeQueue(),
env: {
BUN_LINK_ONLY: "ON",
// ASAN runtime settings — unrelated to build config, affects the
// linked binary's startup during the smoke test.
ASAN_OPTIONS: "allow_user_segv_handler=1:disable_coredump=0:detect_leaks=0",
...getBuildEnv(platform, options),
},
command: `${getBuildCommand(platform, options, "build-bun")} --target bun`,
// link-only downloads artifacts from the sibling build-cpp and
// build-zig steps (derived from BUILDKITE_STEP_KEY) before ninja runs.
command: getBuildCommand(platform, options, "link-only"),
};
}
@@ -677,23 +669,6 @@ function getVerifyBaselineStep(platform, options) {
};
}
/**
* @param {Platform} platform
* @param {PipelineOptions} options
* @returns {Step}
*/
function getBuildBunStep(platform, options) {
return {
key: `${getTargetKey(platform)}-build-bun`,
label: `${getTargetLabel(platform)} - build-bun`,
agents: getCppAgent(platform, options),
retry: getRetry(),
cancel_on_build_failing: isMergeQueue(),
env: getBuildEnv(platform, options),
command: getBuildCommand(platform, options),
};
}
/**
* @typedef {Object} TestOptions
* @property {string} [buildId]

View File

@@ -45,15 +45,14 @@ function formatZigFile() {
function formatTypeScriptFile() {
try {
// Format the TypeScript file
const result = spawnSync(
"./node_modules/.bin/prettier",
["--plugin=prettier-plugin-organize-imports", "--config", ".prettierrc", "--write", filePath],
{
cwd: process.env.CLAUDE_PROJECT_DIR || process.cwd(),
encoding: "utf-8",
},
);
// Format only — NO organize-imports plugin. That plugin strips imports
// it thinks are unused, which breaks split edits (add import → use it
// in next edit). CI's `bun run prettier` runs the plugin, so imports
// still get cleaned up before merge.
const result = spawnSync("./node_modules/.bin/prettier", ["--config", ".prettierrc", "--write", filePath], {
cwd: process.env.CLAUDE_PROJECT_DIR || process.cwd(),
encoding: "utf-8",
});
} catch (error) {}
}

3
.gitattributes vendored
View File

@@ -20,6 +20,9 @@
*.mjs text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
*.mts text eol=lf whitespace=blank-at-eol,-blank-at-eof,-space-before-tab,tab-in-indent,tabwidth=2
# Patch files are line-ending-sensitive — `git apply` rejects CRLF as corrupt.
*.patch text eol=lf
*.lockb binary diff=lockb
.vscode/launch.json linguist-generated

2
.gitignore vendored
View File

@@ -62,7 +62,7 @@
/test.ts
/test.zig
/testdir
build
/build/
build.ninja
bun-binary
bun-mimalloc

View File

@@ -33,20 +33,20 @@
"watch-windows": "bun run zig build check-windows --watch -fincremental --prominent-compile-errors --global-cache-dir build/debug/zig-check-cache --zig-lib-dir vendor/zig/lib",
"bd:v": "./scripts/bd",
"bd": "BUN_DEBUG_QUIET_LOGS=1 ./scripts/bd",
"build:debug": "export COMSPEC=\"C:\\Windows\\System32\\cmd.exe\" && bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Debug -B build/debug --log-level=NOTICE",
"build:debug:fuzzilli": "export COMSPEC=\"C:\\Windows\\System32\\cmd.exe\" && bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Debug -B build/debug-fuzz -DENABLE_FUZZILLI=ON --log-level=NOTICE",
"build:debug:noasan": "export COMSPEC=\"C:\\Windows\\System32\\cmd.exe\" && bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=OFF -B build/debug --log-level=NOTICE",
"build:release": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -B build/release",
"build:ci": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE=ON -DCI=true -B build/release-ci --verbose --fresh",
"build:assert": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_ASSERTIONS=ON -DENABLE_LOGS=ON -B build/release-assert",
"build:asan": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DENABLE_ASSERTIONS=ON -DENABLE_LOGS=OFF -DENABLE_ASAN=ON -DENABLE_LTO=OFF -B build/release-asan",
"build:logs": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DENABLE_LOGS=ON -B build/release-logs",
"build:safe": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DZIG_OPTIMIZE=ReleaseSafe -B build/release-safe",
"build:smol": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=MinSizeRel -B build/release-smol",
"build:local": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Debug -DWEBKIT_LOCAL=ON -B build/debug-local",
"build:release:local": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -DWEBKIT_LOCAL=ON -B build/release-local",
"build:release:with_logs": "cmake . -DCMAKE_BUILD_TYPE=Release -DENABLE_LOGS=true -GNinja -Bbuild-release && ninja -Cbuild-release",
"build:debug-zig-release": "cmake . -DCMAKE_BUILD_TYPE=Release -DZIG_OPTIMIZE=Debug -GNinja -Bbuild-debug-zig-release && ninja -Cbuild-debug-zig-release",
"build:debug": "bun scripts/build.ts --profile=debug",
"build:debug:fuzzilli": "bun scripts/build.ts --profile=debug --fuzzilli=on --build-dir=build/debug-fuzz",
"build:debug:noasan": "bun scripts/build.ts --profile=debug-no-asan",
"build:release": "bun scripts/build.ts --profile=release",
"build:ci": "bun scripts/build.ts --profile=ci-release --build-dir=build/release-ci",
"build:assert": "bun scripts/build.ts --profile=release-assertions --build-dir=build/release-assert",
"build:asan": "bun scripts/build.ts --profile=release-asan --build-dir=build/release-asan",
"build:logs": "bun scripts/build.ts --profile=release --logs=on --build-dir=build/release-logs",
"build:smol": "bun scripts/build.ts --profile=release --build-type=MinSizeRel --build-dir=build/release-smol",
"build:local": "bun scripts/build.ts --profile=debug-local --build-dir=build/debug-local",
"build:release:local": "bun scripts/build.ts --profile=release-local --build-dir=build/release-local",
"build:cmake:debug": "export COMSPEC=\"C:\\Windows\\System32\\cmd.exe\" && bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Debug -B build/cmake-debug --log-level=NOTICE",
"build:cmake:release": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Release -B build/cmake-release",
"build:cmake:local": "bun ./scripts/build.mjs -GNinja -DCMAKE_BUILD_TYPE=Debug -DWEBKIT_LOCAL=ON -B build/cmake-debug-local",
"run:linux": "docker run --rm -v \"$PWD:/root/bun/\" -w /root/bun ghcr.io/oven-sh/bun-development-docker-image",
"css-properties": "bun run src/css/properties/generate_properties.ts",
"uv-posix-stubs": "bun run src/bun.js/bindings/libuv/generate_uv_posix_stubs.ts",

334
scripts/build.ts Normal file
View File

@@ -0,0 +1,334 @@
#!/usr/bin/env bun
/**
* Build entry point — configure + ninja exec.
*
* bun scripts/build.ts --profile=debug
* bun scripts/build.ts --profile=release
* bun scripts/build.ts --profile=debug --asan=off # override a field
* bun scripts/build.ts --profile=debug -- bun-zig # specific ninja target
* bun scripts/build.ts --configure-only # emit ninja, don't run
*
* Replaces scripts/build.mjs. The old CMake build is still available via
* `bun run build:cmake:*` scripts in package.json.
*/
import { spawnSync } from "node:child_process";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { isatty } from "node:tty";
import {
downloadArtifacts,
isCI,
packageAndUpload,
printEnvironment,
spawnWithAnnotations,
startGroup,
uploadArtifacts,
} from "./build/ci.ts";
import { formatConfigUnchanged, type PartialConfig } from "./build/config.ts";
import { configure, type ConfigureResult } from "./build/configure.ts";
import { BuildError } from "./build/error.ts";
import { getProfile } from "./build/profiles.ts";
import { STREAM_FD } from "./build/stream.ts";
// ───────────────────────────────────────────────────────────────────────────
// Main
// ───────────────────────────────────────────────────────────────────────────
try {
await main();
} catch (err) {
if (err instanceof BuildError) {
process.stderr.write(err.format());
process.exit(1);
}
throw err;
}
async function main(): Promise<void> {
// Windows: re-exec inside the VS dev shell if not already there.
// The shell provides PATH (mt.exe, rc.exe, cl.exe), INCLUDE, LIB,
// WindowsSdkDir — things clang-cl can mostly self-detect but nested
// cmake projects can't. Cheap: VSINSTALLDIR check short-circuits on
// subsequent runs in the same terminal.
if (process.platform === "win32" && !process.env.VSINSTALLDIR) {
const vsShell = join(import.meta.dirname, "vs-shell.ps1");
const result = spawnSync(
"pwsh",
["-NoProfile", "-NoLogo", "-File", vsShell, process.argv0, import.meta.filename, ...process.argv.slice(2)],
{ stdio: "inherit" },
);
if (result.error) {
throw new BuildError(`Failed to spawn pwsh`, {
cause: result.error,
hint: "Is PowerShell 7+ (pwsh) installed?",
});
}
process.exit(result.status ?? 1);
}
const args = parseArgs(process.argv.slice(2));
// Resolve PartialConfig: either from --config-file (ninja's generator rule
// replaying a previous configure) or from --profile + overrides (normal use).
const partial: PartialConfig =
args.configFile !== undefined
? loadConfigFile(args.configFile)
: { ...getProfile(args.profile), ...args.overrides };
const ninjaArgv = (cfg: { buildDir: string }) => ["-C", cfg.buildDir, ...args.ninjaArgs, ...args.ninjaTargets];
const ninjaEnv = (env: Record<string, string>) => ({ ...process.env, ...env });
if (isCI) {
// CI: machine/env dump + collapsible groups + annotation-on-failure.
printEnvironment();
const result = (await startGroup("Configure", () => configure(partial))) as ConfigureResult;
if (args.configureOnly) return;
// link-only: download cpp-only + zig-only artifacts before ninja.
if (result.cfg.buildkite && result.cfg.mode === "link-only") {
await startGroup("Download artifacts", () => downloadArtifacts(result.cfg));
}
await startGroup("Build", () =>
spawnWithAnnotations("ninja", ninjaArgv(result.cfg), { label: "ninja", env: ninjaEnv(result.env) }),
);
// cpp-only/zig-only: upload build outputs for downstream link-only.
// link-only: package + upload zips for downstream test steps.
if (result.cfg.buildkite) {
if (result.cfg.mode === "cpp-only" || result.cfg.mode === "zig-only") {
await startGroup("Upload artifacts", () => uploadArtifacts(result.cfg, result.output));
}
if (result.cfg.mode === "link-only") {
await startGroup("Package and upload", () => packageAndUpload(result.cfg, result.output));
}
}
} else {
// Local: configure, then spawn ninja.
const result = await configure(partial);
// Quiet one-liner when configure was a no-op — the full banner only
// prints when build.ninja changed. Timing matters: a regression here
// would otherwise be invisible. Suppressed for ninja's generator-
// rule replay (--config-file) since ninja's [N/M] already says
// "reconfigure" and doubling it is noise.
if (!result.changed && args.configFile === undefined) {
process.stderr.write(formatConfigUnchanged(result.exe, result.elapsed) + "\n");
}
if (args.configureOnly) {
// Hint only for manual --configure-only, not generator replay.
if (args.configFile === undefined) {
process.stderr.write(`run: ninja -C ${result.cfg.buildDir}\n`);
}
return;
}
// FD 3 sideband — only when interactive. stream.ts (wrapping deps +
// zig) writes live output there, bypassing ninja's per-job buffering.
// A human watching a terminal wants to see cmake configure spew and
// zig progress in real time. A log file (scripts/bd, CI) doesn't —
// that live output is noise (hundreds of `-- Looking for header.h`
// lines from cmake). When FD 3 isn't set up, stream.ts falls back to
// stdout which ninja buffers per-job: deps stay quiet until they
// finish or fail, failure logs stay compact.
//
// Ninja's subprocess spawn only touches FDs 0-2; higher fds inherit
// through posix_spawn/CreateProcessA. Passing our stderr fd (2) at
// index STREAM_FD dups it there for the whole ninja process tree.
const stdio: (number | "inherit")[] = ["inherit", "inherit", "inherit"];
if (isatty(2)) {
stdio[STREAM_FD] = 2;
}
const ninja = spawnSync("ninja", ninjaArgv(result.cfg), {
stdio,
env: ninjaEnv(result.env),
});
if (ninja.error) {
process.stderr.write(`Failed to exec ninja: ${ninja.error.message}\nIs ninja in your PATH?\n`);
process.exit(127);
}
// Closing line on success: when restat prunes most of the graph
// (local WebKit no-op shows `[1/555] build WebKit` then silence),
// it's not obvious ninja finished vs. stalled. This disambiguates.
// Always shown — useful for piped/CI too as an end-of-build marker.
if (ninja.status === 0) {
const clear = isatty(2) ? "\r\x1b[K" : "";
process.stderr.write(`${clear}[build] done\n`);
}
process.exit(ninja.status ?? 1);
}
}
/** Load a PartialConfig from JSON (for ninja's generator rule replay). */
function loadConfigFile(path: string): PartialConfig {
try {
return JSON.parse(readFileSync(path, "utf8")) as PartialConfig;
} catch (cause) {
throw new BuildError(`Failed to load config file: ${path}`, { cause });
}
}
// ───────────────────────────────────────────────────────────────────────────
// CLI arg parsing
// ───────────────────────────────────────────────────────────────────────────
interface CliArgs {
profile: string;
/** PartialConfig overrides from --<field>=<value> flags. */
overrides: PartialConfig;
/** Explicit ninja targets after `--`. Empty = use defaults. */
ninjaTargets: string[];
/** Just configure, don't run ninja. */
configureOnly: boolean;
/** Extra ninja args (e.g. -j8, -v). */
ninjaArgs: string[];
/**
* Load PartialConfig from JSON (ninja's generator rule replay).
* Mutually exclusive with --profile/overrides.
*/
configFile: string | undefined;
}
/**
* Parse argv. Format:
* --profile=<name> Profile (required, no default here — caller picks)
* --<field>=<value> Override any PartialConfig boolean/string field
* --configure-only Emit build.ninja, don't run it
* -j<N> / -v / -k<N> Passed through to ninja
* -- <targets...> Explicit ninja targets
*
* Boolean overrides accept: on/off, true/false, yes/no, 1/0.
*/
function parseArgs(argv: string[]): CliArgs {
let profile = "debug";
const overrides: PartialConfig = {};
const ninjaTargets: string[] = [];
const ninjaArgs: string[] = [];
let configureOnly = false;
let configFile: string | undefined;
let inTargets = false;
// PartialConfig fields that are BOOLEANS. Used for value coercion.
// Not exhaustive — add as needed. Unknown --<field> is rejected so you
// notice typos.
const boolFields = new Set([
"lto",
"asan",
"zigAsan",
"assertions",
"logs",
"baseline",
"canary",
"staticSqlite",
"staticLibatomic",
"tinycc",
"valgrind",
"fuzzilli",
"ci",
"buildkite",
]);
// PartialConfig fields that are STRINGS.
const stringFields = new Set([
"os",
"arch",
"abi",
"buildType",
"mode",
"webkit",
"buildDir",
"cacheDir",
"nodejsVersion",
"nodejsAbiVersion",
"zigCommit",
"webkitVersion",
]);
for (const arg of argv) {
if (inTargets) {
ninjaTargets.push(arg);
continue;
}
if (arg === "--") {
inTargets = true;
continue;
}
// Ninja passthrough: -j<N>, -v, -k<N>, -l<N>. Short flags only —
// anything starting with `--` is OURS.
if (/^-[jklv]/.test(arg)) {
ninjaArgs.push(arg);
continue;
}
if (arg === "--configure-only") {
configureOnly = true;
continue;
}
if (arg.startsWith("--config-file=")) {
configFile = arg.slice("--config-file=".length);
configureOnly = true; // --config-file is only used by ninja's regen; never runs ninja
continue;
}
if (arg === "--help" || arg === "-h") {
process.stderr.write(USAGE);
process.exit(0);
}
// --<field>=<value>
const m = arg.match(/^--([a-zA-Z][a-zA-Z0-9-]*)=(.*)$/);
if (!m) {
throw new BuildError(`Unknown argument: ${arg}`, { hint: USAGE });
}
const [, rawKey, value] = m;
const key = rawKey!.replace(/-([a-z])/g, (_, c: string) => c.toUpperCase()); // kebab → camel
if (key === "profile") {
profile = value!;
} else if (boolFields.has(key)) {
(overrides as Record<string, boolean>)[key] = parseBool(value!);
} else if (stringFields.has(key)) {
(overrides as Record<string, string>)[key] = value!;
} else {
throw new BuildError(`Unknown config field: --${rawKey}`, {
hint: `Known fields: profile, ${[...boolFields, ...stringFields].sort().join(", ")}`,
});
}
}
return { profile, overrides, ninjaTargets, ninjaArgs, configureOnly, configFile };
}
function parseBool(v: string): boolean {
const lower = v.toLowerCase();
if (["on", "true", "yes", "1"].includes(lower)) return true;
if (["off", "false", "no", "0"].includes(lower)) return false;
throw new BuildError(`Invalid boolean value: ${v}`, { hint: "Use on/off, true/false, yes/no, or 1/0" });
}
const USAGE = `\
Usage: bun scripts/build.ts [options] [-- ninja-targets...]
Options:
--profile=<name> Build profile (default: debug)
Profiles: debug, debug-local, debug-no-asan,
release, release-local, release-asan,
release-assertions, ci-*
--<field>=<value> Override a config field. Boolean fields take
on/off/true/false/yes/no/1/0.
Fields: asan, lto, assertions, logs, baseline,
canary, valgrind, webkit (prebuilt|local),
buildDir, mode (full|cpp-only|link-only)
--configure-only Emit build.ninja, don't run it
-j<N>, -v, -k<N> Passed through to ninja
-- Everything after is a ninja target
--help Show this help
Examples:
bun scripts/build.ts --profile=debug
bun scripts/build.ts --profile=release --lto=off
bun scripts/build.ts --profile=debug -- bun-zig
bun scripts/build.ts --configure-only
`;

744
scripts/build/bun.ts Normal file
View File

@@ -0,0 +1,744 @@
/**
* The bun executable target — orchestrates everything.
*
* This is where all the phases come together:
* - resolve all deps → lib paths + include dirs
* - emit codegen → generated .cpp/.h/.zig
* - emit zig build → bun-zig.o
* - build PCH from root.h (implicit deps: WebKit libs + all codegen)
* - compile all C/C++ with the PCH
* - link everything → bun-debug (or bun-profile, bun-asan, etc.)
* - smoke test: run `<exe> --revision` to catch load-time failures
*
* ## Build modes
*
* `cfg.mode` controls what we actually produce:
* - "full": everything (default, local dev)
* - "cpp-only": compile to libbun.a, skip zig/link (CI upstream) — TODO(ci-split)
* - "link-only": link pre-built artifacts (CI downstream) — TODO(ci-split)
*
* cpp-only/link-only are for the CI split where C++ and zig build in
* parallel on separate machines then meet for linking.
*/
import { readFileSync } from "node:fs";
import { relative, resolve } from "node:path";
import { emitCodegen, zigFilesGeneratedIntoSrc, type CodegenOutputs } from "./codegen.ts";
import { ar, cc, cxx, link, pch } from "./compile.ts";
import { bunExeName, shouldStrip, type Config } from "./config.ts";
import { generateDepVersionsHeader } from "./depVersionsHeader.ts";
import { allDeps } from "./deps/index.ts";
import { zstd } from "./deps/zstd.ts";
import { assert } from "./error.ts";
import { bunIncludes, computeFlags, extraFlagsFor, linkDepends } from "./flags.ts";
import { writeIfChanged } from "./fs.ts";
import type { Ninja } from "./ninja.ts";
import { quote, slash } from "./shell.ts";
import { computeDepLibs, depSourceStamp, resolveDep, type ResolvedDep } from "./source.ts";
import type { Sources } from "./sources.ts";
import { streamPath } from "./stream.ts";
import { emitZig } from "./zig.ts";
// ───────────────────────────────────────────────────────────────────────────
// Executable naming
// ───────────────────────────────────────────────────────────────────────────
// Re-exported for existing importers (configure.ts, ci.ts). These live
// in config.ts now so flags.ts can use bunExeName without circular import.
export { bunExeName, shouldStrip };
/**
* System libraries to link. Platform-dependent.
*/
function systemLibs(cfg: Config): string[] {
const libs: string[] = [];
if (cfg.linux) {
libs.push("-lc", "-lpthread", "-ldl");
// libatomic: static by default (CI distros ship it), dynamic on Arch-like.
// The static path needs to be the actual file path for lld to find it;
// dynamic uses -l syntax. We emit what CMake does: bare libatomic.a gets
// found in lib search paths, -latomic.so doesn't exist so we use -latomic.
if (cfg.staticLibatomic) {
libs.push("-l:libatomic.a");
} else {
libs.push("-latomic");
}
// Linux local WebKit: link system ICU (prebuilt bundles its own).
// Assumes system ICU is in default lib paths — true on most distros.
if (cfg.webkit === "local") {
libs.push("-licudata", "-licui18n", "-licuuc");
}
}
if (cfg.darwin) {
// icucore: system ICU framework.
// resolv: DNS resolution (getaddrinfo et al).
libs.push("-licucore", "-lresolv");
}
if (cfg.windows) {
// Explicit .lib: these go after /link so no auto-suffixing by the
// clang-cl driver. lld-link auto-appends .lib but link.exe doesn't;
// explicit is portable.
libs.push(
"winmm.lib",
"bcrypt.lib",
"ntdll.lib",
"userenv.lib",
"dbghelp.lib",
"crypt32.lib",
"wsock32.lib", // ws2_32 + wsock32 — wsock32 has TransmitFile (sendfile equiv)
"ws2_32.lib",
"delayimp.lib", // required for /delayload: in release
);
}
return libs;
}
// ───────────────────────────────────────────────────────────────────────────
// Main orchestration
// ───────────────────────────────────────────────────────────────────────────
/**
* Output of `emitBun()`. Paths to the produced artifacts and resolved
* deps — used by configure.ts for mkdir + default-target selection, and
* by ci.ts for artifact upload.
*
* Optional fields are present only when the mode produces them:
* full: exe, strippedExe?, dsym?, zigObjects, objects, deps, codegen
* cpp-only: archive, objects, deps, codegen
* zig-only: zigObjects, deps (zstd), codegen
* link-only: exe, strippedExe?, dsym?
*/
export interface BunOutput {
/** Linked executable (bun-debug, bun-profile). Full/link-only. */
exe?: string;
/** Stripped `bun`. Plain release full/link-only. */
strippedExe?: string | undefined;
/** .dSYM bundle (darwin plain release). Added to default targets so ninja builds it. */
dsym?: string | undefined;
/** libbun.a — all C/C++ objects archived. cpp-only. */
archive?: string;
/** All resolved deps (full libs list). Empty in link-only (paths computed separately). */
deps: ResolvedDep[];
/** All codegen outputs. Not present in link-only. */
codegen?: CodegenOutputs;
/** The zig object file(s). Empty in cpp-only. */
zigObjects: string[];
/** All compiled .o files. Empty in link-only/zig-only. */
objects: string[];
}
/**
* Emit the full bun build graph. Returns the output executable path.
*
* Call after `registerAllRules(n, cfg)`. `sources` is the globbed file
* snapshot from `globAllSources()` — passed in so globbing happens once.
*/
export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
// Split modes get minimal graphs — separate functions.
if (cfg.mode === "zig-only") {
return emitZigOnly(n, cfg, sources);
}
if (cfg.mode === "link-only") {
return emitLinkOnly(n, cfg);
}
const exeName = bunExeName(cfg);
n.comment("════════════════════════════════════════════════════════════════");
n.comment(` Building ${exeName}`);
n.comment("════════════════════════════════════════════════════════════════");
n.blank();
// ─── Step 1: resolve all deps ───
n.comment("─── Dependencies ───");
n.blank();
const deps: ResolvedDep[] = [];
for (const dep of allDeps) {
const resolved = resolveDep(n, cfg, dep);
if (resolved !== null) deps.push(resolved);
}
// Collect all dep lib paths, include dirs, output stamps, and directly-
// compiled source files (deps like picohttpparser that provide .c files
// instead of a .a — we compile those alongside bun's own sources).
const depLibs: string[] = [];
const depIncludes: string[] = [];
const depOutputs: string[] = []; // PCH order-only-deps on these
const depCSources: string[] = [];
for (const d of deps) {
depLibs.push(...d.libs);
depIncludes.push(...d.includes);
depOutputs.push(...d.outputs);
depCSources.push(...d.sources);
}
// ─── Step 2: codegen ───
const codegen = emitCodegen(n, cfg, sources);
// ─── Step 3: zig ───
// zstd source must be FETCHED (not built) before zig runs — build.zig
// @cImports zstd headers. The fetch stamp is the order-only dep.
//
// Filter codegen-into-src .zig files from the glob result — they're
// OUTPUTS of steps above, not inputs to zig build. Leaving them in
// would create a cycle (or a fresh-build error: file doesn't exist yet).
//
// cpp-only: skip zig entirely (runs on a separate CI machine).
let zigObjects: string[] = [];
if (cfg.mode !== "cpp-only") {
const codegenZigSet = new Set(zigFilesGeneratedIntoSrc.map(p => resolve(cfg.cwd, p)));
const zigSources = sources.zig.filter(f => !codegenZigSet.has(f));
zigObjects = emitZig(n, cfg, {
codegenInputs: codegen.zigInputs,
codegenOrderOnly: codegen.zigOrderOnly,
zigSources,
zstdStamp: depSourceStamp(cfg, "zstd"),
});
}
// ─── Step 4: configure-time generated header + assemble flags ───
// bun_dependency_versions.h — written at configure time, not a ninja rule.
// BunProcess.cpp includes it for process.versions. writeIfNotChanged
// semantics so bumping an unrelated dep doesn't recompile everything.
generateDepVersionsHeader(cfg);
const flags = computeFlags(cfg);
// Full include set: bun's own + all dep includes + buildDir (for the
// generated versions header).
const allIncludes = [...bunIncludes(cfg), cfg.buildDir, ...depIncludes];
const includeFlags = allIncludes.map(inc => `-I${inc}`);
const defineFlags = flags.defines.map(d => `-D${d}`);
// Final flag arrays for compile.
const cxxFlagsFull = [...flags.cxxflags, ...includeFlags, ...defineFlags];
const cFlagsFull = [...flags.cflags, ...includeFlags, ...defineFlags];
// ─── Step 5: PCH ───
// In CI, only the cpp-only job uses PCH — full mode skips it since the
// cpp-only artifacts are what actually get used downstream.
//
// Not on Windows: matches cmake (BuildBun.cmake:868 gated on NOT WIN32).
// clang-cl's /Yc//Yu flags exist but the wrapper+stub mechanism here
// is built around clang's -emit-pch model. If Windows PCH is wanted
// later, see compile.ts TODO(windows) for what needs wiring.
const usePch = !cfg.windows && (!cfg.ci || cfg.mode === "cpp-only");
let pchOut: { pch: string; wrapperHeader: string } | undefined;
if (usePch) {
n.comment("─── PCH ───");
n.blank();
// Dep outputs are IMPLICIT inputs (not order-only). The crucial case is
// local WebKit: headers live in buildDir and get REGENERATED by dep_build
// mid-run. At startup, ninja sees old headers via PCH's depfile → thinks
// PCH is fresh. dep_build then regenerates them. cxx fails with "file
// modified since PCH was built". As implicit inputs, restat sees the .a
// changed → PCH rebuilds → one-build convergence. See the pch() docstring.
//
// Codegen stays order-only: those outputs only change if inputs change,
// and inputs don't change mid-build. cppAll (not all) — bake/.zig outputs
// are zig-only; pulling them here would run bake-codegen in cpp-only CI
// mode where it fails on the pinned bun version (see cppAll docstring).
// Scripts that emit undeclared .h also emit a .cpp/.h in cppAll, so they
// still run. cxx transitively waits: cxx → PCH → deps+cppAll.
pchOut = pch(n, cfg, "src/bun.js/bindings/root.h", {
flags: cxxFlagsFull,
implicitInputs: depOutputs,
orderOnlyInputs: codegen.cppAll,
});
}
// ─── Step 6: compile C/C++ ───
n.comment("─── C/C++ compilation ───");
n.blank();
// Source lists: from the pre-globbed snapshot + platform extras.
const cxxSources = [...sources.cxx];
const cSources = [...sources.c];
// Windows-only cpp sources (rescle — PE resource editor for --compile).
if (cfg.windows) {
cxxSources.push(
resolve(cfg.cwd, "src/bun.js/bindings/windows/rescle.cpp"),
resolve(cfg.cwd, "src/bun.js/bindings/windows/rescle-binding.cpp"),
);
}
// Sources provided directly by deps (picohttpparser.c). These are
// declared as implicit outputs of their fetch rules, so ninja knows
// where they come from; we compile them like any other .c file.
cSources.push(...depCSources);
// Codegen .cpp files — compiled like regular sources.
cxxSources.push(...codegen.cppSources);
cxxSources.push(...codegen.bindgenV2Cpp);
// All deps must be ready (headers extracted, libs built) before compile.
// ORDER-ONLY, not implicit: the compiler's .d depfile tracks ACTUAL header
// dependencies on subsequent builds. Order-only ensures first-build ordering;
// after that, touching libJavaScriptCore.a doesn't recompile every .c file
// (.c files don't include JSC headers — depfile knows this).
//
// PCH is different: it has IMPLICIT deps on depOutputs because root.h
// transitively includes WebKit headers, and the PCH encodes those. If
// WebKit headers change (lib rebuilt), PCH must invalidate. The depfile
// mechanism doesn't work for PCH-invalidation because the .cpp's depfile
// says "depends on root.h.pch", not on what root.h.pch was built from.
const depOrderOnly = [...depOutputs, ...codegen.cppAll];
// Compile all .cpp with PCH.
const cxxObjects: string[] = [];
for (const src of cxxSources) {
const relSrc = relative(cfg.cwd, src);
const extraFlags = extraFlagsFor(cfg, relSrc);
const opts: Parameters<typeof cxx>[3] = {
flags: [...cxxFlagsFull, ...extraFlags],
};
if (pchOut !== undefined) {
// PCH has implicit deps on depOutputs. cxx has implicit dep on PCH.
// Transitively: cxx waits for deps. No need for order-only here.
opts.pch = pchOut.pch;
opts.pchHeader = pchOut.wrapperHeader;
} else {
// No PCH (windows) — each cxx needs direct ordering on deps.
// Order-only: depfile tracks actual headers after first build.
opts.orderOnlyInputs = depOrderOnly;
}
cxxObjects.push(cxx(n, cfg, src, opts));
}
// Compile all .c files. No PCH. Order-only on deps for first-build ordering.
const cObjects: string[] = [];
for (const src of cSources) {
cObjects.push(
cc(n, cfg, src, {
flags: cFlagsFull,
orderOnlyInputs: depOrderOnly,
}),
);
}
const allObjects = [...cxxObjects, ...cObjects];
// ─── Step 7: cpp-only → archive and return ───
// CI's build-cpp step: archive all .o into libbun.a, stop. The sibling
// build-zig step produces bun-zig.o independently; build-bun downloads
// both artifacts and links them. Archive name uses the exe name (not
// just "libbun") so asan/debug variants are distinguishable in artifacts.
if (cfg.mode === "cpp-only") {
n.comment("─── Archive (cpp-only) ───");
n.blank();
const archiveName = `${cfg.libPrefix}${exeName}${cfg.libSuffix}`;
const archive = ar(n, cfg, archiveName, allObjects);
n.phony("bun", [archive]);
n.default(["bun"]);
return { archive, deps, codegen, zigObjects, objects: allObjects };
}
// ─── Step 7: link ───
n.comment("─── Link ───");
n.blank();
// Windows resources (.rc → .res): icon, VersionInfo. Compiled at link
// time (not archived in cpp-only) — .res is small and the .rc depends
// on cfg.version which the link step already has. Matches cmake's
// behavior of adding WINDOWS_RESOURCES to add_executable in link-only.
const windowsRes = cfg.windows ? [emitWindowsResources(n, cfg)] : [];
// Full link.
const exe = link(n, cfg, exeName, [...allObjects, ...zigObjects, ...windowsRes], {
libs: depLibs,
flags: [...flags.ldflags, ...systemLibs(cfg), ...manifestLinkFlags(cfg)],
implicitInputs: linkImplicitInputs(cfg),
});
// ─── Step 8: post-link (strip + dsymutil) ───
// Plain release only: produce stripped `bun` alongside `bun-profile`.
// Debug/asan/etc. keep symbols (you want them for debugging).
let strippedExe: string | undefined;
let dsym: string | undefined;
if (shouldStrip(cfg)) {
strippedExe = emitStrip(n, cfg, exe, flags.stripflags);
// darwin: extract debug symbols from the UNSTRIPPED exe into a .dSYM
// bundle. dsymutil reads DWARF from bun-profile, writes bun-profile.dSYM.
// Must run BEFORE stripping could discard sections it needs (we don't
// strip bun-profile itself, only copy → bun, so this is safe).
if (cfg.darwin) {
dsym = emitDsymutil(n, cfg, exe, exeName);
}
}
// Phony `bun` target for convenience — only when strip DIDN'T produce a
// literal file named `bun` (which would collide with the phony). When
// strip runs, `ninja bun` builds the actual stripped file; no phony needed.
if (strippedExe === undefined) {
n.phony("bun", [exe]);
}
// ─── Step 9: smoke test ───
// Run `<exe> --revision`. If it exits non-zero or crashes, something
// broke at load time (missing symbol, static initializer blowup, ABI
// mismatch). Catching this HERE is much better than "CI passes, user
// runs bun, it segfaults".
//
// Linux+ASAN quirk: some systems need ASLR disabled (`setarch -R`) for
// ASAN binaries to run from subprocesses (shadow memory layout conflict
// with ELF_ET_DYN_BASE, see sanitizers/856). We try with setarch first,
// fall back to direct invocation.
emitSmokeTest(n, cfg, exe, exeName);
return { exe, strippedExe, dsym, deps, codegen, zigObjects, objects: allObjects };
}
/**
* zig-only mode: emit just the zig build graph. CI's build-zig step uses
* this to cross-compile bun-zig.o on a linux box for all target platforms
* (zig cross-compiles cleanly; target set via --os/--arch overrides).
*
* Needs:
* - zstd FETCHED (build.zig @cImports its headers) — not built
* - codegen (zig subset: embedFiles, generated .zig modules)
* - zig compiler downloaded + zig build
*
* Does NOT need: any dep built, any cxx, PCH, link. ninja only pulls
* what's depended on — zstd's configure/build rules are emitted but
* unused (its .ref stamp is the only dependency from emitZig).
*/
function emitZigOnly(n: Ninja, cfg: Config, sources: Sources): BunOutput {
n.comment("════════════════════════════════════════════════════════════════");
n.comment(` Building bun-zig.o (zig-only, target: ${cfg.os}-${cfg.arch})`);
n.comment("════════════════════════════════════════════════════════════════");
n.blank();
// Only dep: zstd, for @cImport headers. resolveDep emits its
// fetch/configure/build; emitZig only depends on the fetch stamp.
const zstdDep = resolveDep(n, cfg, zstd);
assert(zstdDep !== null, "zstd resolveDep returned null — should never be skipped");
// Codegen: emitted fully, but only zigInputs/zigOrderOnly are pulled.
// The cpp-related outputs (cppSources, bindgenV2Cpp) have no consumer
// in this graph — ninja skips them.
const codegen = emitCodegen(n, cfg, sources);
const codegenZigSet = new Set(zigFilesGeneratedIntoSrc.map(p => resolve(cfg.cwd, p)));
const zigSources = sources.zig.filter(f => !codegenZigSet.has(f));
const zigObjects = emitZig(n, cfg, {
codegenInputs: codegen.zigInputs,
codegenOrderOnly: codegen.zigOrderOnly,
zigSources,
zstdStamp: depSourceStamp(cfg, "zstd"),
});
n.phony("bun", zigObjects);
n.default(["bun"]);
return { deps: [zstdDep], codegen, zigObjects, objects: [] };
}
/**
* link-only mode: link artifacts downloaded from sibling buildkite steps.
* CI's build-bun step. Build.ts downloads into buildDir BEFORE ninja runs;
* ninja sees the files as source inputs (no build rule — errors cleanly
* if download failed or paths drift).
*
* Expected artifacts (same paths cpp-only/zig-only produced):
* - libbun-profile.a — from cpp-only's ar()
* - bun-zig.o — from zig-only
* - deps/<name>/lib<name>.a — from cpp-only's dep builds
* - cache/webkit-<hash>/lib/... — WebKit prebuilt (same cache path)
*/
function emitLinkOnly(n: Ninja, cfg: Config): BunOutput {
const exeName = bunExeName(cfg);
n.comment("════════════════════════════════════════════════════════════════");
n.comment(` Linking ${exeName} (link-only — artifacts from buildkite)`);
n.comment("════════════════════════════════════════════════════════════════");
n.blank();
// Dep lib paths — computed, not built. Must match cpp-only's output
// paths exactly; computeDepLibs() and emitNestedCmake()'s path logic
// share the same formula. If they drift, link fails with "file not
// found" — loud enough to catch in CI.
const depLibs: string[] = [];
for (const dep of allDeps) {
depLibs.push(...computeDepLibs(cfg, dep));
}
// Archive from cpp-only: same name cpp-only emits (exe name + lib
// prefix/suffix, e.g. libbun-profile.a).
const archive = resolve(cfg.buildDir, `${cfg.libPrefix}${exeName}${cfg.libSuffix}`);
// bun-zig.o from zig-only: same path emitZig writes to.
// Hardcoded filename — emitZig uses "bun-zig.o" regardless of platform
// (zig outputs ELF-like obj format by default; -Dobj_format=obj for
// windows → COFF, but filename stays the same).
const zigObj = resolve(cfg.buildDir, "bun-zig.o");
// Only need ldflags + stripflags (no cflags/cxxflags — no compile).
const flags = computeFlags(cfg);
n.comment("─── Link ───");
n.blank();
// Windows resources: compiled here, not downloaded from cpp-only.
// .res is small; .rc substitution depends on cfg.version which link-only
// knows. Matches cmake's BUN_LINK_ONLY adding WINDOWS_RESOURCES directly.
const windowsRes = cfg.windows ? [emitWindowsResources(n, cfg)] : [];
const exe = link(n, cfg, exeName, [archive, zigObj, ...windowsRes], {
libs: depLibs,
flags: [...flags.ldflags, ...systemLibs(cfg), ...manifestLinkFlags(cfg)],
implicitInputs: linkImplicitInputs(cfg),
});
// Strip + smoke test — same as full mode.
let strippedExe: string | undefined;
let dsym: string | undefined;
if (shouldStrip(cfg)) {
strippedExe = emitStrip(n, cfg, exe, flags.stripflags);
if (cfg.darwin) dsym = emitDsymutil(n, cfg, exe, exeName);
}
if (strippedExe === undefined) n.phony("bun", [exe]);
emitSmokeTest(n, cfg, exe, exeName);
return {
exe,
strippedExe,
dsym,
deps: [], // no ResolvedDep — we only computed lib paths
zigObjects: [zigObj],
objects: [],
};
}
/**
* Smoke test: run the built executable with --revision. If it crashes or
* errors, the build failed — typically means a link-time issue that the
* linker didn't catch (missing symbol only referenced at init, ICU ABI
* mismatch, etc.).
*/
function emitSmokeTest(n: Ninja, cfg: Config, exe: string, exeName: string): void {
const stamp = resolve(cfg.buildDir, `${exeName}.smoke-test-passed`);
// Linux+ASAN: wrap in `setarch <arch> -R` to disable ASLR. Fall back
// to direct invocation if setarch fails (not all systems have it).
// The `|| true` on the outer command isn't there — if BOTH fail, we
// want the rule to error.
const envWrap = "env BUN_DEBUG_QUIET_LOGS=1";
let testCmd: string;
if (cfg.linux && cfg.asan) {
const arch = cfg.x64 ? "x86_64" : "aarch64";
testCmd = `${envWrap} setarch ${arch} -R ${exe} --revision || ${envWrap} ${exe} --revision`;
} else if (cfg.windows) {
// Windows: no setarch, no env wrapper syntax differences matter for
// this simple case. cmd /c handles the pipe.
testCmd = `${exe} --revision`;
} else {
testCmd = `${envWrap} ${exe} --revision`;
}
// stream.ts --console: passthrough + ninja Windows buffering fix.
// sh -c with parens: testCmd may contain `||` (ASAN setarch fallback);
// without grouping, `a || b && touch` parses as `a || (b && touch)` —
// stamp wouldn't get written when setarch succeeds.
const q = (p: string) => quote(p, cfg.windows);
const wrap = `${q(cfg.bun)} ${q(streamPath)} check --console`;
n.rule("smoke_test", {
command: cfg.windows
? `${wrap} cmd /c "${testCmd} && type nul > $out"`
: `${wrap} sh -c '( ${testCmd} ) && touch $out'`,
description: `${exeName} --revision`,
// pool = console: user wants to see the revision output.
pool: "console",
});
n.build({
outputs: [stamp],
rule: "smoke_test",
inputs: [exe],
});
// Phony target — `ninja check` runs the smoke test.
n.phony("check", [stamp]);
}
/**
* Strip the linked executable → plain `bun`. Returns absolute path to
* the stripped output.
*
* Input (bun-profile) is NOT modified — strip writes a new file via `-o`.
* The profile binary keeps its symbols for profiling/debugging release crashes.
*/
function emitStrip(n: Ninja, cfg: Config, inputExe: string, stripflags: string[]): string {
const out = resolve(cfg.buildDir, "bun" + cfg.exeSuffix);
// Windows: strip equivalent is handled at link time (/OPT:REF etc), no
// separate strip binary. The "stripped" bun is just a copy.
if (cfg.windows) {
// Copy as-is. /OPT:REF already applied at link.
n.rule("strip", {
command: `cmd /c "copy /Y $in $out"`,
description: "copy $out (windows: no strip)",
});
} else {
n.rule("strip", {
command: `${quote(cfg.strip, false)} $stripflags $in -o $out`,
description: "strip $out",
});
}
n.build({
outputs: [out],
rule: "strip",
inputs: [inputExe],
vars: cfg.windows ? {} : { stripflags: stripflags.join(" ") },
});
return out;
}
/**
* Extract debug symbols from the linked (unstripped) executable into a
* .dSYM bundle. darwin-only.
*
* Runs dsymutil on bun-profile (which has full DWARF). The .dSYM lets you
* symbolicate crash logs from the stripped `bun` — lldb/Instruments find
* it automatically by UUID.
*/
function emitDsymutil(n: Ninja, cfg: Config, inputExe: string, exeName: string): string {
assert(cfg.darwin, "dsymutil is darwin-only");
assert(cfg.dsymutil !== undefined, "dsymutil not found in toolchain");
const out = resolve(cfg.buildDir, `${exeName}.dSYM`);
// --flat: single-file .dSYM (not a bundle directory). Simpler to upload
// as a CI artifact.
// --keep-function-for-static: keep symbols for static functions (more
// complete backtraces).
// --object-prefix-map: rewrite DWARF path prefixes so debuggers find
// source in the repo root rather than the build machine's absolute path.
// -j: parallelism. Use all cores (dsymutil parallelizes per compile unit).
// CMake uses CMAKE_BUILD_PARALLEL_LEVEL; we use nproc equivalent via
// a subshell.
// stream.ts --console for pool:console consistency (no-op on darwin).
const q = (p: string) => quote(p, false); // darwin-only → posix
const wrap = `${q(cfg.bun)} ${q(streamPath)} dsym --console`;
n.rule("dsymutil", {
command: `${wrap} sh -c '${cfg.dsymutil} $in --flat --keep-function-for-static --object-prefix-map .=${cfg.cwd} -o $out -j $$(sysctl -n hw.ncpu)'`,
description: "dsymutil $out",
// Not restat — dsymutil always writes.
pool: "console", // Can take a while, show progress
});
n.build({
outputs: [out],
rule: "dsymutil",
inputs: [inputExe],
});
return out;
}
// ───────────────────────────────────────────────────────────────────────────
// Windows resources (.rc → .res)
// ───────────────────────────────────────────────────────────────────────────
/**
* Template-substitute windows-app-info.rc and compile it with llvm-rc.
* Returns the path to the .res output (to be linked like an object file).
*
* The .rc file provides:
* - Icon (bun.ico)
* - VS_VERSION_INFO resource (ProductName, FileVersion, CompanyName, ...)
*
* This resource section is what rescle's ResourceUpdater modifies when
* `bun build --compile --windows-title ...` runs. Without it, the copied
* bun.exe has no VersionInfo to update and rescle silently does nothing.
*
* The manifest (longPathAware + SegmentHeap) is embedded at link time via
* /MANIFESTINPUT — see manifestLinkFlags().
*/
function emitWindowsResources(n: Ninja, cfg: Config): string {
assert(cfg.windows, "emitWindowsResources is windows-only");
assert(cfg.rc !== undefined, "llvm-rc not found in toolchain");
// ─── Template substitution (configure time) ───
// The .rc uses @VAR@ cmake-style placeholders. Substitute and write to
// buildDir (not codegenDir — link-only doesn't create codegenDir).
// writeIfChanged → mtime preserved → no spurious rc rebuild when the
// substituted content hasn't changed.
const rcTemplate = resolve(cfg.cwd, "src/windows-app-info.rc");
const ico = resolve(cfg.cwd, "src/bun.ico");
const rcIn = readFileSync(rcTemplate, "utf8");
const [major = "0", minor = "0", patch = "0"] = cfg.version.split(".");
const versionWithTag = cfg.canary ? `${cfg.version}-canary.${cfg.canaryRevision}` : cfg.version;
// slash(): rc parses .rc as C-like source; backslashes in the ICON path
// string would need escaping. Forward slashes work for Windows file APIs.
const rcOut = rcIn
.replace(/@Bun_VERSION_MAJOR@/g, major)
.replace(/@Bun_VERSION_MINOR@/g, minor)
.replace(/@Bun_VERSION_PATCH@/g, patch)
.replace(/@Bun_VERSION_WITH_TAG@/g, versionWithTag)
.replace(/@BUN_ICO_PATH@/g, slash(ico));
const rcFile = resolve(cfg.buildDir, "windows-app-info.rc");
writeIfChanged(rcFile, rcOut);
// ─── Compile .rc → .res (ninja time) ───
// llvm-rc: /FO sets output. `#include "windows.h"` in the .rc resolves
// via the INCLUDE env var set by the VS dev shell (vs-shell.ps1).
const resFile = resolve(cfg.buildDir, "windows-app-info.res");
n.rule("rc", {
command: `${quote(cfg.rc, true)} /FO $out $in`,
description: "rc $out",
});
n.build({
outputs: [resFile],
rule: "rc",
inputs: [rcFile],
// .ico is embedded by rc at compile time — rebuild if it changes.
// The template is NOT tracked here: it's substituted at configure
// time, so template edits need a reconfigure (happens rarely).
implicitInputs: [ico],
});
return resFile;
}
/**
* Linker flags to embed bun.exe.manifest into the executable.
* The manifest enables longPathAware (paths > MAX_PATH) and SegmentHeap
* (Windows 10+ low-fragmentation heap).
*/
function manifestLinkFlags(cfg: Config): string[] {
if (!cfg.windows) return [];
const manifest = resolve(cfg.cwd, "src/bun.exe.manifest");
return [`/MANIFEST:EMBED`, `/MANIFESTINPUT:${manifest}`];
}
/**
* Files the linker reads via ldflags that ninja should track for relinking
* (symbol lists, linker script, manifest). CMake's LINK_DEPENDS equivalent.
*/
function linkImplicitInputs(cfg: Config): string[] {
const files = linkDepends(cfg);
if (cfg.windows) files.push(resolve(cfg.cwd, "src/bun.exe.manifest"));
return files;
}
// ───────────────────────────────────────────────────────────────────────────
// Pre-flight checks
// ───────────────────────────────────────────────────────────────────────────
/**
* Validate config before emitting. Catches obvious problems at configure
* time instead of cryptic build failures later.
*/
export function validateBunConfig(cfg: Config): void {
// All modes now implemented. Kept as a hook for future validation
// (e.g. incompatible option combos).
void cfg;
}

523
scripts/build/ci.ts Normal file
View File

@@ -0,0 +1,523 @@
/**
* CI integration: collapsible log groups, environment dump, Buildkite
* annotations on build failure.
*
* Thin layer over `scripts/utils.mjs` — the same helpers the CMake build
* uses. We import rather than reimplement so CI logs look identical and
* annotation regex stays in one place.
*/
import { spawn as nodeSpawn, spawnSync } from "node:child_process";
import { cpSync, existsSync, mkdirSync, readdirSync, rmSync, statSync } from "node:fs";
import { basename, relative, resolve } from "node:path";
// @ts-ignore — utils.mjs has JSDoc types but no .d.ts
import * as utils from "../utils.mjs";
import { bunExeName, shouldStrip, type BunOutput } from "./bun.ts";
import type { Config } from "./config.ts";
import { WEBKIT_VERSION } from "./deps/webkit.ts";
import { BuildError } from "./error.ts";
import { ZIG_COMMIT } from "./zig.ts";
/** True if running under any CI (env: CI, BUILDKITE, or GITHUB_ACTIONS). */
export const isCI: boolean = utils.isCI;
/** True if running under Buildkite specifically. */
export const isBuildkite: boolean = utils.isBuildkite;
/** True if running under GitHub Actions specifically. */
export const isGithubAction: boolean = utils.isGithubAction;
/**
* Print machine/environment/repository info in collapsible groups.
* Call at the top of a CI run so you can diagnose without SSH access.
*/
export const printEnvironment: () => void = utils.printEnvironment;
/**
* Start a collapsible log group. Buildkite: `--- Title`. GitHub: `::group::`.
* If `fn` is given, runs it and closes the group (handles async).
*/
export const startGroup: (title: string, fn?: () => unknown) => unknown = utils.startGroup;
/** Close the most recent group opened with `startGroup`. */
export const endGroup: () => void = utils.endGroup;
interface SpawnAnnotatedOptions {
/** Working directory for the subprocess. */
cwd?: string;
/** Label for duration printing (defaults to basename of command). */
label?: string;
/** Environment variables for the subprocess. */
env?: NodeJS.ProcessEnv;
}
/**
* Spawn a subprocess with CI output handling. Only call this in CI —
* locally use plain spawnSync for zero-overhead no-ops.
*
* Tees stdout/stderr to the terminal AND a buffer. On non-zero exit,
* parses the buffer for compiler errors (zig/clang/cmake) and posts each
* as a Buildkite annotation. If nothing parseable is found, posts a generic
* "build failed" annotation with the full output. Prints duration at end.
*
* Exits the process with the subprocess's exit code on failure.
* Returns only on success.
*/
export async function spawnWithAnnotations(
command: string,
args: string[],
opts: SpawnAnnotatedOptions = {},
): Promise<void> {
const label = opts.label ?? command;
const child = nodeSpawn(command, args, {
stdio: "pipe",
cwd: opts.cwd,
env: opts.env,
});
// Kill child on parent signals so ninja doesn't linger.
let killedManually = false;
const onKill = () => {
if (!child.killed) {
killedManually = true;
child.kill();
}
};
if (process.platform !== "win32") {
process.once("beforeExit", onKill);
process.once("SIGINT", onKill);
process.once("SIGTERM", onKill);
}
const clearOnKill = () => {
process.off("beforeExit", onKill);
process.off("SIGINT", onKill);
process.off("SIGTERM", onKill);
};
const start = Date.now();
let buffer = "";
// Tee: write to terminal live AND buffer for later annotation parsing.
const stdout = new Promise<void>(resolve => {
child.stdout!.on("end", resolve);
child.stdout!.on("data", (chunk: Buffer) => {
buffer += chunk.toString();
process.stdout.write(chunk);
});
});
const stderr = new Promise<void>(resolve => {
child.stderr!.on("end", resolve);
child.stderr!.on("data", (chunk: Buffer) => {
buffer += chunk.toString();
process.stderr.write(chunk);
});
});
const { exitCode, signalCode, error } = await new Promise<{
exitCode: number | null;
signalCode: NodeJS.Signals | null;
error?: Error;
}>(resolve => {
child.on("error", error => {
clearOnKill();
resolve({ exitCode: null, signalCode: null, error });
});
child.on("exit", (exitCode, signalCode) => {
clearOnKill();
resolve({ exitCode, signalCode });
});
});
await Promise.all([stdout, stderr]);
const elapsed = Date.now() - start;
const elapsedStr =
elapsed > 60000 ? `${(elapsed / 60000).toFixed(2)} minutes` : `${(elapsed / 1000).toFixed(2)} seconds`;
console.log(`${label} took ${elapsedStr}`);
if (error) {
console.error(`Failed to spawn ${command}: ${error.message}`);
process.exit(127);
}
if (exitCode === 0) return;
// ─── Failure: report annotations to Buildkite ───
if (isBuildkite) {
let annotated = false;
try {
// In piped mode, ninja prints ALL command output including successful
// jobs — so the buffer contains dep cmake deprecation warnings from
// vendored CMakeLists.txt we don't control. Keep dep errors (broken
// compiler, bad flags) since those are actionable; drop dep warnings.
const annotatable = buffer
.split("\n")
.filter(line => !/^\[[\w-]+\]\s+CMake (Deprecation )?Warning/i.test(line.replace(/\x1b\[[0-9;]*m/g, "")))
.join("\n");
const { annotations } = utils.parseAnnotations(annotatable);
for (const ann of annotations) {
utils.reportAnnotationToBuildKite({
priority: 10,
label: ann.title || ann.filename,
content: utils.formatAnnotationToHtml(ann),
});
annotated = true;
}
} catch (err) {
console.error("Failed to parse annotations:", err);
}
// Nothing matched the compiler-error regexes → post a generic annotation
// with the full buffered output so there's still a PR-visible signal.
if (!annotated) {
const content = utils.formatAnnotationToHtml({
filename: relative(process.cwd(), import.meta.path),
title: "build failed",
content: buffer,
source: "build",
level: "error",
});
utils.reportAnnotationToBuildKite({
priority: 10,
label: "build failed",
content,
});
}
}
if (signalCode) {
if (!killedManually) console.error(`Command killed: ${signalCode}`);
} else {
console.error(`Command exited: code ${exitCode}`);
}
process.exit(exitCode ?? 1);
}
// ───────────────────────────────────────────────────────────────────────────
// Buildkite artifacts — split-build upload/download
//
// CI splits builds per-platform into three parallel steps:
// build-cpp → libbun.a + all dep libs (this node uploads)
// build-zig → bun-zig.o (this node uploads)
// build-bun → downloads both, links (this node downloads first)
//
// Paths are uploaded RELATIVE TO buildDir. buildkite-agent recreates the
// directory structure on download. The link-only ninja graph expects files
// at the SAME relative paths cpp-only produced them at — computeDepLibs()
// and emitNestedCmake() share the same path formula.
// ───────────────────────────────────────────────────────────────────────────
/**
* Upload build artifacts after a successful cpp-only or zig-only build.
* Runs `buildkite-agent artifact upload` with paths relative to buildDir.
*
* Large archives (libbun-*.a, >1GB) are gzipped — buildkite artifact
* storage is fine but upload/download is faster. link-only gunzips.
*
* ORDER MATTERS: upload dep libs FIRST (some live in cache/ — WebKit
* prebuilt), THEN rm cache + gzip + upload the archive. If cache is
* deleted first, WebKit lib upload fails with "file not found". The
* old cmake had this ordering implicitly — each dep's build uploaded
* its libs immediately; rm only ran when the archive target fired.
*/
export function uploadArtifacts(cfg: Config, output: BunOutput): void {
if (!isBuildkite) {
console.log("Not in Buildkite — skipping artifact upload");
return;
}
if (cfg.mode === "zig-only") {
const paths = output.zigObjects.map(obj => relative(cfg.buildDir, obj));
console.log(`Uploading ${paths.length} zig artifacts...`);
upload(paths, cfg.buildDir);
return;
}
if (cfg.mode !== "cpp-only") {
// full/link-only don't upload split artifacts.
return;
}
// ─── Phase 1: upload dep libs (before we rm anything) ───
const depPaths: string[] = [];
for (const dep of output.deps) {
for (const lib of dep.libs) {
depPaths.push(relative(cfg.buildDir, lib));
}
}
console.log(`Uploading ${depPaths.length} dep libs...`);
upload(depPaths, cfg.buildDir);
// ─── Phase 2: free disk, gzip (posix only), upload archive ───
// CI agents are disk-constrained. Free what we no longer need: codegen/
// (sources already compiled into the archive), obj/ (.o files archived),
// cache/ (WebKit prebuilt — libs uploaded in phase 1, rest is headers
// + tarball we won't touch again).
if (output.archive !== undefined) {
const archiveName = basename(output.archive);
console.log("Cleaning intermediate files to free disk...");
rmSync(cfg.codegenDir, { recursive: true, force: true });
rmSync(resolve(cfg.buildDir, "obj"), { recursive: true, force: true });
rmSync(cfg.cacheDir, { recursive: true, force: true });
// gzip: posix only (matches cmake — only libbun-*.a are gzipped,
// Windows .lib archives uploaded uncompressed). gzip isn't a
// standard Windows tool anyway; the .lib is smaller (PDB is separate).
// downloadArtifacts() only gunzips .gz files it finds, so Windows
// archives pass through unchanged.
if (cfg.windows) {
console.log("Uploading archive (Windows: no gzip)...");
upload([archiveName], cfg.buildDir);
} else {
console.log(`Compressing ${archiveName}...`);
run(["gzip", "-1", archiveName], cfg.buildDir);
console.log("Uploading archive...");
upload([`${archiveName}.gz`], cfg.buildDir);
}
}
}
/**
* Upload via buildkite-agent. Semicolon-joined single arg — the agent
* splits on ";" by default (--delimiter flag, Value: ";"). Second
* positional arg is interpreted as upload DESTINATION, not another path.
*/
function upload(paths: string[], cwd: string): void {
if (paths.length === 0) return;
run(["buildkite-agent", "artifact", "upload", paths.join(";")], cwd);
}
// ───────────────────────────────────────────────────────────────────────────
// Link-only post-link: features.json + link-metadata.json + packaging + upload
//
// The zip contract (matching cmake's BuildBun.cmake packaging — test steps
// download these by exact name):
//
// ${bunTriplet}-profile.zip (plain release)
// └── ${bunTriplet}-profile/
// ├── bun-profile[.exe]
// ├── features.json
// ├── bun-profile.linker-map (linux/mac non-asan)
// ├── bun-profile.pdb (windows)
// └── bun-profile.dSYM (mac)
//
// ${bunTriplet}.zip (stripped, plain release only)
// └── ${bunTriplet}/
// └── bun[.exe]
//
// ${bunTriplet}-asan.zip (asan — single zip, no strip)
// └── ${bunTriplet}-asan/
// ├── bun-asan
// └── features.json
//
// bunTriplet = bun-${os}-${arch}[-musl][-baseline]
//
// Test steps (runner.node.mjs) download '**' from build-bun and pick any
// bun*.zip; baseline-verification step downloads ${triplet}.zip specifically
// and expects ${triplet}/bun inside.
// ───────────────────────────────────────────────────────────────────────────
/**
* Base triplet (bun-os-arch[-musl][-baseline]). Variant suffix (-profile,
* -asan) is added by the caller. Matches ci.mjs getTargetTriplet() and
* cmake's bunTriplet — any drift breaks test-step downloads.
*/
function computeBunTriplet(cfg: Config): string {
let t = `bun-${cfg.os}-${cfg.arch}`;
if (cfg.abi === "musl") t += "-musl";
if (cfg.baseline) t += "-baseline";
return t;
}
/**
* Post-link packaging and upload for link-only mode. Runs AFTER ninja
* succeeds — at that point bun-profile (and stripped bun) exist.
*
* Generates features.json + link-metadata.json, packages into zips,
* uploads. Contract with test steps: see block comment above.
*/
export function packageAndUpload(cfg: Config, output: BunOutput): void {
if (!isBuildkite || cfg.mode !== "link-only") return;
const exe = output.exe;
if (exe === undefined) {
throw new BuildError("link-only packaging: output.exe unset");
}
const buildDir = cfg.buildDir;
const exeName = bunExeName(cfg); // bun-profile, bun-asan, etc.
const bunTriplet = computeBunTriplet(cfg);
// ─── features.json ───
// Run the built bun with features.mjs to dump its feature flags.
// Env vars match cmake's (BuildBun.cmake ~1462).
// No setarch wrapper — cmake doesn't use one for features.mjs either
// (only for the --revision smoke test).
console.log("Generating features.json...");
run([exe, resolve(cfg.cwd, "scripts", "features.mjs")], buildDir, {
BUN_GARBAGE_COLLECTOR_LEVEL: "1",
BUN_DEBUG_QUIET_LOGS: "1",
BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING: "1",
});
// ─── link-metadata.json ───
// Version/webkit/zig info for debugging. Env vars match cmake
// (BuildBun.cmake ~1253). The script reads the ninja link command too.
console.log("Generating link-metadata.json...");
run(
[process.execPath, resolve(cfg.cwd, "scripts", "create-link-metadata.mjs"), buildDir, exeName + cfg.exeSuffix],
cfg.cwd,
{
BUN_VERSION: cfg.version,
WEBKIT_VERSION: WEBKIT_VERSION,
ZIG_COMMIT: ZIG_COMMIT,
// WEBKIT_DOWNLOAD_URL not available directly; we have the version.
// The script handles missing env vars (defaults to "").
},
);
const zipPaths: string[] = [];
// ─── Profile/variant zip ───
// cmake's bunPath: string(REPLACE bun ${bunTriplet} bunPath ${bun})
// where ${bun} is the target name (bun-profile, bun-asan, ...).
// Result: bun-linux-x64-profile, bun-linux-x64-asan, etc.
const bunPath = exeName.replace(/^bun/, bunTriplet);
const files: string[] = [basename(exe), "features.json"];
// Debug symbols / linker map — platform-specific extras.
if (cfg.windows) {
files.push(`${exeName}.pdb`);
} else if (cfg.darwin) {
files.push(`${exeName}.dSYM`);
}
// Linker map: posix non-asan (cmake gate: (APPLE OR LINUX) AND NOT ENABLE_ASAN).
if (cfg.unix && !cfg.asan) {
files.push(`${exeName}.linker-map`);
}
zipPaths.push(makeZip(cfg, bunPath, files));
// ─── Stripped zip ───
// Only for plain release (shouldStrip). Just the stripped `bun` binary.
// cmake: bunStripPath = string(REPLACE bun ${bunTriplet} bunStripPath bun) = bunTriplet.
if (shouldStrip(cfg) && output.strippedExe !== undefined) {
zipPaths.push(makeZip(cfg, bunTriplet, [basename(output.strippedExe)]));
}
// ─── Upload ───
// link-metadata.json uploaded standalone (not in a zip — matches cmake's
// ARTIFACTS ${BUILD_PATH}/link-metadata.json).
console.log(`Uploading ${zipPaths.length} zips + metadata...`);
upload([...zipPaths, "link-metadata.json"], buildDir);
}
/**
* Create a zip at buildDir/${name}.zip containing buildDir/${name}/<files>.
*
* Uses `cmake -E tar cfv x.zip --format=zip` — cmake's cross-platform
* zip wrapper (wraps libarchive). GNU tar (Linux default) DOESN'T support
* --format=zip; bsdtar does but isn't guaranteed on Linux. cmake is
* already a required tool (we use it for nested dep builds), so this
* adds no new dependency. Identical to cmake's own packaging approach
* (BuildBun.cmake:1544).
*
* Files that don't exist are silently skipped (e.g., .pdb on a clean build).
* Returns the zip path relative to buildDir (for the upload call).
*/
function makeZip(cfg: Config, name: string, files: string[]): string {
const buildDir = cfg.buildDir;
const stageDir = resolve(buildDir, name);
const zip = `${name}.zip`;
// Clean previous run (idempotent).
rmSync(stageDir, { recursive: true, force: true });
rmSync(resolve(buildDir, zip), { force: true });
mkdirSync(stageDir, { recursive: true });
// Copy files that exist. Some debug outputs (.pdb, .dSYM, .linker-map)
// are optional depending on build config — skip rather than fail so a
// missing optional file doesn't break packaging.
let copied = 0;
for (const f of files) {
const src = resolve(buildDir, f);
if (!existsSync(src)) {
console.log(` (skip missing: ${f})`);
continue;
}
cpSync(src, resolve(stageDir, basename(f)), { recursive: true });
copied++;
}
console.log(`Creating ${zip} (${copied} files)...`);
// Relative path `name` puts `name/` prefix inside the zip — what test
// steps expect: they extract → `chmod +x ${triplet}/bun`.
run([cfg.cmake, "-E", "tar", "cfv", zip, "--format=zip", name], buildDir);
// Clean up the staging dir.
rmSync(stageDir, { recursive: true, force: true });
return zip;
}
/**
* Download artifacts from sibling buildkite steps before a link-only build.
* Derives sibling step keys from BUILDKITE_STEP_KEY (swap `-build-bun` →
* `-build-cpp` / `-build-zig`). Gunzips any .gz files after download.
*
* Call BEFORE ninja — the downloaded files are ninja's link inputs.
*/
export function downloadArtifacts(cfg: Config): void {
if (cfg.mode !== "link-only") return;
const stepKey = process.env.BUILDKITE_STEP_KEY;
if (stepKey === undefined) {
throw new BuildError("BUILDKITE_STEP_KEY unset", {
hint: "link-only mode requires running inside a Buildkite job",
});
}
// step key is `<target>-build-bun`; siblings are `<target>-build-{cpp,zig}`.
const m = stepKey.match(/^(.+)-build-bun$/);
if (m === null) {
throw new BuildError(`Unexpected BUILDKITE_STEP_KEY: ${stepKey}`, {
hint: "Expected format: <target>-build-bun",
});
}
const targetKey = m[1]!;
for (const suffix of ["cpp", "zig"]) {
const step = `${targetKey}-build-${suffix}`;
console.log(`Downloading artifacts from ${step}...`);
// '*' glob — download everything that step uploaded.
run(["buildkite-agent", "artifact", "download", "*", ".", "--step", step], cfg.buildDir);
}
// Gunzip any compressed archives (libbun-*.a.gz → libbun-*.a).
// -f: overwrite if already decompressed (idempotent re-run).
const gzFiles = existsSync(cfg.buildDir)
? readdirSync(cfg.buildDir).filter(f => f.endsWith(".gz") && statSync(resolve(cfg.buildDir, f)).isFile())
: [];
for (const gz of gzFiles) {
console.log(`Decompressing ${gz}...`);
run(["gunzip", "-f", gz], cfg.buildDir);
}
}
/** Run a command synchronously, throw BuildError on non-zero exit. */
function run(argv: string[], cwd: string, env?: Record<string, string>): void {
const result = spawnSync(argv[0]!, argv.slice(1), {
cwd,
stdio: "inherit",
env: env ? { ...process.env, ...env } : undefined,
});
if (result.error) {
throw new BuildError(`Failed to spawn ${argv[0]}`, { cause: result.error });
}
if (result.status !== 0) {
throw new BuildError(`${argv[0]} exited with code ${result.status}`, {
hint: `Command: ${argv.join(" ")}`,
});
}
}

915
scripts/build/codegen.ts Normal file
View File

@@ -0,0 +1,915 @@
/**
* Code generation as ninja rules.
*
* Each codegen step is a single ninja `build` statement with explicit
* inputs (script + sources) and outputs. All rules set `restat = 1` —
* most codegen scripts use `writeIfNotChanged`, so downstream is pruned
* when output content didn't change.
*
* Source lists come from `cmake/Sources.json` patterns, globbed once at
* configure time via `globAllSources()` — see sources.ts. The expanded
* paths are baked into build.ninja; adding a file picks up on next configure.
*
* bindgenv2 is special: its output set is dynamic (depends on which types
* the .bindv2.ts files export). We invoke it with `--command=list-outputs`
* at CONFIGURE time to get the actual output paths.
*
* ## Undeclared outputs
*
* Several scripts emit MORE files than they report:
* - bindgen.ts emits Generated<Name>.h per namespace (only .cpp declared)
* - bindgenv2 emits Generated<Type>.h per type (list-outputs skips .h)
* - generate-node-errors.ts emits ErrorCode.d.ts (not declared)
* - bundle-modules.ts emits eval/ subdir, BunBuiltinNames+extras.h, etc.
* - cppbind.ts emits cpp.source-links
*
* It WORKS because:
* 1. The declared .cpp outputs guarantee the step runs before compile
* 2. Compilation emits .d depfiles that track the .h files for NEXT build
* 3. PCH order-depends on ALL codegen outputs; every cxx() waits on PCH
* → all codegen completes before any compile, undeclared .h exist
*
* Fixing properly (declaring all outputs) would require patching the
* src/codegen/ scripts to report everything — changing contract with
* existing tooling.
*/
import { spawnSync } from "node:child_process";
import { mkdirSync, readFileSync } from "node:fs";
import { basename, relative, resolve } from "node:path";
import type { Config } from "./config.ts";
import { BuildError, assert } from "./error.ts";
import { writeIfChanged } from "./fs.ts";
import type { Ninja } from "./ninja.ts";
import { quote, quoteArgs } from "./shell.ts";
import type { Sources } from "./sources.ts";
/**
* Codegen outputs that land in `src/` instead of `codegenDir`. The zig
* compiler refuses to import files outside its source tree, so these two
* generated `.zig` files live in `src/bun.js/bindings/` (gitignored).
*
* Consumers of `sources.zig` (the `src/**\/*.zig` glob) must filter these
* out — they're OUTPUTS of codegen, not inputs. bun.ts does this before
* passing the zig list to emitZig().
*
* Paths are relative to repo root. This list is the single source of truth;
* `globAllSources()` does NOT hardcode these.
*/
export const zigFilesGeneratedIntoSrc = [
"src/bun.js/bindings/GeneratedBindings.zig",
"src/bun.js/bindings/GeneratedJS2Native.zig",
] as const;
// The individual emit functions take these four params. Bundled to keep
// signatures short.
interface Ctx {
n: Ninja;
cfg: Config;
sources: Sources;
o: CodegenOutputs;
dirStamp: string;
}
/**
* Read a package.json and return the list of dependency package.json paths
* under node_modules/. Used as outputs of `bun install` — if any are missing,
* install re-runs.
*/
function readPackageDeps(pkgDir: string): string[] {
const pkgPath = resolve(pkgDir, "package.json");
let pkg: { dependencies?: Record<string, string>; devDependencies?: Record<string, string> };
try {
pkg = JSON.parse(readFileSync(pkgPath, "utf8")) as typeof pkg;
} catch (cause) {
throw new BuildError(`Could not parse package.json`, { file: pkgPath, cause });
}
const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
const nodeModules = resolve(pkgDir, "node_modules");
return Object.keys(deps).map(name => resolve(nodeModules, name, "package.json"));
}
// ───────────────────────────────────────────────────────────────────────────
// Ninja rule registration
// ───────────────────────────────────────────────────────────────────────────
/**
* Register ninja rules shared by all codegen steps. Call once before
* emitCodegen().
*/
export function registerCodegenRules(n: Ninja, cfg: Config): void {
// Shell syntax: HOST platform, not target. zig-only cross-compiles on
// a linux box for darwin/windows; these rules run on the linux box.
const hostWin = cfg.host.os === "windows";
const q = (p: string) => quote(p, hostWin);
const bun = q(cfg.bun);
const esbuild = q(cfg.esbuild);
// Generic codegen: `cd <repo-root> && [env VARS] bun <args>`.
// Both `bun run script.ts` and `bun script.ts` go through this — the
// caller puts the `run` subcommand in $args when needed.
//
// restat = 1 because most scripts use writeIfNotChanged(). Scripts that
// don't (generate-jssink, ci_info) always write → restat is a no-op for
// them, no harm.
n.rule("codegen", {
command: hostWin ? `cmd /c "cd /d $cwd && ${bun} $args"` : `cd $cwd && ${bun} $args`,
description: "gen $desc",
restat: true,
});
// esbuild invocations. No restat — esbuild always touches outputs.
// No pool — esbuild is fast and single-threaded per bundle.
n.rule("esbuild", {
command: hostWin ? `cmd /c "cd /d $cwd && ${esbuild} $args"` : `cd $cwd && ${esbuild} $args`,
description: "esbuild $desc",
});
// bun install. Inputs: package.json + bun.lock. Outputs: a stamp file we
// touch on success, plus each node_modules/<dep>/package.json as IMPLICIT
// outputs (so deleting node_modules/ correctly retriggers install).
//
// Why stamp + restat instead of just the node_modules paths as outputs:
// `bun install --frozen-lockfile` with no changes doesn't touch anything.
// If package.json was edited at time T and install ran at T-1day, the
// node_modules files have mtimes from T-1day < T → ninja loops forever.
// Touching the stamp gives ninja something with mtime T to compare against.
// Restat lets implicit outputs keep their old mtimes, pruning downstream.
//
// CMake only tracked package.json as input; we add bun.lock so lockfile
// version bumps actually reinstall.
const touch = hostWin ? "type nul >" : "touch";
n.rule("bun_install", {
command: hostWin
? `cmd /c "cd /d $dir && ${bun} install --frozen-lockfile && ${touch} $stamp"`
: `cd $dir && ${bun} install --frozen-lockfile && ${touch} $stamp`,
description: "install $dir",
restat: true,
// bun install can be memory-hungry and grabs a lockfile; serialize.
pool: "bun_install",
});
n.pool("bun_install", 1);
// Codegen dir stamp — all outputs go into cfg.codegenDir, but the dir must
// exist first. Scripts generally mkdir themselves, but some (esbuild) don't.
n.build({
outputs: [codegenDirStamp(cfg)],
rule: "mkdir_stamp",
inputs: [],
vars: { dir: n.rel(cfg.codegenDir) },
});
// Stamps dir — holds bun_install stamp files.
const stampsDir = resolve(cfg.buildDir, "stamps");
n.build({
outputs: [resolve(stampsDir, ".dir")],
rule: "mkdir_stamp",
inputs: [],
vars: { dir: n.rel(stampsDir) },
});
}
function codegenDirStamp(cfg: Config): string {
return resolve(cfg.codegenDir, ".dir");
}
// ───────────────────────────────────────────────────────────────────────────
// Codegen step emitters
// ───────────────────────────────────────────────────────────────────────────
/**
* All codegen outputs, grouped by consumer. Downstream phases (cpp compile,
* zig build, link) add the appropriate group to their implicit inputs.
*/
export interface CodegenOutputs {
/** All codegen outputs — use for phony target `codegen`. */
all: string[];
/** Outputs that zig `@embedFile`s or imports. */
zigInputs: string[];
/** Outputs that zig needs to exist but doesn't embed (debug bake runtime). */
zigOrderOnly: string[];
/** Generated .cpp files. Compiled alongside handwritten C++ in bun.ts. */
cppSources: string[];
/**
* Generated headers that are #included by hand-written .cpp files.
* The PCH order-depends on all of these, and cxx waits on PCH — so
* they're guaranteed to exist before any compile. Depfile tracking
* handles subsequent changes.
*/
cppHeaders: string[];
/**
* ALL cpp-relevant codegen outputs — the union of cppHeaders, cppSources,
* bindgenV2Cpp. cxx compilation order-depends on THIS (not `all`): cxx
* doesn't need bake.*.js, cpp.zig, runtime.out.js, or any other zig-only
* outputs. Using `all` would pull bake-codegen in cpp-only CI mode, which
* fails on old CI bun versions (bake-codegen shells out to `bun build`
* whose CSS url() handling changed between versions). cmake only wired
* bake outputs into BUN_ZIG_GENERATED_SOURCES, never C++ deps — same here.
*
* The "undeclared .h files" issue (some scripts emit .h alongside their
* declared outputs): those steps also emit a .cpp or .h that IS declared
* here, so they still run before any cxx compile.
*/
cppAll: string[];
/** The bindgenv2 .cpp outputs (compiled separately from handwritten C++). */
bindgenV2Cpp: string[];
/** The bindgenv2 .zig outputs (imported by the zig build). */
bindgenV2Zig: string[];
/**
* Stamp output from `bun install` at repo root.
* The esbuild tool and the cppbind lezer parser live here. Any
* step that uses esbuild (or imports node_modules deps at configure
* time) depends on this.
*/
rootInstall: string;
}
/**
* Emit all codegen steps. Returns grouped outputs for downstream wiring.
*
* Call after registerCodegenRules() and after registerDirStamps() (we use
* the `mkdir_stamp` rule for the codegen dir).
*/
export function emitCodegen(n: Ninja, cfg: Config, sources: Sources): CodegenOutputs {
n.comment("─── Codegen ───");
n.blank();
const dirStamp = codegenDirStamp(cfg);
// ─── Root bun install (provides esbuild + lezer-cpp for cppbind) ───
const rootInstall = emitBunInstall(n, cfg, cfg.cwd);
const o: CodegenOutputs = {
all: [],
zigInputs: [],
zigOrderOnly: [],
cppSources: [],
cppHeaders: [],
cppAll: [],
bindgenV2Cpp: [],
bindgenV2Zig: [],
rootInstall,
};
const ctx: Ctx = { n, cfg, sources, o, dirStamp };
emitBunError(ctx);
emitFallbackDecoder(ctx);
emitRuntimeJs(ctx);
emitNodeFallbacks(ctx);
emitErrorCode(ctx);
emitGeneratedClasses(ctx);
emitCppBind(ctx);
emitCiInfo(ctx);
emitJsModules(ctx);
emitBakeCodegen(ctx);
emitBindgenV2(ctx);
emitBindgen(ctx);
emitJsSink(ctx);
emitObjectLuts(ctx);
n.phony("codegen", o.all);
n.blank();
// Assemble cppAll — the cxx-relevant subset. See field docstring.
o.cppAll = [...new Set([...o.cppHeaders, ...o.cppSources, ...o.bindgenV2Cpp])];
return o;
}
// ───────────────────────────────────────────────────────────────────────────
// Helpers
// ───────────────────────────────────────────────────────────────────────────
/**
* Emit a `bun install` step for a package directory. Returns the stamp file
* path — use it as an implicit input on anything that needs node_modules/.
*
* The stamp is the explicit output; each node_modules/<dep>/package.json is
* an implicit output (so deleting node_modules/ correctly retriggers install,
* and restat prunes downstream when install was a no-op).
*/
function emitBunInstall(n: Ninja, cfg: Config, pkgDir: string): string {
const depPackageJsons = readPackageDeps(pkgDir);
assert(depPackageJsons.length > 0, `package.json has no dependencies: ${pkgDir}/package.json`);
const pkgJson = resolve(pkgDir, "package.json");
const lockfile = resolve(pkgDir, "bun.lock");
// bun.lock is optional (some packages might not have one yet), but if it
// exists it MUST be an input — lockfile bumps reinstall.
const inputs = [pkgJson];
try {
readFileSync(lockfile); // exists check
inputs.push(lockfile);
} catch {
// no lockfile, that's fine
}
// Stamp lives in the build dir, not the package dir — keeps the source
// tree clean and makes `rm -rf build/` fully reset install state.
// Uniqueify by hashing the package dir path (multiple installs possible).
const stampName = pkgDir.replace(/[^A-Za-z0-9]+/g, "_");
const stamp = resolve(cfg.buildDir, "stamps", `install_${stampName}.stamp`);
n.build({
outputs: [stamp],
implicitOutputs: depPackageJsons,
rule: "bun_install",
inputs,
orderOnlyInputs: [resolve(cfg.buildDir, "stamps", ".dir")],
// stamp must be absolute — the command `cd $dir && ... && touch $stamp`
// runs from $dir, not from buildDir. n.rel() would break that.
vars: { dir: pkgDir, stamp },
});
return stamp;
}
/**
/** `--debug=ON` / `--debug=OFF` flag used by several scripts. */
function debugFlag(cfg: Config): string {
return cfg.debug ? "--debug=ON" : "--debug=OFF";
}
/**
* Shell-quote args for a codegen rule command string. These rules wrap in
* `cmd /c` on a Windows HOST, so quoting follows the host shell.
*/
function shJoin(cfg: Config, args: string[]): string {
return quoteArgs(args, cfg.host.os === "windows");
}
// ───────────────────────────────────────────────────────────────────────────
// Individual step emitters
// ───────────────────────────────────────────────────────────────────────────
function emitBunError({ n, cfg, sources, o, dirStamp }: Ctx): void {
const sourceDir = resolve(cfg.cwd, "packages", "bun-error");
const installStamp = emitBunInstall(n, cfg, sourceDir);
const outDir = resolve(cfg.codegenDir, "bun-error");
const outputs = [resolve(outDir, "index.js"), resolve(outDir, "bun-error.css")];
n.build({
outputs,
rule: "esbuild",
inputs: sources.bunError,
// Install stamp as implicit — changing preact version re-bundles.
// Root install as well (esbuild tool lives there).
implicitInputs: [installStamp, o.rootInstall],
orderOnlyInputs: [dirStamp],
vars: {
cwd: sourceDir,
desc: "bun-error",
args: shJoin(cfg, [
"index.tsx",
"bun-error.css",
`--outdir=${outDir}`,
`--define:process.env.NODE_ENV="production"`,
"--minify",
"--bundle",
"--platform=browser",
"--format=esm",
]),
},
});
o.all.push(...outputs);
o.zigInputs.push(...outputs);
}
function emitFallbackDecoder({ n, cfg, o, dirStamp }: Ctx): void {
const src = resolve(cfg.cwd, "src", "fallback.ts");
const out = resolve(cfg.codegenDir, "fallback-decoder.js");
n.build({
outputs: [out],
rule: "esbuild",
inputs: [src],
implicitInputs: [o.rootInstall],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "fallback-decoder.js",
args: shJoin(cfg, [
src,
`--outfile=${out}`,
"--target=esnext",
"--bundle",
"--format=iife",
"--platform=browser",
"--minify",
]),
},
});
o.all.push(out);
o.zigInputs.push(out);
}
function emitRuntimeJs({ n, cfg, o, dirStamp }: Ctx): void {
const src = resolve(cfg.cwd, "src", "runtime.bun.js");
const out = resolve(cfg.codegenDir, "runtime.out.js");
n.build({
outputs: [out],
rule: "esbuild",
inputs: [src],
implicitInputs: [o.rootInstall],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "runtime.out.js",
args: shJoin(cfg, [
src,
`--outfile=${out}`,
`--define:process.env.NODE_ENV="production"`,
"--target=esnext",
"--bundle",
"--format=esm",
"--platform=node",
"--minify",
"--external:/bun:*",
]),
},
});
o.all.push(out);
o.zigInputs.push(out);
}
function emitNodeFallbacks({ n, cfg, sources, o, dirStamp }: Ctx): void {
const sourceDir = resolve(cfg.cwd, "src", "node-fallbacks");
const installStamp = emitBunInstall(n, cfg, sourceDir);
const outDir = resolve(cfg.codegenDir, "node-fallbacks");
// One output per source, same basename.
const outputs = sources.nodeFallbacks.map(s => resolve(outDir, basename(s)));
// The script (build-fallbacks.ts) reads its args as [outdir, ...sources]
// but actually ignores the sources — it does readdirSync(".") to discover
// files. We pass them anyway so ninja tracks them as inputs.
const script = resolve(sourceDir, "build-fallbacks.ts");
n.build({
outputs,
rule: "codegen",
inputs: [script, ...sources.nodeFallbacks],
implicitInputs: [installStamp],
orderOnlyInputs: [dirStamp],
vars: {
cwd: sourceDir,
desc: "node-fallbacks/*.js",
// `bun run build-fallbacks` resolves to `./build-fallbacks.ts` in cwd
args: shJoin(cfg, ["run", "build-fallbacks", outDir, ...sources.nodeFallbacks]),
},
});
o.all.push(...outputs);
o.zigInputs.push(...outputs);
// ─── react-refresh (separate bundle, uses node-fallbacks' node_modules) ───
const rrSrc = resolve(sourceDir, "node_modules", "react-refresh", "cjs", "react-refresh-runtime.development.js");
const rrOut = resolve(outDir, "react-refresh.js");
n.build({
outputs: [rrOut],
rule: "codegen",
inputs: [resolve(sourceDir, "package.json"), resolve(sourceDir, "bun.lock")],
implicitInputs: [installStamp],
orderOnlyInputs: [dirStamp],
vars: {
cwd: sourceDir,
desc: "node-fallbacks/react-refresh.js",
args: shJoin(cfg, [
"build",
rrSrc,
`--outfile=${rrOut}`,
"--target=browser",
"--format=cjs",
"--minify",
`--define:process.env.NODE_ENV="development"`,
]),
},
});
o.all.push(rrOut);
o.zigInputs.push(rrOut);
}
function emitErrorCode({ n, cfg, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "generate-node-errors.ts");
const inputs = [
script,
resolve(cfg.cwd, "src", "bun.js", "bindings", "ErrorCode.ts"),
// ErrorCode.cpp/.h are listed in CMake but the script doesn't read them;
// they're there so changes to the handwritten side (e.g. new error
// category added to the C++ enum) invalidate this step. We include them
// for the same reason.
resolve(cfg.cwd, "src", "bun.js", "bindings", "ErrorCode.cpp"),
resolve(cfg.cwd, "src", "bun.js", "bindings", "ErrorCode.h"),
];
const outputs = [
resolve(cfg.codegenDir, "ErrorCode+List.h"),
resolve(cfg.codegenDir, "ErrorCode+Data.h"),
resolve(cfg.codegenDir, "ErrorCode.zig"),
];
n.build({
outputs,
rule: "codegen",
inputs,
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "ErrorCode.{zig,h}",
args: shJoin(cfg, ["run", script, cfg.codegenDir]),
},
});
o.all.push(...outputs);
o.zigInputs.push(...outputs);
o.cppHeaders.push(outputs[0]!, outputs[1]!);
}
function emitGeneratedClasses({ n, cfg, sources, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "generate-classes.ts");
const outputs = [
resolve(cfg.codegenDir, "ZigGeneratedClasses.h"),
resolve(cfg.codegenDir, "ZigGeneratedClasses.cpp"),
resolve(cfg.codegenDir, "ZigGeneratedClasses+lazyStructureHeader.h"),
resolve(cfg.codegenDir, "ZigGeneratedClasses+DOMClientIsoSubspaces.h"),
resolve(cfg.codegenDir, "ZigGeneratedClasses+DOMIsoSubspaces.h"),
resolve(cfg.codegenDir, "ZigGeneratedClasses+lazyStructureImpl.h"),
resolve(cfg.codegenDir, "ZigGeneratedClasses.zig"),
resolve(cfg.codegenDir, "ZigGeneratedClasses.lut.txt"),
];
n.build({
outputs,
rule: "codegen",
inputs: [script, ...sources.zigGeneratedClasses],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "ZigGeneratedClasses.{zig,cpp,h}",
args: shJoin(cfg, ["run", script, ...sources.zigGeneratedClasses, cfg.codegenDir]),
},
});
o.all.push(...outputs);
o.zigInputs.push(...outputs);
o.cppSources.push(outputs[1]!); // .cpp
o.cppHeaders.push(outputs[0]!, outputs[2]!, outputs[3]!, outputs[4]!, outputs[5]!); // .h files
// .lut.txt is consumed by emitObjectLuts below
}
function emitCppBind({ n, cfg, sources, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "cppbind.ts");
const output = resolve(cfg.codegenDir, "cpp.zig");
// Write the .cpp file list for cppbind to scan. Build system owns the
// glob (sources.ts reads Sources.json); we hand the result to cppbind
// as an explicit input instead of it reading a magic hardcoded path.
// Relative paths, forward slashes — same format cppbind expects.
//
// Written at CONFIGURE time (not via a ninja rule): it's a derived
// manifest from our glob, and we want writeIfChanged semantics so a
// stable .cpp set → unchanged mtime → ninja doesn't re-run cppbind.
// codegenDir may not exist yet on first configure — mkdir it.
mkdirSync(cfg.codegenDir, { recursive: true });
const cxxSourcesFile = resolve(cfg.codegenDir, "cxx-sources.txt");
const cxxSourcesLines = sources.cxx.map(p => relative(cfg.cwd, p).replace(/\\/g, "/"));
writeIfChanged(cxxSourcesFile, cxxSourcesLines.join("\n") + "\n");
n.build({
outputs: [output],
rule: "codegen",
inputs: [script],
// cppbind scans ALL .cpp files for [[ZIG_EXPORT]] annotations. Every
// .cpp is an implicit input so changing an annotation retriggers.
// ~540 files — ninja handles this fine via .ninja_deps stat caching.
// cxxSourcesFile also listed — if the list itself changes (file
// added/removed), that's a different input set.
implicitInputs: [
cxxSourcesFile,
...sources.cxx,
...sources.jsCodegen,
// cppbind auto-runs `bun install` for its lezer-cpp dep if needed,
// but depending on root install ensures that already happened on
// first build (and catches lezer version bumps).
o.rootInstall,
],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "cpp.zig (cppbind)",
// cppbind.ts takes: <srcdir> <codegendir> <cxx-sources>. No `run` —
// direct script invocation (`${BUN_EXECUTABLE} ${script} ...`).
args: shJoin(cfg, [script, resolve(cfg.cwd, "src"), cfg.codegenDir, cxxSourcesFile]),
},
});
o.all.push(output);
o.zigInputs.push(output);
}
function emitCiInfo({ n, cfg, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "ci_info.ts");
const output = resolve(cfg.codegenDir, "ci_info.zig");
// CMake lists JavaScriptCodegenSources as deps here, but ci_info.ts doesn't
// read any of those files — it's a pure static data generator. The CMake
// dep list is wrong (copy-paste from bundle-modules). We use just the script.
n.build({
outputs: [output],
rule: "codegen",
inputs: [script],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "ci_info.zig",
args: shJoin(cfg, [script, output]),
},
});
o.all.push(output);
o.zigInputs.push(output);
}
function emitJsModules({ n, cfg, sources, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "bundle-modules.ts");
// InternalModuleRegistry.cpp is read by the script (for a sanity check).
const extraInput = resolve(cfg.cwd, "src", "bun.js", "bindings", "InternalModuleRegistry.cpp");
// Written into src/ (not codegenDir) — see zigFilesGeneratedIntoSrc at top.
const js2nativeZig = resolve(cfg.cwd, zigFilesGeneratedIntoSrc[1]);
const outputs = [
resolve(cfg.codegenDir, "WebCoreJSBuiltins.cpp"),
resolve(cfg.codegenDir, "WebCoreJSBuiltins.h"),
resolve(cfg.codegenDir, "InternalModuleRegistryConstants.h"),
resolve(cfg.codegenDir, "InternalModuleRegistry+createInternalModuleById.h"),
resolve(cfg.codegenDir, "InternalModuleRegistry+enum.h"),
resolve(cfg.codegenDir, "InternalModuleRegistry+numberOfModules.h"),
resolve(cfg.codegenDir, "NativeModuleImpl.h"),
resolve(cfg.codegenDir, "ResolvedSourceTag.zig"),
resolve(cfg.codegenDir, "SyntheticModuleType.h"),
resolve(cfg.codegenDir, "GeneratedJS2Native.h"),
js2nativeZig,
];
n.build({
outputs,
rule: "codegen",
inputs: [script, ...sources.js, ...sources.jsCodegen, extraInput],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "JS modules (bundle-modules)",
// Note: arg is BUILD_PATH (buildDir), not CODEGEN_PATH. The script
// derives CODEGEN_DIR = join(BUILD_PATH, "codegen") internally.
args: shJoin(cfg, ["run", script, debugFlag(cfg), cfg.buildDir]),
},
});
o.all.push(...outputs);
o.zigInputs.push(...outputs);
o.cppSources.push(outputs[0]!); // WebCoreJSBuiltins.cpp
o.cppHeaders.push(...outputs.filter(p => p.endsWith(".h")));
}
function emitBakeCodegen({ n, cfg, sources, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "bake-codegen.ts");
// InternalModuleRegistry.cpp is listed as a dep in CMake for this step too.
// The script doesn't read it; CMake copy-paste. We skip it.
// CMake only declares bake.client.js and bake.server.js as outputs. The
// script also emits bake.error.js (build.zig embeds it). We declare
// all three.
//
// Debug uses order-only deps on these .js files (loaded at runtime,
// no need to relink zig on change). Release uses implicit (embedded
// via @embedFile, must relink).
const outputs = [
resolve(cfg.codegenDir, "bake.client.js"),
resolve(cfg.codegenDir, "bake.server.js"),
resolve(cfg.codegenDir, "bake.error.js"),
];
n.build({
outputs,
rule: "codegen",
inputs: [script, ...sources.bakeRuntime],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "bake.{client,server,error}.js",
args: shJoin(cfg, ["run", script, debugFlag(cfg), `--codegen-root=${cfg.codegenDir}`]),
},
});
o.all.push(...outputs);
// Debug: read at RUNTIME (not embedded) → zig only needs existence.
// Release: embedded via @embedFile → content changes must rebuild zig.
if (cfg.debug) {
o.zigOrderOnly.push(...outputs);
} else {
o.zigInputs.push(...outputs);
}
}
function emitBindgenV2({ n, cfg, sources, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "bindgenv2", "script.ts");
// The script's output set depends on which NamedTypes the .bindv2.ts files
// export. We run `--command=list-outputs` SYNCHRONOUSLY at configure time
// to get the real list. This is a configure-time dependency on bun +
// sources — same tradeoff CMake makes with execute_process().
//
// If list-outputs fails (e.g. syntax error in a .bindv2.ts file), we fail
// configure immediately with a clear error. Better to catch that here than
// get a cryptic "multiple rules generate <unknown>" from ninja.
const sourcesArg = sources.bindgenV2.join(",");
const listResult = spawnSync(
cfg.bun,
["run", script, "--command=list-outputs", `--sources=${sourcesArg}`, `--codegen-path=${cfg.codegenDir}`],
{ cwd: cfg.cwd, encoding: "utf8" },
);
if (listResult.status !== 0) {
throw new BuildError(`bindgenv2 list-outputs failed (exit ${listResult.status})`, {
file: script,
hint: listResult.stderr?.trim(),
});
}
// Output is semicolon-separated (CMake list format).
const allOutputs = listResult.stdout
.trim()
.split(";")
.filter(p => p.length > 0);
assert(allOutputs.length > 0, "bindgenv2 list-outputs returned no files");
const cppOutputs = allOutputs.filter(p => p.endsWith(".cpp"));
const zigOutputs = allOutputs.filter(p => p.endsWith(".zig"));
const other = allOutputs.filter(p => !p.endsWith(".cpp") && !p.endsWith(".zig"));
assert(other.length === 0, `bindgenv2 emitted unexpected output type: ${other.join(", ")}`);
n.build({
outputs: allOutputs,
rule: "codegen",
inputs: [script, ...sources.bindgenV2, ...sources.bindgenV2Internal],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "bindgenv2",
args: shJoin(cfg, [
"run",
script,
"--command=generate",
`--codegen-path=${cfg.codegenDir}`,
`--sources=${sourcesArg}`,
]),
},
});
o.all.push(...allOutputs);
o.bindgenV2Cpp.push(...cppOutputs);
o.bindgenV2Zig.push(...zigOutputs);
o.zigInputs.push(...zigOutputs);
}
function emitBindgen({ n, cfg, sources, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "bindgen.ts");
// Written into src/ (not codegenDir) — see zigFilesGeneratedIntoSrc at top.
const zigOut = resolve(cfg.cwd, zigFilesGeneratedIntoSrc[0]);
const cppOut = resolve(cfg.codegenDir, "GeneratedBindings.cpp");
// bindgen.ts scans src/ for .bind.ts files itself — this list is only for
// ninja dependency tracking. New .bind.ts files need a reconfigure to be
// picked up (next glob gets them).
n.build({
outputs: [cppOut, zigOut],
rule: "codegen",
inputs: [script, ...sources.bindgen],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: ".bind.ts → GeneratedBindings.{cpp,zig}",
args: shJoin(cfg, ["run", script, debugFlag(cfg), `--codegen-root=${cfg.codegenDir}`]),
},
});
o.all.push(cppOut, zigOut);
o.cppSources.push(cppOut);
o.zigInputs.push(zigOut);
}
function emitJsSink({ n, cfg, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "generate-jssink.ts");
const hashTableScript = resolve(cfg.cwd, "src", "codegen", "create-hash-table.ts");
const perlScript = resolve(cfg.cwd, "src", "codegen", "create_hash_table");
// generate-jssink.ts writes JSSink.{cpp,h,lut.txt}, then internally spawns
// create-hash-table.ts to convert .lut.txt → .lut.h. So all four are outputs
// of this one step (though .lut.txt is really an intermediate — we don't
// expose it).
const outputs = [
resolve(cfg.codegenDir, "JSSink.cpp"),
resolve(cfg.codegenDir, "JSSink.h"),
resolve(cfg.codegenDir, "JSSink.lut.h"),
];
n.build({
outputs,
rule: "codegen",
inputs: [script, hashTableScript, perlScript],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: "JSSink.{cpp,h,lut.h}",
args: shJoin(cfg, ["run", script, cfg.codegenDir]),
},
});
o.all.push(...outputs);
o.cppSources.push(outputs[0]!); // .cpp
o.cppHeaders.push(outputs[1]!, outputs[2]!); // .h + .lut.h
}
/**
* LUT sources → .lut.h outputs. One build statement PER pair, because the
* script takes a single (src, out) pair.
*
* The source .cpp files contain `@begin XXXTable ... @end` blocks that the
* perl script parses into JSC HashTableValue arrays. The TS wrapper adds
* platform-specific #if preprocessing via TARGET_PLATFORM env var.
*/
function emitObjectLuts({ n, cfg, o, dirStamp }: Ctx): void {
const script = resolve(cfg.cwd, "src", "codegen", "create-hash-table.ts");
const perlScript = resolve(cfg.cwd, "src", "codegen", "create_hash_table");
// (source, output) pairs. ZigGeneratedClasses.lut.txt is special: it's
// GENERATED by emitGeneratedClasses, so it's in codegenDir not src/.
const pairs: [src: string, out: string][] = [
[resolve(cfg.cwd, "src/bun.js/bindings/BunObject.cpp"), resolve(cfg.codegenDir, "BunObject.lut.h")],
[resolve(cfg.cwd, "src/bun.js/bindings/ZigGlobalObject.lut.txt"), resolve(cfg.codegenDir, "ZigGlobalObject.lut.h")],
[resolve(cfg.cwd, "src/bun.js/bindings/JSBuffer.cpp"), resolve(cfg.codegenDir, "JSBuffer.lut.h")],
[resolve(cfg.cwd, "src/bun.js/bindings/BunProcess.cpp"), resolve(cfg.codegenDir, "BunProcess.lut.h")],
[
resolve(cfg.cwd, "src/bun.js/bindings/ProcessBindingBuffer.cpp"),
resolve(cfg.codegenDir, "ProcessBindingBuffer.lut.h"),
],
[
resolve(cfg.cwd, "src/bun.js/bindings/ProcessBindingConstants.cpp"),
resolve(cfg.codegenDir, "ProcessBindingConstants.lut.h"),
],
[resolve(cfg.cwd, "src/bun.js/bindings/ProcessBindingFs.cpp"), resolve(cfg.codegenDir, "ProcessBindingFs.lut.h")],
[
resolve(cfg.cwd, "src/bun.js/bindings/ProcessBindingNatives.cpp"),
resolve(cfg.codegenDir, "ProcessBindingNatives.lut.h"),
],
[
resolve(cfg.cwd, "src/bun.js/bindings/ProcessBindingHTTPParser.cpp"),
resolve(cfg.codegenDir, "ProcessBindingHTTPParser.lut.h"),
],
[resolve(cfg.cwd, "src/bun.js/modules/NodeModuleModule.cpp"), resolve(cfg.codegenDir, "NodeModuleModule.lut.h")],
[resolve(cfg.codegenDir, "ZigGeneratedClasses.lut.txt"), resolve(cfg.codegenDir, "ZigGeneratedClasses.lut.h")],
[resolve(cfg.cwd, "src/bun.js/bindings/webcore/JSEvent.cpp"), resolve(cfg.codegenDir, "JSEvent.lut.h")],
];
// create-hash-table.ts reads TARGET_PLATFORM env with process.platform
// fallback. We don't set it — cmake never did either. The preprocessing
// is OS-based (#if OS(WINDOWS) etc.) not arch-based, and bun only
// cross-compiles across arch on the same OS, so host platform == target
// OS. If cross-OS builds are ever added, thread the platform through
// argv here rather than shell env (which isn't portable to cmd.exe).
for (const [src, out] of pairs) {
n.build({
outputs: [out],
rule: "codegen",
inputs: [src],
implicitInputs: [script, perlScript],
orderOnlyInputs: [dirStamp],
vars: {
cwd: cfg.cwd,
desc: basename(out),
args: shJoin(cfg, ["run", script, src, out]),
},
});
o.all.push(out);
o.cppHeaders.push(out);
}
}

473
scripts/build/compile.ts Normal file
View File

@@ -0,0 +1,473 @@
/**
* Compilation constructors.
*
* These are NOT abstractions — they're shortcuts that build a `BuildNode` and
* register it with the Ninja instance. A "library" is just an array of cxx()
* outputs + one ar() output. An executable is cxx() outputs + one link().
*/
import { mkdirSync } from "node:fs";
import { basename, dirname, extname, relative, resolve } from "node:path";
import type { Config } from "./config.ts";
import { assert } from "./error.ts";
import { writeIfChanged } from "./fs.ts";
import type { BuildNode, Ninja, Rule } from "./ninja.ts";
import { quote } from "./shell.ts";
import { streamPath } from "./stream.ts";
// ---------------------------------------------------------------------------
// Rule registration — call once per Ninja instance
// ---------------------------------------------------------------------------
/**
* Register all compilation-related ninja rules.
* Call once before using cxx/cc/pch/link/ar.
*/
export function registerCompileRules(n: Ninja, cfg: Config): void {
// Quote tool paths — ninja passes commands through cmd/sh; a space in a
// toolchain path (e.g. "C:\Program Files\LLVM\bin\clang-cl.exe") would
// split argv without quoting. quote() passes through safe paths unchanged.
const q = (p: string) => quote(p, cfg.windows);
const cc = q(cfg.cc);
const cxx = q(cfg.cxx);
const ar = q(cfg.ar);
const ccacheLauncher = cfg.ccache !== undefined ? `${q(cfg.ccache)} ` : "";
// Depfile handling differs between clang (gcc-style .d) and clang-cl (/showIncludes)
const depfileOpts: Pick<Rule, "depfile" | "deps"> = cfg.windows
? { deps: "msvc" }
: { depfile: "$out.d", deps: "gcc" };
// ─── C++ compile ───
// Note: $cxxflags is set per-build (allows per-file overrides).
n.rule("cxx", {
command: cfg.windows
? `${ccacheLauncher}${cxx} /nologo /showIncludes $cxxflags /c $in /Fo$out`
: `${ccacheLauncher}${cxx} $cxxflags -MMD -MT $out -MF $out.d -c $in -o $out`,
description: "cxx $out",
...depfileOpts,
});
// ─── C++ compile with PCH ───
// PCH is loaded with -include-pch (clang) or /Yu (clang-cl).
// $pch_file is the .pch/.gch output, $pch_header is the wrapper .hxx.
//
// Both -include-pch AND -include (force-include of the wrapper) are passed,
// mirroring CMake's target_precompile_headers(). The force-include re-applies
// `#pragma clang system_header` for the current translation unit's
// preprocessing pass — without it, warnings from PCH-included headers aren't
// suppressed (the pragma's effect is per-preprocessing-pass, not per-AST).
// The -Xclang prefix is required: plain -include doesn't combine with PCH
// on the clang driver, but -Xclang bypasses the driver's sanity check.
n.rule("cxx_pch", {
command: cfg.windows
? `${ccacheLauncher}${cxx} /nologo /showIncludes $cxxflags /Yu$pch_header /Fp$pch_file /c $in /Fo$out`
: `${ccacheLauncher}${cxx} $cxxflags -Winvalid-pch -Xclang -include-pch -Xclang $pch_file -Xclang -include -Xclang $pch_header -MMD -MT $out -MF $out.d -c $in -o $out`,
description: "cxx $out",
...depfileOpts,
});
// ─── C compile ───
n.rule("cc", {
command: cfg.windows
? `${ccacheLauncher}${cc} /nologo /showIncludes $cflags /c $in /Fo$out`
: `${ccacheLauncher}${cc} $cflags -MMD -MT $out -MF $out.d -c $in -o $out`,
description: "cc $out",
...depfileOpts,
});
// ─── PCH compilation ───
// Compiles a header into a precompiled header.
//
// CMake's approach (replicated here): compile an EMPTY stub .cxx as the
// main file, force-include the wrapper .hxx via -Xclang -include, emit
// the PCH via -Xclang -emit-pch. The indirection lets `#pragma clang
// system_header` in the wrapper take effect — that pragma is ignored
// when the file containing it is the MAIN file, but works when the
// file is included. -fpch-instantiate-templates: instantiate templates
// during PCH compilation instead of deferring to each consuming .cpp
// (faster builds, CMake does this too).
// -MD (not -MMD): the wrapper header has `#pragma clang system_header` to
// suppress JSC warnings, which makes everything it transitively includes
// "system" for -MMD purposes. -MMD would give a near-empty depfile; -MD
// tracks all headers so PCH invalidates when WebKit headers change.
n.rule("pch", {
command: cfg.windows
? `${ccacheLauncher}${cxx} /nologo /showIncludes $cxxflags /Yc$pch_header /Fp$out /c $pch_stub /Fo$pch_stub_obj`
: `${ccacheLauncher}${cxx} $cxxflags -Winvalid-pch -fpch-instantiate-templates -Xclang -emit-pch -Xclang -include -Xclang $pch_header -x c++-header -MD -MT $out -MF $out.d -c $in -o $out`,
description: "pch $out",
...depfileOpts,
});
// ─── Link executable ───
// Uses response file because object lists get long (>32k args breaks on windows).
// console pool: link is inherently serial (one exe), takes 30s+ on large
// binaries, and lld prints useful progress (undefined symbol errors,
// --verbose timing). Streaming beats sitting at [N/N] wondering if it hung.
// stream.ts --console: passthrough + ninja Windows buffering fix — see stream.ts.
//
// Windows: -fuse-ld=lld forces lld-link (VS dev shell puts link.exe
// first in PATH, clang-cl would default to it). /link separator —
// everything after passes verbatim to lld-link. Our ldflags are all
// pure linker options (/STACK, /DEF, /OPT, /errorlimit, system libs)
// that clang-cl's driver doesn't recognize.
const wrap = `${q(cfg.bun)} ${q(streamPath)} link --console`;
n.rule("link", {
command: cfg.windows
? `${wrap} ${cxx} /nologo -fuse-ld=lld @$out.rsp /Fe$out /link $ldflags`
: `${wrap} ${cxx} @$out.rsp $ldflags -o $out`,
description: "link $out",
rspfile: "$out.rsp",
rspfile_content: "$in_newline",
pool: "console",
});
// ─── Static library archive ───
n.rule("ar", {
command: cfg.windows ? `${ar} /nologo /out:$out @$out.rsp` : `${ar} rcs $out @$out.rsp`,
description: "ar $out",
rspfile: "$out.rsp",
rspfile_content: "$in_newline",
});
}
// ---------------------------------------------------------------------------
// Compilation constructors
// ---------------------------------------------------------------------------
export interface CompileOpts {
/** Compiler flags (including -I, -D — caller assembles). */
flags: string[];
/** PCH to use (absolute path to .pch/.gch output). */
pch?: string;
/** Original header the PCH was built from (needed for clang-cl /Yu). */
pchHeader?: string;
/**
* Extra implicit deps. Use for generated headers this specific .cpp needs.
* E.g. ErrorCode.cpp depends on ErrorCode+List.h.
*/
implicitInputs?: string[];
/**
* Order-only deps. Must exist before compile, but mtime not tracked.
* The compiler's .d depfile tracks ACTUAL header dependencies on
* subsequent builds — order-only is for "dep libs/headers must be
* extracted before first compile attempts to #include them".
*
* Prefer this over implicitInputs for dep outputs: if you touch
* libJavaScriptCore.a, you don't want every .c file to recompile
* (.c files don't include JSC headers). The depfile knows better.
*/
orderOnlyInputs?: string[];
/** Job pool override. */
pool?: string;
}
/**
* Compile a C++ source file. Returns absolute path to the .o output.
*
* Output path: {buildDir}/obj/{path-from-cwd-with-slashes-flattened}.o
* E.g. src/bun.js/bindings/foo.cpp → obj/src_bun.js_bindings_foo.cpp.o
*/
export function cxx(n: Ninja, cfg: Config, src: string, opts: CompileOpts): string {
assert(
extname(src) === ".cpp" || extname(src) === ".cc" || extname(src) === ".cxx",
`cxx() expects .cpp/.cc/.cxx source, got: ${src}`,
);
return compile(n, cfg, src, opts, "cxx");
}
/**
* Compile a C source file. Returns absolute path to the .o output.
*/
export function cc(n: Ninja, cfg: Config, src: string, opts: Omit<CompileOpts, "pch" | "pchHeader">): string {
assert(extname(src) === ".c", `cc() expects .c source, got: ${src}`);
// C files never use PCH (PCH is C++-only in our build)
return compile(n, cfg, src, opts, "cc");
}
function compile(n: Ninja, cfg: Config, src: string, opts: CompileOpts, lang: "cxx" | "cc"): string {
const absSrc = resolve(cfg.cwd, src);
const out = objectPath(cfg, src);
const rule = opts.pch !== undefined && lang === "cxx" ? "cxx_pch" : lang;
const flagVar = lang === "cxx" ? "cxxflags" : "cflags";
const implicitInputs: string[] = [...(opts.implicitInputs ?? [])];
const vars: Record<string, string> = {
[flagVar]: opts.flags.join(" "),
};
// PCH is always an implicit dep — if it changes, recompile.
if (opts.pch !== undefined) {
assert(opts.pchHeader !== undefined, "cxx with pch requires pchHeader (the wrapper .hxx)");
implicitInputs.push(opts.pch);
vars.pch_file = n.rel(opts.pch);
vars.pch_header = n.rel(opts.pchHeader);
}
const node: BuildNode = {
outputs: [out],
rule,
inputs: [absSrc],
orderOnlyInputs: [objectDirStamp(cfg), ...(opts.orderOnlyInputs ?? [])],
vars,
};
if (implicitInputs.length > 0) node.implicitInputs = implicitInputs;
if (opts.pool !== undefined) node.pool = opts.pool;
n.build(node);
// Record for compile_commands.json
n.addCompileCommand({
directory: cfg.buildDir,
file: absSrc,
output: n.rel(out),
arguments: [
lang === "cxx" ? cfg.cxx : cfg.cc,
...opts.flags,
...(opts.pch !== undefined ? ["-include-pch", n.rel(opts.pch)] : []),
"-c",
absSrc,
"-o",
out,
],
});
return out;
}
/**
* Compile a header into a precompiled header.
* Returns `{ pch, wrapperHeader }` — both paths absolute.
*
* Writes a wrapper .hxx with `#pragma clang system_header` +
* `#include <original>`, compiles
* THAT to a .pch. The pragma marks everything transitively included as a
* system header — warnings from those headers are suppressed even with
* -Werror. This matters for JSC headers (which trigger -Wundefined-var-template
* by design — template statics defined in .cpp, linker resolves).
*
* Consumers should pass BOTH paths to cxx(): the .pch via -include-pch, the
* wrapper via -include. The force-include re-applies the system_header pragma
* for that translation unit's preprocessing pass.
*/
export function pch(
n: Ninja,
cfg: Config,
header: string,
opts: {
flags: string[];
/**
* Files whose change must invalidate the PCH. Typically: dep output
* libs (libJavaScriptCore.a etc.).
*
* Can't be order-only: the depfile tracks headers, but ninja stats at
* startup. Local WebKit headers live in buildDir and get regenerated
* by dep_build MID-RUN. At startup ninja sees old headers → thinks
* PCH is fresh → cxx fails with "file modified since PCH was built"
* → needs a second build. With these implicit, restat propagates the
* lib change to PCH and it rebuilds in the same run.
*
* Cost: PCH also rebuilds on unrelated dep bumps (brotli etc.). Rare
* enough to accept for correctness.
*/
implicitInputs?: string[];
/**
* Must exist before PCH compiles; changes don't invalidate it.
* Codegen outputs go here — they only change when inputs change,
* and inputs don't change mid-build.
*/
orderOnlyInputs?: string[];
},
): { pch: string; wrapperHeader: string } {
// TODO(windows): the clang-cl /Yu rule references $pch_stub / $pch_stub_obj
// that this function doesn't set. Wire them up, then delete this assert.
assert(!cfg.windows, "PCH on Windows not yet wired up", {
hint: "compile.ts pch() doesn't set $pch_stub / $pch_stub_obj for the clang-cl rule",
});
const absHeader = resolve(cfg.cwd, header);
const pchDir = resolve(cfg.buildDir, "pch");
const wrapperHeader = resolve(pchDir, `${basename(header)}.hxx`);
const stubCxx = resolve(pchDir, `${basename(header)}.hxx.cxx`);
const out = resolve(pchDir, `${basename(header)}.hxx.pch`);
// Write the wrapper at configure time. `#pragma clang system_header` must
// be the FIRST non-comment line for clang to honor it.
//
// Both files are configure-time artifacts — their content is fully
// determined by `header`. writeIfNotChanged: avoid touching mtime.
mkdirSync(pchDir, { recursive: true });
writeIfChanged(
wrapperHeader,
[
`/* generated by scripts/build/compile.ts */`,
`#pragma clang system_header`,
`#ifdef __cplusplus`,
`#include "${absHeader}"`,
`#endif`,
``,
].join("\n"),
);
// Stub .cxx — empty. Compiled as the "main file"; wrapper is force-included.
// The pragma is ignored in main files but works in includes, hence this dance.
writeIfChanged(stubCxx, `/* generated by scripts/build/compile.ts */\n`);
n.build({
outputs: [out],
rule: "pch",
// Compile the STUB, force-include the wrapper.
inputs: [stubCxx],
// absHeader + wrapper editing must rebuild PCH. Dep outputs too — see
// the docstring above for why these can't be order-only (startup-stat
// vs mid-build header regeneration). The depfile tracks the REST.
implicitInputs: [absHeader, wrapperHeader, ...(opts.implicitInputs ?? [])],
orderOnlyInputs: [pchDirStamp(cfg), ...(opts.orderOnlyInputs ?? [])],
vars: {
cxxflags: opts.flags.join(" "),
pch_header: n.rel(wrapperHeader),
},
});
return { pch: out, wrapperHeader };
}
// ---------------------------------------------------------------------------
// Link & archive
// ---------------------------------------------------------------------------
export interface LinkOpts {
/** Static libraries to link (absolute paths). Included in $in. */
libs: string[];
/** Linker flags. */
flags: string[];
/**
* Files the link reads that aren't in $in — symbol lists (symbols.def,
* symbols.txt, symbols.dyn), linker scripts (linker.lds), manifests.
* Editing these should trigger relink (cmake's LINK_DEPENDS equivalent).
*/
implicitInputs?: string[];
/** Output linker map to this path (for debugging symbol bloat). */
linkerMapOutput?: string;
}
/**
* Link an executable. Returns absolute path to output (with cfg.exeSuffix
* appended — clang-cl /Fe auto-appends .exe; ninja's output path must match).
*/
export function link(n: Ninja, cfg: Config, out: string, objects: string[], opts: LinkOpts): string {
const absOut = resolve(cfg.buildDir, out + cfg.exeSuffix);
// Linker map is an implicit output (ninja tracks it but not in $out)
const implicitOutputs: string[] = [];
if (opts.linkerMapOutput !== undefined) {
implicitOutputs.push(resolve(cfg.buildDir, opts.linkerMapOutput));
}
const node: BuildNode = {
outputs: [absOut],
rule: "link",
inputs: [...objects, ...opts.libs],
vars: {
ldflags: opts.flags.join(" "),
},
};
if (implicitOutputs.length > 0) node.implicitOutputs = implicitOutputs;
if (opts.implicitInputs !== undefined && opts.implicitInputs.length > 0) {
node.implicitInputs = opts.implicitInputs;
}
n.build(node);
return absOut;
}
/**
* Create a static library. Returns absolute path to output.
*/
export function ar(n: Ninja, cfg: Config, out: string, objects: string[]): string {
const absOut = resolve(cfg.buildDir, out);
n.build({
outputs: [absOut],
rule: "ar",
inputs: objects,
});
return absOut;
}
// ---------------------------------------------------------------------------
// Path computation
// ---------------------------------------------------------------------------
/**
* Compute the .o output path for a source file.
*
* Mirrors the source tree under obj/, so `src/bun.js/bindings/foo.cpp` →
* `obj/src/bun.js/bindings/foo.cpp.o`. Generated sources (codegen .cpp
* files under buildDir) go under `obj/codegen/` to keep a single tree.
*
* Ninja does NOT auto-create parent directories of outputs. Directories
* are created at configure time — each `cxx()`/`cc()` call tracks its
* object's parent dir, and `createObjectDirs()` is called once at the end
* of configure to mkdir the whole tree. Same approach as CMake, which
* pre-creates `CMakeFiles/<target>.dir/` during its generate step.
*/
function objectPath(cfg: Config, src: string): string {
const absSrc = resolve(cfg.cwd, src);
// Normalize to repo-root-relative path. Generated sources (in buildDir)
// get mapped to their buildDir-relative location so `codegen/Foo.cpp`
// stays `codegen/Foo.cpp.o` — no prefix needed since codegen/ doesn't
// collide with any src/ subdir.
let relSrc: string;
if (absSrc.startsWith(cfg.buildDir)) {
relSrc = relative(cfg.buildDir, absSrc);
} else {
relSrc = relative(cfg.cwd, absSrc);
}
return resolve(cfg.buildDir, "obj", relSrc + cfg.objSuffix);
}
/**
* Stamp file for the obj/ directory. Object files depend on this order-only
* so the dir exists before compilation runs.
*/
function objectDirStamp(cfg: Config): string {
return resolve(cfg.buildDir, "obj", ".dir");
}
function pchDirStamp(cfg: Config): string {
return resolve(cfg.buildDir, "pch", ".dir");
}
/**
* Register directory stamp rules. Call once.
*/
export function registerDirStamps(n: Ninja, cfg: Config): void {
const objDir = dirname(objectDirStamp(cfg));
const pchDir = dirname(pchDirStamp(cfg));
// Single rule, mkdir + touch stamp. Configure pre-creates these dirs;
// the rule still runs once to write the stamp ninja tracks. Both sides
// must tolerate "already exists" — posix has -p, cmd doesn't, so
// suppress the error (2>nul) and touch unconditionally (&).
n.rule("mkdir_stamp", {
command: cfg.host.os === "windows" ? `cmd /c "mkdir $dir 2>nul & type nul > $out"` : `mkdir -p $dir && touch $out`,
description: "mkdir $dir",
});
n.build({
outputs: [objectDirStamp(cfg)],
rule: "mkdir_stamp",
inputs: [],
vars: { dir: n.rel(objDir) },
});
n.build({
outputs: [pchDirStamp(cfg)],
rule: "mkdir_stamp",
inputs: [],
vars: { dir: n.rel(pchDir) },
});
}

774
scripts/build/config.ts Normal file
View File

@@ -0,0 +1,774 @@
/**
* Build configuration.
*
* One flat struct. All derived booleans computed once in `resolveConfig()`,
* passed everywhere. No `if(ENABLE_X)` depending on `if(CI)` depending on
* `if(RELEASE)` — the chain is resolved here and the result is a plain value.
*/
import { existsSync, readFileSync } from "node:fs";
import { arch as hostArch, platform as hostPlatform } from "node:os";
import { isAbsolute, join, resolve } from "node:path";
import { NODEJS_ABI_VERSION, NODEJS_VERSION } from "./deps/nodejs-headers.ts";
import { WEBKIT_VERSION } from "./deps/webkit.ts";
import { BuildError, assert } from "./error.ts";
import { clangTargetArch } from "./tools.ts";
import { ZIG_COMMIT } from "./zig.ts";
export type OS = "linux" | "darwin" | "windows";
export type Arch = "x64" | "aarch64";
export type Abi = "gnu" | "musl";
export type BuildType = "Debug" | "Release" | "RelWithDebInfo" | "MinSizeRel";
export type BuildMode = "full" | "cpp-only" | "zig-only" | "link-only";
export type WebKitMode = "prebuilt" | "local";
/**
* Host platform — what's running the build. Distinguish from target
* (Config.os/arch/windows) which is what we're building FOR.
*
* Host vs target matters for zig-only cross-compile: a linux CI box
* can cross-compile bun-zig.o for darwin/windows. Target determines
* zig's triple and compile flags; host determines shell syntax (cmd
* vs sh), quoting, and tool executable suffixes.
*
* For all other modes (full, cpp-only, link-only), host == target
* since we don't cross-compile C++.
*/
export interface Host {
os: OS;
arch: Arch;
}
/**
* Pinned version defaults. Each lives at the top of its own file
* (deps/webkit.ts, zig.ts, deps/nodejs-headers.ts) — look there to bump.
* Overridable via PartialConfig for testing (e.g. trying a WebKit branch).
*/
const versionDefaults = {
nodejsVersion: NODEJS_VERSION,
nodejsAbiVersion: NODEJS_ABI_VERSION,
zigCommit: ZIG_COMMIT,
webkitVersion: WEBKIT_VERSION,
};
/**
* The full resolved build configuration. Every field is concrete — no
* undefined-because-it-depends-on-something-else. This is the single source
* of truth passed to every build function.
*/
export interface Config {
// ─── Target platform ───
os: OS;
arch: Arch;
/** Linux-only. undefined on darwin/windows. */
abi: Abi | undefined;
// ─── Derived platform booleans (computed from os/arch) ───
linux: boolean;
darwin: boolean;
windows: boolean;
/** linux || darwin */
unix: boolean;
x64: boolean;
arm64: boolean;
/**
* What's running the build. Differs from os/arch/windows (target) in
* zig-only cross-compile. Use for: shell syntax in rule commands,
* quoteArgs(), tool executable suffixes. See Host type docs.
*/
host: Host;
// ─── Platform file conventions ───
// Centralized so a new target (or a forgotten .exe) is one edit away.
/** ".exe" on Windows, "" elsewhere. */
exeSuffix: string;
/** ".obj" on Windows, ".o" elsewhere. */
objSuffix: string;
/** "" on Windows, "lib" elsewhere. */
libPrefix: string;
/** ".lib" on Windows, ".a" elsewhere. */
libSuffix: string;
// ─── Build configuration ───
buildType: BuildType;
debug: boolean;
release: boolean;
mode: BuildMode;
// ─── Features (all explicit booleans) ───
lto: boolean;
asan: boolean;
zigAsan: boolean;
assertions: boolean;
logs: boolean;
/** x64-only: target nehalem (no AVX) instead of haswell. */
baseline: boolean;
canary: boolean;
/** MinSizeRel → optimize for size. */
smol: boolean;
staticSqlite: boolean;
staticLibatomic: boolean;
tinycc: boolean;
valgrind: boolean;
fuzzilli: boolean;
// ─── Environment ───
ci: boolean;
buildkite: boolean;
// ─── Dependency modes ───
webkit: WebKitMode;
// ─── Paths (all absolute) ───
/** Repository root. */
cwd: string;
/** Build output directory, e.g. /path/to/bun/build/debug/. */
buildDir: string;
/** Generated code output, e.g. buildDir/codegen/. */
codegenDir: string;
/** Persistent cache for dep tarballs and builds. */
cacheDir: string;
/** Vendored dependencies (gitignored). */
vendorDir: string;
// ─── Toolchain (resolved absolute paths) ───
cc: string;
cxx: string;
ar: string;
/** llvm-ranlib. undefined on windows (llvm-lib indexes itself). */
ranlib: string | undefined;
/** ld.lld on linux, lld-link on windows. May be empty on darwin (clang invokes ld). */
ld: string;
strip: string;
/** darwin-only. */
dsymutil: string | undefined;
zig: string;
/** Self-host bun for codegen. */
bun: string;
esbuild: string;
/** Optional — compiler launcher prefix. */
ccache: string | undefined;
/** cmake executable. Required for nested dep builds. */
cmake: string;
/** cargo executable. undefined when no rust toolchain is available. */
cargo: string | undefined;
/** CARGO_HOME — passed to cargo invocations for reproducibility. */
cargoHome: string | undefined;
/** RUSTUP_HOME — passed to cargo invocations for reproducibility. */
rustupHome: string | undefined;
/** Windows: MSVC link.exe path (to avoid Git's /usr/bin/link shadowing). */
msvcLinker: string | undefined;
/** Windows: llvm-rc for nested cmake (CMAKE_RC_COMPILER). */
rc: string | undefined;
/** Windows: llvm-mt for nested cmake (CMAKE_MT). May be absent in some LLVM distros. */
mt: string | undefined;
// ─── macOS SDK (darwin only, undefined elsewhere) ───
/** e.g. "13.0". Passed to deps as -DCMAKE_OSX_DEPLOYMENT_TARGET. */
osxDeploymentTarget: string | undefined;
/** SDK path from `xcrun --show-sdk-path`. Passed to deps as -DCMAKE_OSX_SYSROOT. */
osxSysroot: string | undefined;
// ─── Versioning ───
/** Bun's own version (from package.json). */
version: string;
/** Git commit of the bun checkout — feeds into zig's -Dsha. */
revision: string;
canaryRevision: string;
/** Node.js compat version. Default in versions.ts; override to test a bump. */
nodejsVersion: string;
nodejsAbiVersion: string;
/** Zig compiler commit. Default in versions.ts; override to test a new compiler. */
zigCommit: string;
/** WebKit commit. Default in versions.ts; override to test a WebKit branch. */
webkitVersion: string;
}
/**
* Partial config — what profiles and CLI flags provide.
* Resolution fills in the rest.
*/
export interface PartialConfig {
os?: OS;
arch?: Arch;
abi?: Abi;
buildType?: BuildType;
mode?: BuildMode;
lto?: boolean;
asan?: boolean;
zigAsan?: boolean;
assertions?: boolean;
logs?: boolean;
baseline?: boolean;
canary?: boolean;
staticSqlite?: boolean;
staticLibatomic?: boolean;
tinycc?: boolean;
valgrind?: boolean;
fuzzilli?: boolean;
ci?: boolean;
buildkite?: boolean;
webkit?: WebKitMode;
buildDir?: string;
cacheDir?: string;
// Version pins (defaults in versions.ts).
nodejsVersion?: string;
nodejsAbiVersion?: string;
zigCommit?: string;
webkitVersion?: string;
}
/**
* Resolved toolchain — found by tool discovery, passed in separately so
* tests can mock it out.
*/
export interface Toolchain {
cc: string;
cxx: string;
ar: string;
ranlib: string | undefined;
ld: string;
strip: string;
dsymutil: string | undefined;
zig: string;
bun: string;
esbuild: string;
ccache: string | undefined;
cmake: string;
/** Cargo executable. Required only if a rust dep (lolhtml) is being built. */
cargo: string | undefined;
/** CARGO_HOME. Set alongside cargo; undefined when cargo is unavailable. */
cargoHome: string | undefined;
/** RUSTUP_HOME. Set alongside cargo; undefined when cargo is unavailable. */
rustupHome: string | undefined;
/**
* Windows only: absolute path to MSVC's link.exe. Set as the cargo linker
* via CARGO_TARGET_<triple>_LINKER to prevent Git Bash's /usr/bin/link
* (the GNU hard-link utility) from shadowing the real linker in PATH.
*/
msvcLinker: string | undefined;
/**
* Windows only: llvm-rc (resource compiler). Passed to nested cmake
* as CMAKE_RC_COMPILER. cmake's own detection usually finds it, but
* that depends on PATH and cmake version — explicit is safer.
*/
rc: string | undefined;
/**
* Windows only: llvm-mt (manifest tool). Passed to nested cmake as
* CMAKE_MT. Optional — some LLVM distributions don't ship llvm-mt;
* when absent, cmake's STATIC_LIBRARY try-compile mode (set in
* source.ts) sidesteps the need.
*/
mt: string | undefined;
}
/**
* Host platform detection. Only used for picking defaults.
*/
export function detectHost(): Host {
const plat = hostPlatform();
const os: OS =
plat === "linux"
? "linux"
: plat === "darwin"
? "darwin"
: plat === "win32"
? "windows"
: (() => {
throw new BuildError(`Unsupported host platform: ${plat}`, {
hint: "Bun builds on linux, darwin, or windows",
});
})();
const a = hostArch();
const arch: Arch =
a === "x64"
? "x64"
: a === "arm64"
? "aarch64"
: (() => {
throw new BuildError(`Unsupported host architecture: ${a}`, { hint: "Bun builds on x64 or arm64" });
})();
return { os, arch };
}
/**
* Detect linux ABI (gnu vs musl) by checking for /etc/alpine-release.
*/
export function detectLinuxAbi(): Abi {
return existsSync("/etc/alpine-release") ? "musl" : "gnu";
}
/**
* Resolve a PartialConfig into a full Config.
*
* This is where all the "X defaults to Y unless Z" chains get resolved into
* concrete values. After this runs, everything downstream sees plain booleans.
*/
export function resolveConfig(partial: PartialConfig, toolchain: Toolchain): Config {
const host = detectHost();
// ─── Target platform ───
const os = partial.os ?? host.os;
// Windows: process.arch can be wrong under emulation (x64 bun on arm64
// hardware). Ask the compiler what it targets — CMake does the same in
// project() to set CMAKE_SYSTEM_PROCESSOR. The found clang's default
// target is what we actually build for.
const compilerArch = os === "windows" ? clangTargetArch(toolchain.cc) : undefined;
const arch = partial.arch ?? compilerArch ?? host.arch;
const abi: Abi | undefined = os === "linux" ? (partial.abi ?? detectLinuxAbi()) : undefined;
const linux = os === "linux";
const darwin = os === "darwin";
const windows = os === "windows";
const unix = linux || darwin;
const x64 = arch === "x64";
const arm64 = arch === "aarch64";
// Platform file conventions — MSVC style on Windows, Unix everywhere else.
const exeSuffix = windows ? ".exe" : "";
const objSuffix = windows ? ".obj" : ".o";
const libPrefix = windows ? "" : "lib";
const libSuffix = windows ? ".lib" : ".a";
// ─── Build type ───
const buildType = partial.buildType ?? "Debug";
const debug = buildType === "Debug";
const release = buildType === "Release" || buildType === "RelWithDebInfo" || buildType === "MinSizeRel";
const smol = buildType === "MinSizeRel";
// ─── Environment ───
// Explicit (not auto-detected from env) — matches CMake's optionx(CI DEFAULT OFF).
// The ci-* profiles set these. Affects build semantics: LTO default, PCH
// skip, macOS min SDK. Log-group/annotation decisions use the runtime env
// detection in ci.ts instead, so running a non-CI profile on a CI machine
// still gets collapsible logs but not CI build flags.
const ci = partial.ci ?? false;
const buildkite = partial.buildkite ?? false;
// ─── Features ───
// Each is resolved exactly once here.
// ASAN: default on for debug builds on arm64 macOS or linux
const asanDefault = debug && ((darwin && arm64) || linux);
const asan = partial.asan ?? asanDefault;
// Zig ASAN follows ASAN unless explicitly overridden
const zigAsan = partial.zigAsan ?? asan;
// Assertions: default on in debug OR asan. ASAN coupling is ABI-critical:
// the -asan WebKit prebuilt is built with ASSERT_ENABLED=1, which gates
// struct fields (RefCountDebugger etc). If bun's C++ isn't also compiled
// with ASSERT_ENABLED=1, the struct layouts mismatch → crashes. CMake's
// build:asan always set ENABLE_ASSERTIONS=ON for this reason.
const assertions = partial.assertions ?? (debug || asan);
// LTO: default on only for CI release linux non-asan non-assertions
const ltoDefault = release && linux && ci && !assertions && !asan;
let lto = partial.lto ?? ltoDefault;
// ASAN and LTO don't mix — ASAN wins (silently, no warn — config is explicit)
if (asan && lto) {
lto = false;
}
// Logs: on by default in debug non-test
const logs = partial.logs ?? debug;
const baseline = partial.baseline ?? false;
const canary = partial.canary ?? true;
const canaryRevision = canary ? "1" : "0";
// Static SQLite: off on Apple (uses system), on elsewhere
const staticSqlite = partial.staticSqlite ?? !darwin;
// Static libatomic: on by default. Arch/Manjaro don't ship libatomic.a —
// those users pass --static-libatomic=off. Not auto-detected: the link
// failure is loud ("cannot find -l:libatomic.a") and the fix is obvious.
const staticLibatomic = partial.staticLibatomic ?? true;
// TinyCC: off on Windows ARM64 (not supported), on elsewhere
const tinycc = partial.tinycc ?? !(windows && arm64);
const valgrind = partial.valgrind ?? false;
const fuzzilli = partial.fuzzilli ?? false;
// ─── Paths ───
const cwd = findRepoRoot();
const defaultBuildDirName = computeBuildDirName({ debug, release, asan, assertions });
const buildDir =
partial.buildDir !== undefined
? isAbsolute(partial.buildDir)
? partial.buildDir
: resolve(cwd, partial.buildDir)
: resolve(cwd, "build", defaultBuildDirName);
const codegenDir = resolve(buildDir, "codegen");
const cacheDir =
partial.cacheDir !== undefined
? isAbsolute(partial.cacheDir)
? partial.cacheDir
: resolve(cwd, partial.cacheDir)
: resolve(buildDir, "cache");
const vendorDir = resolve(cwd, "vendor");
// ─── Validation ───
assert(!baseline || x64, "baseline=true requires arch=x64 (baseline disables AVX which is x64-only)");
assert(!valgrind || linux, "valgrind=true requires os=linux");
assert(!(asan && valgrind), "Cannot enable both asan and valgrind simultaneously");
assert(os !== "linux" || abi !== undefined, "Linux builds require an abi (gnu or musl)");
// ─── Versioning ───
const pkgJsonPath = resolve(cwd, "package.json");
const pkgJson = JSON.parse(readFileSync(pkgJsonPath, "utf8")) as { version: string };
const version = pkgJson.version;
const revision = getGitRevision(cwd);
// Defaults from versions.ts. Override via --webkit-version=<hash> etc.
// to test a branch before bumping the pinned default.
const nodejsVersion = partial.nodejsVersion ?? versionDefaults.nodejsVersion;
const nodejsAbiVersion = partial.nodejsAbiVersion ?? versionDefaults.nodejsAbiVersion;
const zigCommit = partial.zigCommit ?? versionDefaults.zigCommit;
const webkitVersion = partial.webkitVersion ?? versionDefaults.webkitVersion;
// ─── macOS SDK ───
// Must be passed to nested cmake builds or they'll pick the wrong SDK.
// Requires BOTH host and target to be darwin — xcode only exists on
// macOS, and cross-compiling C++/deps to darwin isn't supported (only
// zig cross-compiles, and zig brings its own SDKs).
let osxDeploymentTarget: string | undefined;
let osxSysroot: string | undefined;
if (darwin && host.os === "darwin") {
({ osxDeploymentTarget, osxSysroot } = detectMacosSdk(ci));
}
return {
os,
arch,
abi,
linux,
darwin,
windows,
unix,
x64,
arm64,
host,
exeSuffix,
objSuffix,
libPrefix,
libSuffix,
buildType,
debug,
release,
mode: partial.mode ?? "full",
lto,
asan,
zigAsan,
assertions,
logs,
baseline,
canary,
smol,
staticSqlite,
staticLibatomic,
tinycc,
valgrind,
fuzzilli,
ci,
buildkite,
webkit: partial.webkit ?? "prebuilt",
cwd,
buildDir,
codegenDir,
cacheDir,
vendorDir,
cc: toolchain.cc,
cxx: toolchain.cxx,
ar: toolchain.ar,
ranlib: toolchain.ranlib,
ld: toolchain.ld,
strip: toolchain.strip,
dsymutil: toolchain.dsymutil,
zig: toolchain.zig,
bun: toolchain.bun,
esbuild: toolchain.esbuild,
ccache: toolchain.ccache,
cmake: toolchain.cmake,
cargo: toolchain.cargo,
cargoHome: toolchain.cargoHome,
rustupHome: toolchain.rustupHome,
msvcLinker: toolchain.msvcLinker,
rc: toolchain.rc,
mt: toolchain.mt,
osxDeploymentTarget,
osxSysroot,
version,
revision,
nodejsVersion,
nodejsAbiVersion,
canaryRevision,
zigCommit,
webkitVersion,
};
}
/** Minimum macOS SDK version we support. */
const MIN_OSX_DEPLOYMENT_TARGET = "13.0";
/**
* Detect macOS SDK paths.
*
* - CI: always target the minimum (reproducible builds).
* - Local: target the installed SDK's major version (avoids linker warnings
* about object files built for newer macOS than target).
*
* Fast path: `xcode-select -p` (~5ms) gives the developer dir; from there
* we construct the SDK path and parse the version from the resolved
* symlink. Avoids `xcrun` (~100ms × 2 spawns). Falls back to xcrun only if
* the constructed path doesn't exist (exotic installs).
*/
function detectMacosSdk(ci: boolean): { osxDeploymentTarget: string; osxSysroot: string } {
const { execSync } = require("node:child_process") as typeof import("node:child_process");
const { existsSync, realpathSync } = require("node:fs") as typeof import("node:fs");
// xcode-select -p prints the active developer dir (respects
// `xcode-select --switch` and DEVELOPER_DIR). It's a tiny C binary —
// fast enough to be negligible, unlike xcrun which does a bunch of
// environment discovery.
let devDir: string;
try {
devDir = (process.env.DEVELOPER_DIR ?? execSync("xcode-select -p", { encoding: "utf8" })).trim();
} catch (cause) {
throw new BuildError("xcode-select failed — command line tools not installed?", {
hint: "Run: xcode-select --install",
cause,
});
}
// For full Xcode the dev dir is ".../Developer"; for CLT it's
// "/Library/Developer/CommandLineTools". SDK layout differs:
// Xcode: <dev>/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
// CLT: <dev>/SDKs/MacOSX.sdk
//
// Return the SYMLINK path as sysroot (matches what xcrun returns, and
// what ends up in build.ninja — so swapping SDKs doesn't cause a
// spurious full rebuild). But follow the link to PARSE the version
// from the real basename (e.g. MacOSX14.2.sdk → "14").
const candidates = [`${devDir}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk`, `${devDir}/SDKs/MacOSX.sdk`];
let osxSysroot: string | undefined;
let sdkVersionFromPath: string | undefined;
for (const path of candidates) {
if (existsSync(path)) {
osxSysroot = path; // symlink — matches xcrun's output
const resolved = realpathSync(path);
const m = resolved.match(/MacOSX(\d+)(?:\.\d+)*\.sdk$/);
if (m) sdkVersionFromPath = m[1];
break;
}
}
// Neither layout matched — fall back to xcrun. Rare (custom SDK
// locations via SDKROOT env or similar).
if (osxSysroot === undefined) {
try {
osxSysroot = execSync("xcrun --sdk macosx --show-sdk-path", { encoding: "utf8" }).trim();
} catch (cause) {
throw new BuildError("Failed to find macOS SDK path", {
hint: "Run: xcode-select --install",
cause,
});
}
}
let osxDeploymentTarget: string;
if (ci) {
osxDeploymentTarget = MIN_OSX_DEPLOYMENT_TARGET;
} else if (sdkVersionFromPath !== undefined) {
osxDeploymentTarget = sdkVersionFromPath;
} else {
// Couldn't parse from path (unversioned symlink target?) — ask xcrun.
let sdkVersion: string;
try {
sdkVersion = execSync("xcrun --sdk macosx --show-sdk-version", { encoding: "utf8" }).trim();
} catch (cause) {
throw new BuildError("Failed to find macOS SDK version", {
hint: "Run: xcode-select --install",
cause,
});
}
const major = sdkVersion.match(/^(\d+)/)?.[1];
assert(major !== undefined, `Could not parse macOS SDK version: ${sdkVersion}`);
osxDeploymentTarget = major;
}
// Floor at minimum
if (compareVersionStrings(osxDeploymentTarget, MIN_OSX_DEPLOYMENT_TARGET) < 0) {
throw new BuildError(
`macOS SDK ${osxDeploymentTarget} is older than minimum supported ${MIN_OSX_DEPLOYMENT_TARGET}`,
{ hint: "Update Xcode or Xcode Command Line Tools" },
);
}
return { osxDeploymentTarget, osxSysroot };
}
/** Simple X.Y version comparison. Returns -1, 0, 1. */
function compareVersionStrings(a: string, b: string): number {
const pa = a.split(".").map(Number);
const pb = b.split(".").map(Number);
const len = Math.max(pa.length, pb.length);
for (let i = 0; i < len; i++) {
const ai = pa[i] ?? 0;
const bi = pb[i] ?? 0;
if (ai !== bi) return ai < bi ? -1 : 1;
}
return 0;
}
/**
* Find the repository root by walking up from cwd looking for package.json
* with name "bun". Exported so `resolveToolchain()` in configure.ts can
* resolve paths correctly when invoked from ninja (where cwd = build dir).
*/
export function findRepoRoot(): string {
let dir = process.cwd();
while (true) {
const pkgPath = join(dir, "package.json");
if (existsSync(pkgPath)) {
try {
const pkg = JSON.parse(readFileSync(pkgPath, "utf8")) as { name?: string };
if (pkg.name === "bun") {
return dir;
}
} catch {
// Invalid JSON, keep walking
}
}
const parent = resolve(dir, "..");
if (parent === dir) {
throw new BuildError("Could not find bun repository root", { hint: "Run this from within the bun repository" });
}
dir = parent;
}
}
/**
* Get the current git revision (HEAD sha).
*
* Uses `git rev-parse` rather than reading .git/HEAD directly — the sha
* is baked into the binary and surfaces in bug reports, so correctness
* matters more than the ~20ms spawn. Git's plumbing has edge cases
* (packed-refs, worktrees, symbolic refs) that rev-parse handles for free.
*/
function getGitRevision(cwd: string): string {
// CI env first — authoritative and zero-cost.
const envSha = process.env.BUILDKITE_COMMIT ?? process.env.GITHUB_SHA ?? process.env.GIT_SHA;
if (envSha !== undefined && envSha.length > 0) {
return envSha;
}
try {
const { execSync } = require("node:child_process") as typeof import("node:child_process");
return execSync("git rev-parse HEAD", { cwd, encoding: "utf8" }).trim();
} catch {
return "unknown";
}
}
/**
* Compute build directory name based on config.
* Matches the pattern used by package.json scripts.
*/
function computeBuildDirName(c: { debug: boolean; release: boolean; asan: boolean; assertions: boolean }): string {
if (c.debug) return "debug";
if (c.asan) return "release-asan";
if (c.assertions) return "release-assertions";
return "release";
}
/**
* Name of the output executable (no suffix).
*
* Debug builds: bun-debug. Release with ASAN: bun-asan. Etc.
* The plain `bun` name (without -profile) only exists post-strip.
*
* Lives here (not bun.ts) so flags.ts can use it for linker-map filename
* without a circular import.
*/
export function bunExeName(cfg: Config): string {
if (cfg.debug) return "bun-debug";
// Release variants — suffix encodes which features differ from plain release.
// First match wins.
if (cfg.asan && cfg.valgrind) return "bun-asan-valgrind";
if (cfg.asan) return "bun-asan";
if (cfg.valgrind) return "bun-valgrind";
if (cfg.assertions) return "bun-assertions";
// Plain release: called bun-profile (the stripped one is `bun`).
return "bun-profile";
}
/**
* Whether this config produces a stripped `bun` alongside `bun-profile`.
*
* Only plain release builds strip — not debug (you want symbols), not
* asan/valgrind (strip interferes), not assertions (usually debugging).
*/
export function shouldStrip(cfg: Config): boolean {
return !cfg.debug && !cfg.asan && !cfg.valgrind && !cfg.assertions;
}
// ANSI helpers — no-op when output isn't a TTY (pipe, file, `bd` log).
const useColor = Bun.enableANSIColors && process.stderr.isTTY;
const c = {
dim: (s: string) => (useColor ? `\x1b[2m${s}\x1b[22m` : s),
cyan: (s: string) => (useColor ? `\x1b[36m${s}\x1b[39m` : s),
green: (s: string) => (useColor ? `\x1b[32m${s}\x1b[39m` : s),
};
/**
* Format a config for display (used at configure time).
* `exe` is the output binary name (e.g. "bun-debug" or "bun-profile → bun (stripped)").
*/
export function formatConfig(cfg: Config, exe: string): string {
const label = (s: string) => c.dim(s.padEnd(12));
// Relative build dir with ./ prefix — shorter, copy-pastable.
const { relative: rel, sep } = require("node:path") as typeof import("node:path");
const relBuildDir = `.${sep}${rel(cfg.cwd, cfg.buildDir)}`;
const lines: string[] = [
`[configured] ${c.green(exe)}`,
` ${label("target")} ${cfg.os}-${cfg.arch}${cfg.abi !== undefined ? "-" + cfg.abi : ""}`,
` ${label("build type")} ${cfg.buildType}`,
` ${label("build dir")} ${relBuildDir}`,
// Revision makes it obvious why configure re-ran after a commit
// (the sha changes → zig's -Dsha arg changes → build.ninja differs).
` ${label("revision")} ${cfg.revision === "unknown" ? "unknown" : cfg.revision.slice(0, 10)}`,
];
const features: string[] = [];
if (cfg.lto) features.push("lto");
if (cfg.asan) features.push("asan");
if (cfg.assertions) features.push("assertions");
if (cfg.logs) features.push("logs");
if (cfg.baseline) features.push("baseline");
if (cfg.valgrind) features.push("valgrind");
if (cfg.fuzzilli) features.push("fuzzilli");
if (!cfg.canary) features.push("canary:off");
// Non-default modes — show so you notice when a build is unusual.
if (cfg.webkit !== "prebuilt") features.push(`webkit:${cfg.webkit}`);
if (cfg.mode !== "full") features.push(`mode:${cfg.mode}`);
// Version pin overrides — show a short hash so you catch "forgot to
// revert my WebKit test branch" before the build goes weird.
if (cfg.webkitVersion !== versionDefaults.webkitVersion)
features.push(`webkit-version:${cfg.webkitVersion.slice(0, 10)}`);
if (cfg.zigCommit !== versionDefaults.zigCommit) features.push(`zig-commit:${cfg.zigCommit.slice(0, 10)}`);
if (cfg.nodejsVersion !== versionDefaults.nodejsVersion) features.push(`nodejs:${cfg.nodejsVersion}`);
lines.push(` ${label("features")} ${features.length > 0 ? c.cyan(features.join(", ")) : c.dim("(none)")}`);
return lines.join("\n");
}
/**
* One-line "nothing changed" configure message. Bracketed to match the
* [name] prefix style used by deps/zig.
*/
export function formatConfigUnchanged(exe: string, elapsed: number): string {
return `[configured] ${c.green(exe)} in ${elapsed}ms ${c.dim("(unchanged)")}`;
}

273
scripts/build/configure.ts Normal file
View File

@@ -0,0 +1,273 @@
/**
* Configure: resolve config → emit build.ninja.
*
* Separated from build.ts so configure can be called standalone (just
* regenerate ninja without running the build) and so CI orchestration
* can configure once then run specific targets.
*/
import { mkdirSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { type BunOutput, bunExeName, emitBun, shouldStrip, validateBunConfig } from "./bun.ts";
import {
type Config,
type PartialConfig,
type Toolchain,
detectHost,
findRepoRoot,
formatConfig,
resolveConfig,
} from "./config.ts";
import { BuildError } from "./error.ts";
import { mkdirAll, writeIfChanged } from "./fs.ts";
import { Ninja } from "./ninja.ts";
import { registerAllRules } from "./rules.ts";
import { quote } from "./shell.ts";
import { globAllSources } from "./sources.ts";
import { findBun, findCargo, findMsvcLinker, findSystemTool, resolveLlvmToolchain } from "./tools.ts";
/**
* Full toolchain discovery. Returns absolute paths to all required tools.
*
* Throws BuildError with a hint if a required tool is missing. Optional
* tools (ccache, cargo if no rust deps needed) become `undefined`.
*/
export function resolveToolchain(): Toolchain {
const host = detectHost();
const llvm = resolveLlvmToolchain(host.os, host.arch);
// cmake — required for nested dep builds.
const cmake = findSystemTool("cmake", { required: true, hint: "Install cmake (>= 3.24)" });
if (cmake === undefined) throw new BuildError("unreachable: findSystemTool required=true returned undefined");
// cargo — required for lolhtml. Not found → build will fail at that dep
// with a clear "install rust" hint. We don't hard-fail here because
// someone might be testing a subset that doesn't need lolhtml.
const rust = findCargo(host.os);
// Windows: MSVC link.exe path (to prevent Git Bash's /usr/bin/link
// shadowing). Only needed when cargo builds with the msvc target.
const msvcLinker = host.os === "windows" ? findMsvcLinker(host.arch) : undefined;
// esbuild/zig paths are relative to REPO ROOT, not process.cwd() — when
// ninja's generator rule invokes reconfigure, cwd is the build dir.
const repoRoot = findRepoRoot();
// esbuild — comes from the root bun install. Path is deterministic.
// If not present, the first codegen build will fail with a clear error
// (and the build itself runs `bun install` first via the root install
// stamp, so this path will exist by the time esbuild rules fire).
const esbuild = resolve(repoRoot, "node_modules", ".bin", host.os === "windows" ? "esbuild.exe" : "esbuild");
// zig — lives at vendor/zig/, downloaded by the zig_fetch rule.
// Same deal: path is deterministic, download happens at build time.
const zig = resolve(repoRoot, "vendor", "zig", host.os === "windows" ? "zig.exe" : "zig");
return {
...llvm,
cmake,
zig,
bun: findBun(host.os),
esbuild,
cargo: rust?.cargo,
cargoHome: rust?.cargoHome,
rustupHome: rust?.rustupHome,
msvcLinker,
};
}
export interface ConfigureResult {
cfg: Config;
output: BunOutput;
/** Build.ninja absolute path. */
ninjaFile: string;
/** Env vars the caller should set before spawning ninja. */
env: Record<string, string>;
/** Wall-clock ms for the configure pass. */
elapsed: number;
/** True if build.ninja actually changed (vs an idempotent re-run). */
changed: boolean;
/** Final executable name (e.g. "bun-debug"). For status messages. */
exe: string;
}
/**
* Files that, when changed, should trigger a reconfigure. Globbed at
* configure time — if you add a new build script, it'll be picked up
* on the next reconfigure (since adding a .ts usually means editing
* an existing one to import it).
*
* Excludes runtime-only files (fetch-cli.ts, download.ts, ci.ts) and
* runtime-only scripts — changes to those don't affect the build graph.
*/
function configureInputs(cwd: string): string[] {
const buildDir = resolve(cwd, "scripts", "build");
const excluded = new Set(["fetch-cli.ts", "download.ts", "ci.ts", "stream.ts"]);
// Bun.Glob — node:fs globSync isn't in older bun versions (CI agents pin).
const glob = (pattern: string) => [...new Bun.Glob(pattern).scanSync({ cwd: buildDir })];
const scripts = glob("*.ts")
.filter(f => !excluded.has(f))
.map(f => resolve(buildDir, f));
const deps = glob("deps/*.ts").map(f => resolve(buildDir, f));
return [...scripts, ...deps, resolve(cwd, "cmake", "Sources.json"), resolve(cwd, "package.json")].sort();
}
/**
* Emit the generator rule — makes build.ninja self-rebuilding. When you
* run `ninja` directly and a build script has changed, ninja runs
* reconfigure first, then restarts with the fresh graph.
*
* The original PartialConfig is persisted to configure.json; the regen
* command reads it back via --config-file. This ensures the replay uses
* the exact same profile/overrides as the original configure.
*/
function emitGeneratorRule(n: Ninja, cfg: Config, partial: PartialConfig): void {
const configFile = resolve(cfg.buildDir, "configure.json");
const buildScript = resolve(cfg.cwd, "scripts", "build.ts");
// Persist the partial config. writeIfChanged — same config → no mtime
// bump → no unnecessary regen on identical reconfigures.
// This runs before n.write() (which mkdir's), so ensure dir exists.
mkdirSync(cfg.buildDir, { recursive: true });
writeIfChanged(configFile, JSON.stringify(partial, null, 2) + "\n");
const hostWin = cfg.host.os === "windows";
n.rule("regen", {
command: `${quote(cfg.bun, hostWin)} ${quote(buildScript, hostWin)} --config-file=$in`,
description: "reconfigure",
// generator = 1: exempt from `ninja -t clean`, triggers manifest restart
// when the output (build.ninja) is rebuilt.
generator: true,
// restat: configure uses writeIfChanged on build.ninja. If nothing
// actually changed (unlikely when inputs changed, but possible for
// cosmetic edits), no restart happens.
restat: true,
pool: "console",
});
n.build({
outputs: [resolve(cfg.buildDir, "build.ninja")],
rule: "regen",
inputs: [configFile],
implicitInputs: configureInputs(cfg.cwd),
});
}
/**
* ccache environment to set for compile commands. Points ccache into the
* build dir (not ~/.ccache) so `rm -rf build/` is a complete reset.
*/
function ccacheEnv(cfg: Config): Record<string, string> {
if (cfg.ccache === undefined) return {};
const env: Record<string, string> = {
CCACHE_DIR: resolve(cfg.cacheDir, "ccache"),
// basedir + nohashdir: relativize paths in cache keys so the same
// source at different checkout locations shares cache entries.
CCACHE_BASEDIR: cfg.cwd,
CCACHE_NOHASHDIR: "1",
// Copy-on-write for cache entries — near-free on btrfs/APFS/ReFS.
CCACHE_FILECLONE: "1",
CCACHE_STATSLOG: resolve(cfg.buildDir, "ccache.log"),
};
if (!cfg.ci) {
env.CCACHE_MAXSIZE = "100G";
// Sloppiness: ignore differences that don't affect output. pch_defines:
// PCH can change without the includer's -D list changing. time_macros:
// __TIME__ differs every build. random_seed: -frandom-seed=0 is in our
// flags but ccache doesn't know that. clang_index_store: clangd state.
env.CCACHE_SLOPPINESS = "pch_defines,time_macros,locale,random_seed,clang_index_store,gcno_cwd";
}
return env;
}
/**
* Configure: resolve config → emit build.ninja. Returns the resolved config
* and emitted build info.
*
* `partial` comes from a profile + CLI overrides. If no buildDir is set,
* one is computed from the build type (build/debug, build/release, etc).
*/
export async function configure(partial: PartialConfig): Promise<ConfigureResult> {
const start = performance.now();
const trace = process.env.BUN_BUILD_TRACE === "1";
const mark = (label: string) => {
if (trace) process.stderr.write(` ${label}: ${Math.round(performance.now() - start)}ms\n`);
};
const toolchain = resolveToolchain();
mark("resolveToolchain");
const cfg = resolveConfig(partial, toolchain);
validateBunConfig(cfg);
// Perl check: LUT codegen (create-hash-table.ts) shells out to the
// perl script from JSC. If perl is missing, codegen fails cryptically.
// Check here so the error is at configure time with a clear hint.
// zig-only/link-only don't run LUT codegen — skip the check so split-CI
// steps don't require perl on the zig cross-compile box.
if (cfg.mode === "full" || cfg.mode === "cpp-only") {
if (findSystemTool("perl") === undefined) {
throw new BuildError("perl not found in PATH", {
hint: "LUT codegen (create-hash-table.ts) needs perl. Install it: apt install perl / brew install perl",
});
}
}
mark("validate+perl");
// Glob all source lists — one pass, consistent filesystem snapshot.
const sources = globAllSources(cfg.cwd);
mark("globAllSources");
// Emit ninja.
const n = new Ninja({ buildDir: cfg.buildDir });
registerAllRules(n, cfg);
emitGeneratorRule(n, cfg, partial);
const output = emitBun(n, cfg, sources);
mark("emitBun");
// Default targets. cpp-only sets its own default inside emitBun (archive,
// no smoke test). Full/link-only: `bun` phony (or stripped file) + `check`.
// Release builds produce both bun-profile and stripped bun; `bun` is the
// stripped one. Debug produces bun-debug; `bun` is a phony pointing at it.
// dsym: darwin release only — pulled into defaults so ninja actually builds
// it (no other node depends on it, and unlike cmake's POST_BUILD it doesn't
// auto-trigger).
if (output.exe !== undefined) {
const defaultTarget = output.strippedExe !== undefined ? n.rel(output.strippedExe) : "bun";
const targets = [defaultTarget, "check"];
if (output.dsym !== undefined) targets.push(n.rel(output.dsym));
n.default(targets);
}
// Write build.ninja (only if changed).
const changed = await n.write();
mark("n.write");
// Pre-create all object file parent directories. Ninja doesn't mkdir;
// CMake pre-creates CMakeFiles/<target>.dir/* at generate time, we do
// the same. Derived from output.objects so there's no hidden state —
// the orchestrator already knows every .o path.
mkdirAll(output.objects.map(dirname));
mark("mkdirAll");
const ninjaFile = resolve(cfg.buildDir, "build.ninja");
const elapsed = Math.round(performance.now() - start);
const exe = bunExeName(cfg) + (shouldStrip(cfg) ? " → bun (stripped)" : "");
// Full config print only when build.ninja actually changed (new
// profile, changed flags, new revision, new sources). A no-op
// reconfigure — which happens every run — gets a one-liner from
// build.ts (not here, because we're also called by ninja's generator
// rule and don't want doubled output). CI always prints.
if (changed || cfg.ci) {
process.stderr.write(formatConfig(cfg, exe) + "\n\n");
const codegenCount = output.codegen?.all.length ?? 0;
process.stderr.write(
`${output.deps.length} deps, ${codegenCount} codegen, ${output.objects.length} objects in ${elapsed}ms\n\n`,
);
}
return { cfg, output, ninjaFile, env: ccacheEnv(cfg), elapsed, changed, exe };
}

View File

@@ -0,0 +1,135 @@
/**
* Generates bun_dependency_versions.h — a header with dep version strings
* for `process.versions` and similar runtime introspection.
*
* Version values are DERIVED from `allDeps` — each dep's `source(cfg)`
* provides its commit/identity. Bumping a dep in `deps/<name>.ts`
* automatically updates this header. Single source of truth.
*
* Written at configure time. writeIfChanged semantics so changing an
* unrelated dep doesn't recompile everything via this header's mtime.
*
* Source: cmake/tools/GenerateDependencyVersions.cmake
*/
import { existsSync, mkdirSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import type { Config } from "./config.ts";
import { allDeps } from "./deps/index.ts";
import { writeIfChanged } from "./fs.ts";
import type { Source } from "./source.ts";
/**
* Pull a version identifier from a Source. The shape depends on the kind:
* github-archive → commit hash, prebuilt → identity string. Local/in-tree
* don't have a pinned identifier.
*/
function sourceIdentifier(source: Source): string | undefined {
switch (source.kind) {
case "github-archive":
return source.commit;
case "prebuilt":
return source.identity;
case "local":
case "in-tree":
// User-managed / in-tree — no pinned identifier independent of the
// bun commit. Callers that need a value use cfg.revision.
return undefined;
}
}
/**
* (macro_name, value) pairs. Values that can't be determined are omitted
* from output (CMake behavior — consumers check #ifdef).
*/
function computeVersions(cfg: Config): [string, string][] {
const versions: [string, string][] = [];
// ─── Deps: derived from allDeps source definitions ───
// Single source of truth — bumping deps/<name>.ts updates this.
for (const dep of allDeps) {
if (dep.versionMacro === undefined) continue;
const source = dep.source(cfg);
const id = sourceIdentifier(source);
// WebKit special case: its `prebuilt.identity` includes the suffix
// (e.g. "-debug-asan"), but process.versions should show the clean
// commit hash. Use cfg.webkitVersion instead — it IS the source of
// truth, identity is just version+suffix derived from it.
if (dep.name === "WebKit") {
versions.push([dep.versionMacro, cfg.webkitVersion]);
} else if (id !== undefined) {
versions.push([dep.versionMacro, id]);
}
// local/in-tree with no identifier: omit (consumer does #ifdef)
}
// ─── Non-dep versions ───
versions.push(["BUN_VERSION", cfg.version]);
versions.push(["NODEJS_COMPAT_VERSION", cfg.nodejsVersion]);
// UWS/USOCKETS are vendored at packages/bun-usockets — the bun commit
// IS their version.
versions.push(["UWS", cfg.revision]);
versions.push(["USOCKETS", cfg.revision]);
// Zig: could run `zig version` but that's a subprocess per configure.
// Using zigCommit from config — less human-readable than "0.15.2" but
// more precise (the commit IS what we ship) and stays in sync.
versions.push(["ZIG", cfg.zigCommit]);
// ─── Semantic versions extracted from vendor headers ───
// These can fail if the dep isn't fetched yet — omit on failure.
const libdeflate = extractVersion(
resolve(cfg.vendorDir, "libdeflate", "libdeflate.h"),
/LIBDEFLATE_VERSION_STRING\s+"([^"]+)"/,
);
if (libdeflate !== undefined) versions.push(["LIBDEFLATE_VERSION", libdeflate]);
const zlib = extractVersion(resolve(cfg.vendorDir, "zlib", "zlib.h"), /ZLIB_VERSION\s+"([^"]+)"/);
if (zlib !== undefined) versions.push(["ZLIB_VERSION", zlib]);
return versions;
}
function extractVersion(headerPath: string, regex: RegExp): string | undefined {
if (!existsSync(headerPath)) return undefined;
try {
return readFileSync(headerPath, "utf8").match(regex)?.[1];
} catch {
return undefined;
}
}
/**
* Generate bun_dependency_versions.h at buildDir/. Returns the absolute path.
*/
export function generateDepVersionsHeader(cfg: Config): string {
const outPath = resolve(cfg.buildDir, "bun_dependency_versions.h");
const versions = computeVersions(cfg).filter(([, v]) => v !== "" && v !== "unknown");
const lines: string[] = [
"// Auto-generated by scripts/build/depVersionsHeader.ts. Do not edit.",
"// Version values derived from scripts/build/deps/*.ts source definitions.",
"#ifndef BUN_DEPENDENCY_VERSIONS_H",
"#define BUN_DEPENDENCY_VERSIONS_H",
"",
"#ifdef __cplusplus",
'extern "C" {',
"#endif",
"",
"// Dependency versions",
...versions.map(([name, val]) => `#define BUN_DEP_${name} "${val}"`),
"",
"// C string constants for easy access",
...versions.map(([name, val]) => `static const char* const BUN_VERSION_${name} = "${val}";`),
"",
"#ifdef __cplusplus",
"}",
"#endif",
"",
"#endif // BUN_DEPENDENCY_VERSIONS_H",
"",
];
mkdirSync(cfg.buildDir, { recursive: true });
writeIfChanged(outPath, lines.join("\n"));
return outPath;
}

View File

@@ -0,0 +1,122 @@
# Vendored dependencies
One file per dependency. Each file exports a `Dependency` object that tells
the build system where to fetch the source, how to build it, and what
libraries/headers it provides.
## Adding a dependency
1. Copy `boringssl.ts` (the simplest one) to `<name>.ts`
2. Fill in `name`, `repo`, `commit`, `provides.libs`, `provides.includes`
3. Add `import { <name> } from "./<name>.ts"` + entry in `allDeps` array in `index.ts`
4. `bun run scripts/build/phase3-test.ts` to verify it builds
That's it. For most deps you're done.
**`name` must match the directory on disk** (`vendor/<name>/`). If your repo
is `oven-sh/WebKit`, name it `"WebKit"` — that's what `git clone` creates.
Case-sensitive filesystems enforce this.
**Ordering in `allDeps` matters:**
- Put deps with `fetchDeps: ["X"]` AFTER X in the list
- Link order: deps that PROVIDE symbols go after deps that USE them
## Removing a dependency
1. Delete `<name>.ts`
2. Remove from `allDeps` in `index.ts`
3. If any other dep has `fetchDeps: ["<name>"]`, remove that reference
## Updating a commit
Change the `commit` field. That's it. The build system computes a source
identity hash from `sha256(commit + patch_contents)` — changing the commit
invalidates `.ref`, triggers re-fetch, and everything downstream rebuilds.
## Common fields
```ts
export const mydep: Dependency = {
name: "mydep",
// Source tarball. Fetched from GitHub's archive endpoint (no git history,
// just the files at `commit`). Most deps use this.
//
// Other kinds: `prebuilt` (download pre-compiled .a, e.g. WebKit default),
// `local` (user manages vendor/<name>/ manually — only WebKit uses this
// because its clone is too slow to automate), `in-tree` (source in src/).
source: () => ({ kind: "github-archive", repo: "owner/repo", commit: "..." }),
// Optional: macro name for bun_dependency_versions.h (process.versions).
// Omit if this dep shouldn't appear there.
versionMacro: "MYDEP",
// Optional: .patch files applied after extraction, or overlay files
// copied into source root (e.g. inject a CMakeLists.txt).
patches: ["patches/mydep/fix-something.patch"],
// Optional: deps whose SOURCE must be ready before this one builds
// (for -I cross-dep headers). See libarchive for an example.
fetchDeps: ["zlib"],
// How to build.
build: cfg => ({
kind: "nested-cmake",
args: { MY_OPTION: "ON" },
// targets: [...], // cmake --build --target X. Defaults to lib names.
// extraCFlags: [...], // Appended to CMAKE_C_FLAGS.
// libSubdir: "lib", // If libs land in a subdir of the build dir.
// sourceSubdir: "...", // If CMakeLists.txt isn't at the source root.
// pic: true, // Add -fPIC (and suppress apple -fno-pic).
// buildType: "Release", // Force build type (e.g. lshpack).
}),
// What this dep provides. Paths relative to BUILD dir (libs) or
// SOURCE dir (includes).
provides: cfg => ({
libs: ["mydep"], // bare name → libmydep.a; path with '.' → used as-is
includes: ["include"],
// defines: ["MY_DEP_STATIC=1"], // Preprocessor defines for bun's compile.
}),
// Optional: skip this dep on some platforms.
enabled: cfg => !cfg.windows,
};
```
## Build types
- **`nested-cmake`**: Most deps. Runs `cmake --fresh -B ...` then `cmake --build`.
See `NestedCmakeBuild` in `../source.ts` for all fields.
- **`cargo`**: Rust deps (currently just lolhtml). See `CargoBuild` in `../source.ts`.
- **`none`**: Header-only or prebuilt. No build step; `.ref` stamp is the output.
## Worked examples
- **boringssl.ts** (33 lines) — simplest possible cmake dep
- **zstd.ts** — `sourceSubdir` (CMakeLists.txt not at repo root)
- **libarchive.ts** — `fetchDeps` + `extraCFlags` for cross-dep headers
- **mimalloc.ts** — complex conditional args, lib name depends on config
- **tinycc.ts** — overlay patches (inject a CMakeLists.txt)
- **lolhtml.ts** — cargo build with rustflags
- **sqlite.ts** — in-tree source (lives in `src/`, not `vendor/`)
## How the three-step build works
Each dep becomes three ninja build statements, each with `restat = 1`:
1. **fetch**`vendor/<name>/.ref` stamp
- Downloads tarball, extracts, applies patches
- `.ref` contains `sha256(commit + patches)[:16]`
- restat: if identity unchanged, no write, downstream pruned
2. **configure**`buildDir/deps/<name>/CMakeCache.txt`
- `cmake --fresh -B <dir> -D...`
- `--fresh` drops the cache so stale -D values don't persist
- restat: inner cmake might not touch cache
3. **build**`.a` files
- `cmake --build <dir> --target ...`
- restat: inner ninja no-ops if nothing changed
`restat` is what makes incremental builds fast — if step N was a no-op,
ninja prunes everything after it.

View File

@@ -0,0 +1,31 @@
/**
* BoringSSL — Google's OpenSSL fork. Provides TLS, all crypto primitives,
* and the x509 machinery that node:crypto needs.
*/
import type { Dependency } from "../source.ts";
const BORINGSSL_COMMIT = "4f4f5ef8ebc6e23cbf393428f0ab1b526773f7ac";
export const boringssl: Dependency = {
name: "boringssl",
versionMacro: "BORINGSSL",
source: () => ({
kind: "github-archive",
repo: "oven-sh/boringssl",
commit: BORINGSSL_COMMIT,
}),
build: () => ({
kind: "nested-cmake",
// No explicit targets — defaults to lib names (crypto, ssl, decrepit).
// BoringSSL's cmake targets match its output library names.
args: {},
}),
provides: () => ({
libs: ["crypto", "ssl", "decrepit"],
includes: ["include"],
}),
};

View File

@@ -0,0 +1,51 @@
/**
* Brotli — high-ratio compression. Backs the `br` Content-Encoding in fetch
* and bun's --compress bundler flag.
*/
import type { Dependency, NestedCmakeBuild } from "../source.ts";
// Upstream brotli pins releases by tag, not commit. A retag would change
// what we fetch — if that ever matters, resolve the tag to a sha and pin that.
const BROTLI_COMMIT = "v1.1.0";
export const brotli: Dependency = {
name: "brotli",
source: () => ({
kind: "github-archive",
repo: "google/brotli",
commit: BROTLI_COMMIT,
}),
build: cfg => {
const spec: NestedCmakeBuild = {
kind: "nested-cmake",
args: {
BROTLI_BUILD_TOOLS: "OFF",
BROTLI_EMSCRIPTEN: "OFF",
BROTLI_DISABLE_TESTS: "ON",
},
};
// LTO miscompile: on linux-x64 with AVX (non-baseline), BrotliDecompress
// errors out mid-stream. Root cause unknown — likely an alias-analysis
// issue around brotli's ring-buffer copy hoisting. -fno-lto sidesteps it.
// Linux-only: clang's LTO on darwin/windows has a different codepath.
// x64+non-baseline only: the SSE/AVX path is where the miscompile lives;
// baseline (SSE2-only) doesn't hit it.
if (cfg.linux && cfg.x64 && !cfg.baseline) {
spec.extraCFlags = ["-fno-lto"];
}
return spec;
},
provides: () => ({
// Order matters for static linking: common must come LAST on the link
// line (dec and enc both depend on it — unresolved symbols from dec/enc
// are searched for in later libs).
libs: ["brotlidec", "brotlienc", "brotlicommon"],
includes: ["c/include"],
}),
};

View File

@@ -0,0 +1,45 @@
/**
* c-ares — async DNS resolver. Backs node:dns and the Happy Eyeballs logic
* in bun's HTTP client. Async is the point — libc's getaddrinfo blocks.
*/
import type { Dependency } from "../source.ts";
const CARES_COMMIT = "3ac47ee46edd8ea40370222f91613fc16c434853";
export const cares: Dependency = {
name: "cares",
versionMacro: "C_ARES",
source: () => ({
kind: "github-archive",
repo: "c-ares/c-ares",
commit: CARES_COMMIT,
}),
build: () => ({
kind: "nested-cmake",
targets: ["c-ares"],
// c-ares uses -fPIC internally for worker-thread sharing reasons (its
// thread-local resolver state has to be position-independent on some
// platforms). CARES_STATIC_PIC reflects this; we also set pic: true
// so our flag tracking stays consistent.
pic: true,
args: {
CARES_STATIC: "ON",
CARES_STATIC_PIC: "ON",
CARES_SHARED: "OFF",
CARES_BUILD_TOOLS: "OFF",
// Without this c-ares installs to ${prefix}/lib64 on some linux distros
// (multilib convention). We never install, but it also affects the
// build-tree output path on those systems.
CMAKE_INSTALL_LIBDIR: "lib",
},
libSubdir: "lib",
}),
provides: () => ({
libs: ["cares"],
includes: ["include"],
}),
};

View File

@@ -0,0 +1,36 @@
/**
* HdrHistogram_c — high-dynamic-range latency histogram. Used by bun test's
* per-test timing output and benchmark reporting.
*/
import type { Dependency } from "../source.ts";
const HDRHISTOGRAM_COMMIT = "be60a9987ee48d0abf0d7b6a175bad8d6c1585d1";
export const hdrhistogram: Dependency = {
name: "hdrhistogram",
source: () => ({
kind: "github-archive",
repo: "HdrHistogram/HdrHistogram_c",
commit: HDRHISTOGRAM_COMMIT,
}),
build: () => ({
kind: "nested-cmake",
args: {
HDR_HISTOGRAM_BUILD_SHARED: "OFF",
HDR_HISTOGRAM_BUILD_STATIC: "ON",
// Disables the zlib-dependent log writer. We only need the in-memory
// histogram API — serialization goes through our own code.
HDR_LOG_REQUIRED: "DISABLED",
HDR_HISTOGRAM_BUILD_PROGRAMS: "OFF",
},
libSubdir: "src",
}),
provides: () => ({
libs: ["hdr_histogram_static"],
includes: ["include"],
}),
};

View File

@@ -0,0 +1,56 @@
/**
* Google Highway — portable SIMD intrinsics with runtime dispatch. Used by
* bun's string search (indexOf fastpaths), base64 codec, and the bundler's
* chunk hashing.
*
* Highway compiles every function for multiple targets (SSE2/AVX2/NEON/etc.)
* and picks at runtime. That's why it needs PIC — the dispatch tables are
* function pointers.
*/
import type { Dependency, NestedCmakeBuild } from "../source.ts";
const HIGHWAY_COMMIT = "ac0d5d297b13ab1b89f48484fc7911082d76a93f";
export const highway: Dependency = {
name: "highway",
source: () => ({
kind: "github-archive",
repo: "google/highway",
commit: HIGHWAY_COMMIT,
}),
patches: ["patches/highway/silence-warnings.patch"],
build: cfg => {
const spec: NestedCmakeBuild = {
kind: "nested-cmake",
pic: true,
args: {
HWY_ENABLE_TESTS: "OFF",
HWY_ENABLE_EXAMPLES: "OFF",
HWY_ENABLE_CONTRIB: "OFF",
HWY_ENABLE_INSTALL: "OFF",
},
};
// clang-cl on arm64-windows doesn't define __ARM_NEON even though NEON
// intrinsics work. Highway's cpu-feature detection is gated on the macro,
// so without it you get a scalar-only build. The underlying clang does
// support NEON here — it's a clang-cl frontend quirk.
if (cfg.windows && cfg.arm64) {
spec.extraCFlags = ["-D__ARM_NEON=1"];
spec.extraCxxFlags = ["-D__ARM_NEON=1"];
}
return spec;
},
provides: () => ({
libs: ["hwy"],
// Highway's public header is <hwy/highway.h> but it includes siblings
// via "" paths — need both the root and the hwy/ subdir in -I.
includes: [".", "hwy"],
}),
};

View File

@@ -0,0 +1,85 @@
/**
* All vendored dependencies. Import this to get the full list.
*
* Order matters for two reasons:
* 1. `fetchDeps` relationships — a dep with fetchDeps must come AFTER the
* deps it references, so the referenced .ref stamp node exists in ninja
* when we add order-only edges to it. (zlib before libarchive.)
* 2. Link order — when these libs hit the final link line, static linking
* resolves left-to-right. Deps that PROVIDE symbols should come after
* deps that USE them. This list becomes the link order.
*/
import type { Dependency } from "../source.ts";
import { boringssl } from "./boringssl.ts";
import { brotli } from "./brotli.ts";
import { cares } from "./cares.ts";
import { hdrhistogram } from "./hdrhistogram.ts";
import { highway } from "./highway.ts";
import { libarchive } from "./libarchive.ts";
import { libdeflate } from "./libdeflate.ts";
import { libuv } from "./libuv.ts";
import { lolhtml } from "./lolhtml.ts";
import { lshpack } from "./lshpack.ts";
import { mimalloc } from "./mimalloc.ts";
import { nodejsHeaders } from "./nodejs-headers.ts";
import { picohttpparser } from "./picohttpparser.ts";
import { sqlite } from "./sqlite.ts";
import { tinycc } from "./tinycc.ts";
import { webkit } from "./webkit.ts";
import { zlib } from "./zlib.ts";
import { zstd } from "./zstd.ts";
/**
* All deps in dependency-resolution + link order.
*
* zlib FIRST — libarchive's fetchDeps references it.
* brotli libs in internal dep order (common last on link line).
* boringssl near the end — many things depend on crypto/ssl symbols.
*/
export const allDeps: readonly Dependency[] = [
// Header-only / source-only first — no link order concerns.
picohttpparser,
nodejsHeaders,
zlib,
zstd,
brotli,
libdeflate,
libarchive,
cares,
hdrhistogram,
highway,
libuv,
lolhtml,
lshpack,
mimalloc,
sqlite,
tinycc,
boringssl,
// WebKit LAST in link order — WTF/JSC provide symbols that everything
// above might reference (via JavaScriptCore types in headers).
webkit,
];
// Re-export individuals for direct import when needed.
export {
boringssl,
brotli,
cares,
hdrhistogram,
highway,
libarchive,
libdeflate,
libuv,
lolhtml,
lshpack,
mimalloc,
nodejsHeaders,
picohttpparser,
sqlite,
tinycc,
webkit,
zlib,
zstd,
};

View File

@@ -0,0 +1,89 @@
/**
* libarchive — multi-format archive reader/writer. Bun uses it for tarball
* extraction during `bun install` (npm packages ship as tarballs) and
* `bun pm pack`.
*
* We configure it minimally: only tar + gzip support, everything else
* disabled. No bzip2/lzma/zip/rar/etc. — if someone needs those they can
* use a userland library. Keeping the surface small avoids linking against
* a dozen codec libs we don't need.
*/
import type { Dependency } from "../source.ts";
import { depSourceDir } from "../source.ts";
const LIBARCHIVE_COMMIT = "9525f90ca4bd14c7b335e2f8c84a4607b0af6bdf";
export const libarchive: Dependency = {
name: "libarchive",
versionMacro: "LIBARCHIVE",
source: () => ({
kind: "github-archive",
repo: "libarchive/libarchive",
commit: LIBARCHIVE_COMMIT,
}),
patches: ["patches/libarchive/archive_write_add_filter_gzip.c.patch", "patches/libarchive/CMakeLists.txt.patch"],
// libarchive's configure-time check_include_file("zlib.h") needs zlib's
// headers on disk. We don't LINK zlib into libarchive (ENABLE_ZLIB=OFF) —
// we just need the compile-time knowledge that deflate exists so
// libarchive compiles its gzip filter instead of fork/exec'ing gzip(1).
fetchDeps: ["zlib"],
build: cfg => ({
kind: "nested-cmake",
targets: ["archive_static"],
pic: true,
libSubdir: "libarchive",
// -I into zlib's SOURCE dir (vendor/zlib/). This is why fetchDeps exists:
// the zlib source must be on disk before libarchive's configure runs.
extraCFlags: [`-I${depSourceDir(cfg, "zlib")}`],
args: {
ENABLE_INSTALL: "OFF",
ENABLE_TEST: "OFF",
ENABLE_WERROR: "OFF",
// ─── Codecs we DON'T want ───
// Every ENABLE_X=OFF here is a codec that libarchive would otherwise
// detect from the system and link against. We want a hermetic build:
// tar + gzip, nothing else.
ENABLE_BZip2: "OFF",
ENABLE_CAT: "OFF",
ENABLE_CPIO: "OFF",
ENABLE_UNZIP: "OFF",
ENABLE_EXPAT: "OFF",
ENABLE_ICONV: "OFF",
ENABLE_LIBB2: "OFF",
ENABLE_LibGCC: "OFF",
ENABLE_LIBXML2: "OFF",
ENABLE_WIN32_XMLLITE: "OFF",
ENABLE_LZ4: "OFF",
ENABLE_LZMA: "OFF",
ENABLE_LZO: "OFF",
ENABLE_MBEDTLS: "OFF",
ENABLE_NETTLE: "OFF",
ENABLE_OPENSSL: "OFF",
ENABLE_PCRE2POSIX: "OFF",
ENABLE_PCREPOSIX: "OFF",
ENABLE_ZSTD: "OFF",
// ─── Gzip: "don't link zlib, but trust us, the header exists" ───
// ENABLE_ZLIB=OFF stops libarchive from linking zlib itself (we link
// it at the final bun link step). HAVE_ZLIB_H=ON overrides the
// configure-time detection — without it, libarchive compiles its
// gzip filter as a wrapper around /usr/bin/gzip (fork+exec per
// archive), which is slow and breaks on systems without gzip.
ENABLE_ZLIB: "OFF",
HAVE_ZLIB_H: "ON",
},
}),
provides: () => ({
libs: ["archive"],
includes: ["include"],
}),
};

View File

@@ -0,0 +1,37 @@
/**
* libdeflate — fast deflate/gzip/zlib codec. Faster than zlib for one-shot
* compression (no streaming). Used by Blob.gzip() and bun's .gz asset loader.
*/
import type { Dependency } from "../source.ts";
const LIBDEFLATE_COMMIT = "c8c56a20f8f621e6a966b716b31f1dedab6a41e3";
export const libdeflate: Dependency = {
name: "libdeflate",
versionMacro: "LIBDEFLATE_HASH",
source: () => ({
kind: "github-archive",
repo: "ebiggers/libdeflate",
commit: LIBDEFLATE_COMMIT,
}),
build: () => ({
kind: "nested-cmake",
targets: ["libdeflate_static"],
args: {
LIBDEFLATE_BUILD_STATIC_LIB: "ON",
LIBDEFLATE_BUILD_SHARED_LIB: "OFF",
LIBDEFLATE_BUILD_GZIP: "OFF",
},
}),
// Windows output is `deflatestatic.lib`, unix is `libdeflate.a`. Same code,
// different naming because libdeflate's CMakeLists uses a target-specific
// OUTPUT_NAME on win32 (avoids the windows convention of prefixing "lib").
provides: cfg => ({
libs: [cfg.windows ? "deflatestatic" : "deflate"],
includes: ["."],
}),
};

View File

@@ -0,0 +1,55 @@
/**
* libuv — cross-platform async I/O. Bun uses it on Windows ONLY, for the
* event loop and file I/O (Windows' IOCP model needs a proper abstraction
* layer). On unix, bun's event loop is custom (kqueue/epoll direct).
*
* Built everywhere despite being windows-only at runtime, because node-api
* addons may reference libuv symbols and expect them to link.
*/
import type { Dependency, NestedCmakeBuild } from "../source.ts";
// Latest HEAD as of pin — includes recursion bug fix #4784 (a stack
// overflow in uv__run_timers with many concurrent timers).
const LIBUV_COMMIT = "f3ce527ea940d926c40878ba5de219640c362811";
export const libuv: Dependency = {
name: "libuv",
source: () => ({
kind: "github-archive",
repo: "libuv/libuv",
commit: LIBUV_COMMIT,
}),
patches: ["patches/libuv/fix-win-pipe-cancel-race.patch"],
build: cfg => {
const spec: NestedCmakeBuild = {
kind: "nested-cmake",
targets: ["uv_a"],
args: {
LIBUV_BUILD_SHARED: "OFF",
LIBUV_BUILD_TESTS: "OFF",
LIBUV_BUILD_BENCH: "OFF",
},
};
if (cfg.windows) {
// libuv's windows code has a handful of int-conversion warnings that
// clang-cl elevates. /DWIN32 /D_WINDOWS are what MSVC's cmake preset
// would add automatically; libuv's headers gate win32 paths on them.
spec.extraCFlags = ["/DWIN32", "/D_WINDOWS", "-Wno-int-conversion"];
}
return spec;
},
provides: cfg => ({
// uv_a → libuv.lib on windows (the cmake target sets OUTPUT_NAME=libuv),
// libuv's cmake sets OUTPUT_NAME=libuv on Windows to avoid conflicts
// with system uv.lib. Unix uses the bare name.
libs: [cfg.windows ? "libuv" : "uv"],
includes: ["include"],
}),
};

View File

@@ -0,0 +1,60 @@
/**
* lol-html — Cloudflare's streaming HTML rewriter. Powers `HTMLRewriter` in
* bun and Workers. Rust crate with C FFI bindings.
*
* This is the only cargo-built dep. The C API crate lives under `c-api/`;
* the root is the pure-rust library (which we don't use directly).
*/
import type { CargoBuild, Dependency } from "../source.ts";
const LOLHTML_COMMIT = "e3aa54798602dd27250fafde1b5a66f080046252";
export const lolhtml: Dependency = {
name: "lolhtml",
versionMacro: "LOLHTML",
source: () => ({
kind: "github-archive",
repo: "cloudflare/lol-html",
commit: LOLHTML_COMMIT,
}),
build: cfg => {
const spec: CargoBuild = {
kind: "cargo",
manifestDir: "c-api",
libName: "lolhtml",
};
// On non-Windows we tell rustc to optimize for size and disable unwinding.
// lol-html doesn't catch_unwind anywhere, and the FFI boundary is already
// abort-on-panic (C can't unwind rust frames safely). Dropping unwind
// tables saves ~200KB and force-unwind-tables=no is the knob for that.
//
// Windows REQUIRES unwind tables for SEH — the OS loader refuses to run
// binaries without them on 64-bit. So this is unix-only.
if (!cfg.windows) {
spec.rustflags = ["-Cpanic=abort", "-Cdebuginfo=0", "-Cforce-unwind-tables=no", "-Copt-level=s"];
}
// arm64-windows: cargo defaults to the host triple, but CI builds arm64
// windows binaries on x64 runners. Explicit triple forces the cross-compile.
// (x64-windows doesn't need this — host IS target.)
if (cfg.windows && cfg.arm64) {
spec.rustTarget = "aarch64-pc-windows-msvc";
}
return spec;
},
provides: () => ({
// CargoBuild.libName handles the output path; provides.libs is not
// consulted for cargo deps (emitCargo constructs the path directly).
// We still list it for clarity.
libs: ["lolhtml"],
// No includes — bun's c-api binding header is checked into
// src/bun.js/bindings/, not read from the crate.
includes: [],
}),
};

View File

@@ -0,0 +1,55 @@
/**
* ls-hpack — HPACK header compression for HTTP/2. Litespeed's implementation;
* faster than nghttp2's for our workloads.
*/
import type { Dependency, NestedCmakeBuild } from "../source.ts";
const LSHPACK_COMMIT = "8905c024b6d052f083a3d11d0a169b3c2735c8a1";
export const lshpack: Dependency = {
name: "lshpack",
versionMacro: "LSHPACK",
source: () => ({
kind: "github-archive",
repo: "litespeedtech/ls-hpack",
commit: LSHPACK_COMMIT,
}),
patches: ["patches/lshpack/CMakeLists.txt.patch"],
build: cfg => {
const spec: NestedCmakeBuild = {
kind: "nested-cmake",
args: {
SHARED: "OFF",
LSHPACK_XXH: "ON",
},
// FORCE Release even in debug builds.
//
// lshpack's Debug config adds -fsanitize=address in its own CMakeLists,
// but doesn't link asan — it expects the consuming executable to. Our
// debug link doesn't satisfy those symbols on darwin (the asan runtime
// there is a dylib, not a static lib, so __asan_handle_no_return and
// friends resolve at load time — but lshpack.a references them at link
// time). Forcing Release drops the -fsanitize flag entirely.
//
// If we ever want to asan-test lshpack itself, do it in a separate
// standalone build where asan linking is under our control.
buildType: "Release",
};
if (cfg.windows) {
spec.extraCFlags = ["-w"];
}
return spec;
},
provides: cfg => ({
libs: ["ls-hpack"],
// Windows needs compat/queue for <sys/queue.h> shim (LIST_HEAD/etc. macros
// that don't exist on win32). On unix the real sys/queue.h is used.
includes: cfg.windows ? [".", "compat/queue"] : ["."],
}),
};

View File

@@ -0,0 +1,154 @@
/**
* mimalloc — Microsoft's memory allocator. Bun's global malloc replacement
* on Linux, and the JS heap allocator everywhere.
*/
import type { Dependency, NestedCmakeBuild, Provides } from "../source.ts";
const MIMALLOC_COMMIT = "1beadf9651a7bfdec6b5367c380ecc3fe1c40d1a";
export const mimalloc: Dependency = {
name: "mimalloc",
versionMacro: "MIMALLOC",
source: () => ({
kind: "github-archive",
repo: "oven-sh/mimalloc",
commit: MIMALLOC_COMMIT,
}),
build: cfg => {
const args: Record<string, string> = {
// Always build both the static lib AND the object-library target.
// We link the object file directly on some platforms (see provides()).
MI_BUILD_STATIC: "ON",
MI_BUILD_OBJECT: "ON",
MI_BUILD_SHARED: "OFF",
MI_BUILD_TESTS: "OFF",
// Compile mimalloc's .c files as C++. Required because we link against
// C++ code that uses mimalloc types, and C/C++ ABI can differ (notably
// around structs with trailing flexible arrays).
MI_USE_CXX: "ON",
// Don't walk all heaps on exit. Bun's shutdown is already complicated
// enough without mimalloc traversing every live allocation.
MI_SKIP_COLLECT_ON_EXIT: "ON",
// Disable Transparent Huge Pages. Measured impact:
// bun --eval 1: THP off = 30MB peak, THP on = 52MB peak
// http-hello.js: THP off = 52MB peak, THP on = 74MB peak
// THP trades memory for (sometimes) latency; for a JS runtime the
// memory cost isn't worth it.
MI_NO_THP: "1",
};
const extraCFlags: string[] = [];
const extraCxxFlags: string[] = [];
if (cfg.abi === "musl") {
args.MI_LIBC_MUSL = "ON";
}
// ─── Override behavior (global malloc replacement) ───
// The decision matrix:
// ASAN: always OFF — ASAN interceptors must see the real malloc.
// macOS: OFF — macOS's malloc zones are sufficient and overriding
// causes issues with system frameworks (SecureTransport, etc.)
// that have their own allocator expectations.
// Linux: ON — this is the main win. All malloc/free goes through
// mimalloc, including WebKit's bmalloc when it falls back
// to system malloc.
if (cfg.asan) {
args.MI_TRACK_ASAN = "ON";
args.MI_OVERRIDE = "OFF";
args.MI_OSX_ZONE = "OFF";
args.MI_OSX_INTERPOSE = "OFF";
// Mimalloc's UBSan integration: sets up shadow memory annotations
// so UBSan doesn't false-positive on mimalloc's type punning.
args.MI_DEBUG_UBSAN = "ON";
} else if (cfg.darwin) {
args.MI_OVERRIDE = "OFF";
args.MI_OSX_ZONE = "OFF";
args.MI_OSX_INTERPOSE = "OFF";
} else if (cfg.linux) {
args.MI_OVERRIDE = "ON";
args.MI_OSX_ZONE = "OFF";
args.MI_OSX_INTERPOSE = "OFF";
}
// Windows: use mimalloc's defaults (no override; Windows has its own
// mechanism via the static CRT we link).
if (cfg.debug) {
// Heavy debug checks: guard bytes, freed-memory poisoning, double-free
// detection. Slow but catches memory bugs early.
args.MI_DEBUG_FULL = "ON";
}
if (cfg.valgrind) {
// Mimalloc annotations so valgrind understands its arena layout
// (without this, every mimalloc alloc looks like a leak).
args.MI_TRACK_VALGRIND = "ON";
}
// If mimalloc gets bumped to a version with MI_OPT_ARCH: pass
// MI_NO_OPT_ARCH=ON to stop it setting -march=armv8.1-a on arm64
// (SIGILLs on ARMv8.0 CPUs). Current pin has no arch-detection logic
// so our global -march=armv8-a+crc (via CMAKE_CXX_FLAGS) is sufficient.
// ─── Windows: silence the vendored-C-as-C++ warning flood ───
// MI_USE_CXX=ON means .c files compile as C++. clang-cl then complains
// about every C-ism: old-style casts, zero-as-null, C++98 compat, etc.
// It's noise — mimalloc is correct C, just not idiomatic C++.
if (cfg.windows) {
extraCFlags.push("-w");
extraCxxFlags.push("-w");
}
const spec: NestedCmakeBuild = {
kind: "nested-cmake",
targets: ["mimalloc-static", "mimalloc-obj"],
args,
};
if (extraCFlags.length > 0) spec.extraCFlags = extraCFlags;
if (extraCxxFlags.length > 0) spec.extraCxxFlags = extraCxxFlags;
return spec;
},
provides: cfg => {
// mimalloc's output library name depends on config flags passed to its
// CMake. It appends suffixes based on what's enabled — there's no way
// to override this, so we have to mirror its naming logic.
let libname: string;
if (cfg.windows) {
libname = cfg.debug ? "mimalloc-static-debug" : "mimalloc-static";
} else if (cfg.debug) {
libname = cfg.asan ? "mimalloc-asan-debug" : "mimalloc-debug";
} else {
libname = "mimalloc";
}
// WORKAROUND: Link the object file directly, not the .a.
//
// Linking libmimalloc.a on macOS (and Linux release) produces duplicate
// symbol errors at link time — mimalloc's static.c is a "unity build"
// TU that #includes all other .c files, and libmimalloc.a ALSO contains
// the individually-compiled .o files. The linker pulls both and barfs.
// See https://github.com/microsoft/mimalloc/issues/512.
//
// The fix: link mimalloc-obj's single static.c.o directly. One TU, all
// symbols, no archive index to confuse the linker.
//
// We only do this on apple + linux-release because the CMake build has
// been working this way for years. Debug Linux uses the .a successfully
// (possibly because -g changes symbol visibility in a way that dodges
// the issue, or the debug .a is built differently — haven't dug into it).
const useObjectFile = cfg.darwin || (cfg.linux && cfg.release);
const provides: Provides = {
libs: useObjectFile ? ["CMakeFiles/mimalloc-obj.dir/src/static.c.o"] : [libname],
includes: ["include"],
};
return provides;
},
};

View File

@@ -0,0 +1,43 @@
/**
* Node.js headers — for N-API compatibility.
*
* Downloaded from nodejs.org releases. Headers-only (no libs). After
* extraction we delete `openssl/` and `uv/` subdirs — bun uses BoringSSL
* (not OpenSSL) and its own libuv, and the bundled headers conflict.
*/
import { resolve } from "node:path";
import type { Dependency } from "../source.ts";
/**
* Node.js compat version — reported via process.version, used for headers
* download URL, and passed to zig as -Dreported_nodejs_version.
* Override via `--nodejs-version=X.Y.Z` to test a bump.
*/
export const NODEJS_VERSION = "24.3.0";
/** Node.js NODE_MODULE_VERSION — for native addon ABI compat. */
export const NODEJS_ABI_VERSION = "137";
export const nodejsHeaders: Dependency = {
name: "nodejs",
source: cfg => ({
kind: "prebuilt",
url: `https://nodejs.org/dist/v${cfg.nodejsVersion}/node-v${cfg.nodejsVersion}-headers.tar.gz`,
identity: cfg.nodejsVersion,
// Delete headers that conflict with BoringSSL / our libuv.
// Tarball top-level is `node-v<version>/` (hoisted), inside is `include/node/`.
rmAfterExtract: ["include/node/openssl", "include/node/uv", "include/node/uv.h"],
destDir: resolve(cfg.cacheDir, `nodejs-headers-${cfg.nodejsVersion}`),
}),
build: () => ({ kind: "none" }),
provides: () => ({
libs: [],
// Both include/ and include/node/ — some files use <node/foo.h>,
// some use <foo.h>. CMake adds both.
includes: ["include", "include/node"],
}),
};

View File

@@ -0,0 +1,32 @@
/**
* picohttpparser — tiny HTTP parser. Single .c file, no build system.
*
* No `.a` produced — bun compiles `picohttpparser.c` directly into its
* binary. `provides.sources` tells the build system which files; they're
* declared as implicit outputs of the fetch rule so ninja knows they
* exist once fetch completes (otherwise: "missing and no known rule to
* make it" on fresh checkouts).
*/
import type { Dependency } from "../source.ts";
const PICOHTTPPARSER_COMMIT = "066d2b1e9ab820703db0837a7255d92d30f0c9f5";
export const picohttpparser: Dependency = {
name: "picohttpparser",
versionMacro: "PICOHTTPPARSER",
source: () => ({
kind: "github-archive",
repo: "h2o/picohttpparser",
commit: PICOHTTPPARSER_COMMIT,
}),
build: () => ({ kind: "none" }),
provides: () => ({
libs: [],
includes: ["."],
sources: ["picohttpparser.c"],
}),
};

View File

@@ -0,0 +1,34 @@
/**
* SQLite — embedded SQL database. Backs bun:sqlite.
*
* Source lives IN THE BUN REPO at src/bun.js/bindings/sqlite/ — it's the
* sqlite3 amalgamation (single .c file) plus a small CMakeLists.txt we
* maintain. No fetch step; the source is tracked in git.
*
* Only built when staticSqlite=true. Otherwise bun dlopen()s the system
* sqlite at runtime (macOS ships a recent sqlite; most linux distros don't,
* so static is the default on linux).
*/
import type { Dependency } from "../source.ts";
export const sqlite: Dependency = {
name: "sqlite",
enabled: cfg => cfg.staticSqlite,
source: () => ({
kind: "in-tree",
path: "src/bun.js/bindings/sqlite",
}),
build: () => ({
kind: "nested-cmake",
args: {},
}),
provides: () => ({
libs: ["sqlite3"],
includes: ["."],
}),
};

View File

@@ -0,0 +1,48 @@
/**
* TinyCC — small embeddable C compiler. Powers bun:ffi's JIT-compile path,
* where user-provided C gets compiled and linked at runtime.
*
* Disabled on windows-arm64 (tinycc doesn't have an arm64-coff backend).
*/
import type { Dependency, NestedCmakeBuild } from "../source.ts";
const TINYCC_COMMIT = "12882eee073cfe5c7621bcfadf679e1372d4537b";
export const tinycc: Dependency = {
name: "tinycc",
versionMacro: "TINYCC",
// The cfg.tinycc flag already encodes the windows-arm64 exclusion
// (see config.ts: `tinycc ?? !(windows && arm64)`).
enabled: cfg => cfg.tinycc,
source: () => ({
kind: "github-archive",
repo: "oven-sh/tinycc",
commit: TINYCC_COMMIT,
}),
// Our tinycc fork has no CMakeLists.txt — it uses a configure script. We
// inject one as an overlay file. (The proper fix is to commit this upstream
// to oven-sh/tinycc; see TODO in patches/tinycc/CMakeLists.txt.)
patches: ["patches/tinycc/CMakeLists.txt", "patches/tinycc/tcc.h.patch"],
build: cfg => {
const spec: NestedCmakeBuild = {
kind: "nested-cmake",
args: {},
};
// clang-cl is noisy about tinycc's old-C idioms (implicit int conversions,
// enum coercions). The code is correct; silence it.
if (cfg.windows) {
spec.extraCFlags = ["-w"];
}
return spec;
},
provides: () => ({
libs: ["tcc"],
includes: [],
}),
};

View File

@@ -0,0 +1,301 @@
/**
* WebKit commit — determines prebuilt download URL + what to checkout
* for local mode. Override via `--webkit-version=<hash>` to test a branch.
* From https://github.com/oven-sh/WebKit releases.
*/
export const WEBKIT_VERSION = "00e825523d549a556d75985f486e4954af6ab8c7";
/**
* WebKit (JavaScriptCore) — the JS engine.
*
* Two modes via `cfg.webkit`:
*
* **prebuilt**: Download tarball from oven-sh/WebKit releases. Tarball name
* encodes {os, arch, musl, debug|lto, asan} — each is a separate ABI.
* ASAN MUST match bun's setting: WTF::Vector layout changes with ASAN
* (see WTF/Vector.h:682), so mixing → silent memory corruption.
*
* **local**: Source at `vendor/WebKit/`. User clones manually (clone takes
* 10+ min — too slow for the build system to do). We cmake it like any
* other dep. Headers land in the BUILD dir (generated during configure),
* which is why `provides.includes` returns absolute paths.
*
* ## Implementation notes
*
* - Build dir is `buildDir/deps/webkit/` (generic path), NOT CMake's
* `vendor/WebKit/WebKitBuild/`. Better: consistent, cleaned by `rm -rf
* build/`, separate per-profile.
*
* - Flags: WebKit's own cmake machinery sets compiler flags. We set
* `CMAKE_C_FLAGS: ""` in our args to clear the global dep flags
* (which would otherwise conflict). Dep args go LAST in source.ts,
* so they override.
*
* - Windows local mode: ICU built from source via preBuild hook
* (build-icu.ps1 → msbuild) before cmake configure. Output goes in
* the per-profile build dir, not shared vendor/WebKit/WebKitBuild/icu/
* like the old cmake — avoids debug/release mixing.
*/
import { resolve } from "node:path";
import type { Config } from "../config.ts";
import { slash } from "../shell.ts";
import { type Dependency, type NestedCmakeBuild, type Source, depBuildDir, depSourceDir } from "../source.ts";
// ───────────────────────────────────────────────────────────────────────────
// Prebuilt URL computation
// ───────────────────────────────────────────────────────────────────────────
/**
* Tarball suffix encoding ABI-affecting flags. MUST match the WebKit
* release workflow naming in oven-sh/WebKit's CI.
*/
function prebuiltSuffix(cfg: Config): string {
let s = "";
if (cfg.linux && cfg.abi === "musl") s += "-musl";
// Baseline WebKit artifacts (-march=nehalem, /arch:SSE2 ICU) exist for
// Linux amd64 (glibc + musl) and Windows amd64. No baseline variant for
// arm64 or macOS. Suffix order matches the release asset names:
// bun-webkit-linux-amd64-musl-baseline-lto.tar.gz
if (cfg.baseline && cfg.x64) s += "-baseline";
if (cfg.debug) s += "-debug";
else if (cfg.lto) s += "-lto";
if (cfg.asan) s += "-asan";
return s;
}
function prebuiltUrl(cfg: Config): string {
const os = cfg.windows ? "windows" : cfg.darwin ? "macos" : "linux";
const arch = cfg.arm64 ? "arm64" : "amd64";
const name = `bun-webkit-${os}-${arch}${prebuiltSuffix(cfg)}`;
const version = cfg.webkitVersion;
const tag = version.startsWith("autobuild-") ? version : `autobuild-${version}`;
return `https://github.com/oven-sh/WebKit/releases/download/${tag}/${name}.tar.gz`;
}
/**
* Prebuilt extraction dir. Suffix in the key so switching debug ↔ release
* doesn't reuse a wrong-ABI extraction.
*/
function prebuiltDestDir(cfg: Config): string {
const version16 = cfg.webkitVersion.slice(0, 16);
return resolve(cfg.cacheDir, `webkit-${version16}${prebuiltSuffix(cfg)}`);
}
// ───────────────────────────────────────────────────────────────────────────
// Lib paths — relative to destDir (prebuilt) or buildDir (local)
// ───────────────────────────────────────────────────────────────────────────
/** Build a lib path under the WebKit install's lib/ dir. */
function wkLib(cfg: Config, name: string): string {
return `lib/${cfg.libPrefix}${name}${cfg.libSuffix}`;
}
/**
* Core libs (WTF, JSC) — always present.
*/
function coreLibs(cfg: Config): string[] {
return [wkLib(cfg, "WTF"), wkLib(cfg, "JavaScriptCore")];
}
function bmallocLib(cfg: Config): string {
return wkLib(cfg, "bmalloc");
}
/**
* ICU libs — prebuilt bundles them on linux/windows. macOS uses system ICU.
* Local mode: system ICU on posix (linked via -licu* in bun.ts); built from
* source on Windows (see icuDir/icuLibs).
*/
function prebuiltIcuLibs(cfg: Config): string[] {
if (cfg.windows) {
const d = cfg.debug ? "d" : "";
return [`lib/sicudt${d}.lib`, `lib/sicuin${d}.lib`, `lib/sicuuc${d}.lib`];
}
if (cfg.linux) {
return ["lib/libicudata.a", "lib/libicui18n.a", "lib/libicuuc.a"];
}
return []; // darwin: system ICU
}
// ───────────────────────────────────────────────────────────────────────────
// Windows local mode: ICU built from source via build-icu.ps1
//
// No system ICU on Windows. The script (in vendor/WebKit/) downloads ICU
// source, patches .vcxproj files for static+/MT, runs msbuild. Output goes
// under the WebKit build dir (NOT vendor/WebKit/WebKitBuild/icu/ like the
// old cmake did) — per-profile, so debug/release don't conflate.
// ───────────────────────────────────────────────────────────────────────────
/** Where build-icu.ps1 writes its output. Per-profile via buildDir. */
function icuDir(cfg: Config): string {
return resolve(depBuildDir(cfg, "WebKit"), "icu");
}
/**
* Libs produced by build-icu.ps1. Names are from the script's output
* (sicudt.lib, icuin.lib, icuuc.lib) — no `d` suffix needed since the
* per-profile dir already isolates debug/release.
*/
function localIcuLibs(cfg: Config): string[] {
const dir = icuDir(cfg);
return [resolve(dir, "lib", "sicudt.lib"), resolve(dir, "lib", "icuin.lib"), resolve(dir, "lib", "icuuc.lib")];
}
// ───────────────────────────────────────────────────────────────────────────
// The Dependency
// ───────────────────────────────────────────────────────────────────────────
export const webkit: Dependency = {
name: "WebKit",
versionMacro: "WEBKIT",
source: cfg => {
if (cfg.webkit === "prebuilt") {
const src: Source = {
kind: "prebuilt",
url: prebuiltUrl(cfg),
// Identity = version + suffix. Suffix ensures profile switches
// (debug ↔ release, asan toggle) trigger re-download. Without it,
// same version stamp would skip, leaving the wrong ABI on disk.
identity: `${cfg.webkitVersion}${prebuiltSuffix(cfg)}`,
destDir: prebuiltDestDir(cfg),
};
// macOS: bundled ICU headers conflict with system ICU.
if (cfg.darwin) {
src.rmAfterExtract = ["include/unicode"];
}
return src;
}
// Local: user clones vendor/WebKit/ manually (clone takes 10+ min — the
// one thing the build system doesn't automate). Once cloned, we cmake it
// like any other dep. resolveDep()'s local-mode assert gives a clear
// "clone it yourself" error if missing.
return { kind: "local" };
},
build: cfg => {
if (cfg.webkit === "prebuilt") {
return { kind: "none" };
}
// Local: nested cmake, target=jsc.
//
// CMAKE_C_FLAGS/CMAKE_CXX_FLAGS set to empty: clears the global dep
// flags source.ts would otherwise pass. WebKit's cmake sets its own
// -O/-march/etc.; ours would conflict. Dep args go LAST so they override.
//
// Windows: ICU built from source via preBuild before cmake configure.
// WebKit's cmake finds it via ICU_ROOT. On posix, system ICU is used
// (macOS: Homebrew headers + system libs; Linux: libicu-dev) — cmake
// auto-detects.
const args: Record<string, string> = {
CMAKE_C_FLAGS: "",
CMAKE_CXX_FLAGS: "",
PORT: "JSCOnly",
ENABLE_STATIC_JSC: "ON",
USE_THIN_ARCHIVES: "OFF",
ENABLE_FTL_JIT: "ON",
CMAKE_EXPORT_COMPILE_COMMANDS: "ON",
USE_BUN_JSC_ADDITIONS: "ON",
USE_BUN_EVENT_LOOP: "ON",
ENABLE_BUN_SKIP_FAILING_ASSERTIONS: "ON",
ALLOW_LINE_AND_COLUMN_NUMBER_IN_BUILTINS: "ON",
ENABLE_REMOTE_INSPECTOR: "ON",
ENABLE_MEDIA_SOURCE: "OFF",
ENABLE_MEDIA_STREAM: "OFF",
ENABLE_WEB_RTC: "OFF",
...(cfg.asan ? { ENABLE_SANITIZERS: "address" } : {}),
};
const spec: NestedCmakeBuild = { kind: "nested-cmake", targets: ["jsc"], args };
if (cfg.windows) {
const icu = icuDir(cfg);
const srcDir = depSourceDir(cfg, "WebKit");
// slash(): cmake -D values — see shell.ts.
args.ICU_ROOT = slash(icu);
args.ICU_LIBRARY = slash(resolve(icu, "lib"));
args.ICU_INCLUDE_DIR = slash(resolve(icu, "include"));
// U_STATIC_IMPLEMENTATION: ICU headers default to dllimport; we
// link statically. Matches what the old cmake's SetupWebKit did.
args.CMAKE_C_FLAGS = "/DU_STATIC_IMPLEMENTATION";
args.CMAKE_CXX_FLAGS = "/DU_STATIC_IMPLEMENTATION /clang:-fno-c++-static-destructors";
// Static CRT to match bun + all other deps (we build everything
// with /MTd or /MT). Without this, cmake defaults to /MDd →
// RuntimeLibrary mismatch at link.
args.CMAKE_MSVC_RUNTIME_LIBRARY = cfg.debug ? "MultiThreadedDebug" : "MultiThreaded";
spec.preBuild = {
command: [
"powershell",
"-ExecutionPolicy",
"Bypass",
"-File",
resolve(srcDir, "build-icu.ps1"),
"-Platform",
cfg.x64 ? "x64" : "ARM64",
"-BuildType",
cfg.debug ? "Debug" : "Release",
"-OutputDir",
icu,
],
cwd: srcDir,
outputs: localIcuLibs(cfg),
};
}
return spec;
},
provides: cfg => {
if (cfg.webkit === "prebuilt") {
// Paths relative to prebuilt destDir — emitPrebuilt resolves them.
//
// bmalloc: some historical prebuilts rolled it into JSC. Current
// versions ship it separately on all platforms. Listed here so
// emitPrebuilt declares it as an output — ninja knows fetch creates
// it. If a future version drops libbmalloc.a, you'll get a clear
// "file not found" at link time (not silent omission + cryptic
// undefined symbols).
const libs = [...coreLibs(cfg), ...prebuiltIcuLibs(cfg), bmallocLib(cfg)];
const includes = ["include"];
// Linux/windows: ICU headers under wtf/unicode. macOS: deleted by
// postExtract.
if (!cfg.darwin) includes.push("include/wtf/unicode");
return { libs, includes };
}
// Local: paths relative to BUILD dir (headers generated during build).
// includes uses ABSOLUTE paths via depBuildDir() — source.ts's
// resolve-against-srcDir would point at vendor/WebKit/ (wrong).
const buildDir = depBuildDir(cfg, "WebKit");
// Lib paths: emitNestedCmake resolves these relative to the build dir's
// libSubdir — we set none, so it's buildDir root. But WebKit's libs are
// in lib/. So include the lib/ prefix.
//
// Windows ICU libs are NOT listed here — they're preBuild.outputs,
// which source.ts appends to the resolved libs automatically. Listing
// them here would make dep_build also claim to produce them (dup error).
// Posix uses system ICU (linked via -licu* in bun.ts).
const libs = [...coreLibs(cfg), bmallocLib(cfg)];
const includes = [
// ABSOLUTE — resolved here because they're in the build dir, not src.
buildDir,
resolve(buildDir, "JavaScriptCore", "Headers"),
resolve(buildDir, "JavaScriptCore", "Headers", "JavaScriptCore"),
resolve(buildDir, "JavaScriptCore", "PrivateHeaders"),
resolve(buildDir, "bmalloc", "Headers"),
resolve(buildDir, "WTF", "Headers"),
resolve(buildDir, "JavaScriptCore", "PrivateHeaders", "JavaScriptCore"),
];
// Windows: ICU headers from preBuild output.
if (cfg.windows) includes.push(resolve(icuDir(cfg), "include"));
return { libs, includes };
},
};

View File

@@ -0,0 +1,67 @@
/**
* zlib — the classic deflate library. Cloudflare's fork has SIMD-accelerated
* CRC and hash (2-4x faster compression on modern CPUs). Backs node:zlib
* and the gzip Content-Encoding.
*/
import type { Dependency, NestedCmakeBuild } from "../source.ts";
const ZLIB_COMMIT = "886098f3f339617b4243b286f5ed364b9989e245";
export const zlib: Dependency = {
name: "zlib",
versionMacro: "ZLIB_HASH",
source: () => ({
kind: "github-archive",
repo: "cloudflare/zlib",
commit: ZLIB_COMMIT,
}),
// All patches are unconditional. The remove-machine-x64 patch deletes
// an old workaround for a cmake bug fixed in 2011 — we require cmake
// 3.24+ so the workaround is pure dead weight.
patches: [
"patches/zlib/CMakeLists.txt.patch",
"patches/zlib/deflate.h.patch",
"patches/zlib/ucm.cmake.patch",
"scripts/build/patches/zlib/remove-machine-x64.patch",
],
build: cfg => {
const spec: NestedCmakeBuild = {
kind: "nested-cmake",
targets: ["zlib"],
args: {
BUILD_EXAMPLES: "OFF",
},
};
// Apple clang defines TARGET_OS_* macros that conflict with zlib's own
// platform detection (it has `#ifdef TARGET_OS_MAC` gates that predate
// apple's convention). The flag makes clang stop auto-defining them.
// See https://gitlab.kitware.com/cmake/cmake/-/issues/25755.
if (cfg.darwin) {
spec.extraCFlags = ["-fno-define-target-os-macros"];
spec.extraCxxFlags = ["-fno-define-target-os-macros"];
}
return spec;
},
provides: cfg => {
// zlib's OUTPUT_NAME logic: on unix/mingw → "z", on MSVC → "zlib"
// (or "zlibd" for CMAKE_BUILD_TYPE=Debug — cmake's Windows debug
// suffix convention).
let lib: string;
if (cfg.windows) {
lib = cfg.debug ? "zlibd" : "zlib";
} else {
lib = "z";
}
return {
libs: [lib],
includes: ["."],
};
},
};

View File

@@ -0,0 +1,42 @@
/**
* Zstandard — fast compression with a good ratio/speed tradeoff. Backs
* bun's install cache and the `zstd` Content-Encoding in fetch.
*/
import type { Dependency } from "../source.ts";
const ZSTD_COMMIT = "f8745da6ff1ad1e7bab384bd1f9d742439278e99";
export const zstd: Dependency = {
name: "zstd",
versionMacro: "ZSTD_HASH",
source: () => ({
kind: "github-archive",
repo: "facebook/zstd",
commit: ZSTD_COMMIT,
}),
build: () => ({
kind: "nested-cmake",
targets: ["libzstd_static"],
// zstd's repo root has a Makefile; the cmake build files live under
// build/cmake/. (They support meson too — build/meson/ — but we stick
// with cmake for consistency.)
sourceSubdir: "build/cmake",
args: {
ZSTD_BUILD_STATIC: "ON",
ZSTD_BUILD_PROGRAMS: "OFF",
ZSTD_BUILD_TESTS: "OFF",
ZSTD_BUILD_CONTRIB: "OFF",
},
libSubdir: "lib",
}),
provides: cfg => ({
// Windows: cmake appends "_static" to distinguish from the DLL import lib.
libs: [cfg.windows ? "zstd_static" : "zstd"],
// Headers are in the SOURCE repo at lib/ (zstd.h, zdict.h).
includes: ["lib"],
}),
};

228
scripts/build/download.ts Normal file
View File

@@ -0,0 +1,228 @@
/**
* Download + archive extraction helpers.
*
* Used by the fetch CLIs in source.ts (dep tarballs), webkit.ts (prebuilt
* tarball), zig.ts (compiler zip). Extracted because the retry + temp-then-
* rename logic was copy-pasted three times and the platform-specific
* extraction quirks (tar vs unzip, -m for mtime) were starting to drift.
*
* ## Retry behavior
*
* Exponential backoff (1s → 2s → 4s → 8s → cap at 30s), 5 attempts. GitHub
* releases (our main source) are usually reliable, but CI sees transient
* CDN failures often enough that no-retry means flaky builds. The cap at
* 30s means worst-case we spend ~60s total before giving up.
*
* ## Atomic writes
*
* Download goes to `<dest>.partial`, renamed on success. If download is
* interrupted (ctrl-c, network drop, OOM), no partial file claims to be
* complete. Next build retries from scratch.
*
* ## Why not stream to disk
*
* We buffer the whole response in memory (`arrayBuffer()`) before writing.
* For zig (~50MB) and dep tarballs (~few MB) this is fine. For WebKit
* (~200MB) it's ~200MB peak memory. If that's ever a problem, switch to
* `Bun.write(dest, res)` which streams. Haven't bothered because CI
* machines have GBs of RAM and the whole download is a few seconds.
*/
import { spawnSync } from "node:child_process";
import { existsSync, readFileSync } from "node:fs";
import { mkdir, readdir, rename, rm, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { BuildError, assert } from "./error.ts";
/**
* Download a URL to a file with retry. Atomic: temp file → rename on success.
*
* @param logPrefix Shown in progress/retry messages: `[<logPrefix>] retry 2/5`
*/
export async function downloadWithRetry(url: string, dest: string, logPrefix: string): Promise<void> {
const maxAttempts = 5;
let lastError: unknown;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
if (attempt > 1) {
const backoffMs = Math.min(1000 * Math.pow(2, attempt - 1), 30000);
console.log(`retry ${attempt}/${maxAttempts} in ${backoffMs}ms`);
await new Promise(r => setTimeout(r, backoffMs));
}
try {
const res = await fetch(url, { headers: { "User-Agent": "bun-build-system" } });
if (!res.ok) {
lastError = new BuildError(`HTTP ${res.status} ${res.statusText} for ${url}`);
continue;
}
const tmpPath = `${dest}.partial`;
await rm(tmpPath, { force: true });
const buf = await res.arrayBuffer();
await writeFile(tmpPath, new Uint8Array(buf));
await rename(tmpPath, dest);
return;
} catch (err) {
lastError = err;
}
}
throw new BuildError(`Failed to download after ${maxAttempts} attempts: ${url}`, {
cause: lastError,
hint: "Check network connectivity, or place the file manually at the destination path",
});
}
/**
* Extract a .tar.gz archive with mtime normalization.
*
* `--strip-components=1` removes the top-level dir (github archives always
* have one: `<repo>-<commit>/`).
*
* `-m` sets extracted mtimes to NOW instead of the archive's stored mtimes.
* This is CRITICAL for correct incremental builds: tarballs store commit-time
* mtimes (e.g. 2023), so re-fetching at a new commit gives headers 2024-ish
* mtimes — older than any .o we built yesterday. Downstream ninja staleness
* checks miss the change entirely. With -m, everything extracted is "now",
* so any .o built BEFORE this extraction is correctly stale.
*
* @param stripComponents How many top-level dirs to strip. 1 for github
* archives. 0 for tarballs that are already flat (e.g. prebuilt WebKit
* has `bun-webkit/` that the caller wants to keep for a rename step).
*/
export async function extractTarGz(tarball: string, dest: string, stripComponents = 1): Promise<void> {
const args = ["-xzmf", tarball, "-C", dest];
if (stripComponents > 0) args.push(`--strip-components=${stripComponents}`);
const result = spawnSync("tar", args, {
stdio: ["ignore", "ignore", "pipe"],
encoding: "utf8",
});
if (result.error) {
throw new BuildError(`Failed to spawn tar`, {
hint: "Is `tar` in your PATH? (macOS/linux ship it; Windows 10+ ships bsdtar as tar.exe)",
cause: result.error,
});
}
if (result.status !== 0) {
throw new BuildError(`tar extraction failed (exit ${result.status}): ${result.stderr}`, { file: tarball });
}
const entries = await readdir(dest);
assert(entries.length > 0, `tar extracted nothing from ${tarball}`, { hint: "Tarball may be corrupt" });
}
/**
* Extract a .zip archive with mtime normalization.
*
* Tries `unzip` first (most systems), falls back to `tar` (bsdtar — what
* Windows 10+ ships as tar.exe — handles .zip).
*
* `-DD` (unzip) / `-m` (tar) for the same mtime-fix as extractTarGz.
*
* Does NOT strip top-level dir — zip layouts vary, caller handles hoisting.
*/
export async function extractZip(zipPath: string, dest: string): Promise<void> {
// unzip -o: overwrite, -DD: don't restore timestamps, -d: destination.
const unzipResult = spawnSync("unzip", ["-o", "-DD", "-d", dest, zipPath], {
stdio: ["ignore", "ignore", "pipe"],
encoding: "utf8",
});
if (unzipResult.status === 0) return;
// bsdtar auto-detects .zip. -m: don't preserve mtimes.
const tarResult = spawnSync("tar", ["-xmf", zipPath, "-C", dest], {
stdio: ["ignore", "ignore", "pipe"],
encoding: "utf8",
});
if (tarResult.status === 0) return;
throw new BuildError(
`Failed to extract zip (tried unzip and tar):\n` +
` unzip: ${unzipResult.error?.message ?? `exit ${unzipResult.status}: ${unzipResult.stderr}`}\n` +
` tar: ${tarResult.error?.message ?? `exit ${tarResult.status}: ${tarResult.stderr}`}`,
{ file: zipPath, hint: "Install unzip: apt install unzip / brew install unzip" },
);
}
/**
* Fetch a prebuilt tarball: download + extract + write identity stamp.
*
* Generic mechanism for the `{ kind: "prebuilt" }` Source variant. Download a
* tarball with pre-compiled libraries, extract to `dest/`, write `.identity`
* stamp. On next fetch, if stamp matches, skip download (restat prunes).
*
* Tarball layout assumption: single top-level directory. We extract to a
* staging dir, hoist the single child into `dest/`. Matches GitHub release
* asset conventions (WebKit's `bun-webkit/`, future deps' similar layouts).
* If a tarball has multiple top-level entries, the whole staging dir becomes
* `dest/` (no hoist).
*
* @param identity Written to `dest/.identity`. Changing it triggers re-download.
* @param rmPaths Paths (relative to `dest/`) to delete after extraction.
* Used to remove conflicting headers (WebKit's unicode/, nodejs's openssl/).
* Deleted via fs.rm — no shell, cross-platform.
*/
export async function fetchPrebuilt(
name: string,
url: string,
dest: string,
identity: string,
rmPaths: string[] = [],
): Promise<void> {
const stampPath = resolve(dest, ".identity");
// ─── Short-circuit: already at this identity? ───
if (existsSync(stampPath)) {
const existing = readFileSync(stampPath, "utf8").trim();
if (existing === identity) {
console.log(`up to date`);
return; // restat no-op
}
console.log(`identity changed (was ${existing.slice(0, 16)}, now ${identity.slice(0, 16)}), re-fetching`);
}
console.log(`fetching ${url}`);
// ─── Download ───
const destParent = resolve(dest, "..");
await mkdir(destParent, { recursive: true });
const tarballPath = `${dest}.download.tar.gz`;
await downloadWithRetry(url, tarballPath, name);
// ─── Extract ───
// Wipe dest first — no stale files from a previous version.
// Extract to staging dir, then hoist. We don't extract directly into dest/
// because the tarball's top-level dir name is unpredictable (e.g.
// `bun-webkit/` vs `libfoo-1.2.3/`).
await rm(dest, { recursive: true, force: true });
const stagingDir = `${dest}.staging`;
await rm(stagingDir, { recursive: true, force: true });
await mkdir(stagingDir, { recursive: true });
// stripComponents=0: keep top-level dir for hoisting.
await extractTarGz(tarballPath, stagingDir, 0);
await rm(tarballPath, { force: true });
// Hoist: if single top-level dir, promote its contents to dest.
// If multiple entries (unusual), the staging dir becomes dest.
const entries = await readdir(stagingDir);
assert(entries.length > 0, `tarball extracted nothing`, { file: url });
const hoistFrom = entries.length === 1 ? resolve(stagingDir, entries[0]!) : stagingDir;
await rename(hoistFrom, dest);
await rm(stagingDir, { recursive: true, force: true });
// ─── Post-extract cleanup ───
// Before stamp so failure → next build retries. force:true → no error if
// path already gone (idempotent re-fetch).
for (const p of rmPaths) {
await rm(resolve(dest, p), { recursive: true, force: true });
}
// ─── Write stamp ───
// LAST — if anything above throws, no stamp means next build retries.
await writeFile(stampPath, identity + "\n");
console.log(`extracted to ${dest}`);
}

65
scripts/build/error.ts Normal file
View File

@@ -0,0 +1,65 @@
/**
* All build system errors go through this. Includes context for helpful messages.
*/
export class BuildError extends Error {
readonly hint: string | undefined;
readonly file: string | undefined;
constructor(
message: string,
context?: {
hint?: string;
file?: string;
cause?: unknown;
},
) {
super(message, context?.cause !== undefined ? { cause: context.cause } : undefined);
this.name = "BuildError";
this.hint = context?.hint;
this.file = context?.file;
}
/**
* Format for display to the user.
*/
format(): string {
let out = `error: ${this.message}\n`;
if (this.file !== undefined) {
out += ` at: ${this.file}\n`;
}
if (this.hint !== undefined) {
out += ` hint: ${this.hint}\n`;
}
if (this.cause !== undefined) {
const cause = this.cause instanceof Error ? this.cause.message : String(this.cause);
out += ` cause: ${cause}\n`;
}
return out;
}
}
/**
* Assert a condition, throwing BuildError if false.
*/
export function assert(
condition: unknown,
message: string,
context?: { hint?: string; file?: string },
): asserts condition {
if (!condition) {
throw new BuildError(message, context);
}
}
/**
* Assert a value is defined (not undefined or null).
*/
export function assertDefined<T>(
value: T | undefined | null,
message: string,
context?: { hint?: string; file?: string },
): asserts value is T {
if (value === undefined || value === null) {
throw new BuildError(message, context);
}
}

271
scripts/build/fetch-cli.ts Normal file
View File

@@ -0,0 +1,271 @@
#!/usr/bin/env bun
/**
* Fetch CLI — the single entry point ninja invokes for all downloads.
*
* Ninja rules reference this file via `cfg.bun <this-file> <kind> <args...>`.
* This is BUILD-time code (runs under ninja), not CONFIGURE-time. The
* configure-time modules (source.ts, zig.ts) emit ninja rules that call
* into here but don't execute any of it themselves.
*
* ## Adding a new fetch kind
*
* 1. Write the implementation below (or in download.ts if shared).
* 2. Add a `case` in main() that parses argv and calls it.
* 3. Reference `fetchCliPath` in the ninja rule command.
*
* ## Args format
*
* argv: [bun, fetch-cli.ts, <kind>, ...kind-specific-positional-args]
*
* Positional, not flags — these commands are only invoked by ninja with
* args we control, never by humans. Named flags would be YAGNI.
*/
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import { existsSync, readFileSync } from "node:fs";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { basename, join } from "node:path";
import { downloadWithRetry, extractTarGz, fetchPrebuilt } from "./download.ts";
import { BuildError, assert } from "./error.ts";
import { fetchZig } from "./zig.ts";
/**
* Absolute path to this file. Ninja rules use this in their command strings.
*
* This is a stable way for library modules to build the ninja command
* without knowing where fetch-cli.ts lives — import this constant and
* use it in `command: "${cfg.bun} ${fetchCliPath} <kind> ..."`.
*/
export const fetchCliPath: string = import.meta.filename;
// ───────────────────────────────────────────────────────────────────────────
// Dispatch
// ───────────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
const [, , kind, ...args] = process.argv;
switch (kind) {
case "dep": {
// fetch-cli.ts dep <name> <repo> <commit> <dest> <cache> [...patches]
const [name, repo, commit, dest, cache, ...patches] = args;
assert(name !== undefined && repo !== undefined && commit !== undefined, "dep: missing name/repo/commit");
assert(dest !== undefined && cache !== undefined, "dep: missing dest/cache");
return fetchDep(name, repo, commit, dest, cache, patches);
}
case "prebuilt": {
// fetch-cli.ts prebuilt <name> <url> <dest> <identity> [...rm_paths]
const [name, url, dest, identity, ...rmPaths] = args;
assert(
name !== undefined && url !== undefined && dest !== undefined && identity !== undefined,
"prebuilt: missing name/url/dest/identity",
);
return fetchPrebuilt(name, url, dest, identity, rmPaths);
}
case "zig": {
// fetch-cli.ts zig <url> <dest> <commit>
const [url, dest, commit] = args;
assert(url !== undefined && dest !== undefined && commit !== undefined, "zig: missing url/dest/commit");
return fetchZig(url, dest, commit);
}
case undefined:
case "--help":
case "-h":
process.stderr.write(USAGE);
process.exit(1);
break;
default:
throw new BuildError(`Unknown fetch kind: ${kind}`, { hint: USAGE });
}
}
const USAGE = `\
Usage: bun fetch-cli.ts <kind> <args...>
Kinds:
dep <name> <repo> <commit> <dest> <cache> [...patches]
prebuilt <name> <url> <dest> <identity> [...rm_paths]
zig <url> <dest> <commit>
This is invoked by ninja build rules. You shouldn't need to call it
directly — run ninja targets instead.
`;
// ───────────────────────────────────────────────────────────────────────────
// github-archive dep fetch: download tarball, extract, patch, stamp
// ───────────────────────────────────────────────────────────────────────────
/**
* Fetch a github archive, extract, apply patches, write .ref stamp.
*
* Idempotent: if .ref exists and matches the computed identity, does nothing.
* The ninja rule has restat=1, so a no-op fetch won't trigger downstream.
*
* Tarballs are cached in `cache/` keyed by URL sha256 — downloads are skipped
* if the tarball already exists. Useful when re-extraction is needed after
* a failed patch (you don't re-download).
*/
async function fetchDep(
name: string,
repo: string,
commit: string,
dest: string,
cache: string,
patches: string[],
): Promise<void> {
const refPath = join(dest, ".ref");
// Read patch contents (needed for identity + applying later).
// If a listed patch doesn't exist, that's a bug in the dep definition.
const patchContents: string[] = [];
for (const patch of patches) {
try {
patchContents.push(await readFile(patch, "utf8"));
} catch (cause) {
throw new BuildError(`Patch file not found: ${patch}`, {
hint: `Check the patches list in deps/${name}.ts`,
cause,
});
}
}
const identity = computeSourceIdentity(commit, patchContents);
// Short-circuit: already fetched at this identity?
if (existsSync(refPath)) {
const existing = readFileSync(refPath, "utf8").trim();
if (existing === identity) {
// No-op. Don't touch .ref — restat will see unchanged mtime.
// Printed so the ninja [N/M] line has closure instead of silence.
console.log(`up to date`);
return;
}
// Identity mismatch. Blow it away.
console.log(`source identity changed (was ${existing.slice(0, 8)}, now ${identity.slice(0, 8)})`);
}
console.log(`fetching ${repo}@${commit.slice(0, 8)}`);
// ─── Download (with cache) ───
const url = `https://github.com/${repo}/archive/${commit}.tar.gz`;
const urlHash = createHash("sha256").update(url).digest("hex").slice(0, 16);
const tarballPath = join(cache, `${name}-${urlHash}.tar.gz`);
await mkdir(cache, { recursive: true });
if (!existsSync(tarballPath)) {
await downloadWithRetry(url, tarballPath, name);
}
// ─── Extract ───
// Wipe dest first — we don't want leftover files from a previous version.
await rm(dest, { recursive: true, force: true });
await mkdir(dest, { recursive: true });
// Github archives have a top-level directory <repo>-<commit>/. Strip it.
await extractTarGz(tarballPath, dest);
// ─── Apply patches / overlays ───
for (let i = 0; i < patches.length; i++) {
const p = patches[i]!;
const name = basename(p);
if (p.endsWith(".patch")) {
console.log(`applying ${name}`);
applyPatch(dest, p, patchContents[i]!);
} else {
// Overlay file: copy into source root. Used for e.g. injecting a
// CMakeLists.txt into a project that doesn't have one (tinycc).
console.log(`overlay ${name}`);
await writeFile(join(dest, name), patchContents[i]!);
}
}
// ─── Write stamp ───
// Written LAST — if anything above failed, no stamp means next build retries.
await writeFile(refPath, identity + "\n");
console.log(`done → ${dest}`);
}
/**
* Source identity: sha256(commit + patch_contents)[:16]. This is what goes
* in the .ref stamp. Hashing patch CONTENTS (not paths) means editing a
* patch invalidates the source without a commit bump.
*
* CRLF→LF normalized before hashing: git autocrlf may have converted
* LF→CRLF on Windows checkout. Without normalization, the same patch
* would produce different identities across platforms, triggering
* spurious re-fetches (and worse: `git apply` rejects CRLF patches as
* corrupt, so the re-fetch would fail). The normalized content is also
* what applyPatch() pipes to git — one read, one normalization, used
* for both hashing and applying.
*
* Exported so source.ts can compute the same identity at configure time
* (for the preemptive-delete-on-mismatch check).
*/
export function computeSourceIdentity(commit: string, patchContents: string[]): string {
const h = createHash("sha256");
h.update(commit);
for (const content of patchContents) {
h.update("\0"); // Separator so patch concatenation can't produce collisions.
h.update(normalizeLf(content));
}
return h.digest("hex").slice(0, 16);
}
/** CRLF→LF. Used for patch content before hashing and `git apply`. */
function normalizeLf(s: string): string {
return s.replace(/\r\n/g, "\n");
}
/**
* Apply a patch via `git apply` over stdin.
*
* Normalizes CRLF→LF (same as the identity hash — see computeSourceIdentity)
* so a CRLF-mangled checkout still applies cleanly. --no-index: dest/ is
* not a git repo. --ignore-whitespace / --ignore-space-change: patches are
* authored against upstream which may have different trailing whitespace.
*/
function applyPatch(dest: string, patchPath: string, patchBody: string): void {
const result = spawnSync("git", ["apply", "--ignore-whitespace", "--ignore-space-change", "--no-index", "-"], {
cwd: dest,
input: normalizeLf(patchBody),
stdio: ["pipe", "ignore", "pipe"],
encoding: "utf8",
});
if (result.error) {
throw new BuildError(`Failed to spawn git apply`, { cause: result.error });
}
if (result.status !== 0) {
// If the patch was already applied, the source dir must have been
// partially fetched, which means .ref shouldn't exist, which means
// we should have rm'd the dir. A "cleanly" error here = logic bug.
throw new BuildError(`Patch failed: ${result.stderr}`, {
file: patchPath,
hint: "The patch may be out of date with the pinned commit",
});
}
}
// Only run if this file is the entry point (not imported as a module).
// fetch-cli.ts is ALSO imported by source.ts/zig.ts to get fetchCliPath —
// that import should NOT execute main().
if (import.meta.main) {
try {
await main();
} catch (err) {
// Format BuildError nicely; let anything else bubble to bun's default
// uncaught handler (gets a stack trace, which is what you want for bugs).
if (err instanceof BuildError) {
process.stderr.write(err.format());
process.exit(1);
}
throw err;
}
}

998
scripts/build/flags.ts Normal file
View File

@@ -0,0 +1,998 @@
/**
* Compiler, linker, and define flags for the bun target.
*
* Design: ONE flat table per flag category. Each entry has a `when` predicate
* and a `desc`. To find out why a flag is set, grep for it in this file.
* Related flags (e.g. `-fno-unwind-tables` + `--no-eh-frame-hdr` + strip
* `-R .eh_frame`) live adjacent so their coupling is obvious.
*
* Note: dependency include paths (WebKit, boringssl, etc.) are NOT here —
* they come from each dep's `Provides.includes`. This file covers only flags
* that apply uniformly to bun's own C/C++ sources.
*/
import { join } from "node:path";
import { bunExeName, type Config } from "./config.ts";
import { slash } from "./shell.ts";
export type FlagValue = string | string[] | ((cfg: Config) => string | string[]);
export interface Flag {
/** Flag(s) to emit. Can be a function for flags that interpolate config values. */
flag: FlagValue;
/** Predicate. Omitted = always. */
when?: (cfg: Config) => boolean;
/** Restrict to one language. Omitted = both C and C++. */
lang?: "c" | "cxx";
/** What this flag does. Used by `--explain-flags`. */
desc: string;
}
// ═══════════════════════════════════════════════════════════════════════════
// GLOBAL COMPILER FLAGS
// Applied to BOTH bun's own sources AND forwarded to vendored deps
// via -DCMAKE_C_FLAGS / -DCMAKE_CXX_FLAGS.
// ═══════════════════════════════════════════════════════════════════════════
export const globalFlags: Flag[] = [
// ─── CPU target ───
{
flag: "-mcpu=apple-m1",
when: c => c.darwin && c.arm64,
desc: "Target Apple M1 (works on all Apple Silicon)",
},
{
flag: ["-march=armv8-a+crc", "-mtune=ampere1"],
when: c => c.linux && c.arm64,
desc: "ARM64 Linux: ARMv8-A base + CRC, tuned for Ampere (Graviton-like)",
},
{
flag: ["/clang:-march=armv8-a+crc", "/clang:-mtune=ampere1"],
when: c => c.windows && c.arm64,
desc: "ARM64 Windows: clang-cl prefix required (/clang: passes to clang)",
},
{
flag: "-march=nehalem",
when: c => c.x64 && c.baseline,
desc: "x64 baseline: Nehalem (2008) — no AVX, broadest compatibility",
},
{
flag: "-march=haswell",
when: c => c.x64 && !c.baseline,
desc: "x64 default: Haswell (2013) — AVX2, BMI2 available",
},
// ─── MSVC runtime (Windows) ───
{
flag: "/MTd",
when: c => c.windows && c.debug,
desc: "Static debug MSVC runtime",
},
{
flag: "/MT",
when: c => c.windows && c.release,
desc: "Static MSVC runtime",
},
{
flag: "/U_DLL",
when: c => c.windows,
desc: "Undefine _DLL (we link statically)",
},
// ─── Optimization ───
{
flag: "-O0",
when: c => c.unix && c.debug,
desc: "No optimization (debug)",
},
{
flag: "/Od",
when: c => c.windows && c.debug,
desc: "No optimization (debug)",
},
{
flag: "-Os",
when: c => c.unix && c.smol,
desc: "Optimize for size (MinSizeRel)",
},
{
flag: "/Os",
when: c => c.windows && c.smol,
desc: "Optimize for size (MinSizeRel)",
},
{
flag: "-O3",
when: c => c.unix && c.release && !c.smol,
desc: "Optimize for speed",
},
{
flag: "/O2",
when: c => c.windows && c.release && !c.smol,
desc: "Optimize for speed (MSVC /O2 ≈ clang -O2)",
},
// ─── Debug info ───
{
flag: "/Z7",
when: c => c.windows,
desc: "Emit debug info into .obj (no .pdb during compile)",
},
{
flag: "-gdwarf-4",
when: c => c.darwin,
desc: "DWARF 4 debug info (dsymutil-compatible)",
},
{
// Nix LLVM doesn't support zstd — but we target standard distros.
// Nix users can override via profile if needed.
flag: ["-g3", "-gz=zstd"],
when: c => c.unix && c.debug,
desc: "Full debug info, zstd-compressed",
},
{
flag: "-g1",
when: c => c.unix && c.release,
desc: "Minimal debug info for backtraces",
},
{
flag: "-glldb",
when: c => c.unix,
desc: "Tune debug info for LLDB",
},
// ─── ASAN (global — passed to deps so they link against the same runtime) ───
// Unlike UBSan (bun-target-only below), ASAN must be global: the runtime
// library has to match across all linked objects or you get crashes at init.
{
flag: "-fsanitize=address",
when: c => c.asan,
desc: "AddressSanitizer (also forwarded to deps for ABI consistency)",
},
// ─── C++ language behavior ───
{
flag: "-fno-exceptions",
when: c => c.unix,
desc: "Disable C++ exceptions",
},
{
flag: "/EHsc",
when: c => c.windows,
desc: "Disable C++ exceptions (MSVC: s- disables C++, c- disables C)",
},
{
flag: "-fno-c++-static-destructors",
when: c => c.unix,
lang: "cxx",
desc: "Skip static destructors at exit (JSC assumes this)",
},
{
// /clang: (not -Xclang): single token survives CMAKE_CXX_FLAGS
// re-tokenization in nested dep configures. -Xclang <arg> pair gets
// split by cmake's try_compile → "unknown argument". Also more correct:
// /clang: passes to the driver, -Xclang to cc1.
flag: "/clang:-fno-c++-static-destructors",
when: c => c.windows,
lang: "cxx",
desc: "Skip static destructors at exit (clang-cl syntax)",
},
{
flag: "-fno-rtti",
when: c => c.unix,
desc: "Disable RTTI (no dynamic_cast/typeid)",
},
{
flag: "/GR-",
when: c => c.windows,
desc: "Disable RTTI (MSVC syntax)",
},
// ─── Frame pointers (needed for profiling/backtraces) ───
{
flag: ["-fno-omit-frame-pointer", "-mno-omit-leaf-frame-pointer"],
when: c => c.unix,
desc: "Keep frame pointers (for profiling and backtraces)",
},
{
flag: "/Oy-",
when: c => c.windows,
desc: "Keep frame pointers",
},
// ─── Visibility ───
{
flag: ["-fvisibility=hidden", "-fvisibility-inlines-hidden"],
when: c => c.unix,
desc: "Hidden symbol visibility (explicit exports only)",
},
// ─── Unwinding / exception tables ───
// These go together: -fno-unwind-tables at compile, --no-eh-frame-hdr at
// link (LTO only), and strip -R .eh_frame at post-link (LTO only).
// See linkerFlags and stripFlags below — kept adjacent intentionally.
{
flag: ["-fno-unwind-tables", "-fno-asynchronous-unwind-tables"],
when: c => c.unix,
desc: "Skip unwind tables (we don't use C++ exceptions)",
},
{
// libuv stubs use C23 anonymous parameters
flag: "-Wno-c23-extensions",
when: c => c.unix,
desc: "Allow C23 extensions (libuv stubs use anonymous parameters)",
},
// ─── Sections (enables dead-code stripping at link) ───
{
flag: "-ffunction-sections",
when: c => c.unix,
desc: "One section per function (for --gc-sections)",
},
{
flag: "/Gy",
when: c => c.windows,
desc: "One section per function (COMDAT folding)",
},
{
flag: "-fdata-sections",
when: c => c.unix,
desc: "One section per data item (for --gc-sections)",
},
{
flag: "/Gw",
when: c => c.windows,
desc: "One section per data item",
},
{
// Address-significance table: enables safe ICF at link.
// Macos debug mode + this flag breaks libarchive configure ("pid_t doesn't exist").
flag: "-faddrsig",
when: c => (c.debug && c.linux) || (c.release && c.unix),
desc: "Emit address-significance table (enables safe ICF)",
},
// ─── Windows-specific codegen ───
{
flag: "/GF",
when: c => c.windows,
desc: "String pooling (merge identical string literals)",
},
{
flag: "/GA",
when: c => c.windows,
desc: "Optimize TLS access (assume vars defined in executable)",
},
// ─── Linux-specific codegen ───
{
flag: "-fno-semantic-interposition",
when: c => c.linux,
desc: "Assume no symbol interposition (enables more inlining across TUs)",
},
// ─── Hardening (assertions builds) ───
{
flag: "-fno-delete-null-pointer-checks",
when: c => c.assertions,
desc: "Don't optimize out null checks (hardening)",
},
// ─── Diagnostics ───
{
flag: "-fdiagnostics-color=always",
when: c => c.unix,
desc: "Colored compiler errors",
},
{
flag: "-ferror-limit=100",
when: c => c.unix,
desc: "Stop after 100 errors",
},
{
flag: "/clang:-ferror-limit=100",
when: c => c.windows,
desc: "Stop after 100 errors (clang-cl syntax)",
},
// ─── LTO (compile-side) ───
{
flag: "-flto=full",
when: c => c.unix && c.lto,
desc: "Full link-time optimization (not thin)",
},
{
flag: "-flto",
when: c => c.windows && c.lto,
desc: "Link-time optimization",
},
{
flag: ["-fforce-emit-vtables", "-fwhole-program-vtables"],
when: c => c.unix && c.lto,
lang: "cxx",
desc: "Enable devirtualization across whole program (LTO only)",
},
// ─── Path remapping (CI reproducibility) ───
{
flag: c => [
`-ffile-prefix-map=${c.cwd}=.`,
`-ffile-prefix-map=${c.vendorDir}=vendor`,
`-ffile-prefix-map=${c.cacheDir}=cache`,
],
when: c => c.unix && c.ci,
desc: "Remap source paths in debug info (reproducible builds)",
},
];
// ═══════════════════════════════════════════════════════════════════════════
// BUN-ONLY COMPILER FLAGS
// Applied ONLY to bun's own .c/.cpp files, NOT forwarded to deps.
// This is where -Werror, sanitizer flags, and bun-specific tweaks live.
// ═══════════════════════════════════════════════════════════════════════════
export const bunOnlyFlags: Flag[] = [
// ─── Language standard ───
// WebKit uses gnu++ extensions on Linux; if we don't match, the first
// memory allocation crashes (ABI mismatch in sized delete).
// Not in globalFlags because deps set their own standard.
{
flag: "-std=gnu++23",
when: c => c.linux,
lang: "cxx",
desc: "C++23 with GNU extensions (required to match WebKit's ABI on Linux)",
},
{
flag: "-std=c++23",
when: c => c.darwin,
lang: "cxx",
desc: "C++23 standard",
},
{
flag: "/std:c++23preview",
when: c => c.windows,
lang: "cxx",
desc: "C++23 standard (MSVC flag for clang-cl)",
},
// C standard: gnu17 on unix (cmake: C_STANDARD 17 with C_EXTENSIONS
// default ON → -std=gnu17). Can't go to C23 — MSVC doesn't support it.
// Most .c files are usockets/llhttp which would compile fine without,
// but explicit is better than compiler-default drift.
{
flag: "-std=gnu17",
when: c => c.unix,
lang: "c",
desc: "C17 with GNU extensions (matches cmake's C_STANDARD 17 + default C_EXTENSIONS)",
},
{
flag: "/std:c17",
when: c => c.windows,
lang: "c",
desc: "C17 standard (MSVC-mode flag for clang-cl; no GNU extensions in MSVC mode)",
},
// ─── Sanitizers (bun only — deps would break with -Werror + UBSan) ───
// Note: -fsanitize=address is in globalFlags (deps need ABI consistency).
// UBSan is bun-only because it's stricter and vendored code often violates it.
// Enabled: debug builds (non-musl — musl's implementation hits false positives),
// and release-asan builds (if you're debugging memory you want UBSan too).
{
flag: [
"-fsanitize=null",
"-fno-sanitize-recover=all",
"-fsanitize=bounds",
"-fsanitize=return",
"-fsanitize=nullability-arg",
"-fsanitize=nullability-assign",
"-fsanitize=nullability-return",
"-fsanitize=returns-nonnull-attribute",
"-fsanitize=unreachable",
],
when: c => c.unix && ((c.debug && c.abi !== "musl") || (c.release && c.asan)),
desc: "Undefined-behavior sanitizers",
},
{
flag: ["-fsanitize-coverage=trace-pc-guard", "-DFUZZILLI_ENABLED"],
when: c => c.fuzzilli,
desc: "Fuzzilli coverage instrumentation",
},
// ─── Bun-target-specific ───
{
flag: ["-fconstexpr-steps=2542484", "-fconstexpr-depth=54"],
when: c => c.unix,
lang: "cxx",
desc: "Raise constexpr limits (JSC uses heavy constexpr)",
},
{
flag: ["-fno-pic", "-fno-pie"],
when: c => c.unix,
desc: "No position-independent code (we're a final executable)",
},
// ─── Warnings-as-errors (unix) ───
{
flag: [
"-Werror=return-type",
"-Werror=return-stack-address",
"-Werror=implicit-function-declaration",
"-Werror=uninitialized",
"-Werror=conditional-uninitialized",
"-Werror=suspicious-memaccess",
"-Werror=int-conversion",
"-Werror=nonnull",
"-Werror=move",
"-Werror=sometimes-uninitialized",
"-Wno-c++23-lambda-attributes",
"-Wno-nullability-completeness",
"-Wno-character-conversion",
"-Werror",
],
when: c => c.unix,
desc: "Treat most warnings as errors; suppress known-noisy ones",
},
{
// Debug adds -Werror=unused; release omits it (vars used only in ASSERT)
flag: ["-Werror=unused", "-Wno-unused-function"],
when: c => c.unix && c.debug,
desc: "Warn on unused vars in debug (catches dead code)",
},
{
// Windows: suppress noisy warnings from headers we don't control
flag: [
"-Wno-nullability-completeness",
"-Wno-inconsistent-dllimport",
"-Wno-incompatible-pointer-types",
"-Wno-deprecated-declarations",
"-Wno-character-conversion",
],
when: c => c.windows,
desc: "Suppress noisy warnings from system/dependency headers",
},
];
// ═══════════════════════════════════════════════════════════════════════════
// PREPROCESSOR DEFINES
// -D flags passed to every bun compilation unit.
// ═══════════════════════════════════════════════════════════════════════════
export const defines: Flag[] = [
// ─── Always on ───
{
flag: [
"_HAS_EXCEPTIONS=0",
"LIBUS_USE_OPENSSL=1",
"LIBUS_USE_BORINGSSL=1",
"WITH_BORINGSSL=1",
"STATICALLY_LINKED_WITH_JavaScriptCore=1",
"STATICALLY_LINKED_WITH_BMALLOC=1",
"BUILDING_WITH_CMAKE=1",
"JSC_OBJC_API_ENABLED=0",
"BUN_SINGLE_THREADED_PER_VM_ENTRY_SCOPE=1",
"NAPI_EXPERIMENTAL=ON",
"NOMINMAX",
"IS_BUILD",
"BUILDING_JSCONLY__",
],
desc: "Core bun defines (always on)",
},
{
// Shell-escaped quotes so clang receives literal quotes in the define
// (the preprocessor needs the string to be "24.3.0", not bare 24.3.0).
flag: c => `REPORTED_NODEJS_VERSION=\\"${c.nodejsVersion}\\"`,
desc: "Node.js version string reported by process.version",
},
{
flag: c => `REPORTED_NODEJS_ABI_VERSION=${c.nodejsAbiVersion}`,
desc: "Node.js ABI version (process.versions.modules)",
},
{
// Hardcoded ON — experimental flag not exposed in config
flag: "USE_BUN_MIMALLOC=1",
desc: "Use mimalloc as default allocator",
},
// ─── Config-dependent ───
{
flag: "ASSERT_ENABLED=1",
when: c => c.assertions,
desc: "Enable runtime assertions",
},
{
flag: "BUN_DEBUG=1",
when: c => c.debug,
desc: "Enable debug-only code paths",
},
{
// slash(): path becomes a C string literal — `\U` would be a unicode escape.
flag: c => `BUN_DYNAMIC_JS_LOAD_PATH=\\"${slash(join(c.buildDir, "js"))}\\"`,
when: c => c.debug && !c.ci,
desc: "Hot-reload built-in JS from build dir (dev convenience)",
},
{
// Standard define that disables assert() in libc headers. CMake adds
// this automatically for Release builds; we do it explicitly.
flag: "NDEBUG",
when: c => c.release,
desc: "Disable libc assert() (release builds)",
},
// ─── Platform ───
{
flag: "_DARWIN_NON_CANCELABLE=1",
when: c => c.darwin,
desc: "Use non-cancelable POSIX calls on Darwin",
},
{
flag: ["WIN32", "_WINDOWS", "WIN32_LEAN_AND_MEAN=1", "_CRT_SECURE_NO_WARNINGS", "BORINGSSL_NO_CXX=1"],
when: c => c.windows,
desc: "Standard Windows defines + disable CRT security warnings",
},
{
flag: "U_STATIC_IMPLEMENTATION",
when: c => c.windows,
desc: "ICU static linkage (without this: ABI mismatch → STATUS_STACK_BUFFER_OVERRUN)",
},
{
flag: "U_DISABLE_RENAMING=1",
when: c => c.darwin,
desc: "Disable ICU symbol renaming (using system ICU)",
},
// ─── Feature toggles ───
{
flag: "LAZY_LOAD_SQLITE=0",
when: c => c.staticSqlite,
desc: "SQLite statically linked",
},
{
flag: "LAZY_LOAD_SQLITE=1",
when: c => !c.staticSqlite,
desc: "SQLite loaded at runtime",
},
];
// ═══════════════════════════════════════════════════════════════════════════
// LINKER FLAGS
// For the final bun executable link step only.
// ═══════════════════════════════════════════════════════════════════════════
export const linkerFlags: Flag[] = [
// ─── Sanitizers ───
{
flag: "-fsanitize=address",
when: c => c.unix && c.asan,
desc: "Link ASAN runtime",
},
{
flag: "-fsanitize=null",
when: c => c.unix && c.debug && c.abi !== "musl",
desc: "Link UBSan runtime",
},
{
flag: "-fsanitize=null",
when: c => c.unix && c.release && c.asan,
desc: "Link UBSan runtime (release-asan)",
},
{
flag: "-fsanitize-coverage=trace-pc-guard",
when: c => c.fuzzilli,
desc: "Link fuzzilli coverage runtime",
},
// ─── LTO (link-side) ───
{
flag: ["-flto=full", "-fwhole-program-vtables", "-fforce-emit-vtables"],
when: c => c.unix && c.lto,
desc: "LTO at link time (matches compile-side -flto=full)",
},
// ─── Windows ───
{
flag: ["/STACK:0x1200000,0x200000", "/errorlimit:0"],
when: c => c.windows,
desc: "18MB stack reserve (JSC uses deep recursion), no error limit",
},
{
flag: [
"/LTCG",
"/OPT:REF",
"/OPT:NOICF",
"/DEBUG:FULL",
"/delayload:ole32.dll",
"/delayload:WINMM.dll",
"/delayload:dbghelp.dll",
"/delayload:VCRUNTIME140_1.dll",
"/delayload:WS2_32.dll",
"/delayload:WSOCK32.dll",
"/delayload:ADVAPI32.dll",
"/delayload:IPHLPAPI.dll",
"/delayload:CRYPT32.dll",
],
when: c => c.windows && c.release,
desc: "Release link opts + delay-load non-critical DLLs (faster startup)",
},
// ─── macOS ───
{
flag: ["-Wl,-ld_new", "-Wl,-no_compact_unwind", "-Wl,-stack_size,0x1200000", "-fno-keep-static-consts"],
when: c => c.darwin,
desc: "Use new Apple linker, 18MB stack, skip compact unwind",
},
{
flag: "-Wl,-w",
when: c => c.darwin && c.debug,
desc: "Suppress all linker warnings (workaround: no selective suppress for alignment warnings as of 2025-07)",
},
{
flag: c => ["-dead_strip", "-dead_strip_dylibs", `-Wl,-map,${c.buildDir}/${bunExeName(c)}.linker-map`],
when: c => c.darwin && c.release,
desc: "Dead-code strip + emit linker map",
},
// ─── Linux ───
{
// Wrap old glibc symbols so the binary runs on older glibc
flag: [
"-Wl,--wrap=exp",
"-Wl,--wrap=exp2",
"-Wl,--wrap=expf",
"-Wl,--wrap=fcntl64",
"-Wl,--wrap=gettid",
"-Wl,--wrap=log",
"-Wl,--wrap=log2",
"-Wl,--wrap=log2f",
"-Wl,--wrap=logf",
"-Wl,--wrap=pow",
"-Wl,--wrap=powf",
],
when: c => c.linux && c.abi !== "musl",
desc: "Wrap glibc 2.29+ symbols (portable to older glibc)",
},
{
flag: ["-static-libstdc++", "-static-libgcc"],
when: c => c.linux && c.abi !== "musl",
desc: "Static C++ runtime (don't depend on host libstdc++)",
},
{
flag: ["-lstdc++", "-lgcc"],
when: c => c.linux && c.abi === "musl",
desc: "Dynamic C++ runtime on musl (static unavailable)",
},
{
// Paired with compile-side -fno-unwind-tables above.
// Only in LTO builds — otherwise .eh_frame is needed for backtraces.
flag: "-Wl,--no-eh-frame-hdr",
when: c => c.linux && c.lto,
desc: "Omit eh_frame header (LTO builds; size opt; see stripFlags for matching -R .eh_frame)",
},
{
flag: "-Wl,--eh-frame-hdr",
when: c => c.linux && !c.lto,
desc: "Keep eh_frame header (non-LTO; needed for backtraces)",
},
{
flag: c => `--ld-path=${c.ld}`,
when: c => c.linux,
desc: "Use lld instead of system ld",
},
{
flag: ["-fno-pic", "-Wl,-no-pie"],
when: c => c.linux,
desc: "No PIE (we don't need ASLR; simpler codegen)",
},
{
flag: [
"-Wl,--as-needed",
"-Wl,-z,stack-size=12800000",
"-Wl,--compress-debug-sections=zlib",
"-Wl,-z,lazy",
"-Wl,-z,norelro",
"-Wl,-O2",
"-Wl,--gdb-index",
"-Wl,-z,combreloc",
"-Wl,--sort-section=name",
"-Wl,--hash-style=both",
"-Wl,--build-id=sha1",
],
when: c => c.linux,
desc: "Linux linker tuning: lazy binding, large stack, compressed debug, fast gdb loading",
},
{
flag: "-Wl,--gc-sections",
when: c => c.linux && c.release,
desc: "Garbage-collect unused sections (release only; debug keeps Zig dbHelper symbols)",
},
{
flag: c => ["-Wl,-icf=safe", `-Wl,-Map=${c.buildDir}/${bunExeName(c)}.linker-map`],
when: c => c.linux && c.release && !c.asan && !c.valgrind,
desc: "Safe identical-code-folding + linker map (release only)",
},
// ─── Symbols / exports ───
// These reference files on disk — linkDepends() lists the same paths
// so ninja relinks when they change (cmake's LINK_DEPENDS equivalent).
{
flag: c => `/DEF:${slash(join(c.cwd, "src/symbols.def"))}`,
when: c => c.windows,
desc: "Exported symbol definition (.def format)",
},
{
flag: c => ["-exported_symbols_list", `${c.cwd}/src/symbols.txt`],
when: c => c.darwin,
desc: "Exported symbol list",
},
{
flag: c => [
"-Bsymbolics-functions",
"-rdynamic",
`-Wl,--dynamic-list=${c.cwd}/src/symbols.dyn`,
`-Wl,--version-script=${c.cwd}/src/linker.lds`,
],
when: c => c.linux,
desc: "Dynamic symbol list + version script",
},
];
/**
* Files the linker reads via flags above. Return as implicit inputs so
* ninja relinks when exported symbols / version script change.
* CMake tracks these via set_target_properties LINK_DEPENDS.
*/
export function linkDepends(cfg: Config): string[] {
if (cfg.windows) return [join(cfg.cwd, "src/symbols.def")];
if (cfg.darwin) return [join(cfg.cwd, "src/symbols.txt")];
return [join(cfg.cwd, "src/symbols.dyn"), join(cfg.cwd, "src/linker.lds")];
}
// ═══════════════════════════════════════════════════════════════════════════
// STRIP FLAGS
// For the post-link strip step (release only).
// ═══════════════════════════════════════════════════════════════════════════
/**
* Strip step only runs for plain release builds (bun-profile → bun).
* Not for debug/asan/valgrind/assertions variants — those keep symbols.
*
* Always: --strip-all --strip-debug --discard-all.
* Platform extras remove unwind/exception sections we compile without
* (no -fexceptions, lolhtml built with panic=abort).
*
* Linux section removal: CMake notes llvm-strip doesn't fully delete
* these (leaves [LOAD #2 [R]]), GNU strip does. If size matters and
* llvm-strip's output is larger, swap to /usr/bin/strip for this step.
*/
export const stripFlags: Flag[] = [
{
// Core strip: symbols + debug info + local symbols.
flag: ["--strip-all", "--strip-debug", "--discard-all"],
desc: "Remove symbols, debug info, local symbols",
},
{
flag: [
"--remove-section=__TEXT,__eh_frame",
"--remove-section=__TEXT,__unwind_info",
"--remove-section=__TEXT,__gcc_except_tab",
],
when: c => c.darwin,
desc: "Remove unwind/exception sections (we compile with -fno-exceptions; these come from lolhtml etc., built with panic=abort)",
},
{
// musl: no eh_frame handling differences, but CMake gates on NOT musl so we do too.
// Strip only runs on plain release (shouldStrip gates debug/asan/valgrind/assertions)
// which in CI always has LTO on — in practice paired with --no-eh-frame-hdr.
flag: ["-R", ".eh_frame", "-R", ".gcc_except_table"],
when: c => c.linux && c.abi !== "musl",
desc: "Remove unwind sections (GNU strip required — llvm-strip leaves [LOAD #2 [R]])",
},
];
// ═══════════════════════════════════════════════════════════════════════════
// INCLUDE DIRECTORIES
// Bun's own source tree + build-time generated code.
// Dependency includes (WebKit, boringssl, ...) come from resolveDep().
// ═══════════════════════════════════════════════════════════════════════════
/**
* Bun's source-tree include paths. These are the -I dirs for bun's own code
* (not vendored deps — those come from each dep's `Provides.includes`).
*/
export function bunIncludes(cfg: Config): string[] {
const { cwd, codegenDir, vendorDir } = cfg;
const includes: string[] = [
join(cwd, "packages"),
join(cwd, "packages/bun-usockets"),
join(cwd, "packages/bun-usockets/src"),
join(cwd, "src/bun.js/bindings"),
join(cwd, "src/bun.js/bindings/webcore"),
join(cwd, "src/bun.js/bindings/webcrypto"),
join(cwd, "src/bun.js/bindings/node/crypto"),
join(cwd, "src/bun.js/bindings/node/http"),
join(cwd, "src/bun.js/bindings/sqlite"),
join(cwd, "src/bun.js/bindings/v8"),
join(cwd, "src/bun.js/modules"),
join(cwd, "src/js/builtins"),
join(cwd, "src/napi"),
join(cwd, "src/deps"),
codegenDir,
vendorDir,
join(vendorDir, "picohttpparser"),
join(vendorDir, "zlib"),
// NODEJS_HEADERS_PATH comes from the nodejs dep; added separately
];
if (cfg.windows) {
includes.push(join(cwd, "src/bun.js/bindings/windows"));
} else {
// libuv stubs for unix (real libuv used on windows)
includes.push(join(cwd, "src/bun.js/bindings/libuv"));
}
// musl doesn't ship sys/queue.h (glibc-only BSDism). lshpack bundles
// a compat copy for this case.
if (cfg.linux && cfg.abi === "musl") {
includes.push(join(vendorDir, "lshpack/compat/queue"));
}
return includes;
}
// ═══════════════════════════════════════════════════════════════════════════
// PER-FILE OVERRIDES
// Exceptional files that need different flags than the global set.
// ═══════════════════════════════════════════════════════════════════════════
export interface FileOverride {
/** Source path relative to cfg.cwd. */
file: string;
/** Extra flags appended after the global set. */
extraFlags: FlagValue;
when?: (cfg: Config) => boolean;
desc: string;
}
export const fileOverrides: FileOverride[] = [
{
file: "src/bun.js/bindings/workaround-missing-symbols.cpp",
// -fwhole-program-vtables requires -flto; disabling one requires
// disabling the other or clang errors.
extraFlags: ["-fno-lto", "-fno-whole-program-vtables"],
when: c => c.linux && c.lto && c.abi !== "musl",
desc: "Disable LTO: LLD 21 emits glibc versioned symbols (exp@GLIBC_2.17) into .lto_discard which fails to parse '@'",
},
];
// ═══════════════════════════════════════════════════════════════════════════
// COMPUTED OUTPUT
// ═══════════════════════════════════════════════════════════════════════════
export interface ComputedFlags {
/** C compiler flags (clang -c for .c files). */
cflags: string[];
/** C++ compiler flags (clang++ -c for .cpp files). */
cxxflags: string[];
/** Preprocessor defines, without -D prefix. */
defines: string[];
/** Linker flags for the final bun link. */
ldflags: string[];
/** Strip flags for post-link. */
stripflags: string[];
}
/**
* Resolve a FlagValue to its final string form(s) for a given config.
*/
function resolveFlagValue(v: FlagValue, cfg: Config): string[] {
const resolved = typeof v === "function" ? v(cfg) : v;
return Array.isArray(resolved) ? resolved : [resolved];
}
/**
* Evaluate a Flag table and push into c/cxx output arrays.
*/
function evalTable(table: Flag[], cfg: Config, c: string[], cxx: string[]): void {
for (const f of table) {
if (f.when && !f.when(cfg)) continue;
const flags = resolveFlagValue(f.flag, cfg);
if (f.lang === "c") {
c.push(...flags);
} else if (f.lang === "cxx") {
cxx.push(...flags);
} else {
c.push(...flags);
cxx.push(...flags);
}
}
}
/**
* Evaluate all flag predicates for bun's own source files.
* Combines global flags + bun-only flags.
*/
export function computeFlags(cfg: Config): ComputedFlags {
const cflags: string[] = [];
const cxxflags: string[] = [];
const defs: string[] = [];
const ldflags: string[] = [];
const stripflags: string[] = [];
// Compile: global first, then bun-only
evalTable(globalFlags, cfg, cflags, cxxflags);
evalTable(bunOnlyFlags, cfg, cflags, cxxflags);
// Defines, linker, strip
for (const f of defines) {
if (f.when && !f.when(cfg)) continue;
defs.push(...resolveFlagValue(f.flag, cfg));
}
for (const f of linkerFlags) {
if (f.when && !f.when(cfg)) continue;
ldflags.push(...resolveFlagValue(f.flag, cfg));
}
for (const f of stripFlags) {
if (f.when && !f.when(cfg)) continue;
stripflags.push(...resolveFlagValue(f.flag, cfg));
}
return { cflags, cxxflags, defines: defs, ldflags, stripflags };
}
/**
* Flags forwarded to vendored dependencies via -DCMAKE_C_FLAGS/-DCMAKE_CXX_FLAGS.
* This is ONLY the global table — no -Werror, no bun-specific defines, no UBSan.
*/
export function computeDepFlags(cfg: Config): { cflags: string[]; cxxflags: string[] } {
const cflags: string[] = [];
const cxxflags: string[] = [];
evalTable(globalFlags, cfg, cflags, cxxflags);
return { cflags, cxxflags };
}
/**
* Per-file extra flags lookup. Call after computeFlags() when compiling a
* specific source. Returns extra flags to append (may be empty).
*/
export function extraFlagsFor(cfg: Config, srcRelPath: string): string[] {
for (const o of fileOverrides) {
if (o.file !== srcRelPath) continue;
if (o.when && !o.when(cfg)) continue;
return resolveFlagValue(o.extraFlags, cfg);
}
return [];
}
/**
* Produce a human-readable explanation of all active flags for `--explain-flags`.
* Grouped by flag type, shows each flag alongside its description.
*/
export function explainFlags(cfg: Config): string {
const lines: string[] = [];
const explainTable = (title: string, flags: Flag[]) => {
const active = flags.filter(f => !f.when || f.when(cfg));
if (active.length === 0) return;
lines.push(`\n─── ${title} ───`);
for (const f of active) {
const vals = resolveFlagValue(f.flag, cfg);
const langSuffix = f.lang ? ` [${f.lang}]` : "";
lines.push(` ${vals.join(" ")}${langSuffix}`);
lines.push(` ${f.desc}`);
}
};
explainTable("Global compiler flags (bun + deps)", globalFlags);
explainTable("Bun-only compiler flags", bunOnlyFlags);
explainTable("Defines", defines);
explainTable("Linker flags", linkerFlags);
explainTable("Strip flags", stripFlags);
const overrides = fileOverrides.filter(o => !o.when || o.when(cfg));
if (overrides.length > 0) {
lines.push("\n─── Per-file overrides ───");
for (const o of overrides) {
lines.push(` ${o.file}: ${resolveFlagValue(o.extraFlags, cfg).join(" ")}`);
lines.push(` ${o.desc}`);
}
}
return lines.join("\n");
}

48
scripts/build/fs.ts Normal file
View File

@@ -0,0 +1,48 @@
/**
* Filesystem utilities used at configure time.
*
* Separate from shell.ts because "quote a shell argument" and "write a
* file idempotently" share nothing except being utility functions.
*/
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
/**
* Write `content` to `path` only if different (or file doesn't exist).
* Returns whether a write happened.
*
* Used throughout the build system for configure-time generated files
* (PCH wrapper, dep versions header, build.ninja itself). Preserving
* mtimes on unchanged content is what makes idempotent re-configure
* actually cheap: ninja sees no changes, does nothing. Without this,
* every configure touches everything and ninja at minimum re-stats.
*
* Synchronous because configure is single-threaded and the files are
* small. Async would add await noise for no concurrency benefit.
*/
export function writeIfChanged(path: string, content: string): boolean {
try {
if (readFileSync(path, "utf8") === content) return false;
} catch {
// File doesn't exist (or unreadable) — fall through to write.
}
writeFileSync(path, content);
return true;
}
/**
* Create multiple directories (and their parents). Deduplicates so
* `["a/b/c", "a/b/d"]` only stats/creates `a/b` once.
*
* Used at configure time to pre-create all object-file parent dirs —
* ninja doesn't mkdir, and we don't want N×mkdir syscalls for the same
* directory when compiling N files that share a parent.
*/
export function mkdirAll(dirs: Iterable<string>): void {
const seen = new Set<string>();
for (const dir of dirs) {
if (seen.has(dir)) continue;
mkdirSync(dir, { recursive: true });
seen.add(dir);
}
}

385
scripts/build/ninja.ts Normal file
View File

@@ -0,0 +1,385 @@
/**
* Ninja build file writer.
*
* Reference: https://ninja-build.org/manual.html
*
* The core primitive of the build system. Everything else (compile, link, codegen,
* external builds) is a constructor that produces BuildNodes, which map 1:1 to
* ninja `build` statements.
*/
import { mkdir } from "node:fs/promises";
import { isAbsolute, relative, resolve } from "node:path";
import { BuildError, assert } from "./error.ts";
import { writeIfChanged } from "./fs.ts";
/**
* A ninja `rule` — a reusable command template.
*/
export interface Rule {
/** The shell command. Use $in, $out, and custom vars like $flags. */
command: string;
/** Human-readable description printed during build (e.g. "CXX $out"). */
description?: string;
/** Path to gcc-style depfile ($out.d), enables header dependency tracking. */
depfile?: string;
/** Depfile format. Use "gcc" for clang/gcc, "msvc" for clang-cl. */
deps?: "gcc" | "msvc";
/** Re-stat outputs after command; prunes downstream rebuilds if output unchanged. */
restat?: boolean;
/**
* Marks this as a generator rule. Ninja won't consider itself dirty when only
* the command line of a generator rule changes. Used for the reconfigure rule.
*/
generator?: boolean;
/** Job pool for parallelism control (e.g. "console" for stdout access). */
pool?: string;
/** Response file path. Needed when command line would exceed OS limits. */
rspfile?: string;
/** Content written to rspfile (usually $in or $in_newline). */
rspfile_content?: string;
}
/**
* A ninja `build` statement — the only primitive of the build graph.
*
* Inputs → command (from rule) → outputs.
*/
export interface BuildNode {
/** Files this build produces. Must not be empty. */
outputs: string[];
/** Additional outputs that ninja tracks but that don't appear in $out. */
implicitOutputs?: string[];
/** The rule to use (name of a previously registered rule, or "phony"). */
rule: string;
/** Explicit inputs. Available as $in in the rule command. */
inputs: string[];
/**
* Implicit inputs (ninja `| dep` syntax). Tracked for staleness but not in $in.
* Use for: generated headers, the PCH file, dep library outputs.
*/
implicitInputs?: string[];
/**
* Order-only inputs (ninja `|| dep` syntax). Must exist before this builds,
* but their mtime is ignored. Use for: directory creation, phony groupings.
*/
orderOnlyInputs?: string[];
/** Variable bindings local to this build statement. */
vars?: Record<string, string>;
/** Job pool override (overrides rule's pool). */
pool?: string;
}
/**
* A compile_commands.json entry.
*/
export interface CompileCommand {
directory: string;
file: string;
output: string;
arguments: string[];
}
export interface NinjaOptions {
/** Absolute path to build directory. All paths in build.ninja are relative to this. */
buildDir: string;
/** Minimum ninja version to require. */
ninjaVersion?: string;
}
/**
* Ninja build file writer.
*
* Accumulates rules, build statements, variables, pools. Call `write()` to emit
* `build.ninja` + `compile_commands.json`.
*
* All paths given to this class should be ABSOLUTE. They are converted to
* buildDir-relative at write time via `rel()`.
*/
export class Ninja {
readonly buildDir: string;
private readonly ninjaVersion: string;
private readonly lines: string[] = [];
private readonly ruleNames = new Set<string>();
private readonly outputSet = new Set<string>();
private readonly pools = new Map<string, number>();
private readonly defaults: string[] = [];
private readonly compileCommands: CompileCommand[] = [];
constructor(opts: NinjaOptions) {
assert(isAbsolute(opts.buildDir), `Ninja buildDir must be absolute, got: ${opts.buildDir}`);
this.buildDir = resolve(opts.buildDir);
// 1.9 is the minimum we need — implicit outputs (1.7), console pool
// (1.5), restat (1.0). We don't use dyndep (1.10's headline feature).
// Some CI agents (darwin) ship 1.9 and we don't control their image.
this.ninjaVersion = opts.ninjaVersion ?? "1.9";
}
/**
* Convert an absolute path to buildDir-relative.
* Idempotent on already-relative paths.
*/
rel(path: string): string {
if (!isAbsolute(path)) {
return path;
}
return relative(this.buildDir, path);
}
/** Define a top-level ninja variable. */
variable(name: string, value: string): void {
assert(/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name), `Invalid ninja variable name: ${name}`);
this.lines.push(`${name} = ${ninjaEscapeVarValue(value)}`);
}
/** Add a comment line to the output. */
comment(text: string): void {
for (const line of text.split("\n")) {
this.lines.push(`# ${line}`);
}
}
/** Add a blank line for readability. */
blank(): void {
this.lines.push("");
}
/** Define a ninja pool for parallelism control. */
pool(name: string, depth: number): void {
assert(!this.pools.has(name), `Duplicate pool: ${name}`);
assert(depth >= 1, `Pool depth must be >= 1, got: ${depth}`);
this.pools.set(name, depth);
}
/** Define a ninja rule. */
rule(name: string, spec: Rule): void {
assert(/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name), `Invalid ninja rule name: ${name}`);
assert(!this.ruleNames.has(name), `Duplicate rule: ${name}`);
this.ruleNames.add(name);
this.lines.push(`rule ${name}`);
this.lines.push(` command = ${spec.command}`);
if (spec.description !== undefined) {
this.lines.push(` description = ${spec.description}`);
}
if (spec.depfile !== undefined) {
this.lines.push(` depfile = ${spec.depfile}`);
}
if (spec.deps !== undefined) {
this.lines.push(` deps = ${spec.deps}`);
}
if (spec.restat === true) {
this.lines.push(` restat = 1`);
}
if (spec.generator === true) {
this.lines.push(` generator = 1`);
}
if (spec.pool !== undefined) {
this.lines.push(` pool = ${spec.pool}`);
}
if (spec.rspfile !== undefined) {
this.lines.push(` rspfile = ${spec.rspfile}`);
}
if (spec.rspfile_content !== undefined) {
this.lines.push(` rspfile_content = ${spec.rspfile_content}`);
}
this.lines.push("");
}
/**
* Add a build statement. The core of the graph.
*
* All paths in `node` should be absolute; they are converted to
* buildDir-relative automatically.
*/
build(node: BuildNode): void {
assert(node.outputs.length > 0, `Build node must have at least one output (rule: ${node.rule})`);
assert(node.rule === "phony" || this.ruleNames.has(node.rule), `Unknown rule: ${node.rule}`, {
hint: `Define the rule with ninja.rule("${node.rule}", {...}) first`,
});
// Check for duplicate outputs
const allOuts = [...node.outputs, ...(node.implicitOutputs ?? [])];
for (const out of allOuts) {
const abs = isAbsolute(out) ? resolve(out) : resolve(this.buildDir, out);
if (this.outputSet.has(abs)) {
throw new BuildError(`Duplicate build output: ${out}`, {
hint: "Another build statement already produces this file",
});
}
this.outputSet.add(abs);
}
const outs = node.outputs.map(p => ninjaEscapePath(this.rel(p)));
const implOuts = (node.implicitOutputs ?? []).map(p => ninjaEscapePath(this.rel(p)));
const ins = node.inputs.map(p => ninjaEscapePath(this.rel(p)));
const implIns = (node.implicitInputs ?? []).map(p => ninjaEscapePath(this.rel(p)));
const orderIns = (node.orderOnlyInputs ?? []).map(p => ninjaEscapePath(this.rel(p)));
let line = `build ${outs.join(" ")}`;
if (implOuts.length > 0) {
line += ` | ${implOuts.join(" ")}`;
}
line += `: ${node.rule}`;
if (ins.length > 0) {
line += ` ${ins.join(" ")}`;
}
if (implIns.length > 0) {
line += ` | ${implIns.join(" ")}`;
}
if (orderIns.length > 0) {
line += ` || ${orderIns.join(" ")}`;
}
// Wrap long lines with $\n continuations for readability
this.lines.push(wrapLongLine(line));
if (node.pool !== undefined) {
this.lines.push(` pool = ${node.pool}`);
}
if (node.vars !== undefined) {
for (const [k, v] of Object.entries(node.vars)) {
this.lines.push(` ${k} = ${ninjaEscapeVarValue(v)}`);
}
}
this.lines.push("");
}
/** Shorthand for a phony target (groups other targets). */
phony(name: string, deps: string[]): void {
this.build({
outputs: [name],
rule: "phony",
inputs: deps,
});
}
/**
* Returns an always-dirty phony target. Depending on this forces a rule
* to re-run every build. Useful for nested builds (cmake/cargo) where the
* inner build system tracks its own staleness — we always invoke it, it
* no-ops if nothing changed, `restat=1` on the outer rule prunes downstream.
*
* Emitted lazily on first call; subsequent calls return the same name.
*/
always(): string {
const name = "always";
// outputSet stores absolute paths; phony targets resolve relative to buildDir.
const abs = resolve(this.buildDir, name);
if (!this.outputSet.has(abs)) {
// A phony with no inputs is always dirty (its output file never exists).
this.phony(name, []);
}
return name;
}
/** Mark targets as default (built when running `ninja` with no args). */
default(targets: string[]): void {
for (const t of targets) {
this.defaults.push(this.rel(t));
}
}
/**
* Record a compile command for compile_commands.json.
* Called by `cxx()` and `cc()` in compile.ts.
*/
addCompileCommand(cmd: CompileCommand): void {
this.compileCommands.push(cmd);
}
/**
* Serialize to ninja file content (without writing to disk).
*/
toString(): string {
const header: string[] = [
`# Generated by scripts/build/configure.ts`,
`# DO NOT EDIT — changes will be overwritten on next configure`,
``,
`ninja_required_version = ${this.ninjaVersion}`,
``,
];
const poolLines: string[] = [];
for (const [name, depth] of this.pools) {
poolLines.push(`pool ${name}`);
poolLines.push(` depth = ${depth}`);
poolLines.push("");
}
const defaultLines: string[] =
this.defaults.length > 0 ? [`default ${this.defaults.map(ninjaEscapePath).join(" ")}`, ""] : [];
return [...header, ...poolLines, ...this.lines, ...defaultLines].join("\n");
}
/**
* Write build.ninja and compile_commands.json to buildDir.
*
* Returns `true` if build.ninja content changed (or didn't exist).
* Caller can use this to decide whether to print configure output —
* on an unchanged re-configure (same flags, same sources), stay quiet.
*/
async write(): Promise<boolean> {
await mkdir(this.buildDir, { recursive: true });
// Only write files whose content actually changed — preserves mtimes
// for idempotent re-configures. A ninja run after an unchanged
// reconfigure sees nothing new and stays a true no-op. Without this,
// we'd touch build.ninja every time, which is harmless for ninja
// itself (it tracks content via .ninja_log) but wasteful and makes
// `ls -lt build/` less useful for debugging.
const changed = writeIfChanged(resolve(this.buildDir, "build.ninja"), this.toString());
writeIfChanged(
resolve(this.buildDir, "compile_commands.json"),
JSON.stringify(this.compileCommands, null, 2) + "\n",
);
return changed;
}
}
// ---------------------------------------------------------------------------
// Ninja escaping
//
// Ninja has two escaping contexts:
// 1. Paths in build lines: $ and space must be escaped with $
// 2. Variable values: only $ needs escaping (newlines need $\n but we don't emit those)
// ---------------------------------------------------------------------------
/** Escape a path for use in a `build` line. */
function ninjaEscapePath(path: string): string {
return path.replace(/\$/g, "$$$$").replace(/ /g, "$ ").replace(/:/g, "$:");
}
/** Escape a value for use on the right side of `var = value`. */
function ninjaEscapeVarValue(value: string): string {
return value.replace(/\$/g, "$$$$");
}
/**
* Wrap a long `build` line using ninja's $\n continuation.
* Purely cosmetic — ninja handles arbitrarily long lines, but humans don't.
*/
function wrapLongLine(line: string, width = 120): string {
if (line.length <= width) {
return line;
}
// Split at spaces (that aren't escaped), wrap with $\n + 4-space indent
const parts = line.split(/(?<=[^$]) /);
const out: string[] = [];
let current = parts[0]!;
for (let i = 1; i < parts.length; i++) {
const part = parts[i]!;
if (current.length + 1 + part.length > width) {
out.push(current + " $");
current = " " + part;
} else {
current += " " + part;
}
}
out.push(current);
return out.join("\n");
}

View File

@@ -0,0 +1,25 @@
Remove workaround for CMake bug #11240 (2010, fixed in cmake 2.8.4).
The hardcoded /machine:x64 breaks arm64 windows builds. The workaround is
long obsolete — lib.exe has always set /MACHINE automatically from the object
file architecture; the original bug was in cmake's static lib link command
line construction, not in lib.exe itself.
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -231,13 +231,6 @@
if(UNIX OR MINGW)
set_target_properties(zlib PROPERTIES OUTPUT_NAME z)
endif()
- #============================================================================
- # work around to CMake bug which affects 64-bit Windows
- # see http://public.kitware.com/Bug/view.php?id=11240
- #============================================================================
- if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND MSVC)
- set_target_properties(zlib PROPERTIES STATIC_LIBRARY_FLAGS "/machine:x64")
- endif()
endif()
if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL)

140
scripts/build/profiles.ts Normal file
View File

@@ -0,0 +1,140 @@
/**
* Build profiles — named configuration presets.
*
* Stateless: every `bun run build --profile=X` resolves fresh. No persistence,
* no stickiness. To override a single field, pass CLI flags on top of a profile.
*
* Each profile is a `PartialConfig`; `resolveConfig()` fills the rest with
* defaults derived from the target platform + profile values.
*
* ## Naming convention
*
* `<buildtype>[-<webkit-mode>][-<feature>]`
*
* debug → Debug build, prebuilt WebKit (the default)
* debug-local → Debug build, local WebKit (you cloned vendor/WebKit/)
* release → Release build, prebuilt WebKit, no LTO
* release-local → Release build, local WebKit
* release-assertions → Release + runtime assertions enabled
* release-asan → Release + address sanitizer
* ci-* → CI-specific modes (cpp-only/link-only/full)
*
* If you don't specify a profile, `debug` is used.
*/
import type { PartialConfig } from "./config.ts";
import { BuildError } from "./error.ts";
export type ProfileName = keyof typeof profiles;
export const profiles = {
/** Default local dev: debug + prebuilt WebKit. ASAN defaults on for supported platforms. */
debug: {
buildType: "Debug",
webkit: "prebuilt",
},
/** Debug with local WebKit (user clones vendor/WebKit/). */
"debug-local": {
buildType: "Debug",
webkit: "local",
},
/** Debug without ASAN — faster builds, less safety. */
"debug-no-asan": {
buildType: "Debug",
webkit: "prebuilt",
asan: false,
},
/** Release build for local testing. No LTO (that's CI-only). */
release: {
buildType: "Release",
webkit: "prebuilt",
lto: false,
},
/** Release with local WebKit. */
"release-local": {
buildType: "Release",
webkit: "local",
lto: false,
},
/**
* Release + assertions + logs. RelWithDebInfo → zig gets ReleaseSafe
* (runtime safety checks), matching the old cmake build:assert script.
*/
"release-assertions": {
buildType: "RelWithDebInfo",
webkit: "prebuilt",
assertions: true,
logs: true,
lto: false,
},
/**
* Release + ASAN + assertions. For testing prod-ish builds with
* sanitizer — catches memory bugs that only manifest at -O3. Assertions
* on too (the CMake build:asan did this) since if you're debugging
* memory you probably also want the invariant checks.
*/
"release-asan": {
buildType: "Release",
webkit: "prebuilt",
asan: true,
assertions: true,
},
/** CI: compile C++ to libbun.a only (parallelized with zig build). */
"ci-cpp-only": {
buildType: "Release",
mode: "cpp-only",
ci: true,
buildkite: true,
webkit: "prebuilt",
},
/**
* CI: cross-compile bun-zig.o only. Target platform via --os/--arch
* overrides (zig cross-compiles cleanly; this runs on a fast linux box).
*/
"ci-zig-only": {
buildType: "Release",
mode: "zig-only",
ci: true,
buildkite: true,
webkit: "prebuilt",
},
/** CI: link prebuilt objects downloaded from sibling BuildKite jobs. */
"ci-link-only": {
buildType: "Release",
mode: "link-only",
ci: true,
buildkite: true,
webkit: "prebuilt",
},
/** CI full build with LTO. */
"ci-release": {
buildType: "Release",
ci: true,
buildkite: true,
webkit: "prebuilt",
// lto default resolves to ON (ci + release + linux + !asan + !assertions)
},
} as const satisfies Record<string, PartialConfig>;
/**
* Look up a profile by name.
*/
export function getProfile(name: string): PartialConfig {
if (name in profiles) {
// The const assertion means values are readonly; spread into mutable PartialConfig.
return { ...profiles[name as ProfileName] };
}
throw new BuildError(`Unknown profile: "${name}"`, {
hint: `Available profiles: ${Object.keys(profiles).join(", ")}`,
});
}

53
scripts/build/rules.ts Normal file
View File

@@ -0,0 +1,53 @@
/**
* Convenience wrapper to register all ninja rules in one call.
*
* Ninja requires all rules to be defined before any `build` statement
* references them. Each module that emits build statements has its own
* `registerXxxRules()` function. The order of registration doesn't matter
* (rules are named), but every register call must happen before the first
* emit call.
*
* This wrapper exists so the main configure entry point doesn't need to
* know which rules each phase uses. Call this once, then emit everything.
*
* ## Why not auto-register in each emit function?
*
* Considered. The problem: some rules are SHARED (e.g. dep_configure is
* used by both source.ts deps AND webkit.ts local mode). If each emit
* function auto-registered, we'd need idempotent registration (rule
* already exists → skip). That's not hard, but it makes the "which rule
* lives where" question fuzzy. Explicit registration is clearer.
*/
import { registerCodegenRules } from "./codegen.ts";
import { registerCompileRules, registerDirStamps } from "./compile.ts";
import type { Config } from "./config.ts";
import type { Ninja } from "./ninja.ts";
import { registerDepRules } from "./source.ts";
import { registerZigRules } from "./zig.ts";
/**
* Register every ninja rule. Call once at the top of configure, before
* any `emitXxx()` or `resolveXxx()` calls.
*
* Safe to call even if some rules go unused for a given config — unused
* rules in build.ninja are ignored by ninja.
*/
export function registerAllRules(n: Ninja, cfg: Config): void {
// mkdir_stamp rule + obj/pch dir stamps. Must be first — codegen
// registers its own dir stamp using this rule.
registerDirStamps(n, cfg);
// cxx, cc, pch, link, ar
registerCompileRules(n, cfg);
// dep_fetch, dep_fetch_prebuilt, dep_configure, dep_build, dep_cargo
// WebKit prebuilt uses dep_fetch_prebuilt; local uses dep_configure/dep_build.
registerDepRules(n, cfg);
// codegen, esbuild, bun_install + codegen/stamps dir stamps
registerCodegenRules(n, cfg);
// zig_fetch, zig_build
registerZigRules(n, cfg);
}

98
scripts/build/shell.ts Normal file
View File

@@ -0,0 +1,98 @@
/**
* Shell argument quoting for ninja rule commands.
*
* Ninja executes commands via `/bin/sh -c "<command>"` on unix and
* `cmd /c "<command>"` on windows (when we wrap it that way in rules).
* Arguments with spaces/metacharacters need quoting to survive that layer.
*
* ## Why this is its own file
*
* Every module that emits ninja rules needs to quote args. Before this was
* extracted, we had four slightly-different implementations (source.ts had
* posix-only, codegen.ts had windows-aware, webkit.ts/zig.ts had posix-only
* copies). One implementation, consistently applied, prevents the "works on
* my machine but not CI" class of bug where a path with a space breaks only
* one ninja rule.
*
* ## Quoting rules
*
* POSIX (`/bin/sh`):
* Single-quote the whole thing. Embedded `'` becomes `'\''` (close quote,
* escaped quote, reopen quote). Handles every metachar including `$`, `|`,
* backticks, etc.
*
* Windows (`cmd /c`):
* Double-quote. Embedded `"` becomes `""`. This is cmd's escape convention,
* NOT the C argv convention (`\"`). The distinction matters: if the inner
* executable parses argv itself (most .exe do), cmd unwraps one layer of
* `""` but passes the rest through, so the inner program sees a literal `"`.
* Good enough for paths and values; breaks if you need to nest three layers
* of quoting (you shouldn't).
*
* Known cmd footguns we DON'T handle: `%VAR%` expansion, `^` escape,
* `&`/`|`/`>` redirection. If an argument contains these, double-quoting
* protects SOME but not all. In practice our args are paths + flag values;
* we'd hit this only with very weird file names. If it happens: switch the
* affected rule to invoke via powershell instead of cmd.
*
* ## Safe chars (no quoting needed)
*
* Letters, digits, and a small set of punctuation that's unambiguous on both
* platforms. Keeping safe-chars unquoted makes build.ninja readable — you can
* see `-DFOO=bar` instead of `'-DFOO=bar'`.
*/
/**
* Quote a single argument for a shell command.
*
* @param windows If true, use cmd.exe quoting (`""`). If false, posix (`'`).
* Pass `cfg.windows` from the build config.
*/
export function quote(arg: string, windows: boolean): string {
// Fast path: safe characters only, no quoting needed. Keeps the .ninja
// file legible for the common case (paths without spaces, flag values).
// `\` is safe in cmd (not a metachar) — and posix paths never contain
// it, so including it doesn't affect the posix branch.
if (/^[A-Za-z0-9_@%+=:,./\\\-]+$/.test(arg)) {
return arg;
}
if (windows) {
return `"${arg.replace(/"/g, '""')}"`;
}
return `'${arg.replace(/'/g, `'\\''`)}'`;
}
/**
* Quote an array of arguments and join with spaces.
*
* Convenience for the common "I have argv[] and want a shell command string"
* case — which is basically every ninja rule args var.
*/
export function quoteArgs(args: string[], windows: boolean): string {
return args.map(a => quote(a, windows)).join(" ");
}
/**
* Convert backslashes to forward slashes.
*
* Use when a path will be embedded in a sink that interprets `\` as an
* escape character:
*
* - **CMake -D values**: cmake may write the value verbatim into a generated
* .cmake file, then re-parse it — `\U` in `C:\Users\...` becomes an
* invalid escape. Forward slashes are cmake's native format.
*
* - **C/C++ string literal defines**: `-DFOO=\"C:\Users\..\"` puts the path
* in a `#define` that becomes a string literal at use site. `\U` →
* unicode escape error, `\b`/`\n` → wrong bytes.
*
* Windows file APIs accept forward slashes, so this is safe for any path
* that ends up at CreateFile/fopen. It's NOT safe for paths passed to
* cmd.exe built-ins (cd, del) — those require backslashes — but we avoid
* those anyway.
*
* No-op on posix paths (no backslashes to replace).
*/
export function slash(path: string): string {
return path.replace(/\\/g, "/");
}

1162
scripts/build/source.ts Normal file

File diff suppressed because it is too large Load Diff

141
scripts/build/sources.ts Normal file
View File

@@ -0,0 +1,141 @@
/**
* Source list globbing.
*
* Patterns live in `cmake/Sources.json` (also read by glob-sources.mjs for
* the CMake build — shared single source of truth). `globAllSources()` is
* called on every configure — new/deleted files are picked up automatically.
* All patterns expand in a single pass so there's one consistent filesystem
* snapshot; callers receive a plain struct, no filesystem reads thereafter.
*
* Editing an existing file → ninja handles incrementally (each file is its
* own build edge). Adding/removing a file → next configure re-globs →
* build.ninja differs → `writeIfChanged` writes → ninja sees new graph.
*/
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { BuildError, assert } from "./error.ts";
// Bun.Glob instead of node:fs globSync — the latter isn't in older bun
// versions (CI agents pin bun; we don't control when they bump).
function globSync(pattern: string, opts: { cwd: string }): string[] {
return [...new Bun.Glob(pattern).scanSync({ cwd: opts.cwd })];
}
/**
* All globbed source lists. Each field is absolute paths, sorted.
*
* Field names match the Sources.json `output` keys (minus `.txt` suffix,
* camelCased). If you add a new entry to Sources.json, add a field here
* AND to `fieldMap` below.
*
* TODO(cmake-removal): this interface + fieldMap duplicate Sources.json.
* Once cmake is gone (no more glob-sources.mjs reading the JSON), convert
* Sources.json to inline TS patterns here — field names ARE the keys,
* interface derived via `keyof typeof patterns`. Single source of truth.
*/
export interface Sources {
/** `packages/bun-error/*.{json,ts,tsx,css}` + images */
bunError: string[];
/** `src/node-fallbacks/*.js` */
nodeFallbacks: string[];
/** `src/bun.js/**\/*.classes.ts` — input to generate-classes codegen */
zigGeneratedClasses: string[];
/** `src/js/**\/*.{js,ts}` — built-in modules bundled at build time */
js: string[];
/** `src/codegen/*.ts` — the codegen scripts themselves */
jsCodegen: string[];
/** `src/bake/**` — server-rendering runtime bundled into binary */
bakeRuntime: string[];
/** `src/**\/*.bind.ts` — legacy bindgen input */
bindgen: string[];
/** `src/**\/*.bindv2.ts` — v2 bindgen input */
bindgenV2: string[];
/** `src/codegen/bindgenv2/**\/*.ts` — bindgen v2 generator code */
bindgenV2Internal: string[];
/** `src/**\/*.zig` — NOT filtered; includes codegen-written files (see bun.ts) */
zig: string[];
/** All `*.cpp` compiled into bun (bindings, webcore, v8 shim, usockets) */
cxx: string[];
/** All `*.c` compiled into bun (usockets, llhttp, uv polyfills) */
c: string[];
}
interface SourcePattern {
output: string;
paths: string[];
exclude?: string[];
}
/**
* Glob all source lists from `cmake/Sources.json`. Called once per configure.
*/
export function globAllSources(cwd: string): Sources {
const specPath = resolve(cwd, "cmake", "Sources.json");
let specs: SourcePattern[];
try {
specs = JSON.parse(readFileSync(specPath, "utf8")) as SourcePattern[];
} catch (cause) {
throw new BuildError("Could not read Sources.json", { file: specPath, cause });
}
// Map output name → field name. New entries in Sources.json MUST be added
// here or configure fails — catches drift between the JSON and this struct.
const fieldMap: Record<string, keyof Sources> = {
"BunErrorSources.txt": "bunError",
"NodeFallbacksSources.txt": "nodeFallbacks",
"ZigGeneratedClassesSources.txt": "zigGeneratedClasses",
"JavaScriptSources.txt": "js",
"JavaScriptCodegenSources.txt": "jsCodegen",
"BakeRuntimeSources.txt": "bakeRuntime",
"BindgenSources.txt": "bindgen",
"BindgenV2Sources.txt": "bindgenV2",
"BindgenV2InternalSources.txt": "bindgenV2Internal",
"ZigSources.txt": "zig",
"CxxSources.txt": "cxx",
"CSources.txt": "c",
};
const result = {} as Sources;
const seen = new Set<keyof Sources>();
for (const spec of specs) {
const field = fieldMap[spec.output];
assert(field !== undefined, `Unknown Sources.json entry: ${spec.output}`, {
file: specPath,
hint: `Add a mapping in scripts/build/sources.ts fieldMap and a field to the Sources interface`,
});
const excludes = new Set((spec.exclude ?? []).map(normalize));
const files: string[] = [];
for (const pattern of spec.paths) {
for (const rel of globSync(pattern, { cwd })) {
const normalized = normalize(rel);
if (excludes.has(normalized)) continue;
files.push(resolve(cwd, normalized));
}
}
files.sort((a, b) => a.localeCompare(b));
assert(files.length > 0, `Source list ${spec.output} matched nothing`, {
file: specPath,
hint: `Patterns: ${spec.paths.join(", ")}`,
});
result[field] = files;
seen.add(field);
}
// Verify all fields populated — catches a Sources.json entry being deleted
// without updating this struct.
for (const field of Object.values(fieldMap)) {
assert(seen.has(field), `Sources.json missing entry for ${field}`, { file: specPath });
}
return result;
}
/** Forward slashes, no leading ./ — for exclude-set comparisons. */
function normalize(p: string): string {
return p.replace(/\\/g, "/").replace(/^\.\//, "");
}

392
scripts/build/stream.ts Normal file
View File

@@ -0,0 +1,392 @@
#!/usr/bin/env bun
/**
* Subprocess output wrapper for ninja builds. Two modes:
*
* ## Default: prefix mode (for pooled rules)
*
* Ninja's non-console pools buffer subprocess output until completion.
* Console pool gives live output but forces serial execution. Prefix mode
* threads the needle: runs the real command, prefixes each output line with
* `[name]`, and writes to an inherited FD that bypasses ninja's pipe entirely.
*
* build.ts dups stderr → FD 3 before spawning ninja. Ninja's subprocess
* spawn only touches FDs 0-2; FD 3 is inherited unchanged. Writes to FD 3
* land directly on the terminal. If FD 3 isn't open (direct ninja invocation),
* fall back to stdout — ninja buffers it, build still works.
*
* ## --console mode (for pool=console rules)
*
* stdio inherit — child gets direct TTY. For commands that have their own
* TTY UI (zig's spinner, lld's progress). Ninja defers its own [N/M] while
* the console job owns the terminal.
*
* On Windows, applies a compensation for ninja's stdio buffering bug:
* ninja's \n before console-pool jobs goes through fwrite without fflush
* (line_printer.cc PrintOrBuffer); MSVCRT has no line-buffering, so the \n
* stays in ninja's stdio buffer. Subprocess output (via raw HANDLE) glues
* onto the [N/M] status, and the deferred \n lands as a blank line later.
* Fix: \n before (clean line), \x1b[1A after (absorb the deferred \n).
* Windows+TTY only — posix line-buffers, doesn't need it.
*
* ## Usage (from ninja rule)
*
* # prefix mode, pooled
* command = bun /path/to/stream.ts $name cmake --build $builddir ...
* pool = dep # any depth — output streams live regardless
*
* # console mode
* command = bun /path/to/stream.ts $name --console $zig build obj ...
* pool = console
*
* --cwd=DIR / --env=K=V / --console / --zig-progress go between <name> and
* <command>. They exist so the rule doesn't need `sh -c '...'` (which would
* conflict with shell-quoted ninja vars like $args).
*
* --zig-progress (prefix mode, posix only): sets ZIG_PROGRESS=3, decodes
* zig's binary progress protocol into `[zig] Stage [N/M]` lines. Without it,
* zig sees piped stderr → spinner disabled → silence during compile.
*/
import { spawn, spawnSync } from "node:child_process";
import { createWriteStream, writeSync } from "node:fs";
import { createInterface } from "node:readline";
export const streamPath: string = import.meta.filename;
/**
* File descriptor for bypassing ninja's output buffering. build.ts dups
* its stderr into this FD before spawning ninja; stream.ts writes prefixed
* lines here. Ninja's subprocess spawn only touches FDs 0-2, so this
* flows through unchanged.
*
* Exported so build.ts uses the same number (keep them in sync).
*/
export const STREAM_FD = 3;
/**
* Produce a colored `[name] ` prefix. Color derived from a hash of the
* name — deterministic, so zstd is always the same shade across runs.
*
* Palette: 12 ANSI 256-color codes from gold → green → cyan → blue →
* purple. No reds/pinks (error connotation), no dark blues (illegible
* on black), no white/grey (no contrast). With ~18 deps a few collide —
* the name disambiguates.
*/
function coloredPrefix(name: string): string {
// Override: zig brand orange (hash would give yellow-green).
const overrides: Record<string, number> = { zig: 214 };
const palette = [220, 184, 154, 120, 114, 86, 87, 81, 111, 147, 141, 183];
// fnv-1a — tiny, good-enough distribution for short strings.
let h = 2166136261;
for (let i = 0; i < name.length; i++) {
h ^= name.charCodeAt(i);
h = Math.imul(h, 16777619);
}
const color = overrides[name] ?? palette[(h >>> 0) % palette.length];
// 38;5;N = set foreground to 256-color N. 39 = default foreground.
return `\x1b[38;5;${color}m[${name}]\x1b[39m `;
}
// ───────────────────────────────────────────────────────────────────────────
// CLI — guarded so `import { STREAM_FD } from "./stream.ts"` doesn't run it.
// ───────────────────────────────────────────────────────────────────────────
if (import.meta.main) {
main();
}
function main(): void {
const argv = process.argv.slice(2);
const name = argv.shift();
if (!name) {
process.stderr.write("usage: stream.ts <name> [--cwd=DIR] [--env=K=V ...] <command...>\n");
process.exit(2);
}
// Parse options (stop at first non-flag).
let cwd: string | undefined;
let zigProgress = false;
let consoleMode = false;
const envOverrides: Record<string, string> = {};
while (argv[0]?.startsWith("--")) {
const opt = argv.shift()!;
if (opt.startsWith("--cwd=")) {
cwd = opt.slice(6);
} else if (opt.startsWith("--env=")) {
const kv = opt.slice(6);
const eq = kv.indexOf("=");
if (eq > 0) envOverrides[kv.slice(0, eq)] = kv.slice(eq + 1);
} else if (opt === "--zig-progress") {
zigProgress = true;
} else if (opt === "--console") {
consoleMode = true;
} else {
process.stderr.write(`stream.ts: unknown option ${opt}\n`);
process.exit(2);
}
}
const cmd = argv;
if (cmd.length === 0) {
process.stderr.write("stream.ts: no command given\n");
process.exit(2);
}
// ─── Console mode: passthrough with Windows compensation ───
if (consoleMode) {
const compensate = process.platform === "win32" && process.stderr.isTTY;
if (compensate) process.stderr.write("\n");
const result = spawnSync(cmd[0]!, cmd.slice(1), {
stdio: "inherit",
cwd,
env: Object.keys(envOverrides).length > 0 ? { ...process.env, ...envOverrides } : undefined,
});
if (compensate) process.stderr.write("\x1b[1A");
if (result.error) {
process.stderr.write(`[${name}] spawn failed: ${result.error.message}\n`);
process.exit(127);
}
process.exit(result.status ?? (result.signal ? 1 : 0));
}
// Probe STREAM_FD. If build.ts set it up, it's a dup of the terminal.
// If not (direct ninja invocation, CI without build.ts, etc.), fall
// back to stdout which ninja will buffer — less nice but functional.
//
// TODO(windows): a numeric fd won't work on Windows. Need to inherit
// a HANDLE and open it via `CONOUT$` or CreateFile. The fallback path
// (stdout) works, just buffered. When porting, test stdio[STREAM_FD]
// in build.ts's spawnSync actually inherits on Windows (CreateProcessA
// has bInheritHandles=TRUE in ninja — see subprocess-win32.cc).
let out: NodeJS.WritableStream;
let outFd: number;
try {
writeSync(STREAM_FD, ""); // 0-byte write: throws EBADF if fd isn't open
// autoClose false: the fd is shared across parallel stream.ts procs.
out = createWriteStream("", { fd: STREAM_FD, autoClose: false });
outFd = STREAM_FD;
} catch {
out = process.stdout;
outFd = 1;
}
// "Interactive" = we're on the FD 3 bypass. build.ts only opens FD 3
// when its stderr is a TTY, so this check alone tells us a human is
// watching. Fallback mode (outFd=1, FD 3 not set up) means piped —
// either scripts/bd logging, CI, or direct `ninja` — and in all those
// cases we want the quiet treatment: no colors, no zig progress,
// ninja buffers per-job.
const interactive = outFd === STREAM_FD;
const useColor = interactive && !process.env.NO_COLOR && process.env.TERM !== "dumb";
// Color the prefix so interleaved parallel output is visually separable.
// Hash-to-color: same dep always gets the same color across runs.
const prefix = useColor ? coloredPrefix(name) : `[${name}] `;
// ─── Zig progress IPC (interactive only) ───
// zig's spinner is TTY-only — piped stderr = silence during compile.
// ZIG_PROGRESS=<fd> makes it write a binary protocol to that fd
// instead. We open a pipe at child fd 3, set ZIG_PROGRESS=3, decode
// packets into `[zig] Stage [N/M]` lines.
//
// Needs oven-sh/zig's fix for ziglang/zig#24722 — upstream `zig build`
// strips ZIG_PROGRESS from the build runner's env; the fork forwards
// it through.
//
// Gated on `interactive`: when piped, progress lines are log noise
// (~35 Code Gen lines that clutter failure logs / LLM context). No
// FD 3 setup → zig sees no ZIG_PROGRESS → just start + summary.
const stdio: import("node:child_process").StdioOptions = ["inherit", "pipe", "pipe"];
if (zigProgress && interactive) {
envOverrides.ZIG_PROGRESS = "3";
stdio.push("pipe"); // index 3 = zig's IPC write end
}
const child = spawn(cmd[0]!, cmd.slice(1), {
stdio,
cwd,
env: Object.keys(envOverrides).length > 0 ? { ...process.env, ...envOverrides } : undefined,
});
// Ninja's smart-terminal mode ends each `[N/M] description` status line
// with \r (not \n) so the next status overwrites in place. We write to
// the same terminal via FD 3, so ninja's status can appear BETWEEN any
// two of our lines. `\r\x1b[K` (return-to-col-0 + clear-to-eol) before
// every line guarantees a clean row: if ninja's status is sitting
// there, it's wiped; if the cursor is already at col 0 on an empty
// row (after our own \n), the clear is a no-op. A single leading \n
// (the old approach) only protected the first write — subsequent lines
// could still glue onto a mid-update ninja status.
//
// Only in interactive mode — piped output has no status-line race.
const lead = interactive ? "\r\x1b[K" : "";
const write = (text: string): void => {
out.write(lead + text);
};
// Line-split + prefix + forward. readline handles partial lines at EOF
// correctly (emits the trailing fragment without a newline).
const pump = (stream: NodeJS.ReadableStream): void => {
const rl = createInterface({ input: stream, crlfDelay: Infinity });
rl.on("line", line => write(prefix + line + "\n"));
};
pump(child.stdout!);
pump(child.stderr!);
// stdio[3] only exists if we pushed it above (zigProgress && interactive).
if (child.stdio[3]) {
decodeZigProgress(child.stdio[3] as NodeJS.ReadableStream, text => write(prefix + text + "\n"));
}
// writeSync for final messages: out.write() is async; process.exit()
// terminates before the WriteStream buffer flushes. Sync write ensures
// the last line actually reaches the terminal on error paths.
const writeFinal = (text: string): void => {
writeSync(outFd, lead + prefix + text);
};
child.on("error", err => {
writeFinal(`spawn failed: ${err.message}\n`);
process.exit(127);
});
child.on("close", (code, signal) => {
if (signal) {
writeFinal(`killed by ${signal}\n`);
process.exit(1);
}
process.exit(code ?? 1);
});
}
// ───────────────────────────────────────────────────────────────────────────
// ZIG_PROGRESS protocol decoder
//
// Wire format (vendor/zig/lib/std/Progress.zig writeIpc, all LE):
// 1 byte: N (node count, u8)
// N * 48: Storage[] — per node: u32 completed, u32 total, [40]u8 name
// N * 1: Parent[] — per node: u8 (254=unused 255=root else=index)
//
// Packets arrive ~60ms apart. We pick the most-active counted stage from
// each, throttle to 1/sec, dedupe identical lines. Silence during
// uncounted phases (LLVM Emit Object) matches zig's own non-TTY behavior.
// ───────────────────────────────────────────────────────────────────────────
const STORAGE_SIZE = 48; // u32 + u32 + [40]u8, aligned to 8
const NAME_OFFSET = 8;
const NAME_LEN = 40;
function decodeZigProgress(stream: NodeJS.ReadableStream, emit: (text: string) => void): void {
let buf = Buffer.alloc(0);
let lastText = "";
let lastEmit = 0;
stream.on("data", (chunk: Buffer) => {
buf = Buffer.concat([buf, chunk]);
// Parse complete packets. Packet = 1 + N*48 + N bytes; N is byte 0.
while (buf.length >= 1) {
const n = buf[0]!;
const packetLen = 1 + n * STORAGE_SIZE + n;
if (buf.length < packetLen) break;
const packet = buf.subarray(0, packetLen);
buf = buf.subarray(packetLen);
const text = renderPacket(packet, n);
if (text === null || text === lastText) continue;
// Throttle: counters tick every packet, and `total` GROWS during
// the build (Code Generation: ~130 → ~120k as zig discovers more
// work), so count-based bucketing fails. Time-throttle is stable.
const now = Date.now();
if (now - lastEmit < 1000) continue;
lastText = text;
lastEmit = now;
emit(text);
}
});
}
/**
* Pick a one-line status from the progress tree.
*
* Tree during bun compile (forwarded through the fork's #24722 fix):
* root "" ← frontend, name cleared
* └─ steps [2/5] ← build runner's step counter
* └─ compile obj bun-debug
* ├─ Semantic Analysis [14233] ← completed-only counter
* │ └─ Io.Writer.print__anon_* ← per-symbol noise, c=0 t=0
* ├─ Code Generation [3714/4174] ← active bounded counter
* │ └─ fs.Dir.Walker.next ← per-symbol noise
* └─ Linking [1/11599] ← barely started
*
* Stages run IN PARALLEL and zig reuses node slots — index order is NOT
* tree depth. The distinguisher is counters: stages have them (c and/or
* t > 0), per-symbol noise doesn't (c=0 t=0). Strategy: highest-index
* node with an ACTIVE counter (0 < c < t). Highest index ≈ most recently
* allocated ≈ most relevant. Fall back to completed-only, then
* just-started.
*
* Returns null when no counted node exists (LLVM phase — 60+ seconds
* with no counter). Silence matches zig's own non-TTY behavior.
*/
function renderPacket(packet: Buffer, n: number): string | null {
if (n === 0) return null;
const read = (idx: number) => {
const off = 1 + idx * STORAGE_SIZE;
return {
name: readName(packet, off + NAME_OFFSET),
completed: packet.readUInt32LE(off),
total: packet.readUInt32LE(off + 4),
};
};
const fmt = (name: string, c: number, t: number): string =>
t > 0 ? `${name} [${c}/${t}]` : c > 0 ? `${name} [${c}]` : name;
// total === u32::MAX: node holds an IPC fd (parent's reference to a
// child pipe), not a counter. Never render those.
const isIpc = (t: number) => t === 0xffffffff;
// Pass 1: active bounded counter (0 < c < t). Skips finished stages
// sitting at [976/976] while real work continues elsewhere.
for (let i = n - 1; i >= 0; i--) {
const { name, completed, total } = read(i);
if (isIpc(total) || name.length === 0) continue;
if (total > 0 && completed > 0 && completed < total) {
return fmt(name, completed, total);
}
}
// Pass 2: completed-only counter (Semantic Analysis goes to ~250k
// with no total).
for (let i = n - 1; i >= 0; i--) {
const { name, completed, total } = read(i);
if (isIpc(total) || name.length === 0) continue;
if (completed > 0 && total === 0) {
return fmt(name, completed, total);
}
}
// Pass 3: bounded but not started (c=0, t>0).
for (let i = n - 1; i >= 0; i--) {
const { name, completed, total } = read(i);
if (isIpc(total) || name.length === 0) continue;
if (total > 0) {
return fmt(name, completed, total);
}
}
return null;
}
/** Read a NUL-padded fixed-width name field. */
function readName(buf: Buffer, offset: number): string {
const end = offset + NAME_LEN;
let nul = offset;
while (nul < end && buf[nul]! !== 0) nul++;
return buf.toString("utf8", offset, nul);
}

564
scripts/build/tools.ts Normal file
View File

@@ -0,0 +1,564 @@
/**
* Toolchain discovery.
*
* Finds compilers/tools in PATH + known platform-specific locations.
* Version-checks when a constraint is given. Throws BuildError with a helpful
* hint when a required tool is missing.
*/
import { execSync, spawnSync } from "node:child_process";
import { accessSync, constants, existsSync, readdirSync, statSync } from "node:fs";
import { homedir } from "node:os";
import { delimiter, join } from "node:path";
import type { Arch, OS, Toolchain } from "./config.ts";
import { BuildError } from "./error.ts";
// ───────────────────────────────────────────────────────────────────────────
// Version range checking
// ───────────────────────────────────────────────────────────────────────────
/**
* Parse a version like "21.1.8" out of arbitrary text (tool --version output).
* Returns the first X.Y.Z found, or undefined.
*/
function parseVersion(text: string): string | undefined {
const m = text.match(/(\d+)\.(\d+)\.(\d+)/);
return m ? `${m[1]}.${m[2]}.${m[3]}` : undefined;
}
/**
* Compare two X.Y.Z version strings. Returns -1, 0, 1.
*/
function compareVersions(a: string, b: string): number {
const pa = a.split(".").map(Number);
const pb = b.split(".").map(Number);
for (let i = 0; i < 3; i++) {
const ai = pa[i] ?? 0;
const bi = pb[i] ?? 0;
if (ai !== bi) return ai < bi ? -1 : 1;
}
return 0;
}
/**
* Check if a version satisfies a range string.
*
* Range syntax: `>=X.Y.Z <A.B.C` (space-separated constraints, all must pass).
* Single version without operator = exact match.
* Empty/undefined range = always satisfied.
*/
export function satisfiesRange(version: string, range: string | undefined): boolean {
if (range === undefined || range === "" || range === "ignore") return true;
const v = parseVersion(version);
if (v === undefined) return false;
for (const part of range.split(/\s+/)) {
if (part === "") continue;
const m = part.match(/^(>=|<=|>|<|=)?(\d+\.\d+\.\d+)$/);
if (!m) return false; // malformed range
const op = m[1] ?? "=";
const target = m[2];
if (target === undefined) return false;
const cmp = compareVersions(v, target);
const ok =
op === ">="
? cmp >= 0
: op === ">"
? cmp > 0
: op === "<="
? cmp <= 0
: op === "<"
? cmp < 0
: /* "=" */ cmp === 0;
if (!ok) return false;
}
return true;
}
// ───────────────────────────────────────────────────────────────────────────
// Tool discovery
// ───────────────────────────────────────────────────────────────────────────
export interface ToolSpec {
/** Names to try, in order. On Windows `.exe` is appended automatically. */
names: string[];
/** Extra search paths beyond $PATH. Tried FIRST (more specific). */
paths?: string[];
/** Version constraint, e.g. `">=21.1.0 <22.0.0"`. */
version?: string;
/** How to get the version. `"--version"` (default) or `"version"` (go/zig style). */
versionArg?: string;
/** If true, throws BuildError when not found. */
required: boolean;
/** Extra hint text for the error message. */
hint?: string;
}
/**
* Rejection log for a single tool search — used in error messages.
*/
interface Rejection {
path: string;
reason: string;
}
/**
* Find the bun executable to use for codegen. Prefers ~/.bun/bin/bun over
* process.execPath — CI agents pin an old system bun (/usr/bin/bun), but
* codegen scripts use newer `bun build` CLI flags. cmake did the same
* (SetupBun.cmake: PATHS $ENV{HOME}/.bun/bin before system PATH).
*
* Falls back to process.execPath (the bun running us) — always works for
* local dev where system bun is recent enough.
*/
export function findBun(os: OS): string {
const exe = os === "windows" ? "bun.exe" : "bun";
const userBun = join(homedir(), ".bun", "bin", exe);
if (isExecutable(userBun)) return userBun;
return process.execPath;
}
/**
* Check if a file exists and is executable.
*/
function isExecutable(p: string): boolean {
try {
// Must check isFile(): X_OK on a directory means "traversable", not
// "runnable". Without this, a `cmake/` dir in a PATH entry would shadow
// the real cmake binary.
if (!statSync(p).isFile()) return false;
accessSync(p, constants.X_OK);
return true;
} catch {
return false;
}
}
/**
* Get the version of a tool. Returns the parsed X.Y.Z, or a diagnostic
* string describing why parsing failed (starts with a digit → version,
* otherwise → failure reason for the rejection log).
*/
function getToolVersion(exe: string, versionArg: string): { version: string } | { reason: string } {
// stdio ignore on stdin: on Windows CI the parent's stdin can be a
// handle that blocks the child's CRT init. --version never reads stdin.
// 30s timeout: cold start of a large binary (clang is 100+ MB) through
// Defender scan-on-access can legitimately exceed 5s on a busy CI box.
const result = spawnSync(exe, [versionArg], {
encoding: "utf8",
timeout: 30_000,
stdio: ["ignore", "pipe", "pipe"],
});
if (result.error) {
return { reason: `spawn failed: ${result.error.message}` };
}
// Some tools print version to stderr (e.g. some zig builds). Check both.
const version = parseVersion(result.stdout ?? "") ?? parseVersion(result.stderr ?? "");
if (version !== undefined) return { version };
// Parse failed — include what we saw (truncated) so the error is
// actionable instead of just "could not parse".
const output = ((result.stdout ?? "") + (result.stderr ?? "")).trim().slice(0, 200);
if (result.status !== 0) {
return { reason: `exited ${result.status}: ${output || "(no output)"}` };
}
return { reason: `no X.Y.Z in output: ${output || "(empty)"}` };
}
/**
* Ask clang what arch it targets by default. Parses `Target:` from
* `--version` output. Returns undefined if unparseable.
*
* CMake does this during compiler detection (project()) to set
* CMAKE_SYSTEM_PROCESSOR — that's how the old cmake build knew arm64
* even when cmake.exe itself was x64. process.arch reflects the running
* process (may be emulated); the compiler's target is what we actually
* build for.
*/
export function clangTargetArch(clang: string): Arch | undefined {
const result = spawnSync(clang, ["--version"], {
encoding: "utf8",
timeout: 30_000,
stdio: ["ignore", "pipe", "pipe"],
});
if (result.error || result.status !== 0) return undefined;
const m = (result.stdout ?? "").match(/^Target:\s*(\S+)/m);
if (!m) return undefined;
const triple = m[1]!;
// aarch64-pc-windows-msvc, arm64-apple-darwin, x86_64-unknown-linux-gnu, ...
if (/^(aarch64|arm64)/.test(triple)) return "aarch64";
if (/^(x86_64|x64|amd64)/i.test(triple)) return "x64";
return undefined;
}
/**
* Find a tool. Searches provided paths first, then $PATH.
* Returns the absolute path or undefined (if not required).
*/
export function findTool(spec: ToolSpec): string | undefined {
const exeSuffix = process.platform === "win32" ? ".exe" : "";
const searchPaths = [...(spec.paths ?? []), ...(process.env.PATH ?? "").split(delimiter).filter(p => p.length > 0)];
const versionArg = spec.versionArg ?? "--version";
const rejections: Rejection[] = [];
for (const name of spec.names) {
const candidate = name.endsWith(exeSuffix) ? name : name + exeSuffix;
for (const dir of searchPaths) {
const full = join(dir, candidate);
if (!isExecutable(full)) continue;
if (spec.version !== undefined) {
const v = getToolVersion(full, versionArg);
if ("reason" in v) {
rejections.push({ path: full, reason: v.reason });
continue;
}
if (!satisfiesRange(v.version, spec.version)) {
rejections.push({ path: full, reason: `version ${v.version} does not satisfy ${spec.version}` });
continue;
}
}
return full;
}
}
if (spec.required) {
const primaryName = spec.names[0] ?? "<unknown>";
let msg = `Could not find ${primaryName}`;
if (spec.version !== undefined) msg += ` (version ${spec.version})`;
let hint = spec.hint ?? "";
if (rejections.length > 0) {
hint += (hint ? "\n" : "") + "Found but rejected:\n" + rejections.map(r => ` ${r.path}: ${r.reason}`).join("\n");
}
if (rejections.length === 0 && searchPaths.length > 0) {
hint +=
(hint ? "\n" : "") + `Searched: ${searchPaths.slice(0, 5).join(", ")}${searchPaths.length > 5 ? ", ..." : ""}`;
}
throw new BuildError(msg, hint ? { hint } : {});
}
return undefined;
}
// ───────────────────────────────────────────────────────────────────────────
// LLVM-specific discovery
// ───────────────────────────────────────────────────────────────────────────
/**
* LLVM version constraint. Any version in the same major.minor range is
* accepted (e.g. Alpine 3.23 ships 21.1.2 while we target 21.1.8).
*/
export const LLVM_VERSION = "21.1.8";
const LLVM_MAJOR = "21";
const LLVM_MINOR = "1";
const LLVM_VERSION_RANGE = `>=${LLVM_MAJOR}.${LLVM_MINOR}.0 <${LLVM_MAJOR}.${LLVM_MINOR}.99`;
/**
* Known LLVM install locations per platform. Call ONCE from
* resolveLlvmToolchain — it contains a spawn on macOS (brew --prefix as
* fallback) which takes ~100ms, so calling it per-tool would dominate
* configure time.
*/
function llvmSearchPaths(os: OS, arch: Arch): string[] {
const paths: string[] = [];
if (os === "darwin") {
// Try the arch-default prefix first (correct for standard homebrew
// installs — /opt/homebrew on Apple Silicon, /usr/local on Intel).
// Only spawn `brew --prefix` as a last resort for custom installs —
// brew's startup is slow and this runs on every configure.
const defaultPrefix = arch === "aarch64" ? "/opt/homebrew" : "/usr/local";
let brewPrefix: string;
if (isExecutable(`${defaultPrefix}/bin/brew`)) {
brewPrefix = defaultPrefix;
} else {
try {
brewPrefix = execSync("brew --prefix", { encoding: "utf8", timeout: 3000 }).trim();
} catch {
brewPrefix = defaultPrefix;
}
}
paths.push(`${brewPrefix}/opt/llvm@${LLVM_MAJOR}/bin`);
paths.push(`${brewPrefix}/opt/llvm/bin`);
}
if (os === "windows") {
// Prefer standalone LLVM over VS-bundled
paths.push("C:\\Program Files\\LLVM\\bin");
}
if (os === "linux" || os === "darwin") {
paths.push("/usr/lib/llvm/bin");
// Debian/Ubuntu-style suffixed paths
paths.push(`/usr/lib/llvm-${LLVM_MAJOR}.${LLVM_MINOR}.0/bin`);
paths.push(`/usr/lib/llvm-${LLVM_MAJOR}.${LLVM_MINOR}/bin`);
paths.push(`/usr/lib/llvm-${LLVM_MAJOR}/bin`);
paths.push(`/usr/lib/llvm${LLVM_MAJOR}/bin`);
}
return paths;
}
/**
* Version-suffixed command names (e.g. clang-21, clang-21.1).
* Unix distros often only ship these suffixed versions.
*/
function llvmNameVariants(name: string): string[] {
return [
name,
`${name}-${LLVM_MAJOR}.${LLVM_MINOR}.0`,
`${name}-${LLVM_MAJOR}.${LLVM_MINOR}`,
`${name}-${LLVM_MAJOR}`,
];
}
function llvmInstallHint(os: OS): string {
if (os === "darwin") return `Install with: brew install llvm@${LLVM_MAJOR}`;
if (os === "linux")
return `Install with: apt install clang-${LLVM_MAJOR} lld-${LLVM_MAJOR} (or equivalent for your distro)`;
if (os === "windows") return `Install LLVM ${LLVM_VERSION} from https://github.com/llvm/llvm-project/releases`;
return "";
}
/**
* Find an LLVM tool with version checking. `paths` computed once by the
* caller (contains a slow brew spawn on macOS).
*/
function findLlvmTool(
baseName: string,
paths: string[],
os: OS,
opts: { checkVersion: boolean; required: boolean },
): string | undefined {
const spec: ToolSpec = {
names: llvmNameVariants(baseName),
paths,
required: opts.required,
hint: llvmInstallHint(os),
};
if (opts.checkVersion) spec.version = LLVM_VERSION_RANGE;
return findTool(spec);
}
// ───────────────────────────────────────────────────────────────────────────
// Full toolchain resolution
// ───────────────────────────────────────────────────────────────────────────
/**
* Resolve the entire toolchain for a target.
*
* Call this once at configure time. All tool paths are absolute.
* Throws BuildError if any required tool is missing.
*
* zig/bun/esbuild are resolved separately (they come from cache/, not PATH)
* so pass them in as placeholders for now; they'll be filled by downloaders.
*/
export function resolveLlvmToolchain(
os: OS,
arch: Arch,
): Pick<Toolchain, "cc" | "cxx" | "ar" | "ranlib" | "ld" | "strip" | "dsymutil" | "ccache" | "rc" | "mt"> {
// Compute search paths ONCE. Contains a brew spawn on macOS (~100ms)
// so calling it per-tool would burn ~600ms. Every tool below gets
// the same paths; first-match-wins in findTool means whichever LLVM
// install is highest-priority wins consistently.
const paths = llvmSearchPaths(os, arch);
// clang — version-checked. clang++ is the same binary (hardlink or
// symlink) from the same install; a second version-check spawn would
// just return the same answer. We still locate it separately so the
// "not found" error names the right tool.
const cc = findLlvmTool(os === "windows" ? "clang-cl" : "clang", paths, os, {
checkVersion: true,
required: true,
});
const cxx = findLlvmTool(os === "windows" ? "clang-cl" : "clang++", paths, os, {
checkVersion: false,
required: true,
});
// ar: llvm-ar (or llvm-lib on Windows)
// No version check — ar doesn't always print a parseable version,
// and any ar from the same LLVM install is fine.
const ar = findLlvmTool(os === "windows" ? "llvm-lib" : "llvm-ar", paths, os, {
checkVersion: false,
required: true,
});
// ranlib: llvm-ranlib (unix only — Windows uses llvm-lib which doesn't need it)
// Needed for nested cmake builds (CMAKE_RANLIB). llvm-ar's `s` flag does the
// same thing for our direct archives, but deps may call ranlib explicitly.
let ranlib: string | undefined;
if (os !== "windows") {
ranlib = findLlvmTool("llvm-ranlib", paths, os, {
checkVersion: false,
required: true,
});
}
// ld: ld.lld on Linux (passed as --ld-path=), lld-link on Windows.
// On Darwin clang drives the system linker directly.
let ld: string;
if (os === "windows") {
const found = findLlvmTool("lld-link", paths, os, { checkVersion: false, required: true });
ld = found ?? ""; // unreachable (required=true throws), but keeps types happy
} else if (os === "linux") {
const found = findLlvmTool("ld.lld", paths, os, { checkVersion: true, required: true });
ld = found ?? "";
} else {
ld = ""; // darwin: unused
}
// strip: GNU strip on Linux (more features), llvm-strip elsewhere
let strip: string;
if (os === "linux") {
const found = findTool({
names: ["strip"],
required: true,
hint: "Install binutils for your distro",
});
strip = found ?? "";
} else {
const found = findLlvmTool("llvm-strip", paths, os, { checkVersion: false, required: true });
strip = found ?? "";
}
// dsymutil: darwin only
let dsymutil: string | undefined;
if (os === "darwin") {
dsymutil = findLlvmTool("dsymutil", paths, os, { checkVersion: false, required: true });
}
// rc/mt: windows only. Passed to nested cmake — when CMAKE_C_COMPILER
// is an explicit path, cmake's find_program for these may not search
// the compiler's directory, so we resolve them here and pass
// explicitly. rc is required (cmake's try_compile on windows uses
// it); mt is optional (not all LLVM distros ship it — source.ts sets
// CMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY as fallback).
let rc: string | undefined;
let mt: string | undefined;
if (os === "windows") {
rc = findLlvmTool("llvm-rc", paths, os, { checkVersion: false, required: true });
mt = findLlvmTool("llvm-mt", paths, os, { checkVersion: false, required: false });
}
// ccache: optional. If found, used as compiler launcher.
const ccache = findTool({
names: ["ccache"],
required: false,
});
// These are definitely defined at this point (required=true throws otherwise),
// but TS can't see through that, so assert.
if (cc === undefined || cxx === undefined || ar === undefined) {
throw new BuildError("unreachable: required tool undefined");
}
if (strip === "") {
throw new BuildError("unreachable: strip undefined");
}
return { cc, cxx, ar, ranlib, ld, strip, dsymutil, ccache, rc, mt };
}
/**
* Find an arbitrary system tool (not LLVM-specific).
* Thin wrapper for convenience.
*/
export function findSystemTool(name: string, opts?: { required?: boolean; hint?: string }): string | undefined {
const spec: ToolSpec = {
names: [name],
required: opts?.required ?? false,
};
if (opts?.hint !== undefined) spec.hint = opts.hint;
return findTool(spec);
}
// ───────────────────────────────────────────────────────────────────────────
// Rust toolchain (cargo) — needed for lolhtml only
// ───────────────────────────────────────────────────────────────────────────
export interface CargoToolchain {
cargo: string;
cargoHome: string;
rustupHome: string;
}
/**
* Find cargo + its home directories. Returns undefined if cargo isn't
* installed — caller decides whether to error (only needed when building
* rust deps from source).
*/
export function findCargo(hostOs: OS): CargoToolchain | undefined {
// Resolve CARGO_HOME and RUSTUP_HOME the same way rustup does:
// explicit env var → platform default. We don't probe %PROGRAMFILES%
// for MSI installs — rustup is overwhelmingly the common case.
const home = homedir();
const cargoHome = process.env.CARGO_HOME ?? join(home, ".cargo");
const rustupHome = process.env.RUSTUP_HOME ?? join(home, ".rustup");
// Search $CARGO_HOME/bin BEFORE $PATH. Some systems have an outdated
// distro cargo in /usr/bin that shadows rustup's — we want rustup's.
const cargo = findTool({
names: ["cargo"],
paths: [join(cargoHome, "bin")],
required: false,
});
if (cargo === undefined) return undefined;
// Suppress unused warning for hostOs — kept in signature for future
// host-specific path resolution (e.g. %PROGRAMFILES% probing on win32).
void hostOs;
return { cargo, cargoHome, rustupHome };
}
/**
* Find MSVC's link.exe. Windows only.
*
* Needed because on CI, Git Bash's `/usr/bin/link` (the GNU coreutils
* hard-link utility) can appear in PATH before MSVC's link.exe. Cargo
* invokes `link.exe` to link, and the wrong one silently fails.
*
* We probe the standard VS2022 install layout rather than trusting PATH.
* If VS is installed somewhere non-standard, set the CARGO_TARGET_*_LINKER
* env var yourself.
*/
export function findMsvcLinker(arch: Arch): string | undefined {
// VS2022 standard layout:
// C:/Program Files/Microsoft Visual Studio/2022/<edition>/VC/Tools/MSVC/<ver>/bin/<host>/<target>/link.exe
// Edition is Community|Professional|Enterprise|BuildTools.
const vsBase = "C:/Program Files/Microsoft Visual Studio/2022";
if (!existsSync(vsBase)) return undefined;
// Pick the latest MSVC toolset version across all editions. Usually
// there's only one edition installed, but BuildTools + Community can
// coexist on CI.
let latestVer: string | undefined;
let latestToolset: string | undefined;
for (const edition of readdirSync(vsBase)) {
const msvcDir = join(vsBase, edition, "VC/Tools/MSVC");
if (!existsSync(msvcDir)) continue;
for (const ver of readdirSync(msvcDir)) {
// Lexicographic comparison works for MSVC versions (14.xx.yyyyy).
if (latestVer === undefined || ver > latestVer) {
latestVer = ver;
latestToolset = join(msvcDir, ver);
}
}
}
if (latestToolset === undefined) return undefined;
// For arm64 targets, prefer the native arm64 host linker if available
// (faster), else cross from x64. For x64 targets, use the x64 host.
const candidates: string[] = [];
if (arch === "aarch64") {
candidates.push(join(latestToolset, "bin/HostARM64/arm64/link.exe"));
candidates.push(join(latestToolset, "bin/Hostx64/arm64/link.exe"));
} else {
candidates.push(join(latestToolset, "bin/Hostx64/x64/link.exe"));
}
for (const c of candidates) {
if (existsSync(c)) return c;
}
return undefined;
}

View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"target": "ES2022",
"lib": ["ES2022", "DOM"],
"module": "ESNext",
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"noEmit": true,
"strict": true,
"noUncheckedIndexedAccess": true,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true,
"noImplicitOverride": true,
"exactOptionalPropertyTypes": true,
"skipLibCheck": true,
"types": ["bun"],
"verbatimModuleSyntax": true
},
"include": ["./**/*.ts", "../build.ts"],
"exclude": ["node_modules"]
}

482
scripts/build/zig.ts Normal file
View File

@@ -0,0 +1,482 @@
/**
* Zig toolchain download + zig build step.
*
* Bun uses a FORK of zig at a pinned commit (oven-sh/zig). The compiler
* is downloaded as a prebuilt binary from releases (same pattern as WebKit).
* The downloaded zig includes its own stdlib (vendor/zig/lib/) — we don't
* rely on any system zig.
*
* The zig BUILD is one big `zig build obj` invocation with ~18 -D flags.
* Zig's own build system (build.zig) handles the per-file compilation; our
* ninja rule just invokes it and declares the output. restat lets zig's
* incremental compilation prune downstream when nothing changed.
*
* The compiler download is performed by `fetchZig()` below, invoked by
* ninja via fetch-cli.ts.
*/
import { existsSync, readFileSync, symlinkSync } from "node:fs";
import { mkdir, readdir, rename, rm, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import type { Config } from "./config.ts";
import { downloadWithRetry, extractZip } from "./download.ts";
import { assert } from "./error.ts";
import { fetchCliPath } from "./fetch-cli.ts";
import type { Ninja } from "./ninja.ts";
import { quote, quoteArgs } from "./shell.ts";
import { streamPath } from "./stream.ts";
/**
* Zig compiler commit — determines compiler download + bundled stdlib.
* Override via `--zig-commit=<hash>` to test a new compiler.
* From https://github.com/oven-sh/zig releases.
*/
export const ZIG_COMMIT = "c031cbebf5b063210473ff5204a24ebfb2492c72";
// ───────────────────────────────────────────────────────────────────────────
// Target/optimize/CPU computation
// ───────────────────────────────────────────────────────────────────────────
/**
* Zig target triple. Arch is always `x86_64`/`aarch64` (zig's naming),
* not `x64`/`arm64`.
*/
export function zigTarget(cfg: Config): string {
const arch = cfg.x64 ? "x86_64" : "aarch64";
if (cfg.darwin) return `${arch}-macos-none`;
if (cfg.windows) return `${arch}-windows-msvc`;
// linux: abi is always set (resolveConfig asserts)
assert(cfg.abi !== undefined, "linux build missing abi");
return `${arch}-linux-${cfg.abi}`;
}
/**
* Zig optimize level.
*
* The Windows ReleaseFast → ReleaseSafe downgrade is intentional: since
* Bun 1.1, Windows builds use ReleaseSafe because it caught more crashes.
* This is a load-bearing workaround; don't "fix" it.
*/
export function zigOptimize(cfg: Config): "Debug" | "ReleaseFast" | "ReleaseSafe" | "ReleaseSmall" {
let opt: "Debug" | "ReleaseFast" | "ReleaseSafe" | "ReleaseSmall";
switch (cfg.buildType) {
case "Debug":
opt = "Debug";
break;
case "Release":
opt = cfg.asan ? "ReleaseSafe" : "ReleaseFast";
break;
case "RelWithDebInfo":
opt = "ReleaseSafe";
break;
case "MinSizeRel":
opt = "ReleaseSmall";
break;
}
// Windows: never ReleaseFast. See header comment.
if (cfg.windows && opt === "ReleaseFast") {
opt = "ReleaseSafe";
}
return opt;
}
/**
* Zig CPU target.
*
* arm64: apple_m1 (darwin), cortex_a76 (windows — no ARMv9 windows yet),
* native (linux — no baseline arm64 builds needed).
* x64: nehalem (baseline, pre-AVX), haswell (AVX2).
*/
export function zigCpu(cfg: Config): string {
if (cfg.arm64) {
if (cfg.darwin) return "apple_m1";
if (cfg.windows) return "cortex_a76";
return "native";
}
// x64
return cfg.baseline ? "nehalem" : "haswell";
}
/**
* Whether to download the ReleaseSafe build of the zig COMPILER itself
* (not bun's zig code — this is about the compiler binary).
*
* CI defaults to yes (better error messages on compiler crashes). EXCEPT
* windows-arm64 HOST, where the ReleaseSafe compiler has an LLVM SEH
* epilogue bug that produces broken compiler_rt. Host, not target — the
* compiler runs on the host; zig-only cross-compile runs on linux.
*/
export function zigCompilerSafe(cfg: Config): boolean {
if (cfg.ci && cfg.host.os === "windows" && cfg.host.arch === "aarch64") return false;
return cfg.ci;
}
/**
* Whether codegen outputs should be @embedFile'd into the binary (release)
* or loaded at runtime (debug — faster iteration, no relink on codegen change).
*/
export function codegenEmbed(cfg: Config): boolean {
return cfg.release || cfg.ci;
}
// ───────────────────────────────────────────────────────────────────────────
// Paths
// ───────────────────────────────────────────────────────────────────────────
/**
* Where zig lives. In vendor/ (gitignored), shared across profiles — the
* commit pin is global and changing it affects everything.
*/
function zigPath(cfg: Config): string {
return resolve(cfg.vendorDir, "zig");
}
function zigExecutable(cfg: Config): string {
// Host suffix — zig runs on the host. cfg.exeSuffix is target
// (windows target → .exe), wrong for cross-compile from linux.
const suffix = cfg.host.os === "windows" ? ".exe" : "";
return resolve(zigPath(cfg), "zig" + suffix);
}
/**
* Zig cache directories — where zig stores incremental compilation state.
*/
function zigCacheDirs(cfg: Config): { local: string; global: string } {
return {
local: resolve(cfg.cacheDir, "zig", "local"),
global: resolve(cfg.cacheDir, "zig", "global"),
};
}
/**
* Download URL for the zig compiler binary.
*
* HOST os/arch, not TARGET — the compiler runs on the build machine and
* cross-compiles via -Dtarget.
*
* os-abi: zig binaries are always statically linked (musl on linux, gnu
* on windows), so the abi is fixed per-os.
*/
function zigDownloadUrl(cfg: Config, safe: boolean): string {
const arch = cfg.host.arch === "aarch64" ? "aarch64" : "x86_64";
let osAbi: string;
if (cfg.host.os === "darwin") {
osAbi = "macos-none";
} else if (cfg.host.os === "windows") {
osAbi = "windows-gnu";
} else {
// linux: always musl for the compiler binary (static).
osAbi = "linux-musl";
}
const safeSuffix = safe ? "-ReleaseSafe" : "";
const zipName = `bootstrap-${arch}-${osAbi}${safeSuffix}.zip`;
return `https://github.com/oven-sh/zig/releases/download/autobuild-${cfg.zigCommit}/${zipName}`;
}
// ───────────────────────────────────────────────────────────────────────────
// Ninja rules
// ───────────────────────────────────────────────────────────────────────────
export function registerZigRules(n: Ninja, cfg: Config): void {
const hostWin = cfg.host.os === "windows";
const q = (p: string) => quote(p, hostWin);
const bun = q(cfg.bun);
// Zig fetch wrapped in stream.ts for the [zig] prefix alongside other
// dep fetches. Fast (cache hit <100ms) so pool doesn't matter.
const stream = `${bun} ${q(streamPath)} zig`;
n.rule("zig_fetch", {
command: `${stream} ${bun} ${q(fetchCliPath)} zig $url $dest $commit`,
description: "zig download compiler",
restat: true,
});
// Zig build — the big one. One invocation produces bun-zig.o. Zig's
// own build system handles per-file tracking; restat prunes downstream
// when zig's cache says nothing changed.
//
// Default: --console + pool=console. Zig gets direct TTY, its native
// spinner works. Ninja defers [N/M] while the console job owns the
// terminal, so cxx progress is hidden during zig's compile. Matches
// the old cmake build's behavior.
//
// --zig-progress instead of --console (posix only, set interleave=true):
// decodes ZIG_PROGRESS IPC into `[zig] Stage [N/M]` lines that interleave
// with ninja's [N/M] for cxx — both visible at once. Requires
// oven-sh/zig's fix for ziglang/zig#24722. Windows zig has no IPC in
// our fork (upstream added Feb 2026, not backported).
const interleave = false;
const consoleMode = !interleave || hostWin;
n.rule("zig_build", {
command: `${stream} ${consoleMode ? "--console" : "--zig-progress"} --env=ZIG_LOCAL_CACHE_DIR=$zig_local_cache --env=ZIG_GLOBAL_CACHE_DIR=$zig_global_cache $zig build $step $args`,
description: "zig $step → $out",
...(consoleMode && { pool: "console" }),
restat: true,
});
}
// ───────────────────────────────────────────────────────────────────────────
// Zig build emission
// ───────────────────────────────────────────────────────────────────────────
/**
* Inputs to the zig build step. Assembled by the caller from
* resolved deps + emitted codegen outputs.
*/
export interface ZigBuildInputs {
/**
* Generated files zig needs (content tracked). From CodegenOutputs.zigInputs.
* Changes here must trigger a zig rebuild.
*/
codegenInputs: string[];
/**
* All `*.zig` source files (globbed at configure time, codegen-into-src
* files already filtered out by caller). Implicit inputs for ninja's
* staleness check — zig discovers sources itself, this is just so ninja
* knows when to re-invoke.
*/
zigSources: string[];
/**
* Generated files zig needs to EXIST but doesn't track content of.
* From CodegenOutputs.zigOrderOnly — specifically the bake runtime .js
* files in debug mode (runtime-loaded, not embedded).
*/
codegenOrderOnly: string[];
/**
* zstd source fetch stamp. build.zig `@cImport`s headers from
* vendor/zstd/lib/ directly — doesn't need zstd BUILT, just FETCHED.
* Order-only because the headers don't change often and zig's own
* translate-c caching handles the inner dependency.
*/
zstdStamp: string;
}
/**
* Emit the zig download + zig build steps. Returns the output object file(s).
*
* For normal builds: one `bun-zig.o`. For test builds (future): `bun-test.o`.
* Threaded codegen (LLVM_ZIG_CODEGEN_THREADS > 1) would produce multiple .o
* files, but that's always 0 in practice — deferred.
*/
export function emitZig(n: Ninja, cfg: Config, inputs: ZigBuildInputs): string[] {
n.comment("─── Zig ───");
n.blank();
// ─── Download compiler ───
const zigDest = zigPath(cfg);
const zigExe = zigExecutable(cfg);
const safe = zigCompilerSafe(cfg);
const url = zigDownloadUrl(cfg, safe);
// Commit + safe go into the stamp content, so switching either retriggers.
const stamp = resolve(zigDest, ".zig-commit");
n.build({
outputs: [stamp],
implicitOutputs: [zigExe],
rule: "zig_fetch",
inputs: [],
// Only fetch-cli.ts. This file (zig.ts) has emitZig and other logic
// unrelated to download — editing those shouldn't re-download the
// compiler. The URL/commit are in the rule's vars so changing those
// already retriggers via ninja's command tracking.
implicitInputs: [fetchCliPath],
vars: {
url,
dest: zigDest,
// Safe is encoded in the commit stamp (not just URL) so the CLI
// can short-circuit correctly when safe doesn't change.
commit: `${cfg.zigCommit}${safe ? "-safe" : ""}`,
},
});
n.phony("zig-compiler", [zigExe]);
// ─── Build ───
const cacheDirs = zigCacheDirs(cfg);
const output = resolve(cfg.buildDir, "bun-zig.o");
// Extra embed: scanner-entry.ts is @embedFile'd by the zig code directly.
// A genuinely odd cross-language embed; there's no cleaner way.
const scannerEntry = resolve(cfg.cwd, "src", "install", "PackageManager", "scanner-entry.ts");
// ─── Build args ───
// One -D per feature flag. Each maps directly to a build.zig option.
// Order doesn't matter but we keep it the same as CMake for easy diffing.
const bool = (b: boolean): string => (b ? "true" : "false");
const args: string[] = [
// Cache and lib paths. --zig-lib-dir points at OUR bundled stdlib,
// not any system zig — the compiler and stdlib must match commits.
"--cache-dir",
cacheDirs.local,
"--global-cache-dir",
cacheDirs.global,
"--zig-lib-dir",
resolve(zigDest, "lib"),
"--prefix",
cfg.buildDir,
// Target/optimize/cpu
"-Dobj_format=obj",
`-Dtarget=${zigTarget(cfg)}`,
`-Doptimize=${zigOptimize(cfg)}`,
`-Dcpu=${zigCpu(cfg)}`,
// Feature flags
`-Denable_logs=${bool(cfg.logs)}`,
`-Denable_asan=${bool(cfg.zigAsan)}`,
`-Denable_fuzzilli=${bool(cfg.fuzzilli)}`,
`-Denable_valgrind=${bool(cfg.valgrind)}`,
`-Denable_tinycc=${bool(cfg.tinycc)}`,
// Always ON — bun uses mimalloc as its default allocator. The flag
// exists for experimentation; in practice it's never OFF.
`-Duse_mimalloc=true`,
// Not using threaded codegen — always 0.
`-Dllvm_codegen_threads=0`,
// Versioning
`-Dversion=${cfg.version}`,
`-Dreported_nodejs_version=${cfg.nodejsVersion}`,
`-Dcanary=${cfg.canaryRevision}`,
`-Dcodegen_path=${cfg.codegenDir}`,
`-Dcodegen_embed=${bool(codegenEmbed(cfg))}`,
// Git sha (optional — empty on dirty builds).
...(cfg.revision !== "unknown" && cfg.revision !== "" ? [`-Dsha=${cfg.revision}`] : []),
// Output formatting
"--prominent-compile-errors",
"--summary",
"all",
];
n.build({
outputs: [output],
rule: "zig_build",
inputs: [],
implicitInputs: [
// Compiler itself — rebuild on zig version bump.
zigExe,
// build.zig — the zig build script.
resolve(cfg.cwd, "build.zig"),
// All zig source files (codegen outputs already filtered by caller).
...inputs.zigSources,
// Codegen outputs zig imports/embeds.
...inputs.codegenInputs,
// The odd cross-language embed.
scannerEntry,
],
orderOnlyInputs: [
// zstd headers — must exist for @cImport, but content is tracked by
// zig's translate-c cache, not ninja.
inputs.zstdStamp,
// Debug-mode bake runtime — must exist at runtime-load path, but
// zig doesn't track content (not embedded).
...inputs.codegenOrderOnly,
],
vars: {
zig: zigExe,
step: "obj",
args: quoteArgs(args, cfg.host.os === "windows"),
zig_local_cache: cacheDirs.local,
zig_global_cache: cacheDirs.global,
},
});
n.phony("bun-zig", [output]);
n.blank();
return [output];
}
// ───────────────────────────────────────────────────────────────────────────
// Fetch implementation — invoked by fetch-cli.ts (which ninja calls)
// ───────────────────────────────────────────────────────────────────────────
/**
* Download and extract the zig compiler binary.
*
* Idempotent: if `dest/.zig-commit` matches `commit`, exits without
* touching anything (restat prunes).
*
* The zip has a single top-level dir containing {zig, lib/, doc/, ...}.
* CMake's DownloadUrl.cmake auto-hoists single-child extractions; we do
* the same.
*/
export async function fetchZig(url: string, dest: string, commit: string): Promise<void> {
const stampPath = resolve(dest, ".zig-commit");
// Short-circuit: already at this commit?
if (existsSync(stampPath)) {
const existing = readFileSync(stampPath, "utf8").trim();
if (existing === commit) {
console.log(`up to date`);
return; // restat no-op
}
console.log(`commit changed (was ${existing}, now ${commit}), re-fetching`);
}
console.log(`fetching ${url}`);
// ─── Download ───
const destParent = resolve(dest, "..");
await mkdir(destParent, { recursive: true });
const zipPath = `${dest}.download.zip`;
await downloadWithRetry(url, zipPath, "zig");
// ─── Extract ───
// Wipe dest first — don't want stale files from a previous version.
await rm(dest, { recursive: true, force: true });
// Extract to a temp dir, then find the hoistable top-level dir.
const extractDir = `${dest}.extract`;
await rm(extractDir, { recursive: true, force: true });
await mkdir(extractDir, { recursive: true });
// Use system unzip. Present on all platforms we support (Windows 10+
// has it via PowerShell Expand-Archive, but `tar` also handles .zip
// on bsdtar/Windows tar.exe — use that for consistency).
//
// -m: same mtime fix as tar (zip stores creation timestamps).
// But wait — tar doesn't handle .zip on all platforms reliably.
// `unzip` is more portable for .zip specifically. Check and fall back.
await extractZip(zipPath, extractDir);
await rm(zipPath, { force: true });
// Hoist: zip has one top-level dir (e.g. `zig-linux-x86_64-0.14.0-...`).
const entries = await readdir(extractDir);
assert(entries.length > 0, `zip extracted nothing: ${zipPath}`);
let hoistFrom: string;
if (entries.length === 1) {
hoistFrom = resolve(extractDir, entries[0]!);
} else {
// Multiple top-level entries — zip was already flat.
hoistFrom = extractDir;
}
await rename(hoistFrom, dest);
await rm(extractDir, { recursive: true, force: true });
// ─── Validate ───
const zigExe = resolve(dest, process.platform === "win32" ? "zig.exe" : "zig");
assert(existsSync(zigExe), `zig executable not found after extraction: ${zigExe}`, {
hint: "Archive layout may have changed",
});
assert(existsSync(resolve(dest, "lib")), `zig lib/ dir not found`, {
hint: "Archive may be incomplete",
});
// ─── Editor stability symlinks (unix) ───
// VSCode/neovim zig extensions want a stable `zig.exe`/`zls.exe` path
// even on unix (they probe for both). Create symlinks.
if (process.platform !== "win32") {
try {
symlinkSync("zig", resolve(dest, "zig.exe"));
} catch {}
try {
symlinkSync("zls", resolve(dest, "zls.exe"));
} catch {}
}
// ─── Write stamp ───
await writeFile(stampPath, commit + "\n");
console.log(`extracted to ${dest}`);
}

View File

@@ -751,14 +751,19 @@ async function main() {
|_| |_|
`.slice(1),
);
console.error("Usage: bun src/codegen/cppbind src build/debug/codegen");
console.error("Usage: bun src/codegen/cppbind src build/debug/codegen [cxx-sources.txt]");
process.exit(1);
}
await mkdir(dstDir, { recursive: true });
const parser = cppParser;
const allCppFiles = (await Bun.file("cmake/sources/CxxSources.txt").text())
// Source list: explicit arg from the build system, or fall back to the
// cmake-generated file (until cmake is removed). The build system owns
// globbing and hands us the result — no filesystem dependency outside
// what we're given.
const cxxSourcesPath = args[2] ?? "cmake/sources/CxxSources.txt";
const allCppFiles = (await Bun.file(cxxSourcesPath).text())
.trim()
.split("\n")
.map(q => q.trim())

View File

@@ -269,19 +269,21 @@ it("process.umask()", () => {
});
it("process.versions", () => {
// Expected dependency versions (from CMake-generated header)
// Expected dependency versions — must match scripts/build/deps/*.ts commits.
// These are the ACTUAL commits built into bun (not derived values, so
// bumping a dep requires updating this test too).
const expectedVersions = {
boringssl: "29a2cd359458c9384694b75456026e4b57e3e567",
libarchive: "898dc8319355b7e985f68a9819f182aaed61b53a",
mimalloc: "4c283af60cdae205df5a872530c77e2a6a307d43",
boringssl: "4f4f5ef8ebc6e23cbf393428f0ab1b526773f7ac",
libarchive: "9525f90ca4bd14c7b335e2f8c84a4607b0af6bdf",
mimalloc: "1beadf9651a7bfdec6b5367c380ecc3fe1c40d1a",
picohttpparser: "066d2b1e9ab820703db0837a7255d92d30f0c9f5",
zlib: "886098f3f339617b4243b286f5ed364b9989e245",
tinycc: "ab631362d839333660a265d3084d8ff060b96753",
lolhtml: "8d4c273ded322193d017042d1f48df2766b0f88b",
ares: "d1722e6e8acaf10eb73fa995798a9cd421d9f85e",
libdeflate: "dc76454a39e7e83b68c3704b6e3784654f8d5ac5",
zstd: "794ea1b0afca0f020f4e57b6732332231fb23c70",
lshpack: "3d0f1fc1d6e66a642e7a98c55deb38aa986eb4b0",
tinycc: "12882eee073cfe5c7621bcfadf679e1372d4537b",
lolhtml: "e3aa54798602dd27250fafde1b5a66f080046252",
ares: "3ac47ee46edd8ea40370222f91613fc16c434853",
libdeflate: "c8c56a20f8f621e6a966b716b31f1dedab6a41e3",
zstd: "f8745da6ff1ad1e7bab384bd1f9d742439278e99",
lshpack: "8905c024b6d052f083a3d11d0a169b3c2735c8a1",
};
for (const [name, expectedHash] of Object.entries(expectedVersions)) {