mirror of
https://github.com/oven-sh/bun
synced 2026-02-02 15:08:46 +00:00
1253 lines
33 KiB
JavaScript
Executable File
1253 lines
33 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
/**
|
|
* Build and test Bun on macOS, Linux, and Windows.
|
|
* @link https://buildkite.com/docs/pipelines/defining-steps
|
|
*/
|
|
|
|
import { join } from "node:path";
|
|
import {
|
|
getBootstrapVersion,
|
|
getBuildkiteEmoji,
|
|
getBuildMetadata,
|
|
getBuildNumber,
|
|
getCanaryRevision,
|
|
getCommitMessage,
|
|
getEmoji,
|
|
getEnv,
|
|
getLastSuccessfulBuild,
|
|
getSecret,
|
|
isBuildkite,
|
|
isBuildManual,
|
|
isFork,
|
|
isMainBranch,
|
|
isMergeQueue,
|
|
parseBoolean,
|
|
spawnSafe,
|
|
startGroup,
|
|
toYaml,
|
|
uploadArtifact,
|
|
writeFile,
|
|
} from "../scripts/utils.mjs";
|
|
|
|
/**
|
|
* @typedef {"linux" | "darwin" | "windows" | "freebsd"} Os
|
|
* @typedef {"aarch64" | "x64"} Arch
|
|
* @typedef {"musl"} Abi
|
|
* @typedef {"debian" | "ubuntu" | "alpine" | "amazonlinux"} Distro
|
|
* @typedef {"latest" | "previous" | "oldest" | "eol"} Tier
|
|
* @typedef {"release" | "assert" | "debug" | "asan"} Profile
|
|
*/
|
|
|
|
/**
|
|
* @typedef Target
|
|
* @property {Os} os
|
|
* @property {Arch} arch
|
|
* @property {Abi} [abi]
|
|
* @property {boolean} [baseline]
|
|
* @property {Profile} [profile]
|
|
*/
|
|
|
|
/**
|
|
* @param {Target} target
|
|
* @returns {string}
|
|
*/
|
|
function getTargetKey(target) {
|
|
const { os, arch, abi, baseline, profile } = target;
|
|
let key = `${os}-${arch}`;
|
|
if (abi) {
|
|
key += `-${abi}`;
|
|
}
|
|
if (baseline) {
|
|
key += "-baseline";
|
|
}
|
|
if (profile && profile !== "release") {
|
|
key += `-${profile}`;
|
|
}
|
|
return key;
|
|
}
|
|
|
|
/**
|
|
* @param {Target} target
|
|
* @returns {string}
|
|
*/
|
|
function getTargetLabel(target) {
|
|
const { os, arch, abi, baseline, profile } = target;
|
|
let label = `${getBuildkiteEmoji(os)} ${arch}`;
|
|
if (abi) {
|
|
label += `-${abi}`;
|
|
}
|
|
if (baseline) {
|
|
label += "-baseline";
|
|
}
|
|
if (profile && profile !== "release") {
|
|
label += `-${profile}`;
|
|
}
|
|
return label;
|
|
}
|
|
|
|
/**
|
|
* @typedef Platform
|
|
* @property {Os} os
|
|
* @property {Arch} arch
|
|
* @property {Abi} [abi]
|
|
* @property {boolean} [baseline]
|
|
* @property {Profile} [profile]
|
|
* @property {Distro} [distro]
|
|
* @property {string} release
|
|
* @property {Tier} [tier]
|
|
* @property {string[]} [features]
|
|
*/
|
|
|
|
/**
|
|
* @type {Platform[]}
|
|
*/
|
|
const buildPlatforms = [
|
|
{ os: "darwin", arch: "aarch64", release: "14" },
|
|
{ os: "darwin", arch: "x64", release: "14" },
|
|
{ os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023", features: ["docker"] },
|
|
{ os: "linux", arch: "x64", distro: "amazonlinux", release: "2023", features: ["docker"] },
|
|
{ os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023", features: ["docker"] },
|
|
{ os: "linux", arch: "x64", profile: "asan", distro: "amazonlinux", release: "2023", features: ["docker"] },
|
|
{ os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.22" },
|
|
{ os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.22" },
|
|
{ os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.22" },
|
|
{ os: "windows", arch: "x64", release: "2019" },
|
|
{ os: "windows", arch: "x64", baseline: true, release: "2019" },
|
|
{ os: "freebsd", arch: "x64", release: "14.3" },
|
|
];
|
|
|
|
/**
|
|
* @type {Platform[]}
|
|
*/
|
|
const testPlatforms = [
|
|
{ os: "darwin", arch: "aarch64", release: "14", tier: "latest" },
|
|
{ os: "darwin", arch: "aarch64", release: "13", tier: "previous" },
|
|
{ os: "darwin", arch: "x64", release: "14", tier: "latest" },
|
|
{ os: "darwin", arch: "x64", release: "13", tier: "previous" },
|
|
{ os: "linux", arch: "aarch64", distro: "debian", release: "13", tier: "latest" },
|
|
{ os: "linux", arch: "x64", distro: "debian", release: "13", tier: "latest" },
|
|
{ os: "linux", arch: "x64", baseline: true, distro: "debian", release: "13", tier: "latest" },
|
|
{ os: "linux", arch: "x64", profile: "asan", distro: "debian", release: "13", tier: "latest" },
|
|
{ os: "linux", arch: "aarch64", distro: "ubuntu", release: "25.04", tier: "latest" },
|
|
{ os: "linux", arch: "x64", distro: "ubuntu", release: "25.04", tier: "latest" },
|
|
{ os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "25.04", tier: "latest" },
|
|
{ os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.22", tier: "latest" },
|
|
{ os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.22", tier: "latest" },
|
|
{ os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.22", tier: "latest" },
|
|
{ os: "windows", arch: "x64", release: "2019", tier: "oldest" },
|
|
{ os: "windows", arch: "x64", release: "2019", baseline: true, tier: "oldest" },
|
|
];
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @returns {string}
|
|
*/
|
|
function getPlatformKey(platform) {
|
|
const { distro, release } = platform;
|
|
const target = getTargetKey(platform);
|
|
const version = release.replace(/\./g, "");
|
|
if (distro) {
|
|
return `${target}-${distro}-${version}`;
|
|
}
|
|
return `${target}-${version}`;
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @returns {string}
|
|
*/
|
|
function getPlatformLabel(platform) {
|
|
const { os, arch, baseline, profile, distro, release } = platform;
|
|
let label = `${getBuildkiteEmoji(distro || os)} ${release} ${arch}`;
|
|
if (baseline) {
|
|
label += "-baseline";
|
|
}
|
|
if (profile && profile !== "release") {
|
|
label += `-${profile}`;
|
|
}
|
|
return label;
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @returns {string}
|
|
*/
|
|
function getImageKey(platform) {
|
|
const { os, arch, distro, release, features, abi } = platform;
|
|
const version = release.replace(/\./g, "");
|
|
let key = `${os}-${arch}-${version}`;
|
|
if (distro) {
|
|
key += `-${distro}`;
|
|
}
|
|
if (features?.length) {
|
|
key += `-with-${features.join("-")}`;
|
|
}
|
|
|
|
if (abi) {
|
|
key += `-${abi}`;
|
|
}
|
|
|
|
return key;
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @returns {string}
|
|
*/
|
|
function getImageLabel(platform) {
|
|
const { os, arch, distro, release } = platform;
|
|
return `${getBuildkiteEmoji(distro || os)} ${release} ${arch}`;
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {string}
|
|
*/
|
|
function getImageName(platform, options) {
|
|
const { os } = platform;
|
|
const { buildImages, publishImages } = options;
|
|
|
|
const name = getImageKey(platform);
|
|
|
|
if (buildImages && !publishImages) {
|
|
return `${name}-build-${getBuildNumber()}`;
|
|
}
|
|
|
|
return `${name}-v${getBootstrapVersion(os)}`;
|
|
}
|
|
|
|
/**
|
|
* @param {number} [limit]
|
|
* @link https://buildkite.com/docs/pipelines/command-step#retry-attributes
|
|
*/
|
|
function getRetry() {
|
|
return {
|
|
manual: {
|
|
permit_on_passed: true,
|
|
},
|
|
automatic: false,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @returns {number}
|
|
* @link https://buildkite.com/docs/pipelines/managing-priorities
|
|
*/
|
|
function getPriority() {
|
|
if (isFork()) {
|
|
return -1;
|
|
}
|
|
if (isMainBranch()) {
|
|
return 2;
|
|
}
|
|
if (isMergeQueue()) {
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Agents
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Object} Ec2Options
|
|
* @property {string} instanceType
|
|
* @property {number} cpuCount
|
|
* @property {number} threadsPerCore
|
|
* @property {boolean} dryRun
|
|
*/
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @param {Ec2Options} ec2Options
|
|
* @returns {Agent}
|
|
*/
|
|
function getEc2Agent(platform, options, ec2Options) {
|
|
const { os, arch, abi, distro, release } = platform;
|
|
const { instanceType, cpuCount, threadsPerCore } = ec2Options;
|
|
return {
|
|
os,
|
|
arch,
|
|
abi,
|
|
distro,
|
|
release,
|
|
robobun: true,
|
|
robobun2: true,
|
|
"image-name": getImageName(platform, options),
|
|
"instance-type": instanceType,
|
|
"cpu-count": cpuCount,
|
|
"threads-per-core": threadsPerCore,
|
|
"preemptible": false,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {string}
|
|
*/
|
|
function getCppAgent(platform, options) {
|
|
const { os, arch } = platform;
|
|
|
|
if (os === "darwin") {
|
|
return {
|
|
queue: `build-${os}`,
|
|
os,
|
|
arch,
|
|
};
|
|
}
|
|
|
|
return getEc2Agent(platform, options, {
|
|
instanceType: arch === "aarch64" ? "c8g.4xlarge" : "c7i.4xlarge",
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {string}
|
|
*/
|
|
function getLinkBunAgent(platform, options) {
|
|
const { os, arch } = platform;
|
|
|
|
if (os === "darwin") {
|
|
return {
|
|
queue: `build-${os}`,
|
|
os,
|
|
arch,
|
|
};
|
|
}
|
|
|
|
if (os === "windows") {
|
|
return getEc2Agent(platform, options, {
|
|
instanceType: arch === "aarch64" ? "r8g.large" : "r7i.large",
|
|
});
|
|
}
|
|
|
|
return getEc2Agent(platform, options, {
|
|
instanceType: arch === "aarch64" ? "r8g.xlarge" : "r7i.xlarge",
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @returns {Platform}
|
|
*/
|
|
function getZigPlatform() {
|
|
return {
|
|
os: "linux",
|
|
arch: "aarch64",
|
|
abi: "musl",
|
|
distro: "alpine",
|
|
release: "3.22",
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {Agent}
|
|
*/
|
|
function getZigAgent(_platform, options) {
|
|
return getEc2Agent(getZigPlatform(), options, {
|
|
instanceType: "r8g.large",
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {Agent}
|
|
*/
|
|
function getTestAgent(platform, options) {
|
|
const { os, arch, profile } = platform;
|
|
|
|
if (os === "darwin") {
|
|
return {
|
|
queue: `test-${os}`,
|
|
os,
|
|
arch,
|
|
};
|
|
}
|
|
|
|
// TODO: delete this block when we upgrade to mimalloc v3
|
|
if (os === "windows") {
|
|
return getEc2Agent(platform, options, {
|
|
instanceType: "c7i.2xlarge",
|
|
cpuCount: 2,
|
|
threadsPerCore: 1,
|
|
});
|
|
}
|
|
|
|
if (arch === "aarch64") {
|
|
if (profile === "asan") {
|
|
return getEc2Agent(platform, options, {
|
|
instanceType: "c8g.2xlarge",
|
|
cpuCount: 2,
|
|
threadsPerCore: 1,
|
|
});
|
|
}
|
|
return getEc2Agent(platform, options, {
|
|
instanceType: "c8g.xlarge",
|
|
cpuCount: 2,
|
|
threadsPerCore: 1,
|
|
});
|
|
}
|
|
|
|
if (profile === "asan") {
|
|
return getEc2Agent(platform, options, {
|
|
instanceType: "c7i.2xlarge",
|
|
cpuCount: 2,
|
|
threadsPerCore: 1,
|
|
});
|
|
}
|
|
return getEc2Agent(platform, options, {
|
|
instanceType: "c7i.xlarge",
|
|
cpuCount: 2,
|
|
threadsPerCore: 1,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Steps
|
|
*/
|
|
|
|
/**
|
|
* @param {Target} target
|
|
* @param {PipelineOptions} options
|
|
* @returns {Record<string, string | undefined>}
|
|
*/
|
|
function getBuildEnv(target, options) {
|
|
const { baseline, abi } = target;
|
|
const { canary } = options;
|
|
const revision = typeof canary === "number" ? canary : 1;
|
|
|
|
return {
|
|
ENABLE_BASELINE: baseline ? "ON" : "OFF",
|
|
ENABLE_CANARY: revision > 0 ? "ON" : "OFF",
|
|
CANARY_REVISION: revision,
|
|
ABI: abi === "musl" ? "musl" : undefined,
|
|
CMAKE_VERBOSE_MAKEFILE: "ON",
|
|
CMAKE_TLS_VERIFY: "0",
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Target} target
|
|
* @param {PipelineOptions} options
|
|
* @returns {string}
|
|
*/
|
|
function getBuildCommand(target, options, label) {
|
|
const { profile } = target;
|
|
const buildProfile = profile || "release";
|
|
|
|
if (target.os === "windows" && label === "build-bun") {
|
|
// Only sign release builds, not canary builds (DigiCert charges per signature)
|
|
const enableSigning = !options.canary ? " -DENABLE_WINDOWS_CODESIGNING=ON" : "";
|
|
return `bun run build:${buildProfile}${enableSigning}`;
|
|
}
|
|
|
|
return `bun run build:${buildProfile}`;
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {Step}
|
|
*/
|
|
function getBuildCppStep(platform, options) {
|
|
const command = getBuildCommand(platform, options);
|
|
return {
|
|
key: `${getTargetKey(platform)}-build-cpp`,
|
|
label: `${getTargetLabel(platform)} - build-cpp`,
|
|
agents: getCppAgent(platform, options),
|
|
retry: getRetry(),
|
|
cancel_on_build_failing: isMergeQueue(),
|
|
env: {
|
|
BUN_CPP_ONLY: "ON",
|
|
...getBuildEnv(platform, options),
|
|
},
|
|
// We used to build the C++ dependencies and bun in separate steps.
|
|
// However, as long as the zig build takes longer than both sequentially,
|
|
// it's cheaper to run them in the same step. Can be revisited in the future.
|
|
command: [`${command} --target bun`, `${command} --target dependencies`],
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Target} target
|
|
* @returns {string}
|
|
*/
|
|
function getBuildToolchain(target) {
|
|
const { os, arch, abi, baseline } = target;
|
|
let key = `${os}-${arch}`;
|
|
if (abi) {
|
|
key += `-${abi}`;
|
|
}
|
|
if (baseline) {
|
|
key += "-baseline";
|
|
}
|
|
return key;
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {Step}
|
|
*/
|
|
function getBuildZigStep(platform, options) {
|
|
const toolchain = getBuildToolchain(platform);
|
|
return {
|
|
key: `${getTargetKey(platform)}-build-zig`,
|
|
retry: getRetry(),
|
|
label: `${getTargetLabel(platform)} - build-zig`,
|
|
agents: getZigAgent(platform, options),
|
|
cancel_on_build_failing: isMergeQueue(),
|
|
env: getBuildEnv(platform, options),
|
|
command: `${getBuildCommand(platform, options)} --target bun-zig --toolchain ${toolchain}`,
|
|
timeout_in_minutes: 35,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {Step}
|
|
*/
|
|
function getLinkBunStep(platform, options) {
|
|
return {
|
|
key: `${getTargetKey(platform)}-build-bun`,
|
|
label: `${getTargetLabel(platform)} - build-bun`,
|
|
depends_on: [`${getTargetKey(platform)}-build-cpp`, `${getTargetKey(platform)}-build-zig`],
|
|
agents: getLinkBunAgent(platform, options),
|
|
retry: getRetry(),
|
|
cancel_on_build_failing: isMergeQueue(),
|
|
env: {
|
|
BUN_LINK_ONLY: "ON",
|
|
ASAN_OPTIONS: "allow_user_segv_handler=1:disable_coredump=0:detect_leaks=0",
|
|
...getBuildEnv(platform, options),
|
|
},
|
|
command: `${getBuildCommand(platform, options, "build-bun")} --target bun`,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {Step}
|
|
*/
|
|
function getBuildBunStep(platform, options) {
|
|
return {
|
|
key: `${getTargetKey(platform)}-build-bun`,
|
|
label: `${getTargetLabel(platform)} - build-bun`,
|
|
agents: getCppAgent(platform, options),
|
|
retry: getRetry(),
|
|
cancel_on_build_failing: isMergeQueue(),
|
|
env: getBuildEnv(platform, options),
|
|
command: getBuildCommand(platform, options),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @typedef {Object} TestOptions
|
|
* @property {string} [buildId]
|
|
* @property {string[]} [testFiles]
|
|
* @property {boolean} [dryRun]
|
|
*/
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @param {TestOptions} [testOptions]
|
|
* @returns {Step}
|
|
*/
|
|
function getTestBunStep(platform, options, testOptions = {}) {
|
|
const { os, profile } = platform;
|
|
const { buildId, testFiles } = testOptions;
|
|
|
|
const args = [`--step=${getTargetKey(platform)}-build-bun`];
|
|
if (buildId) {
|
|
args.push(`--build-id=${buildId}`);
|
|
}
|
|
if (testFiles) {
|
|
args.push(...testFiles.map(testFile => `--include=${testFile}`));
|
|
}
|
|
|
|
const depends = [];
|
|
if (!buildId) {
|
|
depends.push(`${getTargetKey(platform)}-build-bun`);
|
|
}
|
|
|
|
return {
|
|
key: `${getPlatformKey(platform)}-test-bun`,
|
|
label: `${getPlatformLabel(platform)} - test-bun`,
|
|
depends_on: depends,
|
|
agents: getTestAgent(platform, options),
|
|
retry: getRetry(),
|
|
cancel_on_build_failing: isMergeQueue(),
|
|
parallelism: os === "darwin" ? 2 : 10,
|
|
timeout_in_minutes: profile === "asan" || os === "windows" ? 45 : 30,
|
|
env: {
|
|
ASAN_OPTIONS: "allow_user_segv_handler=1:disable_coredump=0:detect_leaks=0",
|
|
},
|
|
command:
|
|
os === "windows"
|
|
? `node .\\scripts\\runner.node.mjs ${args.join(" ")}`
|
|
: `./scripts/runner.node.mjs ${args.join(" ")}`,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Platform} platform
|
|
* @param {PipelineOptions} options
|
|
* @returns {Step}
|
|
*/
|
|
function getBuildImageStep(platform, options) {
|
|
const { os, arch, distro, release, features } = platform;
|
|
const { publishImages } = options;
|
|
const action = publishImages ? "publish-image" : "create-image";
|
|
|
|
const command = [
|
|
"node",
|
|
"./scripts/machine.mjs",
|
|
action,
|
|
`--os=${os}`,
|
|
`--arch=${arch}`,
|
|
distro && `--distro=${distro}`,
|
|
`--release=${release}`,
|
|
"--cloud=aws",
|
|
"--ci",
|
|
"--authorized-org=oven-sh",
|
|
];
|
|
for (const feature of features || []) {
|
|
command.push(`--feature=${feature}`);
|
|
}
|
|
|
|
return {
|
|
key: `${getImageKey(platform)}-build-image`,
|
|
label: `${getImageLabel(platform)} - build-image`,
|
|
agents: {
|
|
queue: "build-image",
|
|
},
|
|
env: {
|
|
DEBUG: "1",
|
|
},
|
|
retry: getRetry(),
|
|
cancel_on_build_failing: isMergeQueue(),
|
|
command: command.filter(Boolean).join(" "),
|
|
timeout_in_minutes: 3 * 60,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Platform[]} buildPlatforms
|
|
* @param {PipelineOptions} options
|
|
* @returns {Step}
|
|
*/
|
|
function getReleaseStep(buildPlatforms, options) {
|
|
const { canary } = options;
|
|
const revision = typeof canary === "number" ? canary : 1;
|
|
|
|
return {
|
|
key: "release",
|
|
label: getBuildkiteEmoji("rocket"),
|
|
agents: {
|
|
queue: "test-darwin",
|
|
},
|
|
depends_on: buildPlatforms.filter(p => p.os !== "freebsd").map(platform => `${getTargetKey(platform)}-build-bun`),
|
|
env: {
|
|
CANARY: revision,
|
|
},
|
|
command: ".buildkite/scripts/upload-release.sh",
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {Platform[]} buildPlatforms
|
|
* @returns {Step}
|
|
*/
|
|
function getBenchmarkStep() {
|
|
return {
|
|
key: "benchmark",
|
|
label: "📊",
|
|
agents: {
|
|
queue: "build-image",
|
|
},
|
|
depends_on: `linux-x64-build-bun`,
|
|
command: "node .buildkite/scripts/upload-benchmark.mjs",
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @typedef {Object} Pipeline
|
|
* @property {Step[]} [steps]
|
|
* @property {number} [priority]
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Record<string, string | undefined>} Agent
|
|
*/
|
|
|
|
/**
|
|
* @typedef {GroupStep | CommandStep | BlockStep} Step
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Object} GroupStep
|
|
* @property {string} key
|
|
* @property {string} group
|
|
* @property {Step[]} steps
|
|
* @property {string[]} [depends_on]
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Object} CommandStep
|
|
* @property {string} key
|
|
* @property {string} [label]
|
|
* @property {Record<string, string | undefined>} [agents]
|
|
* @property {Record<string, string | undefined>} [env]
|
|
* @property {string} command
|
|
* @property {string[]} [depends_on]
|
|
* @property {Record<string, string | undefined>} [retry]
|
|
* @property {boolean} [cancel_on_build_failing]
|
|
* @property {boolean} [soft_fail]
|
|
* @property {number} [parallelism]
|
|
* @property {number} [concurrency]
|
|
* @property {string} [concurrency_group]
|
|
* @property {number} [priority]
|
|
* @property {number} [timeout_in_minutes]
|
|
* @link https://buildkite.com/docs/pipelines/command-step
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Object} BlockStep
|
|
* @property {string} key
|
|
* @property {string} block
|
|
* @property {string} [prompt]
|
|
* @property {"passed" | "failed" | "running"} [blocked_state]
|
|
* @property {(SelectInput | TextInput)[]} [fields]
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Object} TextInput
|
|
* @property {string} key
|
|
* @property {string} text
|
|
* @property {string} [default]
|
|
* @property {boolean} [required]
|
|
* @property {string} [hint]
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Object} SelectInput
|
|
* @property {string} key
|
|
* @property {string} select
|
|
* @property {string | string[]} [default]
|
|
* @property {boolean} [required]
|
|
* @property {boolean} [multiple]
|
|
* @property {string} [hint]
|
|
* @property {SelectOption[]} [options]
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Object} SelectOption
|
|
* @property {string} label
|
|
* @property {string} value
|
|
*/
|
|
|
|
/**
|
|
* @typedef {Object} PipelineOptions
|
|
* @property {string | boolean} [skipEverything]
|
|
* @property {string | boolean} [skipBuilds]
|
|
* @property {string | boolean} [skipTests]
|
|
* @property {string | boolean} [forceBuilds]
|
|
* @property {string | boolean} [forceTests]
|
|
* @property {string | boolean} [buildImages]
|
|
* @property {string | boolean} [publishImages]
|
|
* @property {number} [canary]
|
|
* @property {Platform[]} [buildPlatforms]
|
|
* @property {Platform[]} [testPlatforms]
|
|
* @property {string[]} [testFiles]
|
|
*/
|
|
|
|
/**
|
|
* @param {Step} step
|
|
* @param {(string | undefined)[]} dependsOn
|
|
* @returns {Step}
|
|
*/
|
|
function getStepWithDependsOn(step, ...dependsOn) {
|
|
const { depends_on: existingDependsOn = [] } = step;
|
|
return {
|
|
...step,
|
|
depends_on: [...existingDependsOn, ...dependsOn.filter(Boolean)],
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @returns {BlockStep}
|
|
*/
|
|
function getOptionsStep() {
|
|
const booleanOptions = [
|
|
{
|
|
label: `${getEmoji("true")} Yes`,
|
|
value: "true",
|
|
},
|
|
{
|
|
label: `${getEmoji("false")} No`,
|
|
value: "false",
|
|
},
|
|
];
|
|
|
|
return {
|
|
key: "options",
|
|
block: getBuildkiteEmoji("clipboard"),
|
|
blocked_state: "running",
|
|
fields: [
|
|
{
|
|
key: "canary",
|
|
select: "If building, is this a canary build?",
|
|
hint: "If you are building for a release, this should be false",
|
|
required: false,
|
|
default: "true",
|
|
options: booleanOptions,
|
|
},
|
|
{
|
|
key: "skip-builds",
|
|
select: "Do you want to skip the build?",
|
|
hint: "If true, artifacts will be downloaded from the last successful build",
|
|
required: false,
|
|
default: "false",
|
|
options: booleanOptions,
|
|
},
|
|
{
|
|
key: "skip-tests",
|
|
select: "Do you want to skip the tests?",
|
|
required: false,
|
|
default: "false",
|
|
options: booleanOptions,
|
|
},
|
|
{
|
|
key: "force-builds",
|
|
select: "Do you want to force run the build?",
|
|
hint: "If true, the build will run even if no source files have changed",
|
|
required: false,
|
|
default: "false",
|
|
options: booleanOptions,
|
|
},
|
|
{
|
|
key: "force-tests",
|
|
select: "Do you want to force run the tests?",
|
|
hint: "If true, the tests will run even if no test files have changed",
|
|
required: false,
|
|
default: "false",
|
|
options: booleanOptions,
|
|
},
|
|
{
|
|
key: "build-profiles",
|
|
select: "If building, which profiles do you want to build?",
|
|
required: false,
|
|
multiple: true,
|
|
default: ["release"],
|
|
options: [
|
|
{
|
|
label: `${getEmoji("release")} Release`,
|
|
value: "release",
|
|
},
|
|
{
|
|
label: `${getEmoji("assert")} Release with Assertions`,
|
|
value: "assert",
|
|
},
|
|
{
|
|
label: `${getEmoji("asan")} Release with ASAN`,
|
|
value: "asan",
|
|
},
|
|
{
|
|
label: `${getEmoji("debug")} Debug`,
|
|
value: "debug",
|
|
},
|
|
],
|
|
},
|
|
{
|
|
key: "build-platforms",
|
|
select: "If building, which platforms do you want to build?",
|
|
hint: "If this is left blank, all platforms are built",
|
|
required: false,
|
|
multiple: true,
|
|
default: [],
|
|
options: buildPlatforms.map(platform => {
|
|
const { os, arch, abi, baseline } = platform;
|
|
let label = `${getEmoji(os)} ${arch}`;
|
|
if (abi) {
|
|
label += `-${abi}`;
|
|
}
|
|
if (baseline) {
|
|
label += `-baseline`;
|
|
}
|
|
return {
|
|
label,
|
|
value: getTargetKey(platform),
|
|
};
|
|
}),
|
|
},
|
|
{
|
|
key: "test-platforms",
|
|
select: "If testing, which platforms do you want to test?",
|
|
hint: "If this is left blank, all platforms are tested",
|
|
required: false,
|
|
multiple: true,
|
|
default: [],
|
|
options: [...new Map(testPlatforms.map(platform => [getImageKey(platform), platform])).entries()].map(
|
|
([key, platform]) => {
|
|
const { os, arch, abi, distro, release } = platform;
|
|
let label = `${getEmoji(os)} ${arch}`;
|
|
if (abi) {
|
|
label += `-${abi}`;
|
|
}
|
|
if (distro) {
|
|
label += ` ${distro}`;
|
|
}
|
|
if (release) {
|
|
label += ` ${release}`;
|
|
}
|
|
return {
|
|
label,
|
|
value: key,
|
|
};
|
|
},
|
|
),
|
|
},
|
|
{
|
|
key: "test-files",
|
|
text: "If testing, which files do you want to test?",
|
|
hint: "If specified, only run test paths that include the list of strings (e.g. 'test/js', 'test/cli/hot/watch.ts')",
|
|
required: false,
|
|
},
|
|
{
|
|
key: "build-images",
|
|
select: "Do you want to re-build the base images?",
|
|
hint: "This can take 2-3 hours to complete, only do so if you've tested locally",
|
|
required: false,
|
|
default: "false",
|
|
options: booleanOptions,
|
|
},
|
|
{
|
|
key: "publish-images",
|
|
select: "Do you want to re-build and publish the base images?",
|
|
hint: "This can take 2-3 hours to complete, only do so if you've tested locally",
|
|
required: false,
|
|
default: "false",
|
|
options: booleanOptions,
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @returns {Step}
|
|
*/
|
|
function getOptionsApplyStep() {
|
|
const command = getEnv("BUILDKITE_COMMAND");
|
|
return {
|
|
key: "options-apply",
|
|
label: getBuildkiteEmoji("gear"),
|
|
command: `${command} --apply`,
|
|
depends_on: ["options"],
|
|
agents: {
|
|
queue: getEnv("BUILDKITE_AGENT_META_DATA_QUEUE", false),
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @returns {Promise<PipelineOptions | undefined>}
|
|
*/
|
|
async function getPipelineOptions() {
|
|
const isManual = isBuildManual();
|
|
if (isManual && !process.argv.includes("--apply")) {
|
|
return;
|
|
}
|
|
|
|
let filteredBuildPlatforms = buildPlatforms;
|
|
if (isMainBranch()) {
|
|
filteredBuildPlatforms = buildPlatforms.filter(({ profile }) => profile !== "asan");
|
|
}
|
|
|
|
const canary = await getCanaryRevision();
|
|
const buildPlatformsMap = new Map(filteredBuildPlatforms.map(platform => [getTargetKey(platform), platform]));
|
|
const testPlatformsMap = new Map(testPlatforms.map(platform => [getPlatformKey(platform), platform]));
|
|
|
|
if (isManual) {
|
|
const { fields } = getOptionsStep();
|
|
const keys = fields?.map(({ key }) => key) ?? [];
|
|
const values = await Promise.all(keys.map(getBuildMetadata));
|
|
const options = Object.fromEntries(keys.map((key, index) => [key, values[index]]));
|
|
|
|
/**
|
|
* @param {string} value
|
|
* @returns {string[] | undefined}
|
|
*/
|
|
const parseArray = value =>
|
|
value
|
|
?.split("\n")
|
|
?.map(item => item.trim())
|
|
?.filter(Boolean);
|
|
|
|
const buildProfiles = parseArray(options["build-profiles"]);
|
|
const buildPlatformKeys = parseArray(options["build-platforms"]);
|
|
const testPlatformKeys = parseArray(options["test-platforms"]);
|
|
return {
|
|
canary: parseBoolean(options["canary"]) ? canary : 0,
|
|
skipBuilds: parseBoolean(options["skip-builds"]),
|
|
forceBuilds: parseBoolean(options["force-builds"]),
|
|
skipTests: parseBoolean(options["skip-tests"]),
|
|
buildImages: parseBoolean(options["build-images"]),
|
|
publishImages: parseBoolean(options["publish-images"]),
|
|
testFiles: parseArray(options["test-files"]),
|
|
buildPlatforms: buildPlatformKeys?.length
|
|
? buildPlatformKeys.flatMap(key => buildProfiles.map(profile => ({ ...buildPlatformsMap.get(key), profile })))
|
|
: Array.from(buildPlatformsMap.values()),
|
|
testPlatforms: testPlatformKeys?.length
|
|
? testPlatformKeys.flatMap(key => buildProfiles.map(profile => ({ ...testPlatformsMap.get(key), profile })))
|
|
: Array.from(testPlatformsMap.values()),
|
|
dryRun: parseBoolean(options["dry-run"]),
|
|
};
|
|
}
|
|
|
|
const commitMessage = getCommitMessage();
|
|
|
|
/**
|
|
* @param {RegExp} pattern
|
|
* @returns {string | boolean}
|
|
*/
|
|
const parseOption = pattern => {
|
|
const match = pattern.exec(commitMessage);
|
|
if (match) {
|
|
const [, value] = match;
|
|
return value;
|
|
}
|
|
return false;
|
|
};
|
|
|
|
const isCanary =
|
|
!parseBoolean(getEnv("RELEASE", false) || "false") &&
|
|
!/\[(release|build release|release build)\]/i.test(commitMessage);
|
|
return {
|
|
canary: isCanary ? canary : 0,
|
|
skipEverything: parseOption(/\[(skip ci|no ci)\]/i),
|
|
skipBuilds: parseOption(/\[(skip builds?|no builds?|only tests?)\]/i),
|
|
forceBuilds: parseOption(/\[(force builds?)\]/i),
|
|
skipTests: parseOption(/\[(skip tests?|no tests?|only builds?)\]/i),
|
|
buildImages: parseOption(/\[(build images?)\]/i),
|
|
dryRun: parseOption(/\[(dry run)\]/i),
|
|
publishImages: parseOption(/\[(publish images?)\]/i),
|
|
buildPlatforms: Array.from(buildPlatformsMap.values()),
|
|
testPlatforms: Array.from(testPlatformsMap.values()),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* @param {PipelineOptions} [options]
|
|
* @returns {Promise<Pipeline | undefined>}
|
|
*/
|
|
async function getPipeline(options = {}) {
|
|
const priority = getPriority();
|
|
|
|
if (isBuildManual() && !Object.keys(options).length) {
|
|
return {
|
|
priority,
|
|
steps: [getOptionsStep(), getOptionsApplyStep()],
|
|
};
|
|
}
|
|
|
|
const { skipEverything } = options;
|
|
if (skipEverything) {
|
|
return;
|
|
}
|
|
|
|
const { buildPlatforms = [], testPlatforms = [], buildImages, publishImages } = options;
|
|
const imagePlatforms = new Map(
|
|
buildImages || publishImages
|
|
? [...buildPlatforms, ...testPlatforms]
|
|
.filter(({ os }) => os !== "darwin")
|
|
.map(platform => [getImageKey(platform), platform])
|
|
: [],
|
|
);
|
|
|
|
/** @type {Step[]} */
|
|
const steps = [];
|
|
|
|
if (imagePlatforms.size) {
|
|
steps.push({
|
|
key: "build-images",
|
|
group: getBuildkiteEmoji("aws"),
|
|
steps: [...imagePlatforms.values()].map(platform => getBuildImageStep(platform, options)),
|
|
});
|
|
}
|
|
|
|
let { skipBuilds, forceBuilds, dryRun } = options;
|
|
dryRun = dryRun || !!buildImages;
|
|
|
|
/** @type {string | undefined} */
|
|
let buildId;
|
|
if (skipBuilds && !forceBuilds) {
|
|
const lastBuild = await getLastSuccessfulBuild();
|
|
if (lastBuild) {
|
|
const { id } = lastBuild;
|
|
buildId = id;
|
|
} else {
|
|
console.warn("No last successful build found, must force builds...");
|
|
}
|
|
}
|
|
|
|
const includeASAN = !isMainBranch();
|
|
|
|
if (!buildId) {
|
|
let relevantBuildPlatforms = includeASAN
|
|
? buildPlatforms
|
|
: buildPlatforms.filter(({ profile }) => profile !== "asan");
|
|
|
|
// run build-image but no build-bun yet
|
|
relevantBuildPlatforms = relevantBuildPlatforms.filter(({ os }) => os !== "freebsd");
|
|
|
|
steps.push(
|
|
...relevantBuildPlatforms.map(target => {
|
|
const imageKey = getImageKey(target);
|
|
const zigImageKey = getImageKey(getZigPlatform());
|
|
const dependsOn = imagePlatforms.has(zigImageKey) ? [`${zigImageKey}-build-image`] : [];
|
|
if (imagePlatforms.has(imageKey)) {
|
|
dependsOn.push(`${imageKey}-build-image`);
|
|
}
|
|
|
|
const steps = [];
|
|
steps.push(getBuildCppStep(target, options));
|
|
steps.push(getBuildZigStep(target, options));
|
|
steps.push(getLinkBunStep(target, options));
|
|
|
|
return getStepWithDependsOn(
|
|
{
|
|
key: getTargetKey(target),
|
|
group: getTargetLabel(target),
|
|
steps,
|
|
},
|
|
...dependsOn,
|
|
);
|
|
}),
|
|
);
|
|
}
|
|
|
|
if (!isMainBranch()) {
|
|
const { skipTests, forceTests, testFiles } = options;
|
|
if (!skipTests || forceTests) {
|
|
steps.push(
|
|
...testPlatforms.map(target => ({
|
|
key: getTargetKey(target),
|
|
group: getTargetLabel(target),
|
|
steps: [getTestBunStep(target, options, { testFiles, buildId })],
|
|
})),
|
|
);
|
|
}
|
|
}
|
|
|
|
if (isMainBranch()) {
|
|
steps.push(getReleaseStep(buildPlatforms, options));
|
|
}
|
|
steps.push(getBenchmarkStep());
|
|
|
|
/** @type {Map<string, GroupStep>} */
|
|
const stepsByGroup = new Map();
|
|
|
|
for (let i = 0; i < steps.length; i++) {
|
|
const step = steps[i];
|
|
if (!("group" in step)) {
|
|
continue;
|
|
}
|
|
|
|
const { group, steps: groupSteps } = step;
|
|
if (stepsByGroup.has(group)) {
|
|
stepsByGroup.get(group).steps.push(...groupSteps);
|
|
} else {
|
|
stepsByGroup.set(group, step);
|
|
}
|
|
|
|
steps[i] = undefined;
|
|
}
|
|
|
|
return {
|
|
priority,
|
|
steps: [...steps.filter(step => typeof step !== "undefined"), ...Array.from(stepsByGroup.values())],
|
|
};
|
|
}
|
|
|
|
async function main() {
|
|
startGroup("Generating options...");
|
|
const options = await getPipelineOptions();
|
|
if (options) {
|
|
console.log("Generated options:", options);
|
|
}
|
|
|
|
startGroup("Querying GitHub for files...");
|
|
if (options && isBuildkite && !isMainBranch()) {
|
|
/** @type {string[]} */
|
|
let allFiles = [];
|
|
/** @type {string[]} */
|
|
let newFiles = [];
|
|
let prFileCount = 0;
|
|
try {
|
|
console.log("on buildkite: collecting new files from PR");
|
|
const per_page = 50;
|
|
const { BUILDKITE_PULL_REQUEST } = process.env;
|
|
for (let i = 1; i <= 10; i++) {
|
|
const res = await fetch(
|
|
`https://api.github.com/repos/oven-sh/bun/pulls/${BUILDKITE_PULL_REQUEST}/files?per_page=${per_page}&page=${i}`,
|
|
{ headers: { Authorization: `Bearer ${getSecret("GITHUB_TOKEN")}` } },
|
|
);
|
|
const doc = await res.json();
|
|
console.log(`-> page ${i}, found ${doc.length} items`);
|
|
if (doc.length === 0) break;
|
|
for (const { filename, status } of doc) {
|
|
prFileCount += 1;
|
|
allFiles.push(filename);
|
|
if (status !== "added") continue;
|
|
newFiles.push(filename);
|
|
}
|
|
if (doc.length < per_page) break;
|
|
}
|
|
console.log(`- PR ${BUILDKITE_PULL_REQUEST}, ${prFileCount} files, ${newFiles.length} new files`);
|
|
} catch (e) {
|
|
console.error(e);
|
|
}
|
|
if (allFiles.every(filename => filename.startsWith("docs/"))) {
|
|
console.log(`- PR is only docs, skipping tests!`);
|
|
return;
|
|
}
|
|
}
|
|
|
|
startGroup("Generating pipeline...");
|
|
const pipeline = await getPipeline(options);
|
|
if (!pipeline) {
|
|
console.log("Generated pipeline is empty, skipping...");
|
|
return;
|
|
}
|
|
|
|
const content = toYaml(pipeline);
|
|
const contentPath = join(process.cwd(), ".buildkite", "ci.yml");
|
|
writeFile(contentPath, content);
|
|
|
|
console.log("Generated pipeline:");
|
|
console.log(" - Path:", contentPath);
|
|
console.log(" - Size:", (content.length / 1024).toFixed(), "KB");
|
|
|
|
if (isBuildkite) {
|
|
startGroup("Uploading pipeline...");
|
|
try {
|
|
await spawnSafe(["buildkite-agent", "pipeline", "upload", contentPath], { stdio: "inherit" });
|
|
} finally {
|
|
await uploadArtifact(contentPath);
|
|
}
|
|
}
|
|
}
|
|
|
|
await main();
|