diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs old mode 100644 new mode 100755 index fa90de52e3..9bea1e3751 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -8,13 +8,14 @@ import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { + getBootstrapVersion, + getBuildNumber, getCanaryRevision, getChangedFiles, getCommit, getCommitMessage, getLastSuccessfulBuild, getMainBranch, - getRepositoryOwner, getTargetBranch, isBuildkite, isFork, @@ -22,102 +23,162 @@ import { isMergeQueue, printEnvironment, spawnSafe, + toYaml, + uploadArtifact, } from "../scripts/utils.mjs"; -function toYaml(obj, indent = 0) { - const spaces = " ".repeat(indent); - let result = ""; +/** + * @typedef PipelineOptions + * @property {string} [buildId] + * @property {boolean} [buildImages] + * @property {boolean} [publishImages] + * @property {boolean} [skipTests] + */ - for (const [key, value] of Object.entries(obj)) { - if (value === undefined) { - continue; - } +/** + * @param {PipelineOptions} options + */ +function getPipeline(options) { + const { buildId, buildImages, publishImages, skipTests } = options; - if (value === null) { - result += `${spaces}${key}: null\n`; - continue; - } - - if (Array.isArray(value)) { - result += `${spaces}${key}:\n`; - value.forEach(item => { - if (typeof item === "object" && item !== null) { - result += `${spaces}- \n${toYaml(item, indent + 2) - .split("\n") - .map(line => `${spaces} ${line}`) - .join("\n")}\n`; - } else { - result += `${spaces}- ${item}\n`; - } - }); - continue; - } - - if (typeof value === "object") { - result += `${spaces}${key}:\n${toYaml(value, indent + 2)}`; - continue; - } - - if ( - typeof value === "string" && - (value.includes(":") || value.includes("#") || value.includes("'") || value.includes('"') || value.includes("\n")) - ) { - result += `${spaces}${key}: "${value.replace(/"/g, '\\"')}"\n`; - continue; - } - - result += `${spaces}${key}: ${value}\n`; - } - - return result; -} - -function getPipeline(buildId) { /** * Helpers */ - const getKey = platform => { - const { os, arch, abi, baseline } = platform; - - if (abi) { - if (baseline) { - return `${os}-${arch}-${abi}-baseline`; - } - return `${os}-${arch}-${abi}`; + /** + * @param {string} text + * @returns {string} + * @link https://github.com/buildkite/emojis#emoji-reference + */ + const getEmoji = string => { + if (string === "amazonlinux") { + return ":aws:"; } - if (baseline) { - return `${os}-${arch}-baseline`; - } - - return `${os}-${arch}`; + return `:${string}:`; }; - const getLabel = platform => { - const { os, arch, abi, baseline, release } = platform; - let label = release ? `:${os}: ${release} ${arch}` : `:${os}: ${arch}`; + /** + * @typedef {"linux" | "darwin" | "windows"} Os + * @typedef {"aarch64" | "x64"} Arch + * @typedef {"musl"} Abi + */ + + /** + * @typedef Target + * @property {Os} os + * @property {Arch} arch + * @property {Abi} [abi] + * @property {boolean} [baseline] + */ + + /** + * @param {Target} target + * @returns {string} + */ + const getTargetKey = target => { + const { os, arch, abi, baseline } = target; + let key = `${os}-${arch}`; + if (abi) { + key += `-${abi}`; + } + if (baseline) { + key += "-baseline"; + } + return key; + }; + + /** + * @param {Target} target + * @returns {string} + */ + const getTargetLabel = target => { + const { os, arch, abi, baseline } = target; + let label = `${getEmoji(os)} ${arch}`; if (abi) { label += `-${abi}`; } if (baseline) { - label += `-baseline`; + label += "-baseline"; } return label; }; - // https://buildkite.com/docs/pipelines/command-step#retry-attributes - const getRetry = (limit = 3) => { + /** + * @typedef Platform + * @property {Os} os + * @property {Arch} arch + * @property {Abi} [abi] + * @property {boolean} [baseline] + * @property {string} [distro] + * @property {string} release + */ + + /** + * @param {Platform} platform + * @returns {string} + */ + const getPlatformKey = platform => { + const { os, arch, abi, baseline, distro, release } = platform; + const target = getTargetKey({ os, arch, abi, baseline }); + if (distro) { + return `${target}-${distro}-${release.replace(/\./g, "")}`; + } + return `${target}-${release.replace(/\./g, "")}`; + }; + + /** + * @param {Platform} platform + * @returns {string} + */ + const getPlatformLabel = platform => { + const { os, arch, baseline, distro, release } = platform; + let label = `${getEmoji(distro || os)} ${release} ${arch}`; + if (baseline) { + label += "-baseline"; + } + return label; + }; + + /** + * @param {Platform} platform + * @returns {string} + */ + const getImageKey = platform => { + const { os, arch, distro, release } = platform; + if (distro) { + return `${os}-${arch}-${distro}-${release.replace(/\./g, "")}`; + } + return `${os}-${arch}-${release.replace(/\./g, "")}`; + }; + + /** + * @param {Platform} platform + * @returns {string} + */ + const getImageLabel = platform => { + const { os, arch, distro, release } = platform; + return `${getEmoji(distro || os)} ${release} ${arch}`; + }; + + /** + * @param {number} [limit] + * @link https://buildkite.com/docs/pipelines/command-step#retry-attributes + */ + const getRetry = (limit = 0) => { return { automatic: [ - { exit_status: 1, limit: 1 }, - { exit_status: -1, limit }, - { exit_status: 255, limit }, - { signal_reason: "agent_stop", limit }, + { exit_status: 1, limit }, + { exit_status: -1, limit: 3 }, + { exit_status: 255, limit: 3 }, + { signal_reason: "agent_stop", limit: 3 }, ], }; }; - // https://buildkite.com/docs/pipelines/managing-priorities + /** + * @returns {number} + * @link https://buildkite.com/docs/pipelines/managing-priorities + */ const getPriority = () => { if (isFork()) { return -1; @@ -131,132 +192,286 @@ function getPipeline(buildId) { return 0; }; + /** + * @param {Target} target + * @returns {Record} + */ + const getBuildEnv = target => { + const { baseline, abi } = target; + return { + ENABLE_BASELINE: baseline ? "ON" : "OFF", + ABI: abi === "musl" ? "musl" : undefined, + }; + }; + + /** + * @param {Target} target + * @returns {string} + */ + const getBuildToolchain = target => { + const { os, arch, abi, baseline } = target; + let key = `${os}-${arch}`; + if (abi) { + key += `-${abi}`; + } + if (baseline) { + key += "-baseline"; + } + return key; + }; + + /** + * Agents + */ + + /** + * @typedef {Record} Agent + */ + + /** + * @param {Platform} platform + * @returns {boolean} + */ + const isUsingNewAgent = platform => { + const { os, distro } = platform; + if (os === "linux" && distro === "alpine") { + return true; + } + return false; + }; + + /** + * @param {"v1" | "v2"} version + * @param {Platform} platform + * @param {string} [instanceType] + * @returns {Agent} + */ + const getEmphemeralAgent = (version, platform, instanceType) => { + const { os, arch, abi, distro, release } = platform; + if (version === "v1") { + return { + robobun: true, + os, + arch, + distro, + release, + }; + } + let image; + if (distro) { + image = `${os}-${arch}-${distro}-${release}`; + } else { + image = `${os}-${arch}-${release}`; + } + if (buildImages && !publishImages) { + image += `-build-${getBuildNumber()}`; + } else { + image += `-v${getBootstrapVersion()}`; + } + return { + robobun: true, + robobun2: true, + os, + arch, + abi, + distro, + release, + "image-name": image, + "instance-type": instanceType, + }; + }; + + /** + * @param {Target} target + * @returns {Agent} + */ + const getBuildAgent = target => { + const { os, arch, abi } = target; + if (isUsingNewAgent(target)) { + const instanceType = arch === "aarch64" ? "c8g.8xlarge" : "c7i.8xlarge"; + return getEmphemeralAgent("v2", target, instanceType); + } + return { + queue: `build-${os}`, + os, + arch, + abi, + }; + }; + + /** + * @param {Target} target + * @returns {Agent} + */ + const getZigAgent = target => { + const { abi, arch } = target; + // if (abi === "musl") { + // const instanceType = arch === "aarch64" ? "c8g.large" : "c7i.large"; + // return getEmphemeralAgent("v2", target, instanceType); + // } + return { + queue: "build-zig", + }; + }; + + /** + * @param {Platform} platform + * @returns {Agent} + */ + const getTestAgent = platform => { + const { os, arch, release } = platform; + if (isUsingNewAgent(platform)) { + const instanceType = arch === "aarch64" ? "t4g.large" : "t3.large"; + return getEmphemeralAgent("v2", platform, instanceType); + } + if (os === "darwin") { + return { + os, + arch, + release, + queue: "test-darwin", + }; + } + return getEmphemeralAgent("v1", platform); + }; + /** * Steps */ - const getBuildVendorStep = platform => { - const { os, arch, abi, baseline } = platform; + /** + * @typedef Step + * @property {string} key + * @property {string} [label] + * @property {Record} [agents] + * @property {Record} [env] + * @property {string} command + * @property {string[]} [depends_on] + * @property {Record} [retry] + * @property {boolean} [cancel_on_build_failing] + * @property {boolean} [soft_fail] + * @property {number} [parallelism] + * @property {number} [concurrency] + * @property {string} [concurrency_group] + * @property {number} [priority] + * @property {number} [timeout_in_minutes] + * @link https://buildkite.com/docs/pipelines/command-step + */ + /** + * @param {Platform} platform + * @returns {Step} + */ + const getBuildImageStep = platform => { + const { os, arch, distro, release } = platform; + const action = publishImages ? "publish-image" : "create-image"; return { - key: `${getKey(platform)}-build-vendor`, - label: `build-vendor`, + key: `${getImageKey(platform)}-build-image`, + label: `${getImageLabel(platform)} - build-image`, agents: { - os, - arch, - abi, - queue: abi ? `build-${os}-${abi}` : `build-${os}`, + queue: "build-image", }, + env: { + DEBUG: "1", + }, + command: `node ./scripts/machine.mjs ${action} --ci --cloud=aws --os=${os} --arch=${arch} --distro=${distro} --distro-version=${release}`, + }; + }; + + /** + * @param {Target} target + * @returns {Step} + */ + const getBuildVendorStep = target => { + return { + key: `${getTargetKey(target)}-build-vendor`, + label: `${getTargetLabel(target)} - build-vendor`, + agents: getBuildAgent(target), retry: getRetry(), cancel_on_build_failing: isMergeQueue(), - env: { - ENABLE_BASELINE: baseline ? "ON" : "OFF", - }, + env: getBuildEnv(target), command: "bun run build:ci --target dependencies", }; }; - const getBuildCppStep = platform => { - const { os, arch, abi, baseline } = platform; - + /** + * @param {Target} target + * @returns {Step} + */ + const getBuildCppStep = target => { return { - key: `${getKey(platform)}-build-cpp`, - label: `build-cpp`, - agents: { - os, - arch, - abi, - queue: abi ? `build-${os}-${abi}` : `build-${os}`, - }, + key: `${getTargetKey(target)}-build-cpp`, + label: `${getTargetLabel(target)} - build-cpp`, + agents: getBuildAgent(target), retry: getRetry(), cancel_on_build_failing: isMergeQueue(), env: { BUN_CPP_ONLY: "ON", - ENABLE_BASELINE: baseline ? "ON" : "OFF", + ...getBuildEnv(target), }, command: "bun run build:ci --target bun", }; }; - const getBuildZigStep = platform => { - const { os, arch, abi, baseline } = platform; - const toolchain = getKey(platform); - + /** + * @param {Target} target + * @returns {Step} + */ + const getBuildZigStep = target => { + const toolchain = getBuildToolchain(target); return { - key: `${getKey(platform)}-build-zig`, - label: `build-zig`, - agents: { - queue: "build-zig", - }, - retry: getRetry(), + key: `${getTargetKey(target)}-build-zig`, + label: `${getTargetLabel(target)} - build-zig`, + agents: getZigAgent(target), + retry: getRetry(1), // FIXME: Sometimes zig build hangs, so we need to retry once cancel_on_build_failing: isMergeQueue(), - env: { - ENABLE_BASELINE: baseline ? "ON" : "OFF", - }, + env: getBuildEnv(target), command: `bun run build:ci --target bun-zig --toolchain ${toolchain}`, }; }; - const getBuildBunStep = platform => { - const { os, arch, abi, baseline } = platform; - + /** + * @param {Target} target + * @returns {Step} + */ + const getBuildBunStep = target => { return { - key: `${getKey(platform)}-build-bun`, - label: `build-bun`, + key: `${getTargetKey(target)}-build-bun`, + label: `${getTargetLabel(target)} - build-bun`, depends_on: [ - `${getKey(platform)}-build-vendor`, - `${getKey(platform)}-build-cpp`, - `${getKey(platform)}-build-zig`, + `${getTargetKey(target)}-build-vendor`, + `${getTargetKey(target)}-build-cpp`, + `${getTargetKey(target)}-build-zig`, ], - agents: { - os, - arch, - abi, - queue: `build-${os}`, - }, + agents: getBuildAgent(target), retry: getRetry(), cancel_on_build_failing: isMergeQueue(), env: { BUN_LINK_ONLY: "ON", - ENABLE_BASELINE: baseline ? "ON" : "OFF", + ...getBuildEnv(target), }, command: "bun run build:ci --target bun", }; }; + /** + * @param {Platform} platform + * @returns {Step} + */ const getTestBunStep = platform => { - const { os, arch, abi, distro, release } = platform; - - let name; - if (os === "darwin" || os === "windows") { - name = getLabel({ ...platform, release }); - } else { - name = getLabel({ ...platform, os: distro, release }); - } - - let agents; - if (os === "darwin") { - agents = { os, arch, abi, queue: `test-darwin` }; - } else if (os === "windows") { - agents = { os, arch, abi, robobun: true }; - } else { - agents = { os, arch, abi, distro, release, robobun: true }; - } - + const { os } = platform; let command; if (os === "windows") { - command = `node .\\scripts\\runner.node.mjs --step ${getKey(platform)}-build-bun`; + command = `node .\\scripts\\runner.node.mjs --step ${getTargetKey(platform)}-build-bun`; } else { - command = `./scripts/runner.node.mjs --step ${getKey(platform)}-build-bun`; + command = `./scripts/runner.node.mjs --step ${getTargetKey(platform)}-build-bun`; } - let parallelism; if (os === "darwin") { parallelism = 2; } else { parallelism = 10; } - let depends; let env; if (buildId) { @@ -264,21 +479,19 @@ function getPipeline(buildId) { BUILDKITE_ARTIFACT_BUILD_ID: buildId, }; } else { - depends = [`${getKey(platform)}-build-bun`]; + depends = [`${getTargetKey(platform)}-build-bun`]; } - let retry; if (os !== "windows") { // When the runner fails on Windows, Buildkite only detects an exit code of 1. // Because of this, we don't know if the run was fatal, or soft-failed. - retry = getRetry(); + retry = getRetry(1); } - return { - key: `${getKey(platform)}-${distro}-${release.replace(/\./g, "")}-test-bun`, - label: `${name} - test-bun`, + key: `${getPlatformKey(platform)}-test-bun`, + label: `${getPlatformLabel(platform)} - test-bun`, depends_on: depends, - agents, + agents: getTestAgent(platform), retry, cancel_on_build_failing: isMergeQueue(), soft_fail: isMainBranch(), @@ -292,66 +505,145 @@ function getPipeline(buildId) { * Config */ + /** + * @type {Platform[]} + */ const buildPlatforms = [ - { os: "darwin", arch: "aarch64" }, - { os: "darwin", arch: "x64" }, - { os: "linux", arch: "aarch64" }, - // { os: "linux", arch: "aarch64", abi: "musl" }, // TODO: - { os: "linux", arch: "x64" }, - { os: "linux", arch: "x64", baseline: true }, - // { os: "linux", arch: "x64", abi: "musl" }, // TODO: - { os: "windows", arch: "x64" }, - { os: "windows", arch: "x64", baseline: true }, + { os: "darwin", arch: "aarch64", release: "14" }, + { os: "darwin", arch: "x64", release: "14" }, + { os: "linux", arch: "aarch64", distro: "debian", release: "11" }, + { os: "linux", arch: "x64", distro: "debian", release: "11" }, + { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" }, + { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" }, + { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20" }, + { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20" }, + { os: "windows", arch: "x64", release: "2019" }, + { os: "windows", arch: "x64", baseline: true, release: "2019" }, ]; + /** + * @type {Platform[]} + */ const testPlatforms = [ - { os: "darwin", arch: "aarch64", distro: "sonoma", release: "14" }, - { os: "darwin", arch: "aarch64", distro: "ventura", release: "13" }, - { os: "darwin", arch: "x64", distro: "sonoma", release: "14" }, - { os: "darwin", arch: "x64", distro: "ventura", release: "13" }, + { os: "darwin", arch: "aarch64", release: "14" }, + { os: "darwin", arch: "aarch64", release: "13" }, + { os: "darwin", arch: "x64", release: "14" }, + { os: "darwin", arch: "x64", release: "13" }, { os: "linux", arch: "aarch64", distro: "debian", release: "12" }, + // { os: "linux", arch: "aarch64", distro: "debian", release: "11" }, + // { os: "linux", arch: "aarch64", distro: "debian", release: "10" }, + { os: "linux", arch: "x64", distro: "debian", release: "12" }, + // { os: "linux", arch: "x64", distro: "debian", release: "11" }, + // { os: "linux", arch: "x64", distro: "debian", release: "10" }, + { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "12" }, + // { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" }, + // { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "10" }, + // { os: "linux", arch: "aarch64", distro: "ubuntu", release: "24.04" }, { os: "linux", arch: "aarch64", distro: "ubuntu", release: "22.04" }, { os: "linux", arch: "aarch64", distro: "ubuntu", release: "20.04" }, - // { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "edge" }, // TODO: - { os: "linux", arch: "x64", distro: "debian", release: "12" }, + // { os: "linux", arch: "x64", distro: "ubuntu", release: "24.04" }, { os: "linux", arch: "x64", distro: "ubuntu", release: "22.04" }, { os: "linux", arch: "x64", distro: "ubuntu", release: "20.04" }, - { os: "linux", arch: "x64", distro: "debian", release: "12", baseline: true }, - { os: "linux", arch: "x64", distro: "ubuntu", release: "22.04", baseline: true }, - { os: "linux", arch: "x64", distro: "ubuntu", release: "20.04", baseline: true }, - // { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "edge" }, // TODO: - { os: "windows", arch: "x64", distro: "server", release: "2019" }, - { os: "windows", arch: "x64", distro: "server", release: "2019", baseline: true }, + // { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "24.04" }, + { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "22.04" }, + { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "20.04" }, + // { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023" }, + // { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2" }, + // { os: "linux", arch: "x64", distro: "amazonlinux", release: "2023" }, + // { os: "linux", arch: "x64", distro: "amazonlinux", release: "2" }, + // { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023" }, + // { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2" }, + { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" }, + // { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.17" }, + { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20" }, + // { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.17" }, + { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20" }, + // { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.17" }, + { os: "windows", arch: "x64", release: "2019" }, + { os: "windows", arch: "x64", baseline: true, release: "2019" }, ]; + const imagePlatforms = new Map( + [...buildPlatforms, ...testPlatforms] + .filter(platform => buildImages && isUsingNewAgent(platform)) + .map(platform => [getImageKey(platform), platform]), + ); + + /** + * @type {Step[]} + */ + const steps = []; + + if (imagePlatforms.size) { + steps.push({ + group: ":docker:", + steps: [...imagePlatforms.values()].map(platform => getBuildImageStep(platform)), + }); + } + + for (const platform of buildPlatforms) { + const { os, arch, abi, baseline } = platform; + + /** @type {Step[]} */ + const platformSteps = []; + + if (buildImages || !buildId) { + platformSteps.push( + getBuildVendorStep(platform), + getBuildCppStep(platform), + getBuildZigStep(platform), + getBuildBunStep(platform), + ); + } + + if (!skipTests) { + platformSteps.push( + ...testPlatforms + .filter( + testPlatform => + testPlatform.os === os && + testPlatform.arch === arch && + testPlatform.abi === abi && + testPlatform.baseline === baseline, + ) + .map(testPlatform => getTestBunStep(testPlatform)), + ); + } + + if (!platformSteps.length) { + continue; + } + + if (imagePlatforms.has(getImageKey(platform))) { + for (const step of platformSteps) { + if (step.agents?.["image-name"]) { + step.depends_on ??= []; + step.depends_on.push(`${getImageKey(platform)}-build-image`); + } + } + } + + steps.push({ + key: getTargetKey(platform), + group: getTargetLabel(platform), + steps: platformSteps, + }); + } + + if (isMainBranch() && !isFork()) { + steps.push({ + label: ":github:", + agents: { + queue: "test-darwin", + }, + depends_on: buildPlatforms.map(platform => `${getTargetKey(platform)}-build-bun`), + command: ".buildkite/scripts/upload-release.sh", + }); + } + return { priority: getPriority(), - steps: [ - ...buildPlatforms.map(platform => { - const { os, arch, baseline } = platform; - - let steps = [ - ...testPlatforms - .filter(platform => platform.os === os && platform.arch === arch && baseline === platform.baseline) - .map(platform => getTestBunStep(platform)), - ]; - - if (!buildId) { - steps.unshift( - getBuildVendorStep(platform), - getBuildCppStep(platform), - getBuildZigStep(platform), - getBuildBunStep(platform), - ); - } - - return { - key: getKey(platform), - group: getLabel(platform), - steps, - }; - }), - ], + steps, }; } @@ -369,7 +661,6 @@ async function main() { console.log(" - No build found"); } - let changedFiles; if (!isFork()) { console.log("Checking changed files..."); @@ -377,7 +668,7 @@ async function main() { console.log(" - Base Ref:", baseRef); const headRef = lastBuild?.commit_id || getTargetBranch() || getMainBranch(); console.log(" - Head Ref:", headRef); - + changedFiles = await getChangedFiles(undefined, baseRef, headRef); if (changedFiles) { if (changedFiles.length) { @@ -418,6 +709,31 @@ async function main() { } } + console.log("Checking if CI should re-build images..."); + let buildImages; + { + const message = getCommitMessage(); + const match = /\[(build images?|images? build)\]/i.exec(message); + if (match) { + const [, reason] = match; + console.log(" - Yes, because commit message contains:", reason); + buildImages = true; + } + } + + console.log("Checking if CI should publish images..."); + let publishImages; + { + const message = getCommitMessage(); + const match = /\[(publish images?|images? publish)\]/i.exec(message); + if (match) { + const [, reason] = match; + console.log(" - Yes, because commit message contains:", reason); + publishImages = true; + buildImages = true; + } + } + console.log("Checking if build should be skipped..."); let skipBuild; if (!forceBuild) { @@ -434,6 +750,17 @@ async function main() { } } + console.log("Checking if tests should be skipped..."); + let skipTests; + { + const message = getCommitMessage(); + const match = /\[(skip tests?|tests? skip|no tests?|tests? no)\]/i.exec(message); + if (match) { + console.log(" - Yes, because commit message contains:", match[1]); + skipTests = true; + } + } + console.log("Checking if build is a named release..."); let buildRelease; { @@ -447,7 +774,13 @@ async function main() { } console.log("Generating pipeline..."); - const pipeline = getPipeline(lastBuild && skipBuild && !forceBuild ? lastBuild.id : undefined); + const pipeline = getPipeline({ + buildId: lastBuild && skipBuild && !forceBuild ? lastBuild.id : undefined, + buildImages, + publishImages, + skipTests, + }); + const content = toYaml(pipeline); const contentPath = join(process.cwd(), ".buildkite", "ci.yml"); writeFileSync(contentPath, content); @@ -455,14 +788,17 @@ async function main() { console.log("Generated pipeline:"); console.log(" - Path:", contentPath); console.log(" - Size:", (content.length / 1024).toFixed(), "KB"); + if (isBuildkite) { + await uploadArtifact(contentPath); + } if (isBuildkite) { console.log("Setting canary revision..."); const canaryRevision = buildRelease ? 0 : await getCanaryRevision(); - await spawnSafe(["buildkite-agent", "meta-data", "set", "canary", `${canaryRevision}`]); + await spawnSafe(["buildkite-agent", "meta-data", "set", "canary", `${canaryRevision}`], { stdio: "inherit" }); console.log("Uploading pipeline..."); - await spawnSafe(["buildkite-agent", "pipeline", "upload", contentPath]); + await spawnSafe(["buildkite-agent", "pipeline", "upload", contentPath], { stdio: "inherit" }); } } diff --git a/.buildkite/scripts/upload-release.sh b/.buildkite/scripts/upload-release.sh index 5a69f89861..b684dfb4a3 100755 --- a/.buildkite/scripts/upload-release.sh +++ b/.buildkite/scripts/upload-release.sh @@ -202,6 +202,12 @@ function create_release() { bun-linux-x64-profile.zip bun-linux-x64-baseline.zip bun-linux-x64-baseline-profile.zip + bun-linux-aarch64-musl.zip + bun-linux-aarch64-musl-profile.zip + bun-linux-x64-musl.zip + bun-linux-x64-musl-profile.zip + bun-linux-x64-musl-baseline.zip + bun-linux-x64-musl-baseline-profile.zip bun-windows-x64.zip bun-windows-x64-profile.zip bun-windows-x64-baseline.zip diff --git a/ci/linux/Dockerfile b/ci/linux/Dockerfile new file mode 100644 index 0000000000..3b46e73f6c --- /dev/null +++ b/ci/linux/Dockerfile @@ -0,0 +1,18 @@ +ARG IMAGE=debian:11 +FROM $IMAGE +COPY ./scripts/bootstrap.sh /tmp/bootstrap.sh +ENV CI=true +RUN sh /tmp/bootstrap.sh && rm -rf /tmp/* +WORKDIR /workspace/bun +COPY bunfig.toml bunfig.toml +COPY package.json package.json +COPY CMakeLists.txt CMakeLists.txt +COPY cmake/ cmake/ +COPY scripts/ scripts/ +COPY patches/ patches/ +COPY *.zig ./ +COPY src/ src/ +COPY packages/ packages/ +COPY test/ test/ +RUN bun i +RUN bun run build:ci diff --git a/ci/linux/scripts/set-hostname.sh b/ci/linux/scripts/set-hostname.sh new file mode 100644 index 0000000000..e529f74ce0 --- /dev/null +++ b/ci/linux/scripts/set-hostname.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# This script sets the hostname of the current machine. + +execute() { + echo "$ $@" >&2 + if ! "$@"; then + echo "Command failed: $@" >&2 + exit 1 + fi +} + +main() { + if [ "$#" -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 1 + fi + + if [ -f "$(which hostnamectl)" ]; then + execute hostnamectl set-hostname "$1" + else + echo "Error: hostnamectl is not installed." >&2 + exit 1 + fi +} + +main "$@" diff --git a/ci/linux/scripts/start-tailscale.sh b/ci/linux/scripts/start-tailscale.sh new file mode 100644 index 0000000000..3b519bfdf5 --- /dev/null +++ b/ci/linux/scripts/start-tailscale.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# This script starts tailscale on the current machine. + +execute() { + echo "$ $@" >&2 + if ! "$@"; then + echo "Command failed: $@" >&2 + exit 1 + fi +} + +main() { + if [ "$#" -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 1 + fi + + execute tailscale up --reset --ssh --accept-risk=lose-ssh --auth-key="$1" +} + +main "$@" diff --git a/ci/package.json b/ci/package.json index ffb1297dcd..28bd56c959 100644 --- a/ci/package.json +++ b/ci/package.json @@ -2,7 +2,7 @@ "private": true, "scripts": { "bootstrap": "brew install gh jq cirruslabs/cli/tart cirruslabs/cli/sshpass hashicorp/tap/packer && packer init darwin", - "login": "gh auth token | tart login ghcr.io --username $(gh api user --jq .login) --password-stdin", + "login": "token=$(gh auth token); username=$(gh api user --jq .login); echo \"Login as $username...\"; echo \"$token\" | tart login ghcr.io --username \"$username\" --password-stdin; echo \"$token\" | docker login ghcr.io --username \"$username\" --password-stdin", "fetch:image-name": "echo ghcr.io/oven-sh/bun-vm", "fetch:darwin-version": "echo 1", "fetch:macos-version": "sw_vers -productVersion | cut -d. -f1", diff --git a/cmake/Globals.cmake b/cmake/Globals.cmake index 9760101274..106e1285ea 100644 --- a/cmake/Globals.cmake +++ b/cmake/Globals.cmake @@ -105,14 +105,6 @@ else() unsupported(CMAKE_HOST_SYSTEM_NAME) endif() -if(EXISTS "/lib/ld-musl-aarch64.so.1") - set(IS_MUSL ON) -elseif(EXISTS "/lib/ld-musl-x86_64.so.1") - set(IS_MUSL ON) -else() - set(IS_MUSL OFF) -endif() - if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64|ARM64|aarch64|AARCH64") set(HOST_OS "aarch64") elseif(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64|X86_64|x64|X64|amd64|AMD64") @@ -144,6 +136,16 @@ else() set(WARNING WARNING) endif() +if(LINUX) + if(EXISTS "/etc/alpine-release") + set(DEFAULT_ABI "musl") + else() + set(DEFAULT_ABI "gnu") + endif() + + optionx(ABI "musl|gnu" "The ABI to use (e.g. musl, gnu)" DEFAULT ${DEFAULT_ABI}) +endif() + # TODO: This causes flaky zig builds in CI, so temporarily disable it. # if(CI) # set(DEFAULT_VENDOR_PATH ${CACHE_PATH}/vendor) diff --git a/cmake/targets/BuildBun.cmake b/cmake/targets/BuildBun.cmake index d3e8e5d6b1..bb7160ecde 100644 --- a/cmake/targets/BuildBun.cmake +++ b/cmake/targets/BuildBun.cmake @@ -484,14 +484,12 @@ set(BUN_ZIG_OUTPUT ${BUILD_PATH}/bun-zig.o) if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm|ARM|arm64|ARM64|aarch64|AARCH64") - set(IS_ARM64 ON) if(APPLE) set(ZIG_CPU "apple_m1") else() set(ZIG_CPU "native") endif() elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|X86_64|x64|X64|amd64|AMD64") - set(IS_X86_64 ON) if(ENABLE_BASELINE) set(ZIG_CPU "nehalem") else() @@ -761,8 +759,8 @@ if(NOT WIN32) ) if(DEBUG) # TODO: this shouldn't be necessary long term - if (NOT IS_MUSL) - set(ABI_PUBLIC_FLAGS + if (NOT ABI STREQUAL "musl") + target_compile_options(${bun} PUBLIC -fsanitize=null -fsanitize-recover=all -fsanitize=bounds @@ -773,14 +771,9 @@ if(NOT WIN32) -fsanitize=returns-nonnull-attribute -fsanitize=unreachable ) - set(ABI_PRIVATE_FLAGS + target_link_libraries(${bun} PRIVATE -fsanitize=null ) - else() - set(ABI_PUBLIC_FLAGS - ) - set(ABI_PRIVATE_FLAGS - ) endif() target_compile_options(${bun} PUBLIC @@ -798,10 +791,6 @@ if(NOT WIN32) -Wno-unused-function -Wno-nullability-completeness -Werror - ${ABI_PUBLIC_FLAGS} - ) - target_link_libraries(${bun} PRIVATE - ${ABI_PRIVATE_FLAGS} ) else() # Leave -Werror=unused off in release builds so we avoid errors from being used in ASSERT @@ -846,7 +835,9 @@ if(WIN32) /delayload:IPHLPAPI.dll ) endif() -elseif(APPLE) +endif() + +if(APPLE) target_link_options(${bun} PUBLIC -dead_strip -dead_strip_dylibs @@ -856,63 +847,36 @@ elseif(APPLE) -fno-keep-static-consts -Wl,-map,${bun}.linker-map ) -else() - # Try to use lld-16 if available, otherwise fallback to lld - # Cache it so we don't have to re-run CMake to pick it up - if((NOT DEFINED LLD_NAME) AND (NOT CI OR BUN_LINK_ONLY)) - find_program(LLD_EXECUTABLE_NAME lld-${LLVM_VERSION_MAJOR}) +endif() - if(NOT LLD_EXECUTABLE_NAME) - if(CI) - # Ensure we don't use a differing version of lld in CI vs clang - message(FATAL_ERROR "lld-${LLVM_VERSION_MAJOR} not found. Please make sure you have LLVM ${LLVM_VERSION_MAJOR}.x installed and set to lld-${LLVM_VERSION_MAJOR}") - endif() - - # To make it easier for contributors, allow differing versions of lld vs clang/cmake - find_program(LLD_EXECUTABLE_NAME lld) +if(LINUX) + if(NOT ABI STREQUAL "musl") + if(ARCH STREQUAL "aarch64") + target_link_options(${bun} PUBLIC + -Wl,--wrap=fcntl64 + -Wl,--wrap=statx + ) + endif() + + if(ARCH STREQUAL "x64") + target_link_options(${bun} PUBLIC + -Wl,--wrap=fcntl + -Wl,--wrap=fcntl64 + -Wl,--wrap=fstat + -Wl,--wrap=fstat64 + -Wl,--wrap=fstatat + -Wl,--wrap=fstatat64 + -Wl,--wrap=lstat + -Wl,--wrap=lstat64 + -Wl,--wrap=mknod + -Wl,--wrap=mknodat + -Wl,--wrap=stat + -Wl,--wrap=stat64 + -Wl,--wrap=statx + ) endif() - if(NOT LLD_EXECUTABLE_NAME) - message(FATAL_ERROR "LLD not found. Please make sure you have LLVM ${LLVM_VERSION_MAJOR}.x installed and lld is available in your PATH as lld-${LLVM_VERSION_MAJOR}") - endif() - - # normalize to basename so it can be used with -fuse-ld - get_filename_component(LLD_NAME ${LLD_EXECUTABLE_NAME} NAME CACHE) - message(STATUS "Using linker: ${LLD_NAME} (${LLD_EXECUTABLE_NAME})") - elseif(NOT DEFINED LLD_NAME) - set(LLD_NAME lld-${LLVM_VERSION_MAJOR}) - endif() - - if (NOT IS_MUSL) - if (IS_ARM64) - set(ARCH_WRAP_FLAGS - -Wl,--wrap=fcntl64 - -Wl,--wrap=statx - ) - elseif(IS_X86_64) - set(ARCH_WRAP_FLAGS - -Wl,--wrap=fcntl - -Wl,--wrap=fcntl64 - -Wl,--wrap=fstat - -Wl,--wrap=fstat64 - -Wl,--wrap=fstatat - -Wl,--wrap=fstatat64 - -Wl,--wrap=lstat - -Wl,--wrap=lstat64 - -Wl,--wrap=mknod - -Wl,--wrap=mknodat - -Wl,--wrap=stat - -Wl,--wrap=stat64 - -Wl,--wrap=statx - ) - endif() - else() - set(ARCH_WRAP_FLAGS - ) - endif() - - if (NOT IS_MUSL) - set(ABI_WRAP_FLAGS + target_link_options(${bun} PUBLIC -Wl,--wrap=cosf -Wl,--wrap=exp -Wl,--wrap=expf @@ -929,13 +893,10 @@ else() -Wl,--wrap=sinf -Wl,--wrap=tanf ) - else() - set(ABI_WRAP_FLAGS - ) endif() target_link_options(${bun} PUBLIC - -fuse-ld=${LLD_NAME} + --ld-path=${LLD_PROGRAM} -fno-pic -static-libstdc++ -static-libgcc @@ -944,8 +905,6 @@ else() -Wl,--as-needed -Wl,--gc-sections -Wl,-z,stack-size=12800000 - ${ARCH_WRAP_FLAGS} - ${ABI_WRAP_FLAGS} -Wl,--compress-debug-sections=zlib -Wl,-z,lazy -Wl,-z,norelro @@ -1095,12 +1054,12 @@ endif() if(NOT BUN_CPP_ONLY) set(CMAKE_STRIP_FLAGS "") - if (APPLE) + if(APPLE) # We do not build with exceptions enabled. These are generated by lolhtml # and other dependencies. We build lolhtml with abort on panic, so it # shouldn't be including these in the first place. set(CMAKE_STRIP_FLAGS --remove-section=__TEXT,__eh_frame --remove-section=__TEXT,__unwind_info --remove-section=__TEXT,__gcc_except_tab) - elseif(LINUX) + elseif(LINUX AND NOT ABI STREQUAL "musl") # When you use llvm-strip to do this, it doesn't delete it from the binary and instead keeps it as [LOAD #2 [R]] # So, we must use GNU strip to do this. set(CMAKE_STRIP_FLAGS -R .eh_frame -R .gcc_except_table) @@ -1193,10 +1152,12 @@ if(NOT BUN_CPP_ONLY) endif() if(CI) + set(bunTriplet bun-${OS}-${ARCH}) + if(ABI STREQUAL "musl") + set(bunTriplet ${bunTriplet}-musl) + endif() if(ENABLE_BASELINE) - set(bunTriplet bun-${OS}-${ARCH}-baseline) - else() - set(bunTriplet bun-${OS}-${ARCH}) + set(bunTriplet ${bunTriplet}-baseline) endif() string(REPLACE bun ${bunTriplet} bunPath ${bun}) set(bunFiles ${bunExe} features.json) diff --git a/cmake/toolchains/linux-aarch64-musl.cmake b/cmake/toolchains/linux-aarch64-musl.cmake new file mode 100644 index 0000000000..e4a33f709e --- /dev/null +++ b/cmake/toolchains/linux-aarch64-musl.cmake @@ -0,0 +1,6 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(ABI musl) + +set(CMAKE_C_COMPILER_WORKS ON) +set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-aarch64.cmake b/cmake/toolchains/linux-aarch64.cmake index bc23a06302..657594dae8 100644 --- a/cmake/toolchains/linux-aarch64.cmake +++ b/cmake/toolchains/linux-aarch64.cmake @@ -1,5 +1,6 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(ABI gnu) set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-x64-baseline.cmake b/cmake/toolchains/linux-x64-baseline.cmake index f521cfcc4a..73d6bc61e4 100644 --- a/cmake/toolchains/linux-x64-baseline.cmake +++ b/cmake/toolchains/linux-x64-baseline.cmake @@ -1,6 +1,7 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR x64) set(ENABLE_BASELINE ON) +set(ABI gnu) set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-x64-musl-baseline.cmake b/cmake/toolchains/linux-x64-musl-baseline.cmake new file mode 100644 index 0000000000..ea28a1757a --- /dev/null +++ b/cmake/toolchains/linux-x64-musl-baseline.cmake @@ -0,0 +1,7 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR x64) +set(ENABLE_BASELINE ON) +set(ABI musl) + +set(CMAKE_C_COMPILER_WORKS ON) +set(CMAKE_CXX_COMPILER_WORKS ON) \ No newline at end of file diff --git a/cmake/toolchains/linux-x64-musl.cmake b/cmake/toolchains/linux-x64-musl.cmake new file mode 100644 index 0000000000..db4998bba9 --- /dev/null +++ b/cmake/toolchains/linux-x64-musl.cmake @@ -0,0 +1,6 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR x64) +set(ABI musl) + +set(CMAKE_C_COMPILER_WORKS ON) +set(CMAKE_CXX_COMPILER_WORKS ON) diff --git a/cmake/toolchains/linux-x64.cmake b/cmake/toolchains/linux-x64.cmake index 66bc7a592f..4104a1c5df 100644 --- a/cmake/toolchains/linux-x64.cmake +++ b/cmake/toolchains/linux-x64.cmake @@ -1,5 +1,6 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR x64) +set(ABI gnu) set(CMAKE_C_COMPILER_WORKS ON) set(CMAKE_CXX_COMPILER_WORKS ON) diff --git a/cmake/tools/SetupGit.cmake b/cmake/tools/SetupGit.cmake index 8e0f87c312..769735b7b0 100644 --- a/cmake/tools/SetupGit.cmake +++ b/cmake/tools/SetupGit.cmake @@ -29,7 +29,7 @@ execute_process( ) if(NOT GIT_DIFF_RESULT EQUAL 0) - message(${WARNING} "Command failed: ${GIT_DIFF_COMMAND} ${GIT_DIFF_ERROR}") + message(WARNING "Command failed: ${GIT_DIFF_COMMAND} ${GIT_DIFF_ERROR}") return() endif() diff --git a/cmake/tools/SetupLLVM.cmake b/cmake/tools/SetupLLVM.cmake index fb99eff639..5e5fd3a953 100644 --- a/cmake/tools/SetupLLVM.cmake +++ b/cmake/tools/SetupLLVM.cmake @@ -4,7 +4,7 @@ if(NOT ENABLE_LLVM) return() endif() -if(CMAKE_HOST_WIN32 OR CMAKE_HOST_APPLE OR IS_MUSL) +if(CMAKE_HOST_WIN32 OR CMAKE_HOST_APPLE OR ABI STREQUAL "musl") set(DEFAULT_LLVM_VERSION "18.1.8") else() set(DEFAULT_LLVM_VERSION "16.0.6") @@ -52,6 +52,7 @@ if(UNIX) /usr/lib/llvm-${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}/bin /usr/lib/llvm-${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}/bin /usr/lib/llvm-${LLVM_VERSION_MAJOR}/bin + /usr/lib/llvm${LLVM_VERSION_MAJOR}/bin ) endif() endif() @@ -122,6 +123,9 @@ else() find_llvm_command(CMAKE_STRIP llvm-strip) endif() find_llvm_command(CMAKE_RANLIB llvm-ranlib) + if(LINUX) + find_llvm_command(LLD_PROGRAM ld.lld) + endif() if(APPLE) find_llvm_command(CMAKE_DSYMUTIL dsymutil) endif() diff --git a/cmake/tools/SetupWebKit.cmake b/cmake/tools/SetupWebKit.cmake index b71eff33e1..2cdea17edc 100644 --- a/cmake/tools/SetupWebKit.cmake +++ b/cmake/tools/SetupWebKit.cmake @@ -63,7 +63,7 @@ else() message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") endif() -if(IS_MUSL) +if(ABI STREQUAL "musl") set(WEBKIT_SUFFIX "-musl") endif() diff --git a/cmake/tools/SetupZig.cmake b/cmake/tools/SetupZig.cmake index d34c4b53ff..e5a5e574ef 100644 --- a/cmake/tools/SetupZig.cmake +++ b/cmake/tools/SetupZig.cmake @@ -11,7 +11,7 @@ if(APPLE) elseif(WIN32) set(DEFAULT_ZIG_TARGET ${DEFAULT_ZIG_ARCH}-windows-msvc) elseif(LINUX) - if(IS_MUSL) + if(ABI STREQUAL "musl") set(DEFAULT_ZIG_TARGET ${DEFAULT_ZIG_ARCH}-linux-musl) else() set(DEFAULT_ZIG_TARGET ${DEFAULT_ZIG_ARCH}-linux-gnu) diff --git a/scripts/agent.mjs b/scripts/agent.mjs new file mode 100755 index 0000000000..8da3b96e05 --- /dev/null +++ b/scripts/agent.mjs @@ -0,0 +1,230 @@ +#!/usr/bin/env node + +// An agent that starts buildkite-agent and runs others services. + +import { join } from "node:path"; +import { realpathSync } from "node:fs"; +import { + isWindows, + getOs, + getArch, + getKernel, + getAbi, + getAbiVersion, + getDistro, + getDistroVersion, + getHostname, + getCloud, + getCloudMetadataTag, + which, + getEnv, + writeFile, + spawnSafe, +} from "./utils.mjs"; +import { parseArgs } from "node:util"; + +/** + * @param {"install" | "start"} action + */ +async function doBuildkiteAgent(action) { + const username = "buildkite-agent"; + const command = which("buildkite-agent", { required: true }); + + let homePath, cachePath, logsPath, agentLogPath, pidPath; + if (isWindows) { + throw new Error("TODO: Windows"); + } else { + homePath = "/var/lib/buildkite-agent"; + cachePath = "/var/cache/buildkite-agent"; + logsPath = "/var/log/buildkite-agent"; + agentLogPath = join(logsPath, "buildkite-agent.log"); + pidPath = join(logsPath, "buildkite-agent.pid"); + } + + async function install() { + const command = process.execPath; + const args = [realpathSync(process.argv[1]), "start"]; + + if (isOpenRc()) { + const servicePath = "/etc/init.d/buildkite-agent"; + const service = `#!/sbin/openrc-run + name="buildkite-agent" + description="Buildkite Agent" + command=${escape(command)} + command_args=${escape(args.map(escape).join(" "))} + command_user=${escape(username)} + + pidfile=${escape(pidPath)} + start_stop_daemon_args=" \ + --background \ + --make-pidfile \ + --stdout ${escape(agentLogPath)} \ + --stderr ${escape(agentLogPath)}" + + depend() { + need net + use dns logger + } + `; + writeFile(servicePath, service, { mode: 0o755 }); + await spawnSafe(["rc-update", "add", "buildkite-agent", "default"], { stdio: "inherit", privileged: true }); + } + + if (isSystemd()) { + const servicePath = "/etc/systemd/system/buildkite-agent.service"; + const service = ` + [Unit] + Description=Buildkite Agent + After=syslog.target + After=network-online.target + + [Service] + Type=simple + User=${username} + ExecStart=${escape(command)} ${escape(args.map(escape).join(" "))} + RestartSec=5 + Restart=on-failure + KillMode=process + + [Journal] + Storage=persistent + StateDirectory=${escape(agentLogPath)} + + [Install] + WantedBy=multi-user.target + `; + writeFile(servicePath, service); + await spawnSafe(["systemctl", "daemon-reload"], { stdio: "inherit", privileged: true }); + await spawnSafe(["systemctl", "enable", "buildkite-agent"], { stdio: "inherit", privileged: true }); + } + } + + async function start() { + const cloud = await getCloud(); + + let token = getEnv("BUILDKITE_AGENT_TOKEN", false); + if (!token && cloud) { + token = await getCloudMetadataTag("buildkite:token"); + } + + let shell; + if (isWindows) { + const pwsh = which(["pwsh", "powershell"], { required: true }); + shell = `${pwsh} -Command`; + } else { + const sh = which(["bash", "sh"], { required: true }); + shell = `${sh} -c`; + } + + const flags = ["enable-job-log-tmpfile", "no-feature-reporting"]; + const options = { + "name": getHostname(), + "token": token || "xxx", + "shell": shell, + "job-log-path": logsPath, + "build-path": join(homePath, "builds"), + "hooks-path": join(homePath, "hooks"), + "plugins-path": join(homePath, "plugins"), + "experiment": "normalised-upload-paths,resolve-commit-after-checkout,agent-api", + }; + + let ephemeral; + if (cloud) { + const jobId = await getCloudMetadataTag("buildkite:job-uuid"); + if (jobId) { + options["acquire-job"] = jobId; + flags.push("disconnect-after-job"); + ephemeral = true; + } + } + + if (ephemeral) { + options["git-clone-flags"] = "-v --depth=1"; + options["git-fetch-flags"] = "-v --prune --depth=1"; + } else { + options["git-mirrors-path"] = join(cachePath, "git"); + } + + const tags = { + "os": getOs(), + "arch": getArch(), + "kernel": getKernel(), + "abi": getAbi(), + "abi-version": getAbiVersion(), + "distro": getDistro(), + "distro-version": getDistroVersion(), + "cloud": cloud, + }; + + if (cloud) { + const requiredTags = ["robobun", "robobun2"]; + for (const tag of requiredTags) { + const value = await getCloudMetadataTag(tag); + if (typeof value === "string") { + tags[tag] = value; + } + } + } + + options["tags"] = Object.entries(tags) + .filter(([, value]) => value) + .map(([key, value]) => `${key}=${value}`) + .join(","); + + await spawnSafe( + [ + command, + "start", + ...flags.map(flag => `--${flag}`), + ...Object.entries(options).map(([key, value]) => `--${key}=${value}`), + ], + { + stdio: "inherit", + }, + ); + } + + if (action === "install") { + await install(); + } else if (action === "start") { + await start(); + } +} + +/** + * @returns {boolean} + */ +function isSystemd() { + return !!which("systemctl"); +} + +/** + * @returns {boolean} + */ +function isOpenRc() { + return !!which("rc-service"); +} + +function escape(string) { + return JSON.stringify(string); +} + +async function main() { + const { positionals: args } = parseArgs({ + allowPositionals: true, + }); + + if (!args.length || args.includes("install")) { + console.log("Installing agent..."); + await doBuildkiteAgent("install"); + console.log("Agent installed."); + } + + if (args.includes("start")) { + console.log("Starting agent..."); + await doBuildkiteAgent("start"); + console.log("Agent started."); + } +} + +await main(); diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index e09ef4fb6c..480dd91448 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,4 +1,5 @@ #!/bin/sh +# Version: 4 # A script that installs the dependencies needed to build and test Bun. # This should work on macOS and Linux with a POSIX shell. @@ -7,11 +8,10 @@ # https://github.com/oven-sh/bun/issues # If you need to make a change to this script, such as upgrading a dependency, -# increment the version number to indicate that a new image should be built. +# increment the version comment to indicate that a new image should be built. # Otherwise, the existing image will be retroactively updated. -v="3" + pid=$$ -script="$(realpath "$0")" print() { echo "$@" @@ -24,28 +24,41 @@ error() { } execute() { - print "$ $@" >&2 - if ! "$@"; then - error "Command failed: $@" - fi + print "$ $@" >&2 + if ! "$@"; then + error "Command failed: $@" + fi } execute_sudo() { - if [ "$sudo" = "1" ]; then + if [ "$sudo" = "1" ] || [ -z "$can_sudo" ]; then execute "$@" else - execute sudo "$@" + execute sudo -n "$@" fi } -execute_non_root() { - if [ "$sudo" = "1" ]; then - execute sudo -u "$user" "$@" +execute_as_user() { + if [ "$sudo" = "1" ] || [ "$can_sudo" = "1" ]; then + if [ -f "$(which sudo)" ]; then + execute sudo -n -u "$user" /bin/sh -c "$*" + elif [ -f "$(which doas)" ]; then + execute doas -u "$user" /bin/sh -c "$*" + elif [ -f "$(which su)" ]; then + execute su -s /bin/sh "$user" -c "$*" + else + execute /bin/sh -c "$*" + fi else - execute "$@" + execute /bin/sh -c "$*" fi } +grant_to_user() { + path="$1" + execute_sudo chown -R "$user:$group" "$path" +} + which() { command -v "$1" } @@ -73,12 +86,16 @@ fetch() { } download_file() { - url="$1" - filename="${2:-$(basename "$url")}" - path="$(mktemp -d)/$filename" + url="$1" + filename="${2:-$(basename "$url")}" + tmp="$(execute mktemp -d)" + execute chmod 755 "$tmp" - fetch "$url" > "$path" - print "$path" + path="$tmp/$filename" + fetch "$url" > "$path" + execute chmod 644 "$path" + + print "$path" } compare_version() { @@ -96,13 +113,13 @@ append_to_file() { content="$2" if ! [ -f "$file" ]; then - execute mkdir -p "$(dirname "$file")" - execute touch "$file" + execute_as_user mkdir -p "$(dirname "$file")" + execute_as_user touch "$file" fi echo "$content" | while read -r line; do if ! grep -q "$line" "$file"; then - echo "$line" >> "$file" + echo "$line" >>"$file" fi done } @@ -111,7 +128,7 @@ append_to_profile() { content="$1" profiles=".profile .zprofile .bash_profile .bashrc .zshrc" for profile in $profiles; do - file="$HOME/$profile" + file="$home/$profile" if [ "$ci" = "1" ] || [ -f "$file" ]; then append_to_file "$file" "$content" fi @@ -124,172 +141,264 @@ append_to_path() { error "Could not find directory: \"$path\"" fi - append_to_profile "export PATH=\"\$PATH\":$path" - export PATH="$PATH:$path" + append_to_profile "export PATH=\"$path:\$PATH\"" + export PATH="$path:$PATH" } -check_system() { +link_to_bin() { + path="$1" + if ! [ -d "$path" ]; then + error "Could not find directory: \"$path\"" + fi + + for file in "$path"/*; do + if [ -f "$file" ]; then + grant_to_user "$file" + execute_sudo ln -sf "$file" "/usr/bin/$(basename "$file")" + fi + done +} + +check_features() { + print "Checking features..." + + case "$CI" in + true | 1) + ci=1 + print "CI: enabled" + ;; + esac + + case "$@" in + *--ci*) + ci=1 + print "CI: enabled" + ;; + esac +} + +check_operating_system() { + print "Checking operating system..." uname="$(require uname)" - os="$($uname -s)" + os="$("$uname" -s)" case "$os" in Linux*) os="linux" ;; Darwin*) os="darwin" ;; *) error "Unsupported operating system: $os" ;; esac + print "Operating System: $os" - arch="$($uname -m)" + arch="$("$uname" -m)" case "$arch" in x86_64 | x64 | amd64) arch="x64" ;; aarch64 | arm64) arch="aarch64" ;; *) error "Unsupported architecture: $arch" ;; esac + print "Architecture: $arch" - kernel="$(uname -r)" + kernel="$("$uname" -r)" + print "Kernel: $kernel" - if [ "$os" = "darwin" ]; then + case "$os" in + linux) + if [ -f "/etc/alpine-release" ]; then + distro="alpine" + abi="musl" + alpine="$(cat /etc/alpine-release)" + if [ "$alpine" ~ "_" ]; then + release="$(echo "$alpine" | cut -d_ -f1)-edge" + else + release="$alpine" + fi + elif [ -f "/etc/os-release" ]; then + . /etc/os-release + if [ -n "$ID" ]; then + distro="$ID" + fi + if [ -n "$VERSION_ID" ]; then + release="$VERSION_ID" + fi + fi + ;; + darwin) sw_vers="$(which sw_vers)" if [ -f "$sw_vers" ]; then - distro="$($sw_vers -productName)" - release="$($sw_vers -productVersion)" + distro="$("$sw_vers" -productName)" + release="$("$sw_vers" -productVersion)" fi - - if [ "$arch" = "x64" ]; then + case "$arch" in + x64) sysctl="$(which sysctl)" - if [ -f "$sysctl" ] && [ "$($sysctl -n sysctl.proc_translated 2>/dev/null)" = "1" ]; then + if [ -f "$sysctl" ] && [ "$("$sysctl" -n sysctl.proc_translated 2>/dev/null)" = "1" ]; then arch="aarch64" rosetta="1" + print "Rosetta: enabled" fi + ;; + esac + ;; + esac + + if [ -n "$distro" ]; then + print "Distribution: $distro $release" + fi + + case "$os" in + linux) + ldd="$(which ldd)" + if [ -f "$ldd" ]; then + ldd_version="$($ldd --version 2>&1)" + abi_version="$(echo "$ldd_version" | grep -o -E '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -n 1)" + case "$ldd_version" in + *musl*) + abi="musl" + ;; + *GNU* | *GLIBC*) + abi="gnu" + ;; + esac + fi + + if [ -n "$abi" ]; then + print "ABI: $abi $abi_version" + fi + ;; + esac +} + +check_inside_docker() { + if ! [ "$os" = "linux" ]; then + return + fi + print "Checking if inside Docker..." + + if [ -f "/.dockerenv" ]; then + docker=1 + else + if [ -f "/proc/1/cgroup" ]; then + case "$(cat /proc/1/cgroup)" in + */docker/*) + docker=1 + ;; + esac + fi + + if [ -f "/proc/self/mountinfo" ]; then + case "$(cat /proc/self/mountinfo)" in + */docker/*) + docker=1 + ;; + esac fi fi - if [ "$os" = "linux" ] && [ -f /etc/os-release ]; then - . /etc/os-release - if [ -n "$ID" ]; then - distro="$ID" - fi - if [ -n "$VERSION_ID" ]; then - release="$VERSION_ID" - - if [ "$distro" = "alpine" ]; then - if [ "$(echo $release | grep -c '_')" = "1" ]; then - release="edge" - fi - fi - fi + if [ "$docker" = "1" ]; then + print "Docker: enabled" fi +} - if [ "$os" = "linux" ]; then - rpm="$(which rpm)" - if [ -f "$rpm" ]; then - glibc="$($rpm -q glibc --queryformat '%{VERSION}\n')" - else - ldd="$(which ldd)" - awk="$(which awk)" - if [ -f "$ldd" ] && [ -f "$awk" ]; then - glibc="$($ldd --version | $awk 'NR==1{print $NF}')" - fi +check_package_manager() { + print "Checking package manager..." + + case "$os" in + darwin) + if ! [ -f "$(which brew)" ]; then + install_brew fi - fi - - if [ "$os" = "darwin" ]; then - brew="$(which brew)" pm="brew" - fi - - if [ "$os" = "linux" ]; then - apt="$(which apt-get)" - if [ -f "$apt" ]; then + ;; + linux) + if [ -f "$(which apt-get)" ]; then pm="apt" - + elif [ -f "$(which dnf)" ]; then + pm="dnf" + elif [ -f "$(which yum)" ]; then + pm="yum" + elif [ -f "$(which apk)" ]; then + pm="apk" else - dnf="$(which dnf)" - if [ -f "$dnf" ]; then - pm="dnf" - - else - yum="$(which yum)" - if [ -f "$yum" ]; then - pm="yum" - - else - apk="$(which apk)" - if [ -f "$apk" ]; then - pm="apk" - fi - fi - fi - fi - - if [ -z "$pm" ]; then error "No package manager found. (apt, dnf, yum, apk)" fi - fi + ;; + esac + print "Package manager: $pm" + + print "Updating package manager..." + case "$pm" in + apt) + DEBIAN_FRONTEND=noninteractive package_manager update -y + ;; + apk) + package_manager update + ;; + esac +} + +check_user() { + print "Checking user..." if [ -n "$SUDO_USER" ]; then user="$SUDO_USER" else - whoami="$(which whoami)" - if [ -f "$whoami" ]; then - user="$($whoami)" - else - error "Could not determine the current user, set \$USER." - fi + id="$(require id)" + user="$("$id" -un)" + group="$("$id" -gn)" fi + if [ -z "$user" ]; then + error "Could not determine user" + fi + print "User: $user" + print "Group: $group" + + home="$(execute_as_user echo '~')" + if [ -z "$home" ] || [ "$home" = "~" ]; then + error "Could not determine home directory for user: $user" + fi + print "Home: $home" id="$(which id)" if [ -f "$id" ] && [ "$($id -u)" = "0" ]; then sudo=1 - fi - - if [ "$CI" = "true" ]; then - ci=1 - fi - - print "System information:" - if [ -n "$distro" ]; then - print "| Distro: $distro $release" - fi - print "| Operating system: $os" - print "| Architecture: $arch" - if [ -n "$rosetta" ]; then - print "| Rosetta: true" - fi - if [ -n "$glibc" ]; then - print "| Glibc: $glibc" - fi - print "| Package manager: $pm" - print "| User: $user" - if [ -n "$sudo" ]; then - print "| Sudo: true" - fi - if [ -n "$ci" ]; then - print "| CI: true" + print "Sudo: enabled" + elif [ -f "$(which sudo)" ] && [ "$(sudo -n echo 1 2>/dev/null)" = "1" ]; then + can_sudo=1 + print "Sudo: can be used" fi } package_manager() { case "$pm" in - apt) DEBIAN_FRONTEND=noninteractive \ - execute "$apt" "$@" ;; - dnf) execute dnf "$@" ;; - yum) execute "$yum" "$@" ;; + apt) + while ! sudo -n apt-get update -y; do + sleep 1 + done + DEBIAN_FRONTEND=noninteractive execute_sudo apt-get "$@" + ;; + dnf) + case "$distro" in + rhel) + execute_sudo dnf \ + --disableplugin=subscription-manager \ + "$@" + ;; + *) + execute_sudo dnf "$@" + ;; + esac + ;; + yum) + execute_sudo yum "$@" + ;; + apk) + execute_sudo apk "$@" + ;; brew) - if ! [ -f "$brew" ]; then - install_brew - fi - execute_non_root "$brew" "$@" - ;; - apk) execute "$apk" "$@" ;; - *) error "Unsupported package manager: $pm" ;; - esac -} - -update_packages() { - case "$pm" in - apt | apk) - package_manager update - ;; + execute_as_user brew "$@" + ;; + *) + error "Unsupported package manager: $pm" + ;; esac } @@ -310,20 +419,38 @@ check_package() { install_packages() { case "$pm" in apt) - package_manager install --yes --no-install-recommends "$@" + package_manager install \ + --yes \ + --no-install-recommends \ + "$@" ;; dnf) - package_manager install --assumeyes --nodocs --noautoremove --allowerasing "$@" + package_manager install \ + --assumeyes \ + --nodocs \ + --noautoremove \ + --allowerasing \ + "$@" ;; yum) package_manager install -y "$@" ;; brew) - package_manager install --force --formula "$@" - package_manager link --force --overwrite "$@" + package_manager install \ + --force \ + --formula \ + "$@" + package_manager link \ + --force \ + --overwrite \ + "$@" ;; apk) - package_manager add "$@" + package_manager add \ + --no-cache \ + --no-interactive \ + --no-progress \ + "$@" ;; *) error "Unsupported package manager: $pm" @@ -331,24 +458,12 @@ install_packages() { esac } -get_version() { - command="$1" - path="$(which "$command")" - - if [ -f "$path" ]; then - case "$command" in - go | zig) "$path" version ;; - *) "$path" --version ;; - esac - else - print "not found" - fi -} - install_brew() { - bash="$(require bash)" - script=$(download_file "https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh") - NONINTERACTIVE=1 execute_non_root "$bash" "$script" + print "Installing Homebrew..." + + bash="$(require bash)" + script=$(download_file "https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh") + NONINTERACTIVE=1 execute_as_user "$bash" "$script" case "$arch" in x64) @@ -370,78 +485,150 @@ install_brew() { install_common_software() { case "$pm" in - apt) install_packages \ - apt-transport-https \ - software-properties-common - ;; - dnf) install_packages \ - dnf-plugins-core \ - tar - ;; + apt) + install_packages \ + apt-transport-https \ + software-properties-common + ;; + dnf) + install_packages \ + dnf-plugins-core + ;; esac + case "$distro" in + amzn) + install_packages \ + tar + ;; + rhel) + rhel_version="$(execute rpm -E %rhel)" + install_packages \ + "https://dl.fedoraproject.org/pub/epel/epel-release-latest-$rhel_version.noarch.rpm" + ;; + centos) + install_packages \ + epel-release + ;; + esac + + crb="$(which crb)" + if [ -f "$crb" ]; then + execute "$crb" enable + fi + install_packages \ bash \ ca-certificates \ curl \ - jq \ htop \ gnupg \ git \ unzip \ - wget \ - zip + wget install_rosetta install_nodejs install_bun + install_tailscale + install_buildkite +} + +nodejs_version_exact() { + # https://unofficial-builds.nodejs.org/download/release/ + if ! [ "$abi" = "musl" ] && [ -n "$abi_version" ] && ! [ "$(compare_version "$abi_version" "2.27")" = "1" ]; then + print "16.9.1" + else + print "22.9.0" + fi +} + +nodejs_version() { + echo "$(nodejs_version_exact)" | cut -d. -f1 } install_nodejs() { - version="${1:-"22"}" - - if ! [ "$(compare_version "$glibc" "2.27")" = "1" ]; then - version="16" - fi - case "$pm" in dnf | yum) - bash="$(require bash)" - script=$(download_file "https://rpm.nodesource.com/setup_$version.x") - execute "$bash" "$script" + bash="$(require bash)" + script=$(download_file "https://rpm.nodesource.com/setup_$(nodejs_version).x") + execute_sudo "$bash" "$script" ;; apt) - bash="$(require bash)" - script=$(download_file "https://deb.nodesource.com/setup_$version.x") - execute "$bash" "$script" + bash="$(require bash)" + script="$(download_file "https://deb.nodesource.com/setup_$(nodejs_version).x")" + execute_sudo "$bash" "$script" ;; esac - install_packages nodejs + case "$pm" in + apk) + install_packages nodejs npm + ;; + *) + install_packages nodejs + ;; + esac } install_bun() { - if [ "$os" = "linux" ] && [ "$distro" = "alpine" ] && [ "$arch" = "aarch64" ]; then - mkdir -p "$HOME/.bun/bin" - wget -O "$HOME/.bun/bin/bun" https://pub-61e0d0e2da4146a099e4545a59a9f0f7.r2.dev/bun-musl-arm64 - chmod +x "$HOME/.bun/bin/bun" - append_to_path "$HOME/.bun/bin" + case "$os-$abi" in + linux-musl) + case "$arch" in + x64) + exe="$(download_file https://pub-61e0d0e2da4146a099e4545a59a9f0f7.r2.dev/bun-musl-x64)" + ;; + aarch64) + exe="$(download_file https://pub-61e0d0e2da4146a099e4545a59a9f0f7.r2.dev/bun-musl-arm64)" + ;; + esac + execute chmod +x "$exe" + execute mkdir -p "$home/.bun/bin" + execute mv "$exe" "$home/.bun/bin/bun" + execute ln -fs "$home/.bun/bin/bun" "$home/.bun/bin/bunx" + link_to_bin "$home/.bun/bin" return - fi - bash="$(require bash)" - script=$(download_file "https://bun.sh/install") - - version="${1:-"latest"}" - case "$version" in - latest) - execute "$bash" "$script" - ;; - *) - execute "$bash" "$script" -s "$version" ;; esac - append_to_path "$HOME/.bun/bin" + bash="$(require bash)" + script=$(download_file "https://bun.sh/install") + + version="${1:-"latest"}" + case "$version" in + latest) + execute_as_user "$bash" "$script" + ;; + *) + execute_as_user "$bash" "$script" -s "$version" + ;; + esac + + link_to_bin "$home/.bun/bin" +} + +install_cmake() { + case "$os-$pm" in + darwin-* | linux-apk) + install_packages cmake + ;; + linux-*) + sh="$(require sh)" + cmake_version="3.30.5" + case "$arch" in + x64) + url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-x86_64.sh" + ;; + aarch64) + url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-aarch64.sh" + ;; + esac + script=$(download_file "$url") + execute_sudo "$sh" "$script" \ + --skip-license \ + --prefix=/usr + ;; + esac } install_rosetta() { @@ -459,27 +646,56 @@ install_rosetta() { install_build_essentials() { case "$pm" in apt) - install_packages build-essential ninja-build xz-utils pkg-config golang - ;; + install_packages \ + build-essential \ + ninja-build \ + xz-utils \ + pkg-config \ + golang + ;; dnf | yum) - install_packages ninja-build gcc-c++ xz pkg-config golang - ;; + install_packages \ + gcc-c++ \ + xz \ + pkg-config \ + golang + case "$distro" in + rhel) ;; + *) + install_packages ninja-build + ;; + esac + ;; brew) - install_packages ninja pkg-config golang - ;; + install_packages \ + ninja \ + pkg-config \ + golang + ;; apk) - install_packages musl-dev ninja xz - ;; + install_packages \ + build-base \ + linux-headers \ + ninja \ + go \ + xz + ;; + esac + + case "$distro-$pm" in + amzn-dnf) + package_manager groupinstall -y "Development Tools" + ;; esac install_packages \ make \ - cmake \ python3 \ libtool \ ruby \ perl + install_cmake install_llvm install_ccache install_rust @@ -487,185 +703,189 @@ install_build_essentials() { } llvm_version_exact() { - if [ "$os" = "linux" ] && [ "$distro" = "alpine" ]; then + case "$os-$abi" in + darwin-* | windows-* | linux-musl) print "18.1.8" - return - fi - case "$os" in - linux) - print "16.0.6" - ;; - darwin | windows) - print "18.1.8" - ;; - esac + ;; + linux-*) + print "16.0.6" + ;; + esac } llvm_version() { - echo "$(llvm_version_exact)" | cut -d. -f1 + echo "$(llvm_version_exact)" | cut -d. -f1 } install_llvm() { case "$pm" in apt) - bash="$(require bash)" - script=$(download_file "https://apt.llvm.org/llvm.sh") - execute "$bash" "$script" "$(llvm_version)" all + bash="$(require bash)" + script="$(download_file "https://apt.llvm.org/llvm.sh")" + case "$distro-$release" in + ubuntu-24*) + execute_sudo "$bash" "$script" "$(llvm_version)" all -njammy + ;; + *) + execute_sudo "$bash" "$script" "$(llvm_version)" all + ;; + esac + ;; + brew) + install_packages "llvm@$(llvm_version)" + ;; + apk) + install_packages \ + "llvm$(llvm_version)" \ + "clang$(llvm_version)" \ + "scudo-malloc" \ + --repository "http://dl-cdn.alpinelinux.org/alpine/edge/main" + install_packages \ + "lld$(llvm_version)" \ + --repository "http://dl-cdn.alpinelinux.org/alpine/edge/community" ;; - brew) - install_packages "llvm@$(llvm_version)" - ;; - apk) - install_packages "llvm$(llvm_version)-dev" "clang$(llvm_version)-dev" "lld$(llvm_version)-dev" - append_to_path "/usr/lib/llvm$(llvm_version)/bin" - ;; esac } install_ccache() { - case "$pm" in - apt | brew) - install_packages ccache - ;; - esac + case "$pm" in + apt | apk | brew) + install_packages ccache + ;; + esac } install_rust() { - if [ "$os" = "linux" ] && [ "$distro" = "alpine" ]; then - install_packages rust cargo - mkdir -p "$HOME/.cargo/bin" - append_to_path "$HOME/.cargo/bin" - return - fi - sh="$(require sh)" - script=$(download_file "https://sh.rustup.rs") - execute "$sh" "$script" -y - append_to_path "$HOME/.cargo/bin" + case "$pm" in + apk) + install_packages \ + rust \ + cargo + ;; + *) + sh="$(require sh)" + script=$(download_file "https://sh.rustup.rs") + execute_as_user "$sh" "$script" -y + ;; + esac + + # FIXME: This causes cargo to fail to build: + # > error: rustup could not choose a version of cargo to run, + # > because one wasn't specified explicitly, and no default is configured. + # link_to_bin "$home/.cargo/bin" } install_docker() { case "$pm" in brew) - if ! [ -d "/Applications/Docker.app" ]; then - package_manager install docker --cask - fi - ;; - apk) - install_packages docker + if ! [ -d "/Applications/Docker.app" ]; then + package_manager install docker --cask + fi ;; *) - case "$distro-$release" in - amzn-2 | amzn-1) - execute amazon-linux-extras install docker - ;; - amzn-*) - install_packages docker - ;; - *) - sh="$(require sh)" - script=$(download_file "https://get.docker.com") - execute "$sh" "$script" - ;; - esac - ;; + case "$distro-$release" in + amzn-2 | amzn-1) + execute amazon-linux-extras install docker + ;; + amzn-* | alpine-*) + install_packages docker + ;; + *) + sh="$(require sh)" + script=$(download_file "https://get.docker.com") + execute "$sh" "$script" + ;; + esac + ;; esac - systemctl="$(which systemctl)" - if [ -f "$systemctl" ]; then - execute "$systemctl" enable docker - fi + systemctl="$(which systemctl)" + if [ -f "$systemctl" ]; then + execute_sudo "$systemctl" enable docker + fi + + getent="$(which getent)" + if [ -n "$("$getent" group docker)" ]; then + usermod="$(which usermod)" + if [ -f "$usermod" ]; then + execute_sudo "$usermod" -aG docker "$user" + fi + fi } -install_ci_dependencies() { +install_tailscale() { + if [ "$docker" = "1" ]; then + return + fi + + case "$os" in + linux) + sh="$(require sh)" + script=$(download_file "https://tailscale.com/install.sh") + execute "$sh" "$script" + ;; + darwin) + install_packages go + execute_as_user go install tailscale.com/cmd/tailscale{,d}@latest + append_to_path "$home/go/bin" + ;; + esac +} + +create_buildkite_user() { + if ! [ "$ci" = "1" ] || ! [ "$os" = "linux" ]; then + return + fi + + print "Creating Buildkite user..." + user="buildkite-agent" + group="$user" + home="/var/lib/buildkite-agent" + + case "$distro" in + amzn) + install_packages \ + shadow-utils \ + util-linux + ;; + esac + + if [ -z "$(getent passwd "$user")" ]; then + execute_sudo useradd "$user" \ + --system \ + --no-create-home \ + --home-dir "$home" + fi + + if [ -n "$(getent group docker)" ]; then + execute_sudo usermod -aG docker "$user" + fi + + paths="$home /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /var/run/buildkite-agent/buildkite-agent.sock" + for path in $paths; do + execute_sudo mkdir -p "$path" + execute_sudo chown -R "$user:$group" "$path" + done + + files="/var/run/buildkite-agent/buildkite-agent.pid" + for file in $files; do + execute_sudo touch "$file" + execute_sudo chown "$user:$group" "$file" + done +} + +install_buildkite() { if ! [ "$ci" = "1" ]; then return fi - install_tailscale - install_buildkite -} - -install_tailscale() { - case "$os" in - linux) - sh="$(require sh)" - script=$(download_file "https://tailscale.com/install.sh") - execute "$sh" "$script" - ;; - darwin) - install_packages go - execute_non_root go install tailscale.com/cmd/tailscale{,d}@latest - append_to_path "$HOME/go/bin" - ;; - esac -} - -install_buildkite() { - home_dir="/var/lib/buildkite-agent" - config_dir="/etc/buildkite-agent" - config_file="$config_dir/buildkite-agent.cfg" - - if ! [ -d "$home_dir" ]; then - execute_sudo mkdir -p "$home_dir" - fi - - if ! [ -d "$config_dir" ]; then - execute_sudo mkdir -p "$config_dir" - fi - - case "$os" in - linux) - getent="$(require getent)" - if [ -z "$("$getent" passwd buildkite-agent)" ]; then - useradd="$(require useradd)" - execute "$useradd" buildkite-agent \ - --system \ - --no-create-home \ - --home-dir "$home_dir" - fi - - if [ -n "$("$getent" group docker)" ]; then - usermod="$(require usermod)" - execute "$usermod" -aG docker buildkite-agent - fi - - execute chown -R buildkite-agent:buildkite-agent "$home_dir" - execute chown -R buildkite-agent:buildkite-agent "$config_dir" - ;; - darwin) - execute_sudo chown -R "$user:admin" "$home_dir" - execute_sudo chown -R "$user:admin" "$config_dir" - ;; - esac - - if ! [ -f "$config_file" ]; then - cat <"$config_file" -# This is generated by scripts/bootstrap.sh -# https://buildkite.com/docs/agent/v3/configuration - -name="%hostname-%random" -tags="v=$v,os=$os,arch=$arch,distro=$distro,release=$release,kernel=$kernel,glibc=$glibc" - -build-path="$home_dir/builds" -git-mirrors-path="$home_dir/git" -job-log-path="$home_dir/logs" -plugins-path="$config_dir/plugins" -hooks-path="$config_dir/hooks" - -no-ssh-keyscan=true -cancel-grace-period=3600000 # 1 hour -enable-job-log-tmpfile=true -experiment="normalised-upload-paths,resolve-commit-after-checkout,agent-api" -EOF - fi - bash="$(require bash)" - script=$(download_file "https://raw.githubusercontent.com/buildkite/agent/main/install.sh") - execute "$bash" "$script" + script="$(download_file "https://raw.githubusercontent.com/buildkite/agent/main/install.sh")" + tmp_dir="$(execute dirname "$script")" + HOME="$tmp_dir" execute "$bash" "$script" - out_dir="$HOME/.buildkite-agent" - execute_sudo mv -f "$out_dir/bin/buildkite-agent" "/usr/local/bin/buildkite-agent" - execute rm -rf "$out_dir" + out_dir="$tmp_dir/.buildkite-agent" + execute_sudo mv -f "$out_dir/bin/buildkite-agent" "/usr/bin/buildkite-agent" } install_chrome_dependencies() { @@ -738,19 +958,19 @@ install_chrome_dependencies() { xorg-x11-fonts-Type1 \ xorg-x11-utils ;; - apk) - echo # TODO: - ;; esac } main() { - check_system - update_packages - install_common_software - install_build_essentials - install_chrome_dependencies - install_ci_dependencies + check_features "$@" + check_operating_system + check_inside_docker + check_user + check_package_manager + create_buildkite_user + install_common_software + install_build_essentials + install_chrome_dependencies } -main +main "$@" diff --git a/scripts/build.mjs b/scripts/build.mjs old mode 100644 new mode 100755 diff --git a/scripts/features.mjs b/scripts/features.mjs old mode 100644 new mode 100755 diff --git a/scripts/machine.mjs b/scripts/machine.mjs new file mode 100755 index 0000000000..d352ab89ce --- /dev/null +++ b/scripts/machine.mjs @@ -0,0 +1,1201 @@ +#!/usr/bin/env node + +import { inspect, parseArgs } from "node:util"; +import { + $, + getBootstrapVersion, + getBuildNumber, + getSecret, + isCI, + parseArch, + parseOs, + readFile, + spawn, + spawnSafe, + spawnSyncSafe, + startGroup, + tmpdir, + waitForPort, + which, +} from "./utils.mjs"; +import { join, relative, resolve } from "node:path"; +import { homedir } from "node:os"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const docker = { + getPlatform(platform) { + const { os, arch } = platform; + + if (os === "linux" || os === "windows") { + if (arch === "aarch64") { + return `${os}/arm64`; + } else if (arch === "x64") { + return `${os}/amd64`; + } + } + + throw new Error(`Unsupported platform: ${inspect(platform)}`); + }, + + async createMachine(platform) { + const { id } = await docker.getImage(platform); + const platformString = docker.getPlatform(platform); + + const command = ["sleep", "1d"]; + const { stdout } = await spawnSafe(["docker", "run", "--rm", "--platform", platformString, "-d", id, ...command]); + const containerId = stdout.trim(); + + const spawn = async command => { + return spawn(["docker", "exec", containerId, ...command]); + }; + + const spawnSafe = async command => { + return spawnSafe(["docker", "exec", containerId, ...command]); + }; + + const attach = async () => { + const { exitCode, spawnError } = await spawn(["docker", "exec", "-it", containerId, "bash"], { + stdio: "inherit", + }); + + if (exitCode === 0 || exitCode === 130) { + return; + } + + throw spawnError; + }; + + const kill = async () => { + await spawnSafe(["docker", "kill", containerId]); + }; + + return { + spawn, + spawnSafe, + attach, + close: kill, + [Symbol.asyncDispose]: kill, + }; + }, + + async getImage(platform) { + const os = platform["os"]; + const distro = platform["distro"]; + const release = platform["release"] || "latest"; + + let url; + if (os === "linux") { + if (distro === "debian") { + url = `docker.io/library/debian:${release}`; + } else if (distro === "ubuntu") { + url = `docker.io/library/ubuntu:${release}`; + } else if (distro === "amazonlinux") { + url = `public.ecr.aws/amazonlinux/amazonlinux:${release}`; + } else if (distro === "alpine") { + url = `docker.io/library/alpine:${release}`; + } + } + + if (url) { + await spawnSafe(["docker", "pull", "--platform", docker.getPlatform(platform), url]); + const { stdout } = await spawnSafe(["docker", "image", "inspect", url, "--format", "json"]); + const [{ Id }] = JSON.parse(stdout); + return { + id: Id, + name: url, + username: "root", + }; + } + + throw new Error(`Unsupported platform: ${inspect(platform)}`); + }, +}; + +export const aws = { + get name() { + return "aws"; + }, + + /** + * @param {string[]} args + * @returns {Promise} + */ + async spawn(args) { + const aws = which("aws"); + if (!aws) { + throw new Error("AWS CLI is not installed, please install it"); + } + + let env; + if (isCI) { + env = { + AWS_ACCESS_KEY_ID: getSecret("EC2_ACCESS_KEY_ID", { required: true }), + AWS_SECRET_ACCESS_KEY: getSecret("EC2_SECRET_ACCESS_KEY", { required: true }), + AWS_REGION: getSecret("EC2_REGION", { required: false }) || "us-east-1", + }; + } + + const { stdout } = await spawnSafe($`${aws} ${args} --output json`, { env }); + try { + return JSON.parse(stdout); + } catch { + return; + } + }, + + /** + * @param {Record} [options] + * @returns {string[]} + */ + getFilters(options = {}) { + return Object.entries(options) + .filter(([_, value]) => typeof value !== "undefined") + .map(([key, value]) => `Name=${key},Values=${value}`); + }, + + /** + * @param {Record} [options] + * @returns {string[]} + */ + getFlags(options = {}) { + return Object.entries(options) + .filter(([_, value]) => typeof value !== "undefined") + .map(([key, value]) => `--${key}=${value}`); + }, + + /** + * @typedef AwsInstance + * @property {string} InstanceId + * @property {string} ImageId + * @property {string} InstanceType + * @property {string} [PublicIpAddress] + * @property {string} [PlatformDetails] + * @property {string} [Architecture] + * @property {object} [Placement] + * @property {string} [Placement.AvailabilityZone] + * @property {string} LaunchTime + */ + + /** + * @param {Record} [options] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-instances.html + */ + async describeInstances(options) { + const filters = aws.getFilters(options); + const { Reservations } = await aws.spawn($`ec2 describe-instances --filters ${filters}`); + return Reservations.flatMap(({ Instances }) => Instances).sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1)); + }, + + /** + * @param {Record} [options] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/run-instances.html + */ + async runInstances(options) { + const flags = aws.getFlags(options); + const { Instances } = await aws.spawn($`ec2 run-instances ${flags}`); + return Instances.sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1)); + }, + + /** + * @param {...string} instanceIds + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/stop-instances.html + */ + async stopInstances(...instanceIds) { + await aws.spawn($`ec2 stop-instances --no-hibernate --force --instance-ids ${instanceIds}`); + }, + + /** + * @param {...string} instanceIds + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/terminate-instances.html + */ + async terminateInstances(...instanceIds) { + await aws.spawn($`ec2 terminate-instances --instance-ids ${instanceIds}`); + }, + + /** + * @param {"instance-running" | "instance-stopped" | "instance-terminated"} action + * @param {...string} instanceIds + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait.html + */ + async waitInstances(action, ...instanceIds) { + await aws.spawn($`ec2 wait ${action} --instance-ids ${instanceIds}`); + }, + + /** + * @typedef AwsImage + * @property {string} ImageId + * @property {string} Name + * @property {string} State + * @property {string} CreationDate + */ + + /** + * @param {Record} [options] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-images.html + */ + async describeImages(options = {}) { + const { ["owner-alias"]: owners, ...filterOptions } = options; + const filters = aws.getFilters(filterOptions); + if (owners) { + filters.push(`--owners=${owners}`); + } + const { Images } = await aws.spawn($`ec2 describe-images --filters ${filters}`); + return Images.sort((a, b) => (a.CreationDate < b.CreationDate ? 1 : -1)); + }, + + /** + * @param {Record} [options] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/create-image.html + */ + async createImage(options) { + const flags = aws.getFlags(options); + try { + const { ImageId } = await aws.spawn($`ec2 create-image ${flags}`); + return ImageId; + } catch (error) { + const match = /already in use by AMI (ami-[a-z0-9]+)/i.exec(inspect(error)); + if (!match) { + throw error; + } + const [, existingImageId] = match; + await aws.spawn($`ec2 deregister-image --image-id ${existingImageId}`); + const { ImageId } = await aws.spawn($`ec2 create-image ${flags}`); + return ImageId; + } + }, + + /** + * @param {Record} options + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/copy-image.html + */ + async copyImage(options) { + const flags = aws.getFlags(options); + const { ImageId } = await aws.spawn($`ec2 copy-image ${flags}`); + return ImageId; + }, + + /** + * @param {"image-available"} action + * @param {...string} imageIds + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait/image-available.html + */ + async waitImage(action, ...imageIds) { + while (true) { + try { + await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`); + return; + } catch (error) { + if (!/max attempts exceeded/i.test(inspect(error))) { + throw error; + } + } + } + }, + + /** + * @param {AwsImage | string} imageOrImageId + * @returns {Promise} + */ + async getAvailableImage(imageOrImageId) { + let imageId = imageOrImageId; + if (typeof imageOrImageId === "object") { + const { ImageId, State } = imageOrImageId; + if (State === "available") { + return imageOrImageId; + } + imageId = ImageId; + } + + await aws.waitImage("image-available", imageId); + const [availableImage] = await aws.describeImages({ + "state": "available", + "image-id": imageId, + }); + + if (!availableImage) { + throw new Error(`Failed to find available image: ${imageId}`); + } + + return availableImage; + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async getBaseImage(options) { + const { os, arch, distro, distroVersion } = options; + + let name, owner; + if (os === "linux") { + if (!distro || distro === "debian") { + owner = "amazon"; + name = `debian-${distroVersion || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-*`; + } else if (distro === "ubuntu") { + owner = "099720109477"; + name = `ubuntu/images/hvm-ssd*/ubuntu-*-${distroVersion || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-server-*`; + } else if (distro === "amazonlinux") { + owner = "amazon"; + if (distroVersion === "1") { + // EOL + } else if (distroVersion === "2") { + name = `amzn2-ami-hvm-*-${arch === "aarch64" ? "arm64" : "x86_64"}-gp2`; + } else { + name = `al${distroVersion || "*"}-ami-*-${arch === "aarch64" ? "arm64" : "x86_64"}`; + } + } else if (distro === "alpine") { + owner = "538276064493"; + name = `alpine-${distroVersion || "*"}.*-${arch === "aarch64" ? "aarch64" : "x86_64"}-uefi-cloudinit-*`; + } else if (distro === "centos") { + owner = "aws-marketplace"; + name = `CentOS-Stream-ec2-${distroVersion || "*"}-*.${arch === "aarch64" ? "aarch64" : "x86_64"}-*`; + } + } else if (os === "windows") { + if (!distro || distro === "server") { + owner = "amazon"; + name = `Windows_Server-${distroVersion || "*"}-English-Full-Base-*`; + } + } + + if (!name) { + throw new Error(`Unsupported platform: ${inspect(options)}`); + } + + const baseImages = await aws.describeImages({ + "state": "available", + "owner-alias": owner, + "name": name, + }); + + if (!baseImages.length) { + throw new Error(`No base image found: ${inspect(options)}`); + } + + const [baseImage] = baseImages; + return aws.getAvailableImage(baseImage); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { arch, imageId, instanceType, metadata } = options; + + /** @type {AwsImage} */ + let image; + if (imageId) { + image = await aws.getAvailableImage(imageId); + } else { + image = await aws.getBaseImage(options); + } + + const { ImageId, Name, RootDeviceName, BlockDeviceMappings } = image; + const blockDeviceMappings = BlockDeviceMappings.map(device => { + const { DeviceName } = device; + if (DeviceName === RootDeviceName) { + return { + ...device, + Ebs: { + VolumeSize: getDiskSize(options), + }, + }; + } + return device; + }); + + const username = getUsername(Name); + const userData = getCloudInit({ ...options, username }); + + let tagSpecification = []; + if (metadata) { + tagSpecification = ["instance", "volume"].map(resourceType => { + return { + ResourceType: resourceType, + Tags: Object.entries(metadata).map(([Key, Value]) => ({ Key, Value: String(Value) })), + }; + }); + } + + const [instance] = await aws.runInstances({ + ["image-id"]: ImageId, + ["instance-type"]: instanceType || (arch === "aarch64" ? "t4g.large" : "t3.large"), + ["user-data"]: userData, + ["block-device-mappings"]: JSON.stringify(blockDeviceMappings), + ["metadata-options"]: JSON.stringify({ + "HttpTokens": "optional", + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "enabled", + "InstanceMetadataTags": "enabled", + }), + ["tag-specifications"]: JSON.stringify(tagSpecification), + }); + + return aws.toMachine(instance, { ...options, username }); + }, + + /** + * @param {AwsInstance} instance + * @param {MachineOptions} [options] + * @returns {Machine} + */ + toMachine(instance, options = {}) { + let { InstanceId, ImageId, InstanceType, Placement, PublicIpAddress } = instance; + + const connect = async () => { + if (!PublicIpAddress) { + await aws.waitInstances("instance-running", InstanceId); + const [{ PublicIpAddress: IpAddress }] = await aws.describeInstances({ + ["instance-id"]: InstanceId, + }); + PublicIpAddress = IpAddress; + } + + const { username, sshKeys } = options; + const identityPaths = sshKeys + ?.filter(({ privatePath }) => existsSync(privatePath)) + ?.map(({ privatePath }) => privatePath); + + return { hostname: PublicIpAddress, username, identityPaths }; + }; + + const spawn = async (command, options) => { + const connectOptions = await connect(); + return spawnSsh({ ...connectOptions, command }, options); + }; + + const spawnSafe = async (command, options) => { + const connectOptions = await connect(); + return spawnSshSafe({ ...connectOptions, command }, options); + }; + + const attach = async () => { + const connectOptions = await connect(); + await spawnSshSafe({ ...connectOptions }); + }; + + const upload = async (source, destination) => { + const connectOptions = await connect(); + await spawnScp({ ...connectOptions, source, destination }); + }; + + const snapshot = async name => { + await aws.stopInstances(InstanceId); + await aws.waitInstances("instance-stopped", InstanceId); + const imageId = await aws.createImage({ + ["instance-id"]: InstanceId, + ["name"]: name || `${InstanceId}-snapshot-${Date.now()}`, + }); + await aws.waitImage("image-available", imageId); + return imageId; + }; + + const terminate = async () => { + await aws.terminateInstances(InstanceId); + }; + + return { + cloud: "aws", + id: InstanceId, + imageId: ImageId, + instanceType: InstanceType, + region: Placement?.AvailabilityZone, + get publicIp() { + return PublicIpAddress; + }, + spawn, + spawnSafe, + upload, + attach, + snapshot, + close: terminate, + [Symbol.asyncDispose]: terminate, + }; + }, +}; + +const google = { + async createMachine(platform) { + const image = await google.getImage(platform); + const { id: imageId, username } = image; + + const authorizedKeys = await getAuthorizedKeys(); + const sshKeys = authorizedKeys?.map(key => `${username}:${key}`).join("\n") ?? ""; + + const { os, ["instance-type"]: type } = platform; + const instanceType = type || "e2-standard-4"; + + let metadata = `ssh-keys=${sshKeys}`; + if (os === "windows") { + metadata += `,sysprep-specialize-script-cmd=googet -noconfirm=true install google-compute-engine-ssh,enable-windows-ssh=TRUE`; + } + + const [{ id, networkInterfaces }] = await google.createInstances({ + ["zone"]: "us-central1-a", + ["image"]: imageId, + ["machine-type"]: instanceType, + ["boot-disk-auto-delete"]: true, + // ["boot-disk-size"]: "10GB", + // ["boot-disk-type"]: "pd-standard", + ["metadata"]: metadata, + }); + + const publicIp = () => { + for (const { accessConfigs } of networkInterfaces) { + for (const { natIP } of accessConfigs) { + return natIP; + } + } + throw new Error(`Failed to find public IP for instance: ${id}`); + }; + + const spawn = command => { + const hostname = publicIp(); + return spawnSsh({ hostname, username, command }); + }; + + const spawnSafe = command => { + const hostname = publicIp(); + return spawnSshSafe({ hostname, username, command }); + }; + + const attach = async () => { + const hostname = publicIp(); + await spawnSshSafe({ hostname, username }); + }; + + const terminate = async () => { + await google.deleteInstance(id); + }; + + return { + spawn, + spawnSafe, + attach, + close: terminate, + [Symbol.asyncDispose]: terminate, + }; + }, + + async getImage(platform) { + const { os, arch, distro, release } = platform; + const architecture = arch === "aarch64" ? "ARM64" : "X86_64"; + + let name; + let username; + if (os === "linux") { + if (distro === "debian") { + name = `debian-${release}-*`; + username = "admin"; + } else if (distro === "ubuntu") { + name = `ubuntu-${release.replace(/\./g, "")}-*`; + username = "ubuntu"; + } + } else if (os === "windows" && arch === "x64") { + if (distro === "server") { + name = `windows-server-${release}-dc-core-*`; + username = "administrator"; + } + } + + if (name && username) { + const images = await google.listImages({ name, architecture }); + if (images.length) { + const [image] = images; + const { name, selfLink } = image; + return { + id: selfLink, + name, + username, + }; + } + } + + throw new Error(`Unsupported platform: ${inspect(platform)}`); + }, + + async listImages(options = {}) { + const filter = Object.entries(options) + .map(([key, value]) => [value.includes("*") ? `${key}~${value}` : `${key}=${value}`]) + .join(" AND "); + const filters = filter ? ["--filter", filter] : []; + const { stdout } = await spawnSafe(["gcloud", "compute", "images", "list", ...filters, "--format", "json"]); + const images = JSON.parse(stdout); + return images.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + }, + + async listInstances(options = {}) { + const filter = Object.entries(options) + .map(([key, value]) => [value.includes("*") ? `${key}~${value}` : `${key}=${value}`]) + .join(" AND "); + const filters = filter ? ["--filter", filter] : []; + const { stdout } = await spawnSafe(["gcloud", "compute", "instances", "list", ...filters, "--format", "json"]); + const instances = JSON.parse(stdout); + return instances.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + }, + + async createInstances(options = {}) { + const flags = Object.entries(options).flatMap(([key, value]) => + typeof value === "boolean" ? `--${key}` : `--${key}=${value}`, + ); + const randomId = "i-" + Math.random().toString(36).substring(2, 15); + const { stdout } = await spawnSafe([ + "gcloud", + "compute", + "instances", + "create", + randomId, + ...flags, + "--format", + "json", + ]); + const instances = JSON.parse(stdout); + return instances.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + }, + + async deleteInstance(instanceId) { + await spawnSafe(["gcloud", "compute", "instances", "delete", instanceId, "--zone", "us-central1-a", "--quiet"]); + }, +}; + +/** + * @typedef CloudInit + * @property {string} [distro] + * @property {SshKey[]} [sshKeys] + * @property {string} [username] + * @property {string} [password] + */ + +/** + * @param {CloudInit} cloudInit + * @returns {string} + */ +function getCloudInit(cloudInit) { + const username = cloudInit["username"] || "root"; + const password = cloudInit["password"] || crypto.randomUUID(); + const authorizedKeys = JSON.stringify(cloudInit["sshKeys"]?.map(({ publicKey }) => publicKey) || []); + + let sftpPath = "/usr/lib/openssh/sftp-server"; + switch (cloudInit["distro"]) { + case "alpine": + sftpPath = "/usr/lib/ssh/sftp-server"; + break; + case "amazonlinux": + case "rhel": + case "centos": + sftpPath = "/usr/libexec/openssh/sftp-server"; + break; + } + + // https://cloudinit.readthedocs.io/en/stable/ + return `#cloud-config + + package_update: true + packages: + - curl + - ca-certificates + - openssh-server + + write_files: + - path: /etc/ssh/sshd_config + content: | + PermitRootLogin yes + PasswordAuthentication yes + Subsystem sftp ${sftpPath} + + chpasswd: + expire: false + list: | + root:${password} + ${username}:${password} + + disable_root: false + + ssh_pwauth: true + ssh_authorized_keys: ${authorizedKeys} + `; +} + +/** + * @param {string} distro + * @returns {string} + */ +function getUsername(distro) { + if (/windows/i.test(distro)) { + return "administrator"; + } + + if (/alpine|centos/i.test(distro)) { + return "root"; + } + + if (/debian/i.test(distro)) { + return "admin"; + } + + if (/ubuntu/i.test(distro)) { + return "ubuntu"; + } + + if (/amazon|amzn|al\d+|rhel/i.test(distro)) { + return "ec2-user"; + } + + throw new Error(`Unsupported distro: ${distro}`); +} + +/** + * @param {MachineOptions} options + * @returns {number} + */ +function getDiskSize(options) { + const { os, diskSizeGb } = options; + + if (diskSizeGb) { + return diskSizeGb; + } + + return os === "windows" ? 50 : 30; +} + +/** + * @typedef SshKey + * @property {string} privatePath + * @property {string} publicPath + * @property {string} publicKey + */ + +/** + * @returns {SshKey} + */ +function createSshKey() { + const sshPath = join(homedir(), ".ssh"); + if (!existsSync(sshPath)) { + mkdirSync(sshPath, { recursive: true }); + } + + const name = `id_rsa_${crypto.randomUUID()}`; + const privatePath = join(sshPath, name); + const publicPath = join(sshPath, `${name}.pub`); + spawnSyncSafe(["ssh-keygen", "-t", "rsa", "-b", "4096", "-f", privatePath, "-N", ""], { stdio: "inherit" }); + + if (!existsSync(privatePath) || !existsSync(publicPath)) { + throw new Error(`Failed to generate SSH key: ${privatePath} / ${publicPath}`); + } + + const sshAgent = which("ssh-agent"); + const sshAdd = which("ssh-add"); + if (sshAgent && sshAdd) { + spawnSyncSafe(["sh", "-c", `eval $(${sshAgent} -s) && ${sshAdd} ${privatePath}`], { stdio: "inherit" }); + } + + return { + privatePath, + publicPath, + get publicKey() { + return readFile(publicPath, { cache: true }); + }, + }; +} + +/** + * @returns {SshKey[]} + */ +function getSshKeys() { + const homePath = homedir(); + const sshPath = join(homePath, ".ssh"); + + /** @type {SshKey[]} */ + const sshKeys = []; + if (existsSync(sshPath)) { + const sshFiles = readdirSync(sshPath, { withFileTypes: true }); + const publicPaths = sshFiles + .filter(entry => entry.isFile() && entry.name.endsWith(".pub")) + .map(({ name }) => join(sshPath, name)); + + sshKeys.push( + ...publicPaths.map(publicPath => ({ + publicPath, + privatePath: publicPath.replace(/\.pub$/, ""), + get publicKey() { + return readFile(publicPath, { cache: true }); + }, + })), + ); + } + + if (!sshKeys.length) { + sshKeys.push(createSshKey()); + } + + return sshKeys; +} + +/** + * @typedef SshOptions + * @property {string} hostname + * @property {number} [port] + * @property {string} [username] + * @property {string[]} [command] + * @property {string[]} [identityPaths] + * @property {number} [retries] + */ + +/** + * @param {SshOptions} options + * @param {object} [spawnOptions] + * @returns {Promise} + */ +async function spawnSsh(options, spawnOptions = {}) { + const { hostname, port, username, identityPaths, command } = options; + await waitForPort({ hostname, port: port || 22 }); + + const ssh = ["ssh", hostname, "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"]; + if (port) { + ssh.push("-p", port); + } + if (username) { + ssh.push("-l", username); + } + if (identityPaths) { + ssh.push(...identityPaths.flatMap(path => ["-i", path])); + } + const stdio = command ? "pipe" : "inherit"; + if (command) { + ssh.push(...command); + } + + return spawn(ssh, { stdio, ...spawnOptions }); +} + +/** + * @param {SshOptions} options + * @param {object} [spawnOptions] + * @returns {Promise} + */ +async function spawnSshSafe(options, spawnOptions = {}) { + const { hostname, port, username, identityPaths, command } = options; + await waitForPort({ hostname, port: port || 22 }); + + const ssh = ["ssh", hostname, "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"]; + if (port) { + ssh.push("-p", port); + } + if (username) { + ssh.push("-l", username); + } + if (identityPaths) { + ssh.push(...identityPaths.flatMap(path => ["-i", path])); + } + const stdio = command ? "pipe" : "inherit"; + if (command) { + ssh.push(...command); + } + + return spawnSafe(ssh, { stdio, ...spawnOptions }); +} + +/** + * @typedef ScpOptions + * @property {string} hostname + * @property {string} source + * @property {string} destination + * @property {string[]} [identityPaths] + * @property {string} [port] + * @property {string} [username] + * @property {number} [retries] + */ + +/** + * @param {ScpOptions} options + * @returns {Promise} + */ +async function spawnScp(options) { + const { hostname, port, username, identityPaths, source, destination, retries = 10 } = options; + await waitForPort({ hostname, port: port || 22 }); + + const command = ["scp", "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"]; + if (port) { + command.push("-P", port); + } + if (identityPaths) { + command.push(...identityPaths.flatMap(path => ["-i", path])); + } + command.push(resolve(source)); + if (username) { + command.push(`${username}@${hostname}:${destination}`); + } else { + command.push(`${hostname}:${destination}`); + } + + let cause; + for (let i = 0; i < retries; i++) { + const result = await spawn(command, { stdio: "inherit" }); + const { exitCode, stderr } = result; + if (exitCode === 0) { + return; + } + + cause = stderr.trim() || undefined; + if (/(bad configuration option)|(no such file or directory)/i.test(stderr)) { + break; + } + await new Promise(resolve => setTimeout(resolve, Math.pow(2, i) * 1000)); + } + + throw new Error(`SCP failed: ${source} -> ${username}@${hostname}:${destination}`, { cause }); +} + +/** + * @typedef Cloud + * @property {string} name + * @property {(options: MachineOptions) => Promise} createMachine + */ + +/** + * @param {string} name + * @returns {Cloud} + */ +function getCloud(name) { + switch (name) { + case "aws": + return aws; + } + throw new Error(`Unsupported cloud: ${name}`); +} + +/** + * @typedef Machine + * @property {string} cloud + * @property {string} [name] + * @property {string} id + * @property {string} imageId + * @property {string} instanceType + * @property {string} region + * @property {string} [publicIp] + * @property {(command: string[]) => Promise} spawn + * @property {(command: string[]) => Promise} spawnSafe + * @property {(source: string, destination: string) => Promise} upload + * @property {() => Promise} attach + * @property {() => Promise} snapshot + * @property {() => Promise} close + */ + +/** + * @typedef {"linux" | "darwin" | "windows"} Os + * @typedef {"aarch64" | "x64"} Arch + */ + +/** + * @typedef MachineOptions + * @property {Cloud} cloud + * @property {Os} os + * @property {Arch} arch + * @property {string} distro + * @property {string} [distroVersion] + * @property {string} [imageId] + * @property {string} [imageName] + * @property {number} [cpuCount] + * @property {number} [memoryGb] + * @property {number} [diskSizeGb] + * @property {boolean} [persistent] + * @property {boolean} [detached] + * @property {Record} [tags] + * @property {boolean} [bootstrap] + * @property {boolean} [ci] + * @property {SshKey[]} [sshKeys] + */ + +async function main() { + const { positionals } = parseArgs({ + allowPositionals: true, + strict: false, + }); + + const [command] = positionals; + if (!/^(ssh|create-image|publish-image)$/.test(command)) { + const scriptPath = relative(process.cwd(), fileURLToPath(import.meta.url)); + throw new Error(`Usage: ./${scriptPath} [ssh|create-image|publish-image] [options]`); + } + + const { values: args } = parseArgs({ + allowPositionals: true, + options: { + "cloud": { type: "string", default: "aws" }, + "os": { type: "string", default: "linux" }, + "arch": { type: "string", default: "x64" }, + "distro": { type: "string", default: "debian" }, + "distro-version": { type: "string" }, + "instance-type": { type: "string" }, + "image-id": { type: "string" }, + "image-name": { type: "string" }, + "cpu-count": { type: "string" }, + "memory-gb": { type: "string" }, + "disk-size-gb": { type: "string" }, + "persistent": { type: "boolean" }, + "detached": { type: "boolean" }, + "tag": { type: "string", multiple: true }, + "ci": { type: "boolean" }, + "no-bootstrap": { type: "boolean" }, + "buildkite-token": { type: "string" }, + "tailscale-authkey": { type: "string" }, + }, + }); + + /** @type {MachineOptions} */ + const options = { + cloud: getCloud(args["cloud"]), + os: parseOs(args["os"]), + arch: parseArch(args["arch"]), + distro: args["distro"], + distroVersion: args["distro-version"], + instanceType: args["instance-type"], + imageId: args["image-id"], + imageName: args["image-name"], + tags: { + "robobun": "true", + "robobun2": "true", + "buildkite:token": args["buildkite-token"], + "tailscale:authkey": args["tailscale-authkey"], + ...Object.fromEntries(args["tag"]?.map(tag => tag.split("=")) ?? []), + }, + cpuCount: parseInt(args["cpu-count"]) || undefined, + memoryGb: parseInt(args["memory-gb"]) || undefined, + diskSizeGb: parseInt(args["disk-size-gb"]) || undefined, + persistent: !!args["persistent"], + detached: !!args["detached"], + bootstrap: args["no-bootstrap"] !== true, + ci: !!args["ci"], + sshKeys: getSshKeys(), + }; + + const { cloud, detached, bootstrap, ci, os, arch, distro, distroVersion } = options; + const name = `${os}-${arch}-${distro}-${distroVersion}`; + + let bootstrapPath, agentPath; + if (bootstrap) { + bootstrapPath = resolve(import.meta.dirname, "bootstrap.sh"); + if (!existsSync(bootstrapPath)) { + throw new Error(`Script not found: ${bootstrapPath}`); + } + if (ci) { + const npx = which("bunx") || which("npx"); + if (!npx) { + throw new Error("Executable not found: bunx or npx"); + } + const entryPath = resolve(import.meta.dirname, "agent.mjs"); + const tmpPath = mkdtempSync(join(tmpdir(), "agent-")); + agentPath = join(tmpPath, "agent.mjs"); + await spawnSafe($`${npx} esbuild ${entryPath} --bundle --platform=node --format=esm --outfile=${agentPath}`); + } + } + + /** @type {Machine} */ + const machine = await startGroup("Creating machine...", async () => { + console.log("Creating machine:", JSON.parse(JSON.stringify(options))); + const result = await cloud.createMachine(options); + console.log("Created machine:", result); + return result; + }); + + if (!detached) { + let closing; + for (const event of ["beforeExit", "SIGINT", "SIGTERM"]) { + process.on(event, () => { + if (!closing) { + closing = true; + machine.close().finally(() => { + if (event !== "beforeExit") { + process.exit(1); + } + }); + } + }); + } + } + + try { + await startGroup("Connecting...", async () => { + const command = os === "windows" ? ["cmd", "/c", "ver"] : ["uname", "-a"]; + await machine.spawnSafe(command, { stdio: "inherit" }); + }); + + if (bootstrapPath) { + const remotePath = "/tmp/bootstrap.sh"; + const args = ci ? ["--ci"] : []; + await startGroup("Running bootstrap...", async () => { + await machine.upload(bootstrapPath, remotePath); + await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" }); + }); + } + + if (agentPath) { + const tmpPath = "/tmp/agent.mjs"; + const remotePath = "/var/lib/buildkite-agent/agent.mjs"; + await startGroup("Installing agent...", async () => { + await machine.upload(agentPath, tmpPath); + const command = []; + { + const { exitCode } = await machine.spawn(["sudo", "echo", "1"], { stdio: "ignore" }); + if (exitCode === 0) { + command.unshift("sudo"); + } + } + await machine.spawnSafe([...command, "cp", tmpPath, remotePath]); + { + const { stdout } = await machine.spawn(["node", "-v"]); + const version = parseInt(stdout.trim().replace(/^v/, "")); + if (isNaN(version) || version < 20) { + command.push("bun"); + } else { + command.push("node"); + } + } + await machine.spawnSafe([...command, remotePath, "install"], { stdio: "inherit" }); + }); + } + + if (command === "create-image" || command === "publish-image") { + let suffix; + if (command === "publish-image") { + suffix = `v${getBootstrapVersion()}`; + } else if (isCI) { + suffix = `build-${getBuildNumber()}`; + } else { + suffix = `draft-${Date.now()}`; + } + const label = `${name}-${suffix}`; + await startGroup("Creating image...", async () => { + console.log("Creating image:", label); + const result = await machine.snapshot(label); + console.log("Created image:", result); + }); + } + + if (command === "ssh") { + await machine.attach(); + } + } catch (error) { + if (isCI) { + throw error; + } + console.error(error); + try { + await machine.attach(); + } catch (error) { + console.error(error); + } + } finally { + if (!detached) { + await machine.close(); + } + } +} + +await main(); diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs index 898b596a50..792c825ac1 100755 --- a/scripts/runner.node.mjs +++ b/scripts/runner.node.mjs @@ -39,7 +39,7 @@ import { } from "./utils.mjs"; import { userInfo } from "node:os"; -const cwd = dirname(import.meta.dirname); +const cwd = import.meta.dirname ? dirname(import.meta.dirname) : process.cwd(); const testsPath = join(cwd, "test"); const spawnTimeout = 5_000; @@ -232,7 +232,7 @@ async function runTests() { if (testRunner === "bun") { await runTest(title, () => spawnBunTest(execPath, testPath, { cwd: vendorPath })); } else { - const testRunnerPath = join(import.meta.dirname, "..", "test", "runners", `${testRunner}.ts`); + const testRunnerPath = join(cwd, "test", "runners", `${testRunner}.ts`); if (!existsSync(testRunnerPath)) { throw new Error(`Unsupported test runner: ${testRunner}`); } @@ -632,7 +632,7 @@ function parseTestStdout(stdout, testPath) { const removeStart = lines.length - skipCount; const removeCount = skipCount - 2; const omitLine = `${getAnsi("gray")}... omitted ${removeCount} tests ...${getAnsi("reset")}`; - lines = lines.toSpliced(removeStart, removeCount, omitLine); + lines.splice(removeStart, removeCount, omitLine); } skipCount = 0; } @@ -1133,6 +1133,13 @@ function addPath(...paths) { return paths.join(":"); } +/** + * @returns {string | undefined} + */ +function getTestLabel() { + return getBuildLabel()?.replace(" - test-bun", ""); +} + /** * @param {TestResult | TestResult[]} result * @param {boolean} concise @@ -1140,7 +1147,7 @@ function addPath(...paths) { */ function formatTestToMarkdown(result, concise) { const results = Array.isArray(result) ? result : [result]; - const buildLabel = getBuildLabel(); + const buildLabel = getTestLabel(); const buildUrl = getBuildUrl(); const platform = buildUrl ? `${buildLabel}` : buildLabel; @@ -1273,7 +1280,7 @@ function reportAnnotationToBuildKite({ label, content, style = "error", priority const cause = error ?? signal ?? `code ${status}`; throw new Error(`Failed to create annotation: ${label}`, { cause }); } - const buildLabel = getBuildLabel(); + const buildLabel = getTestLabel(); const buildUrl = getBuildUrl(); const platform = buildUrl ? `${buildLabel}` : buildLabel; let errorMessage = `
${label} - annotation error on ${platform}`; diff --git a/scripts/utils.mjs b/scripts/utils.mjs old mode 100644 new mode 100755 index de7ec0b686..41196942b0 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -3,9 +3,18 @@ import { spawn as nodeSpawn, spawnSync as nodeSpawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { appendFileSync, existsSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; -import { writeFile } from "node:fs/promises"; -import { hostname, tmpdir as nodeTmpdir, userInfo } from "node:os"; +import { + appendFileSync, + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { connect } from "node:net"; +import { hostname, tmpdir as nodeTmpdir, userInfo, release } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { normalize as normalizeWindows } from "node:path/win32"; @@ -53,8 +62,9 @@ export function getSecret(name, options = { required: true, redact: true }) { command.push("--skip-redaction"); } - const { error, stdout: secret } = spawnSync(command); - if (error || !secret.trim()) { + const { error, stdout } = spawnSync(command); + const secret = stdout.trim(); + if (error || !secret) { const orgId = getEnv("BUILDKITE_ORGANIZATION_SLUG", false); const clusterId = getEnv("BUILDKITE_CLUSTER_ID", false); @@ -106,8 +116,8 @@ export function setEnv(name, value) { * @property {string} [cwd] * @property {number} [timeout] * @property {Record} [env] - * @property {string} [stdout] - * @property {string} [stderr] + * @property {string} [stdin] + * @property {boolean} [privileged] */ /** @@ -119,20 +129,93 @@ export function setEnv(name, value) { * @property {Error} [error] */ +/** + * @param {TemplateStringsArray} strings + * @param {...any} values + * @returns {string[]} + */ +export function $(strings, ...values) { + const result = []; + for (let i = 0; i < strings.length; i++) { + result.push(...strings[i].trim().split(/\s+/).filter(Boolean)); + if (i < values.length) { + const value = values[i]; + if (Array.isArray(value)) { + result.push(...value); + } else if (typeof value === "string") { + if (result.at(-1)?.endsWith("=")) { + result[result.length - 1] += value; + } else { + result.push(value); + } + } + } + } + return result; +} + +/** @type {string[] | undefined} */ +let priviledgedCommand; + +/** + * @param {string[]} command + * @param {SpawnOptions} options + */ +function parseCommand(command, options) { + if (options?.privileged) { + return [...getPrivilegedCommand(), ...command]; + } + return command; +} + +/** + * @returns {string[]} + */ +function getPrivilegedCommand() { + if (typeof priviledgedCommand !== "undefined") { + return priviledgedCommand; + } + + if (isWindows) { + return (priviledgedCommand = []); + } + + const sudo = ["sudo", "-n"]; + const { error: sudoError } = spawnSync([...sudo, "true"]); + if (!sudoError) { + return (priviledgedCommand = sudo); + } + + const su = ["su", "-s", "sh", "root", "-c"]; + const { error: suError } = spawnSync([...su, "true"]); + if (!suError) { + return (priviledgedCommand = su); + } + + const doas = ["doas", "-u", "root"]; + const { error: doasError } = spawnSync([...doas, "true"]); + if (!doasError) { + return (priviledgedCommand = doas); + } + + return (priviledgedCommand = []); +} + /** * @param {string[]} command * @param {SpawnOptions} options * @returns {Promise} */ export async function spawn(command, options = {}) { - debugLog("$", ...command); + const [cmd, ...args] = parseCommand(command, options); + debugLog("$", cmd, ...args); - const [cmd, ...args] = command; + const stdin = options["stdin"]; const spawnOptions = { cwd: options["cwd"] ?? process.cwd(), timeout: options["timeout"] ?? undefined, env: options["env"] ?? undefined, - stdio: ["ignore", "pipe", "pipe"], + stdio: [stdin ? "pipe" : "ignore", "pipe", "pipe"], ...options, }; @@ -145,6 +228,16 @@ export async function spawn(command, options = {}) { const result = new Promise((resolve, reject) => { const subprocess = nodeSpawn(cmd, args, spawnOptions); + if (typeof stdin !== "undefined") { + subprocess.stdin?.on("error", error => { + if (error.code !== "EPIPE") { + reject(error); + } + }); + subprocess.stdin?.write(stdin); + subprocess.stdin?.end(); + } + subprocess.stdout?.on("data", chunk => { stdout += chunk; }); @@ -215,9 +308,9 @@ export async function spawnSafe(command, options) { * @returns {SpawnResult} */ export function spawnSync(command, options = {}) { - debugLog("$", ...command); + const [cmd, ...args] = parseCommand(command, options); + debugLog("$", cmd, ...args); - const [cmd, ...args] = command; const spawnOptions = { cwd: options["cwd"] ?? process.cwd(), timeout: options["timeout"] ?? undefined, @@ -245,8 +338,8 @@ export function spawnSync(command, options = {}) { } else { exitCode = status ?? 1; signalCode = signal || undefined; - stdout = stdoutBuffer.toString(); - stderr = stderrBuffer.toString(); + stdout = stdoutBuffer?.toString(); + stderr = stderrBuffer?.toString(); } if (exitCode !== 0 && isWindows) { @@ -258,7 +351,7 @@ export function spawnSync(command, options = {}) { if (error || signalCode || exitCode !== 0) { const description = command.map(arg => (arg.includes(" ") ? `"${arg.replace(/"/g, '\\"')}"` : arg)).join(" "); - const cause = error || stderr.trim() || stdout.trim() || undefined; + const cause = error || stderr?.trim() || stdout?.trim() || undefined; if (signalCode) { error = new Error(`Command killed with ${signalCode}: ${description}`, { cause }); @@ -670,7 +763,7 @@ export async function curl(url, options = {}) { try { if (filename && ok) { const buffer = await response.arrayBuffer(); - await writeFile(filename, new Uint8Array(buffer)); + writeFile(filename, new Uint8Array(buffer)); } else if (arrayBuffer && ok) { body = await response.arrayBuffer(); } else if (json && ok) { @@ -735,7 +828,7 @@ export function readFile(filename, options = {}) { } const relativePath = relative(process.cwd(), absolutePath); - debugLog("cat", relativePath); + debugLog("$", "cat", relativePath); let content; try { @@ -752,6 +845,51 @@ export function readFile(filename, options = {}) { return content; } +/** + * @param {string} filename + * @param {string | Buffer} content + * @param {object} [options] + * @param {number} [options.mode] + */ +export function writeFile(filename, content, options = {}) { + const parent = dirname(filename); + if (!existsSync(parent)) { + mkdirSync(parent, { recursive: true }); + } + + writeFileSync(filename, content); + + if (options["mode"]) { + chmodSync(filename, options["mode"]); + } +} + +/** + * @param {string | string[]} command + * @param {object} [options] + * @param {boolean} [options.required] + * @returns {string | undefined} + */ +export function which(command, options = {}) { + const commands = Array.isArray(command) ? command : [command]; + const path = getEnv("PATH", false) || ""; + const binPaths = path.split(isWindows ? ";" : ":"); + + for (const binPath of binPaths) { + for (const command of commands) { + const commandPath = join(binPath, command); + if (existsSync(commandPath)) { + return commandPath; + } + } + } + + if (options["required"]) { + const description = commands.join(" or "); + throw new Error(`Command not found: ${description}`); + } +} + /** * @param {string} [cwd] * @param {string} [base] @@ -840,7 +978,7 @@ export function getBuildUrl() { */ export function getBuildLabel() { if (isBuildkite) { - const label = getEnv("BUILDKITE_GROUP_LABEL", false) || getEnv("BUILDKITE_LABEL", false); + const label = getEnv("BUILDKITE_LABEL", false) || getEnv("BUILDKITE_GROUP_LABEL", false); if (label) { return label; } @@ -854,6 +992,22 @@ export function getBuildLabel() { } } +/** + * @returns {number} + */ +export function getBootstrapVersion() { + if (isWindows) { + return 0; // TODO + } + const scriptPath = join(import.meta.dirname, "bootstrap.sh"); + const scriptContent = readFile(scriptPath, { cache: true }); + const match = /# Version: (\d+)/.exec(scriptContent); + if (match) { + return parseInt(match[1]); + } + return 0; +} + /** * @typedef {object} BuildArtifact * @property {string} [job] @@ -1027,6 +1181,17 @@ export async function getLastSuccessfulBuild() { } } +/** + * @param {string} filename + * @param {string} [cwd] + */ +export async function uploadArtifact(filename, cwd) { + if (isBuildkite) { + const relativePath = relative(cwd ?? process.cwd(), filename); + await spawnSafe(["buildkite-agent", "artifact", "upload", relativePath], { cwd, stdio: "inherit" }); + } +} + /** * @param {string} string * @returns {string} @@ -1035,6 +1200,17 @@ export function stripAnsi(string) { return string.replace(/\u001b\[\d+m/g, ""); } +/** + * @param {string} string + * @returns {string} + */ +export function escapeYaml(string) { + if (/[:"{}[\],&*#?|\-<>=!%@`]/.test(string)) { + return `"${string.replace(/"/g, '\\"')}"`; + } + return string; +} + /** * @param {string} string * @returns {string} @@ -1173,24 +1349,79 @@ export function getArch() { return parseArch(process.arch); } +/** + * @returns {string} + */ +export function getKernel() { + const kernel = release(); + const match = /(\d+)\.(\d+)(?:\.(\d+))?/.exec(kernel); + + if (match) { + const [, major, minor, patch] = match; + if (patch) { + return `${major}.${minor}.${patch}`; + } + return `${major}.${minor}`; + } + + return kernel; +} + /** * @returns {"musl" | "gnu" | undefined} */ export function getAbi() { - if (isLinux) { - const arch = getArch() === "x64" ? "x86_64" : "aarch64"; - const muslLibPath = `/lib/ld-musl-${arch}.so.1`; - if (existsSync(muslLibPath)) { + if (!isLinux) { + return; + } + + if (existsSync("/etc/alpine-release")) { + return "musl"; + } + + const arch = getArch() === "x64" ? "x86_64" : "aarch64"; + const muslLibPath = `/lib/ld-musl-${arch}.so.1`; + if (existsSync(muslLibPath)) { + return "musl"; + } + + const gnuLibPath = `/lib/ld-linux-${arch}.so.2`; + if (existsSync(gnuLibPath)) { + return "gnu"; + } + + const { error, stdout } = spawnSync(["ldd", "--version"]); + if (!error) { + if (/musl/i.test(stdout)) { return "musl"; } - - const gnuLibPath = `/lib/ld-linux-${arch}.so.2`; - if (existsSync(gnuLibPath)) { + if (/gnu|glibc/i.test(stdout)) { return "gnu"; } } } +/** + * @returns {string | undefined} + */ +export function getAbiVersion() { + if (!isLinux) { + return; + } + + const { error, stdout } = spawnSync(["ldd", "--version"]); + if (!error) { + const match = /(\d+)\.(\d+)(?:\.(\d+))?/.exec(stdout); + if (match) { + const [, major, minor, patch] = match; + if (patch) { + return `${major}.${minor}.${patch}`; + } + return `${major}.${minor}`; + } + } +} + /** * @typedef {object} Target * @property {"darwin" | "linux" | "windows"} os @@ -1360,17 +1591,24 @@ export async function downloadTarget(target, release) { } /** - * @returns {string | undefined} + * @returns {string} */ -export function getTailscaleIp() { - let tailscale = "tailscale"; +export function getTailscale() { if (isMacOS) { const tailscaleApp = "/Applications/Tailscale.app/Contents/MacOS/tailscale"; if (existsSync(tailscaleApp)) { - tailscale = tailscaleApp; + return tailscaleApp; } } + return "tailscale"; +} + +/** + * @returns {string | undefined} + */ +export function getTailscaleIp() { + const tailscale = getTailscale(); const { error, stdout } = spawnSync([tailscale, "ip", "--1"]); if (!error) { return stdout.trim(); @@ -1419,7 +1657,31 @@ export function getUsername() { } /** - * @returns {string} + * @typedef {object} User + * @property {string} username + * @property {number} uid + * @property {number} gid + */ + +/** + * @param {string} username + * @returns {Promise} + */ +export async function getUser(username) { + if (isWindows) { + throw new Error("TODO: Windows"); + } + + const [uid, gid] = await Promise.all([ + spawnSafe(["id", "-u", username]).then(({ stdout }) => parseInt(stdout.trim())), + spawnSafe(["id", "-g", username]).then(({ stdout }) => parseInt(stdout.trim())), + ]); + + return { username, uid, gid }; +} + +/** + * @returns {string | undefined} */ export function getDistro() { if (isMacOS) { @@ -1427,6 +1689,11 @@ export function getDistro() { } if (isLinux) { + const alpinePath = "/etc/alpine-release"; + if (existsSync(alpinePath)) { + return "alpine"; + } + const releasePath = "/etc/os-release"; if (existsSync(releasePath)) { const releaseFile = readFile(releasePath, { cache: true }); @@ -1438,10 +1705,8 @@ export function getDistro() { const { error, stdout } = spawnSync(["lsb_release", "-is"]); if (!error) { - return stdout.trim(); + return stdout.trim().toLowerCase(); } - - return "Linux"; } if (isWindows) { @@ -1449,17 +1714,13 @@ export function getDistro() { if (!error) { return stdout.trim(); } - - return "Windows"; } - - return `${process.platform} ${process.arch}`; } /** * @returns {string | undefined} */ -export function getDistroRelease() { +export function getDistroVersion() { if (isMacOS) { const { error, stdout } = spawnSync(["sw_vers", "-productVersion"]); if (!error) { @@ -1468,6 +1729,16 @@ export function getDistroRelease() { } if (isLinux) { + const alpinePath = "/etc/alpine-release"; + if (existsSync(alpinePath)) { + const release = readFile(alpinePath, { cache: true }).trim(); + if (release.includes("_")) { + const [version] = release.split("_"); + return `${version}-edge`; + } + return release; + } + const releasePath = "/etc/os-release"; if (existsSync(releasePath)) { const releaseFile = readFile(releasePath, { cache: true }); @@ -1491,6 +1762,231 @@ export function getDistroRelease() { } } +/** + * @typedef {"aws" | "google"} Cloud + */ + +/** @type {Cloud | undefined} */ +let detectedCloud; + +/** + * @returns {Promise} + */ +export async function isAws() { + if (typeof detectedCloud === "string") { + return detectedCloud === "aws"; + } + + async function checkAws() { + if (isLinux) { + const kernel = release(); + if (kernel.endsWith("-aws")) { + return true; + } + + const { error: systemdError, stdout } = await spawn(["systemd-detect-virt"]); + if (!systemdError) { + if (stdout.includes("amazon")) { + return true; + } + } + + const dmiPath = "/sys/devices/virtual/dmi/id/board_asset_tag"; + if (existsSync(dmiPath)) { + const dmiFile = readFileSync(dmiPath, { encoding: "utf-8" }); + if (dmiFile.startsWith("i-")) { + return true; + } + } + } + + if (isWindows) { + const executionEnv = getEnv("AWS_EXECUTION_ENV", false); + if (executionEnv === "EC2") { + return true; + } + + const { error: powershellError, stdout } = await spawn([ + "powershell", + "-Command", + "Get-CimInstance -ClassName Win32_ComputerSystem | Select-Object Manufacturer", + ]); + if (!powershellError) { + return stdout.includes("Amazon"); + } + } + + const instanceId = await getCloudMetadata("instance-id", "google"); + if (instanceId) { + return true; + } + } + + if (await checkAws()) { + detectedCloud = "aws"; + return true; + } +} + +/** + * @returns {Promise} + */ +export async function isGoogleCloud() { + if (typeof detectedCloud === "string") { + return detectedCloud === "google"; + } + + async function detectGoogleCloud() { + if (isLinux) { + const vendorPaths = [ + "/sys/class/dmi/id/sys_vendor", + "/sys/class/dmi/id/bios_vendor", + "/sys/class/dmi/id/product_name", + ]; + + for (const vendorPath of vendorPaths) { + if (existsSync(vendorPath)) { + const vendorFile = readFileSync(vendorPath, { encoding: "utf-8" }); + if (vendorFile.includes("Google")) { + return true; + } + } + } + } + + const instanceId = await getCloudMetadata("id", "google"); + if (instanceId) { + return true; + } + } + + if (await detectGoogleCloud()) { + detectedCloud = "google"; + return true; + } +} + +/** + * @returns {Promise} + */ +export async function getCloud() { + if (typeof detectedCloud === "string") { + return detectedCloud; + } + + if (await isAws()) { + return "aws"; + } + + if (await isGoogleCloud()) { + return "google"; + } +} + +/** + * @param {string | Record} name + * @param {Cloud} [cloud] + * @returns {Promise} + */ +export async function getCloudMetadata(name, cloud) { + cloud ??= await getCloud(); + if (!cloud) { + return; + } + + if (typeof name === "object") { + name = name[cloud]; + } + + let url; + let headers; + if (cloud === "aws") { + url = new URL(name, "http://169.254.169.254/latest/meta-data/"); + } else if (cloud === "google") { + url = new URL(name, "http://metadata.google.internal/computeMetadata/v1/instance/"); + headers = { "Metadata-Flavor": "Google" }; + } else { + throw new Error(`Unsupported cloud: ${inspect(cloud)}`); + } + + const { error, body } = await curl(url, { headers, retries: 0 }); + if (error) { + return; + } + + return body.trim(); +} + +/** + * @param {string} tag + * @param {Cloud} [cloud] + * @returns {Promise} + */ +export function getCloudMetadataTag(tag, cloud) { + const metadata = { + "aws": `tags/instance/${tag}`, + }; + + return getCloudMetadata(metadata, cloud); +} + +/** + * @param {string} name + * @returns {Promise} + */ +export async function getBuildMetadata(name) { + if (isBuildkite) { + const { error, stdout } = await spawn(["buildkite-agent", "meta-data", "get", name]); + if (!error) { + const value = stdout.trim(); + if (value) { + return value; + } + } + } +} + +/** + * @typedef ConnectOptions + * @property {string} hostname + * @property {number} port + * @property {number} [retries] + */ + +/** + * @param {ConnectOptions} options + * @returns {Promise} + */ +export async function waitForPort(options) { + const { hostname, port, retries = 10 } = options; + + let cause; + for (let i = 0; i < retries; i++) { + if (cause) { + await new Promise(resolve => setTimeout(resolve, Math.pow(2, i) * 1000)); + } + + const connected = new Promise((resolve, reject) => { + const socket = connect({ host: hostname, port }); + socket.on("connect", () => { + socket.destroy(); + resolve(); + }); + socket.on("error", error => { + socket.destroy(); + reject(error); + }); + }); + + try { + return await connected; + } catch (error) { + cause = error; + } + } + + return cause; +} /** * @returns {Promise} */ @@ -1536,6 +2032,52 @@ export function getGithubUrl() { return new URL(getEnv("GITHUB_SERVER_URL", false) || "https://github.com"); } +/** + * @param {object} obj + * @param {number} indent + * @returns {string} + */ +export function toYaml(obj, indent = 0) { + const spaces = " ".repeat(indent); + let result = ""; + for (const [key, value] of Object.entries(obj)) { + if (value === undefined) { + continue; + } + if (value === null) { + result += `${spaces}${key}: null\n`; + continue; + } + if (Array.isArray(value)) { + result += `${spaces}${key}:\n`; + value.forEach(item => { + if (typeof item === "object" && item !== null) { + result += `${spaces}- \n${toYaml(item, indent + 2) + .split("\n") + .map(line => `${spaces} ${line}`) + .join("\n")}\n`; + } else { + result += `${spaces}- ${item}\n`; + } + }); + continue; + } + if (typeof value === "object") { + result += `${spaces}${key}:\n${toYaml(value, indent + 2)}`; + continue; + } + if ( + typeof value === "string" && + (value.includes(":") || value.includes("#") || value.includes("'") || value.includes('"') || value.includes("\n")) + ) { + result += `${spaces}${key}: "${value.replace(/"/g, '\\"')}"\n`; + continue; + } + result += `${spaces}${key}: ${value}\n`; + } + return result; +} + /** * @param {string} title * @param {function} [fn] @@ -1575,11 +2117,13 @@ export function printEnvironment() { startGroup("Machine", () => { console.log("Operating System:", getOs()); console.log("Architecture:", getArch()); + console.log("Kernel:", getKernel()); if (isLinux) { console.log("ABI:", getAbi()); + console.log("ABI Version:", getAbiVersion()); } console.log("Distro:", getDistro()); - console.log("Release:", getDistroRelease()); + console.log("Distro Version:", getDistroVersion()); console.log("Hostname:", getHostname()); if (isCI) { console.log("Tailscale IP:", getTailscaleIp()); diff --git a/src/cli/install.sh b/src/cli/install.sh index 08a0817f6d..f32e073258 100644 --- a/src/cli/install.sh +++ b/src/cli/install.sh @@ -78,6 +78,14 @@ case $platform in ;; esac +case "$target" in +'linux'*) + if [ -f /etc/alpine-release ]; then + target="$target-musl" + fi + ;; +esac + if [[ $target = darwin-x64 ]]; then # Is this process running in Rosetta? # redirect stderr to devnull to avoid error message when not running in Rosetta @@ -91,19 +99,20 @@ GITHUB=${GITHUB-"https://github.com"} github_repo="$GITHUB/oven-sh/bun" -if [[ $target = darwin-x64 ]]; then - # If AVX2 isn't supported, use the -baseline build +# If AVX2 isn't supported, use the -baseline build +case "$target" in +'darwin-x64'*) if [[ $(sysctl -a | grep machdep.cpu | grep AVX2) == '' ]]; then - target=darwin-x64-baseline + target="$target-baseline" fi -fi - -if [[ $target = linux-x64 ]]; then + ;; +'linux-x64'*) # If AVX2 isn't supported, use the -baseline build if [[ $(cat /proc/cpuinfo | grep avx2) = '' ]]; then - target=linux-x64-baseline + target="$target-baseline" fi -fi + ;; +esac exe_name=bun