#!/usr/bin/env node // An agent that starts buildkite-agent and runs others services. import { parseArgs } from "node:util"; import { join, relative } from "node:path"; import { existsSync, readdirSync, realpathSync } from "node:fs"; import { getEnv, getUser, getUsername, parseOs, readFile, startGroup } from "./utils.mjs"; import { rm, mkdir, writeFile, mkdtemp, which, spawn, spawnSafe, spawnSsh, spawnSshSafe, spawnScp } from "./utils.mjs"; import { isPosix, isLinux, isMacOS, isWindows, getCloud, getCloudMetadataTag } from "./utils.mjs"; import { getOs, getArch, getKernel, getAbi, getAbiVersion, getDistro, getDistroVersion } from "./utils.mjs"; /** * @returns {string} */ function getAgentName() { let name = `${getOs()}-${getArch()}`; if (isLinux) { name += `-${getDistro()}-${getDistroVersion()}`; } else { name += `-${getDistroVersion()}`; } return name; } /** * @typedef {Object} AgentLocation * @property {string} agentPath * @property {string} homePath * @property {string} configPath * @property {string} cachePath * @property {string} logsPath * @property {string} agentLogPath * @property {string} tmpPath */ /** * @param {"windows" | "linux" | "darwin"} [os] * @param {string} [username] * @returns {AgentLocation} */ function getAgentLocation(os = getOs(), username = getUsername()) { const agentPath = which("buildkite-agent", { required: true }); if (os === "windows") { const homePath = "C:\\buildkite-agent"; return { agentPath, homePath, configPath: join(homePath, "buildkite-agent.cfg"), cachePath: join(homePath, "cache"), logsPath: join(homePath, "logs"), agentLogPath: join(homePath, "logs", "buildkite-agent.log"), tmpPath: "C:\\Windows\\TEMP", }; } if (os === "darwin") { const userPath = `/Users/${username === "root" ? "administrator" : username}`; return { agentPath, // FIXME: Library/Application Support/buildkite-agent // causes issues with the space in the path, fix this later. homePath: join(userPath, "Library/Services/buildkite-agent"), configPath: join(userPath, "Library/Preferences/buildkite-agent.cfg"), cachePath: join(userPath, "Library/Caches/buildkite-agent"), logsPath: join(userPath, "Library/Logs/buildkite-agent"), agentLogPath: join(userPath, "Library/Logs/buildkite-agent/buildkite-agent.log"), tmpPath: "/tmp", }; } return { agentPath, homePath: "/var/lib/buildkite-agent", configPath: "/etc/buildkite-agent/buildkite-agent.cfg", cachePath: "/var/cache/buildkite-agent", logsPath: "/var/log/buildkite-agent", agentLogPath: "/var/log/buildkite-agent/buildkite-agent.log", tmpPath: "/tmp", }; } /** * @param {Record} [options] * @returns {string} */ function getAgentConfig(options = {}) { const lines = Object.entries(options).map(([key, value]) => `${key}=${escape(value)}`); return `# Generated by scripts/agent.mjs # https://buildkite.com/docs/agent/v3/configuration ${lines.join("\n")} `; } /** * @param {AgentLocation} location * @returns {Promise} * @link https://buildkite.com/docs/agent/v3/cli-start */ async function getAgentCommand(location) { const { agentPath, homePath, configPath, cachePath, logsPath } = location; const cloud = getCloud(); const command = [agentPath, "start"]; let name = getAgentName(); if (existsSync(configPath)) { command.push("--config", configPath); if (readFile(configPath).includes("spawn=")) { name += "-%spawn"; } } command.push("--name", name); // If the agent token is set, use it. // If this is not set, the agent will fail to start. const agentToken = getEnv("BUILDKITE_AGENT_TOKEN", false); if (agentToken) { command.push("--token", agentToken); } else if (cloud) { const agentToken = await getCloudMetadataTag("buildkite:token"); if (agentToken) { command.push("--token", agentToken); } } // For ephemeral agents, they can be assigned a specific job ID to run. // This prevents them from being assigned to other jobs. let ephemeral = false; if (cloud) { const jobId = await getCloudMetadataTag("buildkite:job-uuid"); if (jobId) { command.push("--disconnect-after-job", "--acquire-job", jobId); ephemeral = true; } } // If the agent is ephemeral, add extra flags to speed up the agent. if (ephemeral) { command.push("--git-clone-flags", "-v --depth=1"); command.push("--git-fetch-flags", "-v --prune --depth=1"); } // On Windows, use Command Prompt, since it's much faster on startup // and it propogates the exit code of the command, which PowerShell does not. // On macOS and Linux, use plain sh -l so ~/.profile is sourced. if (isWindows) { const cmd = which("cmd", { required: true }); command.push("--shell", `"${cmd}" /S /C`); } else { const sh = which("sh", { required: true }); command.push("--shell", `${sh} -elc`); } // Ensure that paths are set correctly. command.push("--job-log-path", logsPath); command.push("--build-path", join(homePath, "builds")); command.push("--hooks-path", join(homePath, "hooks")); command.push("--plugins-path", join(homePath, "plugins")); if (!ephemeral) { command.push("--git-mirrors-path", join(cachePath, "git")); } // Enable various feature flags that are not required, but are useful. command.push( "--enable-job-log-tmpfile", "--no-feature-reporting", "--experiment", "normalised-upload-paths,resolve-commit-after-checkout,agent-api", ); // Define the tags that will be used to identify the agent. // Steps can use these tags to specify which agent to run on. const tags = { "os": getOs(), "arch": getArch(), "posix": isPosix, "windows": isWindows, "kernel": getKernel(), "abi": getAbi(), "abi-version": getAbiVersion(), "distro": getDistro(), "distro-version": getDistroVersion(), "cloud": getCloud(), "ephemeral": ephemeral, // Defined for legacy reasons. "release": parseInt(getDistroVersion()) || undefined, }; // Steps add these tags to tell robobun that it should create an agent. // If that is the case, these tags need to be added to the metadata. if (cloud) { const extraTags = ["robobun", "robobun2"]; for (const tag of extraTags) { const value = await getCloudMetadataTag(tag); if (typeof value === "string") { metadata[tag] = value; } } } // Add the tags to the command. command.push( "--tags", Object.entries(tags) .filter(([, value]) => typeof value !== "undefined" && value !== null) .map(([key, value]) => `${key}=${value}`) .join(","), ); return command; } /** * @param {ServiceType} [type] * @returns {Service} */ export function getAgentService(type) { return getService("buildkite-agent", type); } /** * @param {AgentLocation} location * @param {ServiceType} [type] * @returns {Promise} */ export async function createAgentService(location, type) { const service = getAgentService(type); const { type: serviceType } = service; const { homePath, logsPath, agentLogPath, configPath } = location; // Instead of running the agentPath directly, call this script with the "exec" arguments. // This allows the agent command to be generated on each startup, instead of at install time, // which is important because tags can change (for example, a macOS machine upgraded to a new release). const { execPath, argv } = process; const scriptPath = realpathSync(argv[1]); const extraArgs = argv.slice(2); const args = [scriptPath, "exec", ...extraArgs]; if (serviceType === "openrc") { const pidPath = join(logsPath, "buildkite-agent.pid"); const serviceConfig = `#!/sbin/openrc-run name="buildkite-agent" description="Buildkite Agent" command=${escape(execPath)} command_args=${escape(args.map(escape).join(" "))} command_user=buildkite-agent pidfile=${escape(pidPath)} start_stop_daemon_args=" \\ --background \\ --make-pidfile \\ --stdout ${escape(agentLogPath)} \\ --stderr ${escape(agentLogPath)}" depend() { need net use dns logger } `; await service.install(serviceConfig); } if (serviceType === "systemd") { const serviceConfig = ` [Unit] Description=Buildkite Agent After=syslog.target After=network-online.target [Service] Type=simple User=buildkite-agent ExecStart=${escape(execPath)} ${args.map(escape).join(" ")} RestartSec=5 Restart=on-failure KillMode=process [Journal] Storage=persistent StateDirectory=${escape(agentLogPath)} [Install] WantedBy=multi-user.target `; await service.install(serviceConfig); } if (serviceType === "nssm") { const serviceConfig = { command: [execPath, ...args], options: { "AppDirectory": homePath, "AppStdout": agentLogPath, "AppStderr": agentLogPath, }, }; await service.install(serviceConfig); } if (serviceType === "plist") { const username = getUsername(); const serviceConfig = ` Label buildkite-agent UserName ${username === "root" ? "administrator" : username} EnvironmentVariables PATH ${process.env.PATH} KeepAlive SuccessfulExit ProcessType Interactive ProgramArguments ${execPath} ${scriptPath} exec RunAtLoad StandardErrorPath ${agentLogPath} StandardOutPath ${agentLogPath} WorkingDirectory ${homePath} WatchPaths ${configPath} `; await service.install(serviceConfig); } return service; } /** * @param {AgentLocation} location */ export async function startAgent(location) { const command = await getAgentCommand(location); await spawnSafe(command, { stdio: "inherit" }); } /** * @param {AgentLocation} location */ export async function cleanAgent(location) { const { homePath, cachePath, tmpPath } = location; const buildPath = join(homePath, "builds"); // Remove the items in the directory, but not the directory itself. for (const parentPath of [buildPath, cachePath, tmpPath]) { try { const entries = readdirSync(parentPath, { encoding: "utf8" }); for (const entry of entries) { const entryPath = join(parentPath, entry); try { rm(entryPath); } catch (error) { console.error(`Failed to remove ${entryPath}:`, error); } } } catch (error) { console.error(`Failed to clean ${parentPath}:`, error); } } } /** * @returns {Promise} */ async function getRebootService() { if (!isMacOS) { return; } // Reboot every morning at 6am PT. // Randomize the minute of the hour to avoid thundering herd. const hourOfDay = 6; const minuteOfHour = Math.floor(Math.random() * 60); const service = getService("reboot", "plist"); const serviceConfig = ` Label reboot ProgramArguments /sbin/shutdown -r now StartCalendarInterval Hour ${hourOfDay} Minute ${minuteOfHour} `; await service.install(serviceConfig); await service.enable(); return service; } /** * @typedef {"systemd" | "openrc" | "nssm" | "plist"} ServiceType */ /** * @typedef {Object} Service * @property {string} name * @property {ServiceType} type * @property {(config: string) => Promise} install * @property {() => Promise} status * @property {() => Promise} enable * @property {() => Promise} disable * @property {() => Promise} start * @property {() => Promise} stop * @property {() => Promise} restart * @property {() => Promise} uninstall */ /** * @param {string} name * @param {ServiceType} [type] * @returns {Service} */ function getService(name, type) { const spawnOptions = { stdio: "inherit", privileged: true }; // https://docs.alpinelinux.org/user-handbook/0.1a/Working/openrc.html if (type === "openrc" || getDistro() === "alpine") { const rcService = which("rc-service", { required: true }); const rcUpdate = which("rc-update", { required: true }); return { name, type: "openrc", async install(config) { const servicePath = `/etc/init.d/${name}`; writeFile(servicePath, config, { mode: 0o755 }); }, async status() { await spawnSafe([rcService, "status", name], spawnOptions); }, async enable() { await spawnSafe([rcUpdate, "add", name, "default"], spawnOptions); }, async disable() { await spawnSafe([rcUpdate, "delete", name, "default"], spawnOptions); }, async start() { await spawnSafe([rcService, "start", name], spawnOptions); }, async stop() { await spawnSafe([rcService, "stop", name], spawnOptions); }, async restart() { await spawnSafe([rcService, "restart", name], spawnOptions); }, async uninstall() { await spawnSafe([rcService, "remove", name], spawnOptions); }, }; } if (type === "systemd" || isLinux) { const systemctl = which("systemctl", { required: true }); return { name, type: "systemd", async install(config) { const servicePath = `/etc/systemd/system/${name}.service`; writeFile(servicePath, config, { mode: 0o644 }); await spawnSafe([systemctl, "daemon-reload"], spawnOptions); await spawnSafe([systemctl, "enable", name], spawnOptions); }, async status() { await spawnSafe([systemctl, "status", name], spawnOptions); }, async enable() { await spawnSafe([systemctl, "enable", name], spawnOptions); }, async disable() { await spawnSafe([systemctl, "disable", name], spawnOptions); }, async start() { await spawnSafe([systemctl, "start", name], spawnOptions); }, async stop() { await spawnSafe([systemctl, "stop", name], spawnOptions); }, async restart() { await spawnSafe([systemctl, "restart", name], spawnOptions); }, async uninstall() { await spawnSafe([systemctl, "disable", name], spawnOptions); await spawnSafe([systemctl, "stop", name], spawnOptions); await spawnSafe([systemctl, "remove", name], spawnOptions); }, }; } // https://nssm.cc/commands if (type === "nssm" || isWindows) { const nssm = which("nssm", { required: true }); return { name, type: "nssm", async install(config) { const { command, options } = config; await spawnSafe([nssm, "install", name, ...command], spawnOptions); if (options) { for (const [key, value] of Object.entries(options)) { await spawnSafe([nssm, "set", name, key, value], spawnOptions); } } }, async status() { await spawnSafe([nssm, "get", name, "State"], spawnOptions); }, async enable() { await spawnSafe([nssm, "set", name, "Start", "SERVICE_AUTO_START"], spawnOptions); }, async disable() { await spawnSafe([nssm, "set", name, "Start", "SERVICE_DISABLED"], spawnOptions); }, async start() { await spawnSafe([nssm, "start", name], spawnOptions); }, async stop() { await spawnSafe([nssm, "stop", name], spawnOptions); }, async restart() { await spawnSafe([nssm, "restart", name], spawnOptions); }, async uninstall() { await spawnSafe([nssm, "remove", name, "confirm"], spawnOptions); }, }; } // https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPSystemStartup/Chapters/Introduction.html if (type === "plist" || isMacOS) { const launchctl = which("launchctl", { required: true }); const serviceId = `system/${name}`; return { name, type: "plist", async install(config) { const servicePath = `/Library/LaunchDaemons/${name}.plist`; writeFile(servicePath, config, { mode: 0o644 }); await spawnSafe(["chown", "root:wheel", servicePath], spawnOptions); const plutil = which("plutil"); if (plutil) { await spawnSafe([plutil, "-lint", servicePath], spawnOptions); } // For some reason, it must be unloaded before it can be loaded, otherwise: // Load failed: 5: Input/output error await spawnSafe([launchctl, "unload", servicePath], spawnOptions); const { error, exitCode } = await spawn([launchctl, "load", servicePath], spawnOptions); if (!(exitCode === 0 || exitCode === 3)) { throw error; } }, async status() { await spawnSafe([launchctl, "print", serviceId], spawnOptions); }, async enable() { await spawnSafe([launchctl, "enable", serviceId], spawnOptions); }, async disable() { await spawnSafe([launchctl, "disable", serviceId], spawnOptions); }, async start() { const { error, exitCode } = await spawn([launchctl, "start", serviceId], spawnOptions); if (!(exitCode === 0 || exitCode === 3)) { throw error; } }, async stop() { await spawnSafe([launchctl, "stop", serviceId], spawnOptions); }, async restart() { await spawnSafe([launchctl, "stop", serviceId], spawnOptions); await spawnSafe([launchctl, "start", serviceId], spawnOptions); }, async uninstall() { await spawnSafe([launchctl, "unload", serviceId], spawnOptions); }, }; } throw new Error(`Unsupported service type: ${type}`); } /** * @param {string} string * @returns {string} */ function escape(string) { return JSON.stringify(string); } /** * @param {import("./utils.mjs").SshOptions} sshOptions * @returns {Promise<"windows" | "linux" | "darwin">} */ async function getOsSsh(sshOptions) { try { const { error: unameError, stdout: uname } = await spawnSsh({ ...sshOptions, command: ["uname", "-s"] }); if (!unameError) { return parseOs(uname); } const { error: cmdError, stdout: cmd } = await spawnSsh({ ...sshOptions, command: ["cmd", "/c", "ver"] }); if (!cmdError) { return parseOs(cmd); } throw unameError || cmdError; } catch (cause) { const { hostname } = sshOptions; throw new Error(`Failed to determine the machine's platform: ${hostname}`, { cause }); } } /** * @param {import("./utils.mjs").SshOptions} sshOptions * @returns {Promise} */ async function getSudoCommand(sshOptions) { const { password } = sshOptions; const { exitCode } = await spawnSsh({ ...sshOptions, command: ["sudo", "echo", "1"] }); if (exitCode === 0) { return ["sudo"]; } if (password) { const { exitCode } = await spawnSsh({ ...sshOptions, command: [ "sh", "-c", `echo '${password}' | sudo -S sh -c 'echo \"%admin ALL=(ALL) NOPASSWD: ALL\" | tee -a /etc/sudoers.d/nopasswd'`, ], }); if (exitCode === 0) { return ["sudo"]; } } return []; } /** * @param {import("./utils.mjs").SshOptions} sshOptions * @returns {Promise} */ async function uploadBootstrap(sshOptions) { const os = await getOsSsh(sshOptions); const { tmpPath } = getAgentLocation(os); const filename = os === "windows" ? "bootstrap.ps1" : "bootstrap.sh"; const scriptPath = join(import.meta.dirname, filename); const bootstrapPath = join(tmpPath, filename); await spawnScp({ ...sshOptions, source: scriptPath, destination: bootstrapPath, }); if (os === "windows") { return ["powershell", "-ExecutionPolicy", "Bypass", "-File", bootstrapPath, "-CI"]; } const _ = await getSudoCommand(sshOptions); return ["sh", bootstrapPath, "--ci"]; } /** * @param {import("./utils.mjs").SshOptions} sshOptions * @returns {Promise} */ async function uploadScript(sshOptions) { const os = await getOsSsh(sshOptions); const runnerPath = which(["bunx", "npx"], { required: true }); const scriptPath = realpathSync(process.argv[1]); const localTmpScriptPath = mkdtemp("agent-", "agent.mjs"); await spawnSafe([ runnerPath, "esbuild", scriptPath, "--bundle", "--platform=node", "--format=esm", `--outfile=${localTmpScriptPath}`, ]); const { username } = sshOptions; const location = getAgentLocation(os, username); const { tmpPath, homePath } = location; const tmpScriptPath = join(tmpPath, "agent.mjs"); await spawnScp({ ...sshOptions, source: localTmpScriptPath, destination: tmpScriptPath, }); const command = []; if (os !== "windows") { const sudoCommand = await getSudoCommand(sshOptions); command.push(...sudoCommand); } await spawnSshSafe({ ...sshOptions, command: [...command, "mkdir", "-p", homePath], }); const agentScriptPath = join(homePath, "agent.mjs"); await spawnSshSafe({ ...sshOptions, command: [...command, "cp", tmpScriptPath, agentScriptPath], }); const { stdout: nodeStdout } = await spawnSsh({ ...sshOptions, command: ["node", "-v"] }); const nodeVersion = parseInt(nodeStdout.trim().replace(/^v/, "")); if (isNaN(nodeVersion) || nodeVersion < 20) { command.push("bun"); } else { command.push("node"); } return [...command, agentScriptPath]; } async function main() { const { positionals: args, values: options } = parseArgs({ allowPositionals: true, options: { "hostname": { type: "string", multiple: true }, "username": { type: "string" }, "password": { type: "string" }, "bootstrap": { type: "boolean" }, "token": { type: "string" }, "queue": { type: "string" }, "spawn": { type: "string" }, "tailscale-authkey": { type: "string" }, }, }); if (!args.length) { const scriptPath = relative(process.cwd(), process.argv[1]); console.error(`Usage: ${scriptPath} [install|enable|disable|start|stop|restart|uninstall]`); console.error(); console.error(`Options:`); console.error(` --hostname=string: The hostname of the machine to connect to (can specify multiple).`); console.error(` --username=string: The username of the machine to connect to.`); console.error(` --password=string: The password of the machine to connect to.`); console.error(` --bootstrap: If true, run the bootstrap script on the machine.`); console.error(` --token=string: The Buildkite token to use.`); console.error(` --queue=string: The name of the Buildkite queue to use (e.g. "build-zig").`); console.error(` --spawn=number: The number of agents to run on the machine (default: 1).`); console.error(` --tailscale-authkey=string: The Tailscale authkey to use.`); console.error(); console.error(`Examples:`); console.error(`1. Install the agent on the local machine:`); console.error(` ${scriptPath} install`); console.error(); console.error(`2. Install the agent on a remote machine:`); console.error(` ${scriptPath} --hostname=127.0.0.1 --username=admin install`); console.error(); console.error(`3. Start the agent on a remote machine:`); console.error(` ${scriptPath} --hostname=127.0.0.1 --username=admin start`); console.error(); console.error(`4. Stop the agent on a remote machine (with a password):`); console.error(` ${scriptPath} --hostname=127.0.0.1 --username=administrator --password=admin stop`); process.exit(1); } const { hostname: hostnames, username, password, bootstrap, ...agentOptions } = options; // When a hostname is defined, connect to the machine, // bundle and upload this script, then run it on that machine. if (hostnames?.length) { for (const hostname of hostnames) { const sshOptions = { hostname, username, password }; if (bootstrap) { await startGroup(`Running bootstrap script on ${hostname}...`, async () => { const command = await uploadBootstrap(sshOptions); await spawnSshSafe( { ...sshOptions, command, }, { stdio: "inherit", }, ); }); } await startGroup(`Uploading script to ${hostname}...`, async () => { const command = await uploadScript(sshOptions); const commandWithoutSsh = [ ...command, ...args, ...Object.entries(agentOptions).map(([key, value]) => `--${key}=${value}`), ]; await spawnSshSafe( { ...sshOptions, command: commandWithoutSsh, }, { stdio: "inherit", }, ); }); } // Return after running the script on the remote machine. return; } const { uid, gid } = getUser(); const fileOptions = { mode: 0o777, uid, gid }; const location = getAgentLocation(); const { "tailscale-authkey": tailscaleAuthkey, ...agentConfig } = agentOptions; const { tmpPath, agentPath, configPath, agentLogPath, ...locationPaths } = location; if (args.includes("clean") || args.includes("exec")) { await startGroup("Cleaning agent...", async () => { await cleanAgent(location); }); } if (args.includes("exec")) { return startGroup("Running agent...", async () => { await startAgent(location); }); } if (args.includes("install")) { for (const path of Object.values(locationPaths)) { await startGroup(`Creating directory: ${path}`, async () => { mkdir(path, fileOptions); }); } if (!existsSync(configPath) || Object.values(agentConfig).some(Boolean)) { await startGroup(`Creating config: ${configPath}`, async () => { const configFile = getAgentConfig(agentConfig); writeFile(configPath, configFile, fileOptions); }); } if (tailscaleAuthkey) { await startGroup("Setting up Tailscale...", async () => { const tailscale = which("tailscale", { required: true }); const hostname = getAgentName(); await spawnSafe( [tailscale, "up", "--accept-risk=all", "--ssh", "--hostname", hostname, "--auth-key", tailscaleAuthkey], { stdio: "inherit", }, ); }); } await startGroup("Installing agent...", async () => { const service = await createAgentService(location); const { name, type } = service; console.log(`Created agent: ${name} (${type})`); }); if (isMacOS) { await startGroup("Installing reboot service...", async () => { const rebootService = await getRebootService(); if (rebootService) { const { name, type } = rebootService; console.log(`Created service: ${name} (${type})`); } }); } } else { const service = getAgentService(); const { name, type } = service; console.log(`Found agent: ${name} (${type})`); for (const arg of args) { startGroup(`Running command: ${arg}...`, async () => { if (service[arg]) { await service[arg](); } else { throw new Error(`Unsupported command: ${arg}`); } }); } } } await main();