ci: Expand automated build images to Debian, Ubuntu, and Amazon Linux (#15250)

This commit is contained in:
Ashcon Partovi
2024-11-19 18:31:15 -08:00
committed by GitHub
parent ff4eccc3b4
commit 708ed00705
7 changed files with 629 additions and 116 deletions

View File

@@ -234,8 +234,8 @@ function getPipeline(options) {
* @returns {boolean}
*/
const isUsingNewAgent = platform => {
const { os, distro } = platform;
if (os === "linux" && distro === "alpine") {
const { os } = platform;
if (os === "linux") {
return true;
}
return false;
@@ -359,6 +359,21 @@ function getPipeline(options) {
* @link https://buildkite.com/docs/pipelines/command-step
*/
/**
* @param {Platform} platform
* @param {string} [step]
* @returns {string[]}
*/
const getDependsOn = (platform, step) => {
if (imagePlatforms.has(getImageKey(platform))) {
const key = `${getImageKey(platform)}-build-image`;
if (key !== step) {
return [key];
}
}
return [];
};
/**
* @param {Platform} platform
* @returns {Step}
@@ -375,81 +390,85 @@ function getPipeline(options) {
env: {
DEBUG: "1",
},
retry: getRetry(),
command: `node ./scripts/machine.mjs ${action} --ci --cloud=aws --os=${os} --arch=${arch} --distro=${distro} --distro-version=${release}`,
};
};
/**
* @param {Target} target
* @param {Platform} platform
* @returns {Step}
*/
const getBuildVendorStep = target => {
const getBuildVendorStep = platform => {
return {
key: `${getTargetKey(target)}-build-vendor`,
label: `${getTargetLabel(target)} - build-vendor`,
agents: getBuildAgent(target),
key: `${getTargetKey(platform)}-build-vendor`,
label: `${getTargetLabel(platform)} - build-vendor`,
depends_on: getDependsOn(platform),
agents: getBuildAgent(platform),
retry: getRetry(),
cancel_on_build_failing: isMergeQueue(),
env: getBuildEnv(target),
env: getBuildEnv(platform),
command: "bun run build:ci --target dependencies",
};
};
/**
* @param {Target} target
* @param {Platform} platform
* @returns {Step}
*/
const getBuildCppStep = target => {
const getBuildCppStep = platform => {
return {
key: `${getTargetKey(target)}-build-cpp`,
label: `${getTargetLabel(target)} - build-cpp`,
agents: getBuildAgent(target),
key: `${getTargetKey(platform)}-build-cpp`,
label: `${getTargetLabel(platform)} - build-cpp`,
depends_on: getDependsOn(platform),
agents: getBuildAgent(platform),
retry: getRetry(),
cancel_on_build_failing: isMergeQueue(),
env: {
BUN_CPP_ONLY: "ON",
...getBuildEnv(target),
...getBuildEnv(platform),
},
command: "bun run build:ci --target bun",
};
};
/**
* @param {Target} target
* @param {Platform} platform
* @returns {Step}
*/
const getBuildZigStep = target => {
const toolchain = getBuildToolchain(target);
const getBuildZigStep = platform => {
const toolchain = getBuildToolchain(platform);
return {
key: `${getTargetKey(target)}-build-zig`,
label: `${getTargetLabel(target)} - build-zig`,
agents: getZigAgent(target),
key: `${getTargetKey(platform)}-build-zig`,
label: `${getTargetLabel(platform)} - build-zig`,
depends_on: getDependsOn(platform),
agents: getZigAgent(platform),
retry: getRetry(1), // FIXME: Sometimes zig build hangs, so we need to retry once
cancel_on_build_failing: isMergeQueue(),
env: getBuildEnv(target),
env: getBuildEnv(platform),
command: `bun run build:ci --target bun-zig --toolchain ${toolchain}`,
};
};
/**
* @param {Target} target
* @param {Platform} platform
* @returns {Step}
*/
const getBuildBunStep = target => {
const getBuildBunStep = platform => {
return {
key: `${getTargetKey(target)}-build-bun`,
label: `${getTargetLabel(target)} - build-bun`,
key: `${getTargetKey(platform)}-build-bun`,
label: `${getTargetLabel(platform)} - build-bun`,
depends_on: [
`${getTargetKey(target)}-build-vendor`,
`${getTargetKey(target)}-build-cpp`,
`${getTargetKey(target)}-build-zig`,
`${getTargetKey(platform)}-build-vendor`,
`${getTargetKey(platform)}-build-cpp`,
`${getTargetKey(platform)}-build-zig`,
],
agents: getBuildAgent(target),
agents: getBuildAgent(platform),
retry: getRetry(),
cancel_on_build_failing: isMergeQueue(),
env: {
BUN_LINK_ONLY: "ON",
...getBuildEnv(target),
...getBuildEnv(platform),
},
command: "bun run build:ci --target bun",
};
@@ -473,8 +492,8 @@ function getPipeline(options) {
} else {
parallelism = 10;
}
let depends;
let env;
let depends = [];
if (buildId) {
env = {
BUILDKITE_ARTIFACT_BUILD_ID: buildId,
@@ -488,14 +507,20 @@ function getPipeline(options) {
// Because of this, we don't know if the run was fatal, or soft-failed.
retry = getRetry(1);
}
let soft_fail;
if (isMainBranch()) {
soft_fail = true;
} else {
soft_fail = [{ exit_status: 2 }];
}
return {
key: `${getPlatformKey(platform)}-test-bun`,
label: `${getPlatformLabel(platform)} - test-bun`,
depends_on: depends,
depends_on: [...depends, ...getDependsOn(platform)],
agents: getTestAgent(platform),
retry,
cancel_on_build_failing: isMergeQueue(),
soft_fail: isMainBranch(),
soft_fail,
parallelism,
command,
env,
@@ -531,14 +556,14 @@ function getPipeline(options) {
{ os: "darwin", arch: "x64", release: "14" },
{ os: "darwin", arch: "x64", release: "13" },
{ os: "linux", arch: "aarch64", distro: "debian", release: "12" },
// { os: "linux", arch: "aarch64", distro: "debian", release: "11" },
// { os: "linux", arch: "aarch64", distro: "debian", release: "10" },
{ os: "linux", arch: "aarch64", distro: "debian", release: "11" },
{ os: "linux", arch: "aarch64", distro: "debian", release: "10" },
{ os: "linux", arch: "x64", distro: "debian", release: "12" },
// { os: "linux", arch: "x64", distro: "debian", release: "11" },
// { os: "linux", arch: "x64", distro: "debian", release: "10" },
{ os: "linux", arch: "x64", distro: "debian", release: "11" },
{ os: "linux", arch: "x64", distro: "debian", release: "10" },
{ os: "linux", arch: "x64", baseline: true, distro: "debian", release: "12" },
// { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" },
// { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "10" },
{ os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" },
{ os: "linux", arch: "x64", baseline: true, distro: "debian", release: "10" },
// { os: "linux", arch: "aarch64", distro: "ubuntu", release: "24.04" },
{ os: "linux", arch: "aarch64", distro: "ubuntu", release: "22.04" },
{ os: "linux", arch: "aarch64", distro: "ubuntu", release: "20.04" },
@@ -548,11 +573,11 @@ function getPipeline(options) {
// { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "24.04" },
{ os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "22.04" },
{ os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "20.04" },
// { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023" },
// { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2" },
// { os: "linux", arch: "x64", distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "x64", distro: "amazonlinux", release: "2023" },
// { os: "linux", arch: "x64", distro: "amazonlinux", release: "2" },
// { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023" },
// { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2" },
{ os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" },
// { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.17" },
@@ -615,15 +640,6 @@ function getPipeline(options) {
continue;
}
if (imagePlatforms.has(getImageKey(platform))) {
for (const step of platformSteps) {
if (step.agents?.["image-name"]) {
step.depends_on ??= [];
step.depends_on.push(`${getImageKey(platform)}-build-image`);
}
}
}
steps.push({
key: getTargetKey(platform),
group: getTargetLabel(platform),

View File

@@ -32,7 +32,10 @@ async function doBuildkiteAgent(action) {
let homePath, cachePath, logsPath, agentLogPath, pidPath;
if (isWindows) {
throw new Error("TODO: Windows");
homePath = "C:\\buildkite-agent";
cachePath = join(homePath, "cache");
logsPath = join(homePath, "logs");
agentLogPath = join(logsPath, "buildkite-agent.log");
} else {
homePath = "/var/lib/buildkite-agent";
cachePath = "/var/cache/buildkite-agent";
@@ -45,6 +48,19 @@ async function doBuildkiteAgent(action) {
const command = process.execPath;
const args = [realpathSync(process.argv[1]), "start"];
if (isWindows) {
const serviceCommand = [
"New-Service",
"-Name",
"buildkite-agent",
"-StartupType",
"Automatic",
"-BinaryPathName",
`${escape(command)} ${escape(args.map(escape).join(" "))}`,
];
await spawnSafe(["powershell", "-Command", serviceCommand.join(" ")], { stdio: "inherit" });
}
if (isOpenRc()) {
const servicePath = "/etc/init.d/buildkite-agent";
const service = `#!/sbin/openrc-run
@@ -81,7 +97,7 @@ async function doBuildkiteAgent(action) {
[Service]
Type=simple
User=${username}
ExecStart=${escape(command)} ${escape(args.map(escape).join(" "))}
ExecStart=${escape(command)} ${args.map(escape).join(" ")}
RestartSec=5
Restart=on-failure
KillMode=process

339
scripts/bootstrap.ps1 Executable file
View File

@@ -0,0 +1,339 @@
# Version: 4
# A powershell script that installs the dependencies needed to build and test Bun.
# This should work on Windows 10 or newer.
# If this script does not work on your machine, please open an issue:
# https://github.com/oven-sh/bun/issues
# If you need to make a change to this script, such as upgrading a dependency,
# increment the version comment to indicate that a new image should be built.
# Otherwise, the existing image will be retroactively updated.
param (
[Parameter(Mandatory = $false)]
[switch]$CI = $false,
[Parameter(Mandatory = $false)]
[switch]$Optimize = $CI
)
function Execute-Command {
$command = $args -join ' '
Write-Output "$ $command"
& $args[0] $args[1..$args.Length]
if ((-not $?) -or ($LASTEXITCODE -ne 0 -and $null -ne $LASTEXITCODE)) {
throw "Command failed: $command"
}
}
function Which {
param ([switch]$Required = $false)
foreach ($command in $args) {
$result = Get-Command $command -ErrorAction SilentlyContinue
if ($result -and $result.Path) {
return $result.Path
}
}
if ($Required) {
$commands = $args -join ', '
throw "Command not found: $commands"
}
}
function Install-Chocolatey {
if (Which choco) {
return
}
Write-Output "Installing Chocolatey..."
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
iex -Command ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
Refresh-Path
}
function Refresh-Path {
$paths = @(
[System.Environment]::GetEnvironmentVariable("Path", "Machine"),
[System.Environment]::GetEnvironmentVariable("Path", "User"),
[System.Environment]::GetEnvironmentVariable("Path", "Process")
)
$uniquePaths = $paths |
Where-Object { $_ } |
ForEach-Object { $_.Split(';', [StringSplitOptions]::RemoveEmptyEntries) } |
Where-Object { $_ -and (Test-Path $_) } |
Select-Object -Unique
$env:Path = ($uniquePaths -join ';').TrimEnd(';')
if ($env:ChocolateyInstall) {
Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 -ErrorAction SilentlyContinue
}
}
function Add-To-Path {
$absolutePath = Resolve-Path $args[0]
$currentPath = [Environment]::GetEnvironmentVariable("Path", "Machine")
if ($currentPath -like "*$absolutePath*") {
return
}
$newPath = $currentPath.TrimEnd(";") + ";" + $absolutePath
if ($newPath.Length -ge 2048) {
Write-Warning "PATH is too long, removing duplicate and old entries..."
$paths = $currentPath.Split(';', [StringSplitOptions]::RemoveEmptyEntries) |
Where-Object { $_ -and (Test-Path $_) } |
Select-Object -Unique
$paths += $absolutePath
$newPath = $paths -join ';'
while ($newPath.Length -ge 2048 -and $paths.Count -gt 1) {
$paths = $paths[1..$paths.Count]
$newPath = $paths -join ';'
}
}
Write-Output "Adding $absolutePath to PATH..."
[Environment]::SetEnvironmentVariable("Path", $newPath, "Machine")
Refresh-Path
}
function Install-Package {
param (
[Parameter(Mandatory = $true, Position = 0)]
[string]$Name,
[Parameter(Mandatory = $false)]
[string]$Command = $Name,
[Parameter(Mandatory = $false)]
[string]$Version,
[Parameter(Mandatory = $false)]
[switch]$Force = $false,
[Parameter(Mandatory = $false)]
[string[]]$ExtraArgs = @()
)
if (-not $Force `
-and (Which $Command) `
-and (-not $Version -or (& $Command --version) -like "*$Version*")) {
return
}
Write-Output "Installing $Name..."
$flags = @(
"--yes",
"--accept-license",
"--no-progress",
"--force"
)
if ($Version) {
$flags += "--version=$Version"
}
Execute-Command choco install $Name @flags @ExtraArgs
Refresh-Path
}
function Install-Packages {
foreach ($package in $args) {
Install-Package -Name $package
}
}
function Install-Common-Software {
Install-Chocolatey
Install-Pwsh
Install-Git
Install-Packages curl 7zip
Install-NodeJs
Install-Bun
Install-Cygwin
if ($CI) {
Install-Tailscale
Install-Buildkite
}
}
function Install-Pwsh {
Install-Package powershell-core -Command pwsh
if ($CI) {
$shellPath = (Which pwsh -Required)
New-ItemProperty `
-Path "HKLM:\\SOFTWARE\\OpenSSH" `
-Name DefaultShell `
-Value $shellPath `
-PropertyType String `
-Force
}
}
function Install-Git {
Install-Packages git
if ($CI) {
Execute-Command git config --system --add safe.directory "*"
Execute-Command git config --system core.autocrlf false
Execute-Command git config --system core.eol lf
Execute-Command git config --system core.longpaths true
}
}
function Install-NodeJs {
Install-Package nodejs -Command node -Version "22.9.0"
}
function Install-Bun {
Install-Package bun -Version "1.1.30"
}
function Install-Cygwin {
Install-Package cygwin
Add-To-Path "C:\tools\cygwin\bin"
}
function Install-Tailscale {
Install-Package tailscale
}
function Install-Buildkite {
if (Which buildkite-agent) {
return
}
Write-Output "Installing Buildkite agent..."
$env:buildkiteAgentToken = "xxx"
iex ((New-Object System.Net.WebClient).DownloadString("https://raw.githubusercontent.com/buildkite/agent/main/install.ps1"))
Refresh-Path
}
function Install-Build-Essentials {
# Install-Visual-Studio
Install-Packages `
cmake `
make `
ninja `
ccache `
python `
golang `
nasm `
ruby `
mingw
Install-Rust
Install-Llvm
}
function Install-Visual-Studio {
$components = @(
"Microsoft.VisualStudio.Workload.NativeDesktop",
"Microsoft.VisualStudio.Component.Windows10SDK.18362",
"Microsoft.VisualStudio.Component.Windows11SDK.22000",
"Microsoft.VisualStudio.Component.Windows11Sdk.WindowsPerformanceToolkit",
"Microsoft.VisualStudio.Component.VC.ASAN", # C++ AddressSanitizer
"Microsoft.VisualStudio.Component.VC.ATL", # C++ ATL for latest v143 build tools (x86 & x64)
"Microsoft.VisualStudio.Component.VC.DiagnosticTools", # C++ Diagnostic Tools
"Microsoft.VisualStudio.Component.VC.CLI.Support", # C++/CLI support for v143 build tools (Latest)
"Microsoft.VisualStudio.Component.VC.CoreIde", # C++ core features
"Microsoft.VisualStudio.Component.VC.Redist.14.Latest" # C++ 2022 Redistributable Update
)
$arch = (Get-WmiObject Win32_Processor).Architecture
if ($arch -eq 9) {
$components += @(
"Microsoft.VisualStudio.Component.VC.Tools.x86.x64", # MSVC v143 build tools (x86 & x64)
"Microsoft.VisualStudio.Component.VC.Modules.x86.x64" # MSVC v143 C++ Modules for latest v143 build tools (x86 & x64)
)
} elseif ($arch -eq 5) {
$components += @(
"Microsoft.VisualStudio.Component.VC.Tools.ARM64", # MSVC v143 build tools (ARM64)
"Microsoft.VisualStudio.Component.UWP.VC.ARM64" # C++ Universal Windows Platform support for v143 build tools (ARM64/ARM64EC)
)
}
$packageParameters = $components | ForEach-Object { "--add $_" }
Install-Package visualstudio2022community `
-ExtraArgs "--package-parameters '--add Microsoft.VisualStudio.Workload.NativeDesktop --includeRecommended --includeOptional'"
}
function Install-Rust {
if (Which rustc) {
return
}
Write-Output "Installing Rust..."
$rustupInit = "$env:TEMP\rustup-init.exe"
(New-Object System.Net.WebClient).DownloadFile("https://win.rustup.rs/", $rustupInit)
Execute-Command $rustupInit -y
Add-To-Path "$env:USERPROFILE\.cargo\bin"
}
function Install-Llvm {
Install-Package llvm `
-Command clang-cl `
-Version "18.1.8"
Add-To-Path "C:\Program Files\LLVM\bin"
}
function Optimize-System {
Disable-Windows-Defender
Disable-Windows-Threat-Protection
Disable-Windows-Services
Disable-Power-Management
Uninstall-Windows-Defender
}
function Disable-Windows-Defender {
Write-Output "Disabling Windows Defender..."
Set-MpPreference -DisableRealtimeMonitoring $true
Add-MpPreference -ExclusionPath "C:\", "D:\"
}
function Disable-Windows-Threat-Protection {
$itemPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows Advanced Threat Protection"
if (Test-Path $itemPath) {
Write-Output "Disabling Windows Threat Protection..."
Set-ItemProperty -Path $itemPath -Name "ForceDefenderPassiveMode" -Value 1 -Type DWORD
}
}
function Uninstall-Windows-Defender {
Write-Output "Uninstalling Windows Defender..."
Uninstall-WindowsFeature -Name Windows-Defender
}
function Disable-Windows-Services {
$services = @(
"WSearch", # Windows Search
"wuauserv", # Windows Update
"DiagTrack", # Connected User Experiences and Telemetry
"dmwappushservice", # WAP Push Message Routing Service
"PcaSvc", # Program Compatibility Assistant
"SysMain" # Superfetch
)
foreach ($service in $services) {
Stop-Service $service -Force
Set-Service $service -StartupType Disabled
}
}
function Disable-Power-Management {
Write-Output "Disabling power management features..."
powercfg /setactive 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c # High performance
powercfg /change monitor-timeout-ac 0
powercfg /change monitor-timeout-dc 0
powercfg /change standby-timeout-ac 0
powercfg /change standby-timeout-dc 0
powercfg /change hibernate-timeout-ac 0
powercfg /change hibernate-timeout-dc 0
}
Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
if ($Optimize) {
Optimize-System
}
Install-Common-Software
Install-Build-Essentials

View File

@@ -1,5 +1,5 @@
#!/bin/sh
# Version: 4
# Version: 5
# A script that installs the dependencies needed to build and test Bun.
# This should work on macOS and Linux with a POSIX shell.
@@ -327,7 +327,8 @@ check_package_manager() {
print "Updating package manager..."
case "$pm" in
apt)
DEBIAN_FRONTEND=noninteractive package_manager update -y
export DEBIAN_FRONTEND=noninteractive
package_manager update -y
;;
apk)
package_manager update
@@ -373,7 +374,7 @@ package_manager() {
while ! sudo -n apt-get update -y; do
sleep 1
done
DEBIAN_FRONTEND=noninteractive execute_sudo apt-get "$@"
execute_sudo apt-get "$@"
;;
dnf)
case "$distro" in
@@ -569,6 +570,22 @@ install_nodejs() {
install_packages nodejs
;;
esac
# Some distros do not install the node headers by default.
# These are needed for certain FFI tests, such as: `cc.test.ts`
case "$distro" in
alpine | amzn)
install_nodejs_headers
;;
esac
}
install_nodejs_headers() {
headers_tar="$(download_file "https://nodejs.org/download/release/v$(nodejs_version_exact)/node-v$(nodejs_version_exact)-headers.tar.gz")"
headers_dir="$(dirname "$headers_tar")"
execute tar -xzf "$headers_tar" -C "$headers_dir"
headers_include="$headers_dir/node-v$(nodejs_version_exact)/include"
execute_sudo cp -R "$headers_include/" "/usr"
}
install_bun() {
@@ -959,6 +976,13 @@ install_chrome_dependencies() {
xorg-x11-utils
;;
esac
case "$distro" in
amzn)
install_packages \
mesa-libgbm
;;
esac
}
main() {

View File

@@ -17,6 +17,7 @@ import {
tmpdir,
waitForPort,
which,
escapePowershell,
} from "./utils.mjs";
import { join, relative, resolve } from "node:path";
import { homedir } from "node:os";
@@ -119,7 +120,7 @@ export const aws = {
/**
* @param {string[]} args
* @returns {Promise<any>}
* @returns {Promise<unknown>}
*/
async spawn(args) {
const aws = which("aws");
@@ -136,7 +137,14 @@ export const aws = {
};
}
const { stdout } = await spawnSafe($`${aws} ${args} --output json`, { env });
const { error, stdout } = await spawn($`${aws} ${args} --output json`, { env });
if (error) {
if (/max attempts exceeded/i.test(inspect(error))) {
return this.spawn(args);
}
throw error;
}
try {
return JSON.parse(stdout);
} catch {
@@ -286,16 +294,7 @@ export const aws = {
* @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait/image-available.html
*/
async waitImage(action, ...imageIds) {
while (true) {
try {
await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`);
return;
} catch (error) {
if (!/max attempts exceeded/i.test(inspect(error))) {
throw error;
}
}
}
await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`);
},
/**
@@ -386,7 +385,7 @@ export const aws = {
* @returns {Promise<Machine>}
*/
async createMachine(options) {
const { arch, imageId, instanceType, metadata } = options;
const { os, arch, imageId, instanceType, tags } = options;
/** @type {AwsImage} */
let image;
@@ -411,14 +410,18 @@ export const aws = {
});
const username = getUsername(Name);
const userData = getCloudInit({ ...options, username });
let userData = getUserData({ ...options, username });
if (os === "windows") {
userData = `<powershell>${userData}</powershell><powershellArguments>-ExecutionPolicy Unrestricted -NoProfile -NonInteractive</powershellArguments><persist>false</persist>`;
}
let tagSpecification = [];
if (metadata) {
if (tags) {
tagSpecification = ["instance", "volume"].map(resourceType => {
return {
ResourceType: resourceType,
Tags: Object.entries(metadata).map(([Key, Value]) => ({ Key, Value: String(Value) })),
Tags: Object.entries(tags).map(([Key, Value]) => ({ Key, Value: String(Value) })),
};
});
}
@@ -435,6 +438,7 @@ export const aws = {
"InstanceMetadataTags": "enabled",
}),
["tag-specifications"]: JSON.stringify(tagSpecification),
["key-name"]: "ashcon-bun",
});
return aws.toMachine(instance, { ...options, username });
@@ -672,6 +676,14 @@ const google = {
* @property {string} [password]
*/
function getUserData(cloudInit) {
const { os } = cloudInit;
if (os === "windows") {
return getWindowsStartupScript(cloudInit);
}
return getCloudInit(cloudInit);
}
/**
* @param {CloudInit} cloudInit
* @returns {string}
@@ -722,6 +734,80 @@ function getCloudInit(cloudInit) {
`;
}
/**
* @param {CloudInit} cloudInit
* @returns {string}
*/
function getWindowsStartupScript(cloudInit) {
const { sshKeys } = cloudInit;
const authorizedKeys = sshKeys.filter(({ publicKey }) => publicKey).map(({ publicKey }) => publicKey);
return `
$ErrorActionPreference = "Stop"
Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
function Install-Ssh {
$sshService = Get-WindowsCapability -Online | Where-Object Name -like 'OpenSSH.Server*'
if ($sshService.State -ne "Installed") {
Write-Output "Installing OpenSSH server..."
Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
}
$pwshPath = Get-Command pwsh -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path
if (-not $pwshPath) {
$pwshPath = Get-Command powershell -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path
}
if (-not (Get-Service -Name sshd -ErrorAction SilentlyContinue)) {
Write-Output "Enabling OpenSSH server..."
Set-Service -Name sshd -StartupType Automatic
Start-Service sshd
}
if ($pwshPath) {
Write-Output "Setting default shell to $pwshPath..."
New-ItemProperty -Path "HKLM:\\SOFTWARE\\OpenSSH" -Name DefaultShell -Value $pwshPath -PropertyType String -Force
}
$firewallRule = Get-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -ErrorAction SilentlyContinue
if (-not $firewallRule) {
Write-Output "Configuring firewall..."
New-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22
}
$sshPath = "C:\\ProgramData\\ssh"
if (-not (Test-Path $sshPath)) {
Write-Output "Creating SSH directory..."
New-Item -Path $sshPath -ItemType Directory
}
$authorizedKeysPath = Join-Path $sshPath "administrators_authorized_keys"
$authorizedKeys = @(${authorizedKeys.map(key => `"${escapePowershell(key)}"`).join("\n")})
if (-not (Test-Path $authorizedKeysPath) -or (Get-Content $authorizedKeysPath) -ne $authorizedKeys) {
Write-Output "Adding SSH keys..."
Set-Content -Path $authorizedKeysPath -Value $authorizedKeys
}
$sshdConfigPath = Join-Path $sshPath "sshd_config"
$sshdConfig = @"
PasswordAuthentication no
PubkeyAuthentication yes
AuthorizedKeysFile $authorizedKeysPath
Subsystem sftp sftp-server.exe
"@
if (-not (Test-Path $sshdConfigPath) -or (Get-Content $sshdConfigPath) -ne $sshdConfig) {
Write-Output "Writing SSH configuration..."
Set-Content -Path $sshdConfigPath -Value $sshdConfig
}
Write-Output "Restarting SSH server..."
Restart-Service sshd
}
Install-Ssh
`;
}
/**
* @param {string} distro
* @returns {string}
@@ -824,7 +910,7 @@ function getSshKeys() {
publicPath,
privatePath: publicPath.replace(/\.pub$/, ""),
get publicKey() {
return readFile(publicPath, { cache: true });
return readFile(publicPath, { cache: true }).trim();
},
})),
);
@@ -1030,7 +1116,7 @@ async function main() {
"cloud": { type: "string", default: "aws" },
"os": { type: "string", default: "linux" },
"arch": { type: "string", default: "x64" },
"distro": { type: "string", default: "debian" },
"distro": { type: "string" },
"distro-version": { type: "string" },
"instance-type": { type: "string" },
"image-id": { type: "string" },
@@ -1080,7 +1166,7 @@ async function main() {
let bootstrapPath, agentPath;
if (bootstrap) {
bootstrapPath = resolve(import.meta.dirname, "bootstrap.sh");
bootstrapPath = resolve(import.meta.dirname, os === "windows" ? "bootstrap.ps1" : "bootstrap.sh");
if (!existsSync(bootstrapPath)) {
throw new Error(`Script not found: ${bootstrapPath}`);
}
@@ -1127,38 +1213,56 @@ async function main() {
});
if (bootstrapPath) {
const remotePath = "/tmp/bootstrap.sh";
const args = ci ? ["--ci"] : [];
await startGroup("Running bootstrap...", async () => {
await machine.upload(bootstrapPath, remotePath);
await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" });
});
if (os === "windows") {
const remotePath = "C:\\Windows\\Temp\\bootstrap.ps1";
const args = ci ? ["-CI"] : [];
await startGroup("Running bootstrap...", async () => {
await machine.upload(bootstrapPath, remotePath);
await machine.spawnSafe(["powershell", remotePath, ...args], { stdio: "inherit" });
});
} else {
const remotePath = "/tmp/bootstrap.sh";
const args = ci ? ["--ci"] : [];
await startGroup("Running bootstrap...", async () => {
await machine.upload(bootstrapPath, remotePath);
await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" });
});
}
}
if (agentPath) {
const tmpPath = "/tmp/agent.mjs";
const remotePath = "/var/lib/buildkite-agent/agent.mjs";
await startGroup("Installing agent...", async () => {
await machine.upload(agentPath, tmpPath);
const command = [];
{
const { exitCode } = await machine.spawn(["sudo", "echo", "1"], { stdio: "ignore" });
if (exitCode === 0) {
command.unshift("sudo");
if (os === "windows") {
// TODO
// const remotePath = "C:\\Windows\\Temp\\agent.mjs";
// await startGroup("Installing agent...", async () => {
// await machine.upload(agentPath, remotePath);
// await machine.spawnSafe(["node", remotePath, "install"], { stdio: "inherit" });
// });
} else {
const tmpPath = "/tmp/agent.mjs";
const remotePath = "/var/lib/buildkite-agent/agent.mjs";
await startGroup("Installing agent...", async () => {
await machine.upload(agentPath, tmpPath);
const command = [];
{
const { exitCode } = await machine.spawn(["sudo", "echo", "1"], { stdio: "ignore" });
if (exitCode === 0) {
command.unshift("sudo");
}
}
}
await machine.spawnSafe([...command, "cp", tmpPath, remotePath]);
{
const { stdout } = await machine.spawn(["node", "-v"]);
const version = parseInt(stdout.trim().replace(/^v/, ""));
if (isNaN(version) || version < 20) {
command.push("bun");
} else {
command.push("node");
await machine.spawnSafe([...command, "cp", tmpPath, remotePath]);
{
const { stdout } = await machine.spawn(["node", "-v"]);
const version = parseInt(stdout.trim().replace(/^v/, ""));
if (isNaN(version) || version < 20) {
command.push("bun");
} else {
command.push("node");
}
}
}
await machine.spawnSafe([...command, remotePath, "install"], { stdio: "inherit" });
});
await machine.spawnSafe([...command, remotePath, "install"], { stdio: "inherit" });
});
}
}
if (command === "create-image" || command === "publish-image") {

View File

@@ -872,14 +872,16 @@ export function writeFile(filename, content, options = {}) {
*/
export function which(command, options = {}) {
const commands = Array.isArray(command) ? command : [command];
const executables = isWindows ? commands.flatMap(name => [name, `${name}.exe`, `${name}.cmd`]) : commands;
const path = getEnv("PATH", false) || "";
const binPaths = path.split(isWindows ? ";" : ":");
for (const binPath of binPaths) {
for (const command of commands) {
const commandPath = join(binPath, command);
if (existsSync(commandPath)) {
return commandPath;
for (const executable of executables) {
const executablePath = join(binPath, executable);
if (existsSync(executablePath)) {
return executablePath;
}
}
}
@@ -1249,6 +1251,14 @@ export function escapeCodeBlock(string) {
return string.replace(/`/g, "\\`");
}
/**
* @param {string} string
* @returns {string}
*/
export function escapePowershell(string) {
return string.replace(/'/g, "''").replace(/`/g, "``");
}
/**
* @returns {string}
*/
@@ -1280,14 +1290,6 @@ export function tmpdir() {
return nodeTmpdir();
}
/**
* @param {string} string
* @returns {string}
*/
function escapePowershell(string) {
return string.replace(/'/g, "''").replace(/`/g, "``");
}
/**
* @param {string} filename
* @param {string} [output]
@@ -1697,7 +1699,7 @@ export function getDistro() {
const releasePath = "/etc/os-release";
if (existsSync(releasePath)) {
const releaseFile = readFile(releasePath, { cache: true });
const match = releaseFile.match(/ID=\"(.*)\"/);
const match = releaseFile.match(/^ID=\"?(.*)\"?/m);
if (match) {
return match[1];
}
@@ -1742,7 +1744,7 @@ export function getDistroVersion() {
const releasePath = "/etc/os-release";
if (existsSync(releasePath)) {
const releaseFile = readFile(releasePath, { cache: true });
const match = releaseFile.match(/VERSION_ID=\"(.*)\"/);
const match = releaseFile.match(/^VERSION_ID=\"?(.*)\"?/m);
if (match) {
return match[1];
}

View File

@@ -288,21 +288,33 @@ pub const FFI = struct {
}
}
} else if (Environment.isLinux) {
// On Debian/Ubuntu, the lib and include paths are suffixed with {arch}-linux-gnu
// e.g. x86_64-linux-gnu or aarch64-linux-gnu
// On Alpine and RHEL-based distros, the paths are not suffixed
if (Environment.isX64) {
if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/include/x86_64-linux-gnu").isTrue()) {
cached_default_system_include_dir = "/usr/include/x86_64-linux-gnu";
} else if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/include").isTrue()) {
cached_default_system_include_dir = "/usr/include";
}
if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/lib/x86_64-linux-gnu").isTrue()) {
cached_default_system_library_dir = "/usr/lib/x86_64-linux-gnu";
} else if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/lib64").isTrue()) {
cached_default_system_library_dir = "/usr/lib64";
}
} else if (Environment.isAarch64) {
if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/include/aarch64-linux-gnu").isTrue()) {
cached_default_system_include_dir = "/usr/include/aarch64-linux-gnu";
} else if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/include").isTrue()) {
cached_default_system_include_dir = "/usr/include";
}
if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/lib/aarch64-linux-gnu").isTrue()) {
cached_default_system_library_dir = "/usr/lib/aarch64-linux-gnu";
} else if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/lib64").isTrue()) {
cached_default_system_library_dir = "/usr/lib64";
}
}
}