From 708ed00705b1e9f4caab0c810fde13176ceea4ec Mon Sep 17 00:00:00 2001 From: Ashcon Partovi Date: Tue, 19 Nov 2024 18:31:15 -0800 Subject: [PATCH] ci: Expand automated build images to Debian, Ubuntu, and Amazon Linux (#15250) --- .buildkite/ci.mjs | 118 +++++++------- scripts/agent.mjs | 20 ++- scripts/bootstrap.ps1 | 339 +++++++++++++++++++++++++++++++++++++++++ scripts/bootstrap.sh | 30 +++- scripts/machine.mjs | 196 ++++++++++++++++++------ scripts/utils.mjs | 30 ++-- src/bun.js/api/ffi.zig | 12 ++ 7 files changed, 629 insertions(+), 116 deletions(-) create mode 100755 scripts/bootstrap.ps1 diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index 092bc38426..db696943c7 100755 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -234,8 +234,8 @@ function getPipeline(options) { * @returns {boolean} */ const isUsingNewAgent = platform => { - const { os, distro } = platform; - if (os === "linux" && distro === "alpine") { + const { os } = platform; + if (os === "linux") { return true; } return false; @@ -359,6 +359,21 @@ function getPipeline(options) { * @link https://buildkite.com/docs/pipelines/command-step */ + /** + * @param {Platform} platform + * @param {string} [step] + * @returns {string[]} + */ + const getDependsOn = (platform, step) => { + if (imagePlatforms.has(getImageKey(platform))) { + const key = `${getImageKey(platform)}-build-image`; + if (key !== step) { + return [key]; + } + } + return []; + }; + /** * @param {Platform} platform * @returns {Step} @@ -375,81 +390,85 @@ function getPipeline(options) { env: { DEBUG: "1", }, + retry: getRetry(), command: `node ./scripts/machine.mjs ${action} --ci --cloud=aws --os=${os} --arch=${arch} --distro=${distro} --distro-version=${release}`, }; }; /** - * @param {Target} target + * @param {Platform} platform * @returns {Step} */ - const getBuildVendorStep = target => { + const getBuildVendorStep = platform => { return { - key: `${getTargetKey(target)}-build-vendor`, - label: `${getTargetLabel(target)} - build-vendor`, - agents: getBuildAgent(target), + key: `${getTargetKey(platform)}-build-vendor`, + label: `${getTargetLabel(platform)} - build-vendor`, + depends_on: getDependsOn(platform), + agents: getBuildAgent(platform), retry: getRetry(), cancel_on_build_failing: isMergeQueue(), - env: getBuildEnv(target), + env: getBuildEnv(platform), command: "bun run build:ci --target dependencies", }; }; /** - * @param {Target} target + * @param {Platform} platform * @returns {Step} */ - const getBuildCppStep = target => { + const getBuildCppStep = platform => { return { - key: `${getTargetKey(target)}-build-cpp`, - label: `${getTargetLabel(target)} - build-cpp`, - agents: getBuildAgent(target), + key: `${getTargetKey(platform)}-build-cpp`, + label: `${getTargetLabel(platform)} - build-cpp`, + depends_on: getDependsOn(platform), + agents: getBuildAgent(platform), retry: getRetry(), cancel_on_build_failing: isMergeQueue(), env: { BUN_CPP_ONLY: "ON", - ...getBuildEnv(target), + ...getBuildEnv(platform), }, command: "bun run build:ci --target bun", }; }; /** - * @param {Target} target + * @param {Platform} platform * @returns {Step} */ - const getBuildZigStep = target => { - const toolchain = getBuildToolchain(target); + const getBuildZigStep = platform => { + const toolchain = getBuildToolchain(platform); return { - key: `${getTargetKey(target)}-build-zig`, - label: `${getTargetLabel(target)} - build-zig`, - agents: getZigAgent(target), + key: `${getTargetKey(platform)}-build-zig`, + label: `${getTargetLabel(platform)} - build-zig`, + depends_on: getDependsOn(platform), + agents: getZigAgent(platform), retry: getRetry(1), // FIXME: Sometimes zig build hangs, so we need to retry once cancel_on_build_failing: isMergeQueue(), - env: getBuildEnv(target), + env: getBuildEnv(platform), command: `bun run build:ci --target bun-zig --toolchain ${toolchain}`, }; }; /** - * @param {Target} target + * @param {Platform} platform * @returns {Step} */ - const getBuildBunStep = target => { + const getBuildBunStep = platform => { return { - key: `${getTargetKey(target)}-build-bun`, - label: `${getTargetLabel(target)} - build-bun`, + key: `${getTargetKey(platform)}-build-bun`, + label: `${getTargetLabel(platform)} - build-bun`, depends_on: [ - `${getTargetKey(target)}-build-vendor`, - `${getTargetKey(target)}-build-cpp`, - `${getTargetKey(target)}-build-zig`, + `${getTargetKey(platform)}-build-vendor`, + `${getTargetKey(platform)}-build-cpp`, + `${getTargetKey(platform)}-build-zig`, ], - agents: getBuildAgent(target), + agents: getBuildAgent(platform), retry: getRetry(), cancel_on_build_failing: isMergeQueue(), env: { BUN_LINK_ONLY: "ON", - ...getBuildEnv(target), + ...getBuildEnv(platform), }, command: "bun run build:ci --target bun", }; @@ -473,8 +492,8 @@ function getPipeline(options) { } else { parallelism = 10; } - let depends; let env; + let depends = []; if (buildId) { env = { BUILDKITE_ARTIFACT_BUILD_ID: buildId, @@ -488,14 +507,20 @@ function getPipeline(options) { // Because of this, we don't know if the run was fatal, or soft-failed. retry = getRetry(1); } + let soft_fail; + if (isMainBranch()) { + soft_fail = true; + } else { + soft_fail = [{ exit_status: 2 }]; + } return { key: `${getPlatformKey(platform)}-test-bun`, label: `${getPlatformLabel(platform)} - test-bun`, - depends_on: depends, + depends_on: [...depends, ...getDependsOn(platform)], agents: getTestAgent(platform), retry, cancel_on_build_failing: isMergeQueue(), - soft_fail: isMainBranch(), + soft_fail, parallelism, command, env, @@ -531,14 +556,14 @@ function getPipeline(options) { { os: "darwin", arch: "x64", release: "14" }, { os: "darwin", arch: "x64", release: "13" }, { os: "linux", arch: "aarch64", distro: "debian", release: "12" }, - // { os: "linux", arch: "aarch64", distro: "debian", release: "11" }, - // { os: "linux", arch: "aarch64", distro: "debian", release: "10" }, + { os: "linux", arch: "aarch64", distro: "debian", release: "11" }, + { os: "linux", arch: "aarch64", distro: "debian", release: "10" }, { os: "linux", arch: "x64", distro: "debian", release: "12" }, - // { os: "linux", arch: "x64", distro: "debian", release: "11" }, - // { os: "linux", arch: "x64", distro: "debian", release: "10" }, + { os: "linux", arch: "x64", distro: "debian", release: "11" }, + { os: "linux", arch: "x64", distro: "debian", release: "10" }, { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "12" }, - // { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" }, - // { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "10" }, + { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" }, + { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "10" }, // { os: "linux", arch: "aarch64", distro: "ubuntu", release: "24.04" }, { os: "linux", arch: "aarch64", distro: "ubuntu", release: "22.04" }, { os: "linux", arch: "aarch64", distro: "ubuntu", release: "20.04" }, @@ -548,11 +573,11 @@ function getPipeline(options) { // { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "24.04" }, { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "22.04" }, { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "20.04" }, - // { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023" }, + { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023" }, // { os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2" }, - // { os: "linux", arch: "x64", distro: "amazonlinux", release: "2023" }, + { os: "linux", arch: "x64", distro: "amazonlinux", release: "2023" }, // { os: "linux", arch: "x64", distro: "amazonlinux", release: "2" }, - // { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023" }, + { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023" }, // { os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2" }, { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" }, // { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.17" }, @@ -615,15 +640,6 @@ function getPipeline(options) { continue; } - if (imagePlatforms.has(getImageKey(platform))) { - for (const step of platformSteps) { - if (step.agents?.["image-name"]) { - step.depends_on ??= []; - step.depends_on.push(`${getImageKey(platform)}-build-image`); - } - } - } - steps.push({ key: getTargetKey(platform), group: getTargetLabel(platform), diff --git a/scripts/agent.mjs b/scripts/agent.mjs index 8da3b96e05..84af695374 100755 --- a/scripts/agent.mjs +++ b/scripts/agent.mjs @@ -32,7 +32,10 @@ async function doBuildkiteAgent(action) { let homePath, cachePath, logsPath, agentLogPath, pidPath; if (isWindows) { - throw new Error("TODO: Windows"); + homePath = "C:\\buildkite-agent"; + cachePath = join(homePath, "cache"); + logsPath = join(homePath, "logs"); + agentLogPath = join(logsPath, "buildkite-agent.log"); } else { homePath = "/var/lib/buildkite-agent"; cachePath = "/var/cache/buildkite-agent"; @@ -45,6 +48,19 @@ async function doBuildkiteAgent(action) { const command = process.execPath; const args = [realpathSync(process.argv[1]), "start"]; + if (isWindows) { + const serviceCommand = [ + "New-Service", + "-Name", + "buildkite-agent", + "-StartupType", + "Automatic", + "-BinaryPathName", + `${escape(command)} ${escape(args.map(escape).join(" "))}`, + ]; + await spawnSafe(["powershell", "-Command", serviceCommand.join(" ")], { stdio: "inherit" }); + } + if (isOpenRc()) { const servicePath = "/etc/init.d/buildkite-agent"; const service = `#!/sbin/openrc-run @@ -81,7 +97,7 @@ async function doBuildkiteAgent(action) { [Service] Type=simple User=${username} - ExecStart=${escape(command)} ${escape(args.map(escape).join(" "))} + ExecStart=${escape(command)} ${args.map(escape).join(" ")} RestartSec=5 Restart=on-failure KillMode=process diff --git a/scripts/bootstrap.ps1 b/scripts/bootstrap.ps1 new file mode 100755 index 0000000000..eda27d917a --- /dev/null +++ b/scripts/bootstrap.ps1 @@ -0,0 +1,339 @@ +# Version: 4 +# A powershell script that installs the dependencies needed to build and test Bun. +# This should work on Windows 10 or newer. + +# If this script does not work on your machine, please open an issue: +# https://github.com/oven-sh/bun/issues + +# If you need to make a change to this script, such as upgrading a dependency, +# increment the version comment to indicate that a new image should be built. +# Otherwise, the existing image will be retroactively updated. + +param ( + [Parameter(Mandatory = $false)] + [switch]$CI = $false, + [Parameter(Mandatory = $false)] + [switch]$Optimize = $CI +) + +function Execute-Command { + $command = $args -join ' ' + Write-Output "$ $command" + + & $args[0] $args[1..$args.Length] + + if ((-not $?) -or ($LASTEXITCODE -ne 0 -and $null -ne $LASTEXITCODE)) { + throw "Command failed: $command" + } +} + +function Which { + param ([switch]$Required = $false) + + foreach ($command in $args) { + $result = Get-Command $command -ErrorAction SilentlyContinue + if ($result -and $result.Path) { + return $result.Path + } + } + + if ($Required) { + $commands = $args -join ', ' + throw "Command not found: $commands" + } +} + +function Install-Chocolatey { + if (Which choco) { + return + } + + Write-Output "Installing Chocolatey..." + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 + iex -Command ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + Refresh-Path +} + +function Refresh-Path { + $paths = @( + [System.Environment]::GetEnvironmentVariable("Path", "Machine"), + [System.Environment]::GetEnvironmentVariable("Path", "User"), + [System.Environment]::GetEnvironmentVariable("Path", "Process") + ) + $uniquePaths = $paths | + Where-Object { $_ } | + ForEach-Object { $_.Split(';', [StringSplitOptions]::RemoveEmptyEntries) } | + Where-Object { $_ -and (Test-Path $_) } | + Select-Object -Unique + $env:Path = ($uniquePaths -join ';').TrimEnd(';') + + if ($env:ChocolateyInstall) { + Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 -ErrorAction SilentlyContinue + } +} + +function Add-To-Path { + $absolutePath = Resolve-Path $args[0] + $currentPath = [Environment]::GetEnvironmentVariable("Path", "Machine") + if ($currentPath -like "*$absolutePath*") { + return + } + + $newPath = $currentPath.TrimEnd(";") + ";" + $absolutePath + if ($newPath.Length -ge 2048) { + Write-Warning "PATH is too long, removing duplicate and old entries..." + + $paths = $currentPath.Split(';', [StringSplitOptions]::RemoveEmptyEntries) | + Where-Object { $_ -and (Test-Path $_) } | + Select-Object -Unique + + $paths += $absolutePath + $newPath = $paths -join ';' + while ($newPath.Length -ge 2048 -and $paths.Count -gt 1) { + $paths = $paths[1..$paths.Count] + $newPath = $paths -join ';' + } + } + + Write-Output "Adding $absolutePath to PATH..." + [Environment]::SetEnvironmentVariable("Path", $newPath, "Machine") + Refresh-Path +} + +function Install-Package { + param ( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Name, + [Parameter(Mandatory = $false)] + [string]$Command = $Name, + [Parameter(Mandatory = $false)] + [string]$Version, + [Parameter(Mandatory = $false)] + [switch]$Force = $false, + [Parameter(Mandatory = $false)] + [string[]]$ExtraArgs = @() + ) + + if (-not $Force ` + -and (Which $Command) ` + -and (-not $Version -or (& $Command --version) -like "*$Version*")) { + return + } + + Write-Output "Installing $Name..." + $flags = @( + "--yes", + "--accept-license", + "--no-progress", + "--force" + ) + if ($Version) { + $flags += "--version=$Version" + } + + Execute-Command choco install $Name @flags @ExtraArgs + Refresh-Path +} + +function Install-Packages { + foreach ($package in $args) { + Install-Package -Name $package + } +} + +function Install-Common-Software { + Install-Chocolatey + Install-Pwsh + Install-Git + Install-Packages curl 7zip + Install-NodeJs + Install-Bun + Install-Cygwin + if ($CI) { + Install-Tailscale + Install-Buildkite + } +} + +function Install-Pwsh { + Install-Package powershell-core -Command pwsh + + if ($CI) { + $shellPath = (Which pwsh -Required) + New-ItemProperty ` + -Path "HKLM:\\SOFTWARE\\OpenSSH" ` + -Name DefaultShell ` + -Value $shellPath ` + -PropertyType String ` + -Force + } +} + +function Install-Git { + Install-Packages git + + if ($CI) { + Execute-Command git config --system --add safe.directory "*" + Execute-Command git config --system core.autocrlf false + Execute-Command git config --system core.eol lf + Execute-Command git config --system core.longpaths true + } +} + +function Install-NodeJs { + Install-Package nodejs -Command node -Version "22.9.0" +} + +function Install-Bun { + Install-Package bun -Version "1.1.30" +} + +function Install-Cygwin { + Install-Package cygwin + Add-To-Path "C:\tools\cygwin\bin" +} + +function Install-Tailscale { + Install-Package tailscale +} + +function Install-Buildkite { + if (Which buildkite-agent) { + return + } + + Write-Output "Installing Buildkite agent..." + $env:buildkiteAgentToken = "xxx" + iex ((New-Object System.Net.WebClient).DownloadString("https://raw.githubusercontent.com/buildkite/agent/main/install.ps1")) + Refresh-Path +} + +function Install-Build-Essentials { + # Install-Visual-Studio + Install-Packages ` + cmake ` + make ` + ninja ` + ccache ` + python ` + golang ` + nasm ` + ruby ` + mingw + Install-Rust + Install-Llvm +} + +function Install-Visual-Studio { + $components = @( + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.Component.Windows10SDK.18362", + "Microsoft.VisualStudio.Component.Windows11SDK.22000", + "Microsoft.VisualStudio.Component.Windows11Sdk.WindowsPerformanceToolkit", + "Microsoft.VisualStudio.Component.VC.ASAN", # C++ AddressSanitizer + "Microsoft.VisualStudio.Component.VC.ATL", # C++ ATL for latest v143 build tools (x86 & x64) + "Microsoft.VisualStudio.Component.VC.DiagnosticTools", # C++ Diagnostic Tools + "Microsoft.VisualStudio.Component.VC.CLI.Support", # C++/CLI support for v143 build tools (Latest) + "Microsoft.VisualStudio.Component.VC.CoreIde", # C++ core features + "Microsoft.VisualStudio.Component.VC.Redist.14.Latest" # C++ 2022 Redistributable Update + ) + + $arch = (Get-WmiObject Win32_Processor).Architecture + if ($arch -eq 9) { + $components += @( + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", # MSVC v143 build tools (x86 & x64) + "Microsoft.VisualStudio.Component.VC.Modules.x86.x64" # MSVC v143 C++ Modules for latest v143 build tools (x86 & x64) + ) + } elseif ($arch -eq 5) { + $components += @( + "Microsoft.VisualStudio.Component.VC.Tools.ARM64", # MSVC v143 build tools (ARM64) + "Microsoft.VisualStudio.Component.UWP.VC.ARM64" # C++ Universal Windows Platform support for v143 build tools (ARM64/ARM64EC) + ) + } + + $packageParameters = $components | ForEach-Object { "--add $_" } + Install-Package visualstudio2022community ` + -ExtraArgs "--package-parameters '--add Microsoft.VisualStudio.Workload.NativeDesktop --includeRecommended --includeOptional'" +} + +function Install-Rust { + if (Which rustc) { + return + } + + Write-Output "Installing Rust..." + $rustupInit = "$env:TEMP\rustup-init.exe" + (New-Object System.Net.WebClient).DownloadFile("https://win.rustup.rs/", $rustupInit) + Execute-Command $rustupInit -y + Add-To-Path "$env:USERPROFILE\.cargo\bin" +} + +function Install-Llvm { + Install-Package llvm ` + -Command clang-cl ` + -Version "18.1.8" + Add-To-Path "C:\Program Files\LLVM\bin" +} + +function Optimize-System { + Disable-Windows-Defender + Disable-Windows-Threat-Protection + Disable-Windows-Services + Disable-Power-Management + Uninstall-Windows-Defender +} + +function Disable-Windows-Defender { + Write-Output "Disabling Windows Defender..." + Set-MpPreference -DisableRealtimeMonitoring $true + Add-MpPreference -ExclusionPath "C:\", "D:\" +} + +function Disable-Windows-Threat-Protection { + $itemPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows Advanced Threat Protection" + if (Test-Path $itemPath) { + Write-Output "Disabling Windows Threat Protection..." + Set-ItemProperty -Path $itemPath -Name "ForceDefenderPassiveMode" -Value 1 -Type DWORD + } +} + +function Uninstall-Windows-Defender { + Write-Output "Uninstalling Windows Defender..." + Uninstall-WindowsFeature -Name Windows-Defender +} + +function Disable-Windows-Services { + $services = @( + "WSearch", # Windows Search + "wuauserv", # Windows Update + "DiagTrack", # Connected User Experiences and Telemetry + "dmwappushservice", # WAP Push Message Routing Service + "PcaSvc", # Program Compatibility Assistant + "SysMain" # Superfetch + ) + + foreach ($service in $services) { + Stop-Service $service -Force + Set-Service $service -StartupType Disabled + } +} + +function Disable-Power-Management { + Write-Output "Disabling power management features..." + powercfg /setactive 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c # High performance + powercfg /change monitor-timeout-ac 0 + powercfg /change monitor-timeout-dc 0 + powercfg /change standby-timeout-ac 0 + powercfg /change standby-timeout-dc 0 + powercfg /change hibernate-timeout-ac 0 + powercfg /change hibernate-timeout-dc 0 +} + +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force +if ($Optimize) { + Optimize-System +} + +Install-Common-Software +Install-Build-Essentials + diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 480dd91448..c5f59ca116 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Version: 4 +# Version: 5 # A script that installs the dependencies needed to build and test Bun. # This should work on macOS and Linux with a POSIX shell. @@ -327,7 +327,8 @@ check_package_manager() { print "Updating package manager..." case "$pm" in apt) - DEBIAN_FRONTEND=noninteractive package_manager update -y + export DEBIAN_FRONTEND=noninteractive + package_manager update -y ;; apk) package_manager update @@ -373,7 +374,7 @@ package_manager() { while ! sudo -n apt-get update -y; do sleep 1 done - DEBIAN_FRONTEND=noninteractive execute_sudo apt-get "$@" + execute_sudo apt-get "$@" ;; dnf) case "$distro" in @@ -569,6 +570,22 @@ install_nodejs() { install_packages nodejs ;; esac + + # Some distros do not install the node headers by default. + # These are needed for certain FFI tests, such as: `cc.test.ts` + case "$distro" in + alpine | amzn) + install_nodejs_headers + ;; + esac +} + +install_nodejs_headers() { + headers_tar="$(download_file "https://nodejs.org/download/release/v$(nodejs_version_exact)/node-v$(nodejs_version_exact)-headers.tar.gz")" + headers_dir="$(dirname "$headers_tar")" + execute tar -xzf "$headers_tar" -C "$headers_dir" + headers_include="$headers_dir/node-v$(nodejs_version_exact)/include" + execute_sudo cp -R "$headers_include/" "/usr" } install_bun() { @@ -959,6 +976,13 @@ install_chrome_dependencies() { xorg-x11-utils ;; esac + + case "$distro" in + amzn) + install_packages \ + mesa-libgbm + ;; + esac } main() { diff --git a/scripts/machine.mjs b/scripts/machine.mjs index d352ab89ce..1048d6abff 100755 --- a/scripts/machine.mjs +++ b/scripts/machine.mjs @@ -17,6 +17,7 @@ import { tmpdir, waitForPort, which, + escapePowershell, } from "./utils.mjs"; import { join, relative, resolve } from "node:path"; import { homedir } from "node:os"; @@ -119,7 +120,7 @@ export const aws = { /** * @param {string[]} args - * @returns {Promise} + * @returns {Promise} */ async spawn(args) { const aws = which("aws"); @@ -136,7 +137,14 @@ export const aws = { }; } - const { stdout } = await spawnSafe($`${aws} ${args} --output json`, { env }); + const { error, stdout } = await spawn($`${aws} ${args} --output json`, { env }); + if (error) { + if (/max attempts exceeded/i.test(inspect(error))) { + return this.spawn(args); + } + throw error; + } + try { return JSON.parse(stdout); } catch { @@ -286,16 +294,7 @@ export const aws = { * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait/image-available.html */ async waitImage(action, ...imageIds) { - while (true) { - try { - await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`); - return; - } catch (error) { - if (!/max attempts exceeded/i.test(inspect(error))) { - throw error; - } - } - } + await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`); }, /** @@ -386,7 +385,7 @@ export const aws = { * @returns {Promise} */ async createMachine(options) { - const { arch, imageId, instanceType, metadata } = options; + const { os, arch, imageId, instanceType, tags } = options; /** @type {AwsImage} */ let image; @@ -411,14 +410,18 @@ export const aws = { }); const username = getUsername(Name); - const userData = getCloudInit({ ...options, username }); + + let userData = getUserData({ ...options, username }); + if (os === "windows") { + userData = `${userData}-ExecutionPolicy Unrestricted -NoProfile -NonInteractivefalse`; + } let tagSpecification = []; - if (metadata) { + if (tags) { tagSpecification = ["instance", "volume"].map(resourceType => { return { ResourceType: resourceType, - Tags: Object.entries(metadata).map(([Key, Value]) => ({ Key, Value: String(Value) })), + Tags: Object.entries(tags).map(([Key, Value]) => ({ Key, Value: String(Value) })), }; }); } @@ -435,6 +438,7 @@ export const aws = { "InstanceMetadataTags": "enabled", }), ["tag-specifications"]: JSON.stringify(tagSpecification), + ["key-name"]: "ashcon-bun", }); return aws.toMachine(instance, { ...options, username }); @@ -672,6 +676,14 @@ const google = { * @property {string} [password] */ +function getUserData(cloudInit) { + const { os } = cloudInit; + if (os === "windows") { + return getWindowsStartupScript(cloudInit); + } + return getCloudInit(cloudInit); +} + /** * @param {CloudInit} cloudInit * @returns {string} @@ -722,6 +734,80 @@ function getCloudInit(cloudInit) { `; } +/** + * @param {CloudInit} cloudInit + * @returns {string} + */ +function getWindowsStartupScript(cloudInit) { + const { sshKeys } = cloudInit; + const authorizedKeys = sshKeys.filter(({ publicKey }) => publicKey).map(({ publicKey }) => publicKey); + + return ` + $ErrorActionPreference = "Stop" + Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force + + function Install-Ssh { + $sshService = Get-WindowsCapability -Online | Where-Object Name -like 'OpenSSH.Server*' + if ($sshService.State -ne "Installed") { + Write-Output "Installing OpenSSH server..." + Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 + } + + $pwshPath = Get-Command pwsh -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path + if (-not $pwshPath) { + $pwshPath = Get-Command powershell -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path + } + + if (-not (Get-Service -Name sshd -ErrorAction SilentlyContinue)) { + Write-Output "Enabling OpenSSH server..." + Set-Service -Name sshd -StartupType Automatic + Start-Service sshd + } + + if ($pwshPath) { + Write-Output "Setting default shell to $pwshPath..." + New-ItemProperty -Path "HKLM:\\SOFTWARE\\OpenSSH" -Name DefaultShell -Value $pwshPath -PropertyType String -Force + } + + $firewallRule = Get-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -ErrorAction SilentlyContinue + if (-not $firewallRule) { + Write-Output "Configuring firewall..." + New-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 + } + + $sshPath = "C:\\ProgramData\\ssh" + if (-not (Test-Path $sshPath)) { + Write-Output "Creating SSH directory..." + New-Item -Path $sshPath -ItemType Directory + } + + $authorizedKeysPath = Join-Path $sshPath "administrators_authorized_keys" + $authorizedKeys = @(${authorizedKeys.map(key => `"${escapePowershell(key)}"`).join("\n")}) + if (-not (Test-Path $authorizedKeysPath) -or (Get-Content $authorizedKeysPath) -ne $authorizedKeys) { + Write-Output "Adding SSH keys..." + Set-Content -Path $authorizedKeysPath -Value $authorizedKeys + } + + $sshdConfigPath = Join-Path $sshPath "sshd_config" + $sshdConfig = @" + PasswordAuthentication no + PubkeyAuthentication yes + AuthorizedKeysFile $authorizedKeysPath + Subsystem sftp sftp-server.exe +"@ + if (-not (Test-Path $sshdConfigPath) -or (Get-Content $sshdConfigPath) -ne $sshdConfig) { + Write-Output "Writing SSH configuration..." + Set-Content -Path $sshdConfigPath -Value $sshdConfig + } + + Write-Output "Restarting SSH server..." + Restart-Service sshd + } + + Install-Ssh + `; +} + /** * @param {string} distro * @returns {string} @@ -824,7 +910,7 @@ function getSshKeys() { publicPath, privatePath: publicPath.replace(/\.pub$/, ""), get publicKey() { - return readFile(publicPath, { cache: true }); + return readFile(publicPath, { cache: true }).trim(); }, })), ); @@ -1030,7 +1116,7 @@ async function main() { "cloud": { type: "string", default: "aws" }, "os": { type: "string", default: "linux" }, "arch": { type: "string", default: "x64" }, - "distro": { type: "string", default: "debian" }, + "distro": { type: "string" }, "distro-version": { type: "string" }, "instance-type": { type: "string" }, "image-id": { type: "string" }, @@ -1080,7 +1166,7 @@ async function main() { let bootstrapPath, agentPath; if (bootstrap) { - bootstrapPath = resolve(import.meta.dirname, "bootstrap.sh"); + bootstrapPath = resolve(import.meta.dirname, os === "windows" ? "bootstrap.ps1" : "bootstrap.sh"); if (!existsSync(bootstrapPath)) { throw new Error(`Script not found: ${bootstrapPath}`); } @@ -1127,38 +1213,56 @@ async function main() { }); if (bootstrapPath) { - const remotePath = "/tmp/bootstrap.sh"; - const args = ci ? ["--ci"] : []; - await startGroup("Running bootstrap...", async () => { - await machine.upload(bootstrapPath, remotePath); - await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" }); - }); + if (os === "windows") { + const remotePath = "C:\\Windows\\Temp\\bootstrap.ps1"; + const args = ci ? ["-CI"] : []; + await startGroup("Running bootstrap...", async () => { + await machine.upload(bootstrapPath, remotePath); + await machine.spawnSafe(["powershell", remotePath, ...args], { stdio: "inherit" }); + }); + } else { + const remotePath = "/tmp/bootstrap.sh"; + const args = ci ? ["--ci"] : []; + await startGroup("Running bootstrap...", async () => { + await machine.upload(bootstrapPath, remotePath); + await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" }); + }); + } } if (agentPath) { - const tmpPath = "/tmp/agent.mjs"; - const remotePath = "/var/lib/buildkite-agent/agent.mjs"; - await startGroup("Installing agent...", async () => { - await machine.upload(agentPath, tmpPath); - const command = []; - { - const { exitCode } = await machine.spawn(["sudo", "echo", "1"], { stdio: "ignore" }); - if (exitCode === 0) { - command.unshift("sudo"); + if (os === "windows") { + // TODO + // const remotePath = "C:\\Windows\\Temp\\agent.mjs"; + // await startGroup("Installing agent...", async () => { + // await machine.upload(agentPath, remotePath); + // await machine.spawnSafe(["node", remotePath, "install"], { stdio: "inherit" }); + // }); + } else { + const tmpPath = "/tmp/agent.mjs"; + const remotePath = "/var/lib/buildkite-agent/agent.mjs"; + await startGroup("Installing agent...", async () => { + await machine.upload(agentPath, tmpPath); + const command = []; + { + const { exitCode } = await machine.spawn(["sudo", "echo", "1"], { stdio: "ignore" }); + if (exitCode === 0) { + command.unshift("sudo"); + } } - } - await machine.spawnSafe([...command, "cp", tmpPath, remotePath]); - { - const { stdout } = await machine.spawn(["node", "-v"]); - const version = parseInt(stdout.trim().replace(/^v/, "")); - if (isNaN(version) || version < 20) { - command.push("bun"); - } else { - command.push("node"); + await machine.spawnSafe([...command, "cp", tmpPath, remotePath]); + { + const { stdout } = await machine.spawn(["node", "-v"]); + const version = parseInt(stdout.trim().replace(/^v/, "")); + if (isNaN(version) || version < 20) { + command.push("bun"); + } else { + command.push("node"); + } } - } - await machine.spawnSafe([...command, remotePath, "install"], { stdio: "inherit" }); - }); + await machine.spawnSafe([...command, remotePath, "install"], { stdio: "inherit" }); + }); + } } if (command === "create-image" || command === "publish-image") { diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 5f2f2ee3c3..17a27da7a7 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -872,14 +872,16 @@ export function writeFile(filename, content, options = {}) { */ export function which(command, options = {}) { const commands = Array.isArray(command) ? command : [command]; + const executables = isWindows ? commands.flatMap(name => [name, `${name}.exe`, `${name}.cmd`]) : commands; + const path = getEnv("PATH", false) || ""; const binPaths = path.split(isWindows ? ";" : ":"); for (const binPath of binPaths) { - for (const command of commands) { - const commandPath = join(binPath, command); - if (existsSync(commandPath)) { - return commandPath; + for (const executable of executables) { + const executablePath = join(binPath, executable); + if (existsSync(executablePath)) { + return executablePath; } } } @@ -1249,6 +1251,14 @@ export function escapeCodeBlock(string) { return string.replace(/`/g, "\\`"); } +/** + * @param {string} string + * @returns {string} + */ +export function escapePowershell(string) { + return string.replace(/'/g, "''").replace(/`/g, "``"); +} + /** * @returns {string} */ @@ -1280,14 +1290,6 @@ export function tmpdir() { return nodeTmpdir(); } -/** - * @param {string} string - * @returns {string} - */ -function escapePowershell(string) { - return string.replace(/'/g, "''").replace(/`/g, "``"); -} - /** * @param {string} filename * @param {string} [output] @@ -1697,7 +1699,7 @@ export function getDistro() { const releasePath = "/etc/os-release"; if (existsSync(releasePath)) { const releaseFile = readFile(releasePath, { cache: true }); - const match = releaseFile.match(/ID=\"(.*)\"/); + const match = releaseFile.match(/^ID=\"?(.*)\"?/m); if (match) { return match[1]; } @@ -1742,7 +1744,7 @@ export function getDistroVersion() { const releasePath = "/etc/os-release"; if (existsSync(releasePath)) { const releaseFile = readFile(releasePath, { cache: true }); - const match = releaseFile.match(/VERSION_ID=\"(.*)\"/); + const match = releaseFile.match(/^VERSION_ID=\"?(.*)\"?/m); if (match) { return match[1]; } diff --git a/src/bun.js/api/ffi.zig b/src/bun.js/api/ffi.zig index 8153adee7f..6e6c166099 100644 --- a/src/bun.js/api/ffi.zig +++ b/src/bun.js/api/ffi.zig @@ -288,21 +288,33 @@ pub const FFI = struct { } } } else if (Environment.isLinux) { + // On Debian/Ubuntu, the lib and include paths are suffixed with {arch}-linux-gnu + // e.g. x86_64-linux-gnu or aarch64-linux-gnu + // On Alpine and RHEL-based distros, the paths are not suffixed + if (Environment.isX64) { if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/include/x86_64-linux-gnu").isTrue()) { cached_default_system_include_dir = "/usr/include/x86_64-linux-gnu"; + } else if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/include").isTrue()) { + cached_default_system_include_dir = "/usr/include"; } if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/lib/x86_64-linux-gnu").isTrue()) { cached_default_system_library_dir = "/usr/lib/x86_64-linux-gnu"; + } else if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/lib64").isTrue()) { + cached_default_system_library_dir = "/usr/lib64"; } } else if (Environment.isAarch64) { if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/include/aarch64-linux-gnu").isTrue()) { cached_default_system_include_dir = "/usr/include/aarch64-linux-gnu"; + } else if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/include").isTrue()) { + cached_default_system_include_dir = "/usr/include"; } if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/lib/aarch64-linux-gnu").isTrue()) { cached_default_system_library_dir = "/usr/lib/aarch64-linux-gnu"; + } else if (bun.sys.directoryExistsAt(std.fs.cwd(), "/usr/lib64").isTrue()) { + cached_default_system_library_dir = "/usr/lib64"; } } }