Compare commits

...

9 Commits

Author SHA1 Message Date
Meghan Denny
778998cbe5 [build images] 2025-12-02 14:07:56 -08:00
Meghan Denny
059e595930 [build images] 2025-12-02 11:56:11 -08:00
Meghan Denny
427174bb79 [build images] 2025-12-01 23:00:08 -08:00
Meghan Denny
6c920ee304 [build images] 2025-12-01 21:00:23 -08:00
Meghan Denny
99cf16d999 [build images] 2025-12-01 20:17:43 -08:00
Meghan Denny
1a99bbaef7 [build images] 2025-12-01 20:04:22 -08:00
Meghan Denny
ae90e5dcbb [build images] 2025-12-01 19:47:25 -08:00
Meghan Denny
af7675d11a [build images] 2025-12-01 18:32:45 -08:00
Meghan Denny
87b7425630 ci: rework amazonlinux 2023 setup 2025-12-01 18:23:59 -08:00
7 changed files with 61 additions and 737 deletions

View File

@@ -1,186 +0,0 @@
ARG LLVM_VERSION="19"
ARG REPORTED_LLVM_VERSION="19.1.7"
ARG OLD_BUN_VERSION="1.1.38"
ARG BUILDKITE_AGENT_TAGS="queue=linux,os=linux,arch=${TARGETARCH}"
FROM --platform=$BUILDPLATFORM ubuntu:20.04 as base-arm64
FROM --platform=$BUILDPLATFORM ubuntu:20.04 as base-amd64
FROM base-$TARGETARCH as base
ARG LLVM_VERSION
ARG OLD_BUN_VERSION
ARG TARGETARCH
ARG REPORTED_LLVM_VERSION
ENV DEBIAN_FRONTEND=noninteractive \
CI=true \
DOCKER=true
RUN echo "Acquire::Queue-Mode \"host\";" > /etc/apt/apt.conf.d/99-apt-queue-mode.conf \
&& echo "Acquire::Timeout \"120\";" >> /etc/apt/apt.conf.d/99-apt-timeout.conf \
&& echo "Acquire::Retries \"3\";" >> /etc/apt/apt.conf.d/99-apt-retries.conf \
&& echo "APT::Install-Recommends \"false\";" >> /etc/apt/apt.conf.d/99-apt-install-recommends.conf \
&& echo "APT::Install-Suggests \"false\";" >> /etc/apt/apt.conf.d/99-apt-install-suggests.conf
RUN apt-get update && apt-get install -y --no-install-recommends \
wget curl git python3 python3-pip ninja-build \
software-properties-common apt-transport-https \
ca-certificates gnupg lsb-release unzip \
libxml2-dev ruby ruby-dev bison gawk perl make golang \
&& add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-get update \
&& apt-get install -y gcc-13 g++-13 libgcc-13-dev libstdc++-13-dev \
libasan6 libubsan1 libatomic1 libtsan0 liblsan0 \
libgfortran5 libc6-dev \
&& wget https://apt.llvm.org/llvm.sh \
&& chmod +x llvm.sh \
&& ./llvm.sh ${LLVM_VERSION} all \
&& rm llvm.sh
RUN --mount=type=tmpfs,target=/tmp \
cmake_version="3.30.5" && \
if [ "$TARGETARCH" = "arm64" ]; then \
cmake_url="https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-aarch64.sh"; \
else \
cmake_url="https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-linux-x86_64.sh"; \
fi && \
wget -O /tmp/cmake.sh "$cmake_url" && \
sh /tmp/cmake.sh --skip-license --prefix=/usr
RUN --mount=type=tmpfs,target=/tmp \
sccache_version="0.12.0" && \
arch=$(uname -m) && \
sccache_url="https://github.com/mozilla/sccache/releases/download/v${sccache_version}/sccache-v${sccache_version}-${arch}-unknown-linux-musl.tar.gz" && \
wget -O /tmp/sccache.tar.gz "$sccache_url" && \
tar -xzf /tmp/sccache.tar.gz -C /tmp && \
install -m755 /tmp/sccache-v${sccache_version}-${arch}-unknown-linux-musl/sccache /usr/local/bin
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \
--slave /usr/bin/g++ g++ /usr/bin/g++-13 \
--slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-13 \
--slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-13 \
--slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-13
RUN echo "ARCH_PATH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64-linux-gnu" || echo "x86_64-linux-gnu")" >> /etc/environment \
&& echo "BUN_ARCH=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "x64")" >> /etc/environment
ENV LD_LIBRARY_PATH=/usr/lib/gcc/${ARCH_PATH}/13:/usr/lib/${ARCH_PATH} \
LIBRARY_PATH=/usr/lib/gcc/${ARCH_PATH}/13:/usr/lib/${ARCH_PATH} \
CPLUS_INCLUDE_PATH=/usr/include/c++/13:/usr/include/${ARCH_PATH}/c++/13 \
C_INCLUDE_PATH=/usr/lib/gcc/${ARCH_PATH}/13/include
RUN if [ "$TARGETARCH" = "arm64" ]; then \
export ARCH_PATH="aarch64-linux-gnu"; \
else \
export ARCH_PATH="x86_64-linux-gnu"; \
fi \
&& mkdir -p /usr/lib/gcc/${ARCH_PATH}/13 \
&& ln -sf /usr/lib/${ARCH_PATH}/libstdc++.so.6 /usr/lib/gcc/${ARCH_PATH}/13/ \
&& echo "/usr/lib/gcc/${ARCH_PATH}/13" > /etc/ld.so.conf.d/gcc-13.conf \
&& echo "/usr/lib/${ARCH_PATH}" >> /etc/ld.so.conf.d/gcc-13.conf \
&& ldconfig
RUN for f in /usr/lib/llvm-${LLVM_VERSION}/bin/*; do ln -sf "$f" /usr/bin; done \
&& ln -sf /usr/bin/clang-${LLVM_VERSION} /usr/bin/clang \
&& ln -sf /usr/bin/clang++-${LLVM_VERSION} /usr/bin/clang++ \
&& ln -sf /usr/bin/lld-${LLVM_VERSION} /usr/bin/lld \
&& ln -sf /usr/bin/lldb-${LLVM_VERSION} /usr/bin/lldb \
&& ln -sf /usr/bin/clangd-${LLVM_VERSION} /usr/bin/clangd \
&& ln -sf /usr/bin/llvm-ar-${LLVM_VERSION} /usr/bin/llvm-ar \
&& ln -sf /usr/bin/ld.lld /usr/bin/ld \
&& ln -sf /usr/bin/clang /usr/bin/cc \
&& ln -sf /usr/bin/clang++ /usr/bin/c++
ENV CC="clang" \
CXX="clang++" \
AR="llvm-ar-${LLVM_VERSION}" \
RANLIB="llvm-ranlib-${LLVM_VERSION}" \
LD="lld-${LLVM_VERSION}"
RUN --mount=type=tmpfs,target=/tmp \
bash -c '\
set -euxo pipefail && \
source /etc/environment && \
echo "Downloading bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip from https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip" && \
curl -fsSL https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/bun-v${OLD_BUN_VERSION}/bun-linux-$BUN_ARCH.zip -o /tmp/bun.zip && \
unzip /tmp/bun.zip -d /tmp/bun && \
mv /tmp/bun/*/bun /usr/bin/bun && \
chmod +x /usr/bin/bun'
ENV LLVM_VERSION=${REPORTED_LLVM_VERSION}
WORKDIR /workspace
FROM --platform=$BUILDPLATFORM base as buildkite
ARG BUILDKITE_AGENT_TAGS
# Install Rust nightly
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
&& export PATH=$HOME/.cargo/bin:$PATH \
&& rustup install nightly \
&& rustup default nightly
RUN ARCH=$(if [ "$TARGETARCH" = "arm64" ]; then echo "arm64"; else echo "amd64"; fi) && \
echo "Downloading buildkite" && \
curl -fsSL "https://github.com/buildkite/agent/releases/download/v3.87.0/buildkite-agent-linux-${ARCH}-3.87.0.tar.gz" -o /tmp/buildkite-agent.tar.gz && \
mkdir -p /tmp/buildkite-agent && \
tar -xzf /tmp/buildkite-agent.tar.gz -C /tmp/buildkite-agent && \
mv /tmp/buildkite-agent/buildkite-agent /usr/bin/buildkite-agent
RUN mkdir -p /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /etc/buildkite-agent /var/lib/buildkite-agent/cache/bun
# The following is necessary to configure buildkite to use a stable
# checkout directory. sccache hashes absolute paths into its cache keys,
# so if buildkite uses a different checkout path each time (which it does
# by default), sccache will be useless.
RUN mkdir -p -m 755 /var/lib/buildkite-agent/hooks && \
cat <<'EOF' > /var/lib/buildkite-agent/hooks/environment
#!/bin/sh
set -efu
export BUILDKITE_BUILD_CHECKOUT_PATH=/var/lib/buildkite-agent/build
EOF
RUN chmod 744 /var/lib/buildkite-agent/hooks/environment
COPY ../*/agent.mjs /var/bun/scripts/
ENV BUN_INSTALL_CACHE=/var/lib/buildkite-agent/cache/bun
ENV BUILDKITE_AGENT_TAGS=${BUILDKITE_AGENT_TAGS}
WORKDIR /var/bun/scripts
ENV PATH=/root/.cargo/bin:$PATH
CMD ["bun", "/var/bun/scripts/agent.mjs", "start"]
FROM --platform=$BUILDPLATFORM base as bun-build-linux-local
ARG LLVM_VERSION
WORKDIR /workspace/bun
COPY . /workspace/bun
# Install Rust nightly
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
&& export PATH=$HOME/.cargo/bin:$PATH \
&& rustup install nightly \
&& rustup default nightly
ENV PATH=/root/.cargo/bin:$PATH
ENV LLVM_VERSION=${REPORTED_LLVM_VERSION}
RUN --mount=type=tmpfs,target=/workspace/bun/build \
ls -la \
&& bun run build:release \
&& mkdir -p /target \
&& cp -r /workspace/bun/build/release/bun /target/bun

View File

@@ -1,122 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Check if running as root
if [ "$EUID" -ne 0 ]; then
echo "error: must run as root"
exit 1
fi
# Check OS compatibility
if ! command -v dnf &> /dev/null; then
echo "error: this script requires dnf (RHEL/Fedora/CentOS)"
exit 1
fi
# Ensure /tmp/agent.mjs, /tmp/Dockerfile are present
if [ ! -f /tmp/agent.mjs ] || [ ! -f /tmp/Dockerfile ]; then
# Print each missing file
if [ ! -f /tmp/agent.mjs ]; then
echo "error: /tmp/agent.mjs is missing"
fi
if [ ! -f /tmp/Dockerfile ]; then
echo "error: /tmp/Dockerfile is missing"
fi
exit 1
fi
# Install Docker
dnf update -y
dnf install -y docker
systemctl enable docker
systemctl start docker || {
echo "error: failed to start Docker"
exit 1
}
# Create builder
docker buildx create --name builder --driver docker-container --bootstrap --use || {
echo "error: failed to create Docker buildx builder"
exit 1
}
# Set up Docker to start on boot
cat << 'EOF' > /etc/systemd/system/buildkite-agent.service
[Unit]
Description=Buildkite Docker Container
After=docker.service network-online.target
Requires=docker.service network-online.target
[Service]
TimeoutStartSec=0
Restart=always
RestartSec=5
ExecStartPre=-/usr/bin/docker stop buildkite
ExecStartPre=-/usr/bin/docker rm buildkite
ExecStart=/usr/bin/docker run \
--name buildkite \
--restart=unless-stopped \
--network host \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /tmp:/tmp \
buildkite:latest
[Install]
WantedBy=multi-user.target
EOF
echo "Building Buildkite image"
# Clean up any previous build artifacts
rm -rf /tmp/fakebun
mkdir -p /tmp/fakebun/scripts /tmp/fakebun/.buildkite
# Copy required files
cp /tmp/agent.mjs /tmp/fakebun/scripts/ || {
echo "error: failed to copy agent.mjs"
exit 1
}
cp /tmp/Dockerfile /tmp/fakebun/.buildkite/Dockerfile || {
echo "error: failed to copy Dockerfile"
exit 1
}
cd /tmp/fakebun || {
echo "error: failed to change directory"
exit 1
}
# Build the Buildkite image
docker buildx build \
--platform $(uname -m | sed 's/aarch64/linux\/arm64/;s/x86_64/linux\/amd64/') \
--tag buildkite:latest \
--target buildkite \
-f .buildkite/Dockerfile \
--load \
. || {
echo "error: Docker build failed"
exit 1
}
# Create container to ensure image is cached in AMI
docker container create \
--name buildkite \
--restart=unless-stopped \
buildkite:latest || {
echo "error: failed to create buildkite container"
exit 1
}
# Reload systemd to pick up new service
systemctl daemon-reload
# Enable the service, but don't start it yet
systemctl enable buildkite-agent || {
echo "error: failed to enable buildkite-agent service"
exit 1
}
echo "Bootstrap complete"
echo "To start the Buildkite agent, run: "
echo " systemctl start buildkite-agent"

View File

@@ -96,7 +96,6 @@ function getTargetLabel(target) {
* @property {Distro} [distro]
* @property {string} release
* @property {Tier} [tier]
* @property {string[]} [features]
*/
/**
@@ -105,10 +104,10 @@ function getTargetLabel(target) {
const buildPlatforms = [
{ os: "darwin", arch: "aarch64", release: "14" },
{ os: "darwin", arch: "x64", release: "14" },
{ os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023", features: ["docker"] },
{ os: "linux", arch: "x64", distro: "amazonlinux", release: "2023", features: ["docker"] },
{ os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023", features: ["docker"] },
{ os: "linux", arch: "x64", profile: "asan", distro: "amazonlinux", release: "2023", features: ["docker"] },
{ os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "x64", distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "x64", profile: "asan", distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.22" },
{ os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.22" },
{ os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.22" },
@@ -125,6 +124,10 @@ const testPlatforms = [
{ os: "darwin", arch: "aarch64", release: "13", tier: "previous" },
{ os: "darwin", arch: "x64", release: "14", tier: "latest" },
{ os: "darwin", arch: "x64", release: "13", tier: "previous" },
{ os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "x64", distro: "amazonlinux", release: "2023" },
{ os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023" },
// { os: "linux", arch: "x64", profile: "asan", distro: "amazonlinux", release: "2023" }, // TODO
{ os: "linux", arch: "aarch64", distro: "debian", release: "13", tier: "latest" },
{ os: "linux", arch: "x64", distro: "debian", release: "13", tier: "latest" },
{ os: "linux", arch: "x64", baseline: true, distro: "debian", release: "13", tier: "latest" },
@@ -174,20 +177,15 @@ function getPlatformLabel(platform) {
* @returns {string}
*/
function getImageKey(platform) {
const { os, arch, distro, release, features, abi } = platform;
const { os, arch, distro, release, abi } = platform;
const version = release.replace(/\./g, "");
let key = `${os}-${arch}-${version}`;
if (distro) {
key += `-${distro}`;
}
if (features?.length) {
key += `-with-${features.join("-")}`;
}
if (abi) {
key += `-${abi}`;
}
return key;
}
@@ -606,7 +604,7 @@ function getTestBunStep(platform, options, testOptions = {}) {
* @returns {Step}
*/
function getBuildImageStep(platform, options) {
const { os, arch, distro, release, features } = platform;
const { os, arch, distro, release } = platform;
const { publishImages } = options;
const action = publishImages ? "publish-image" : "create-image";
@@ -622,9 +620,6 @@ function getBuildImageStep(platform, options) {
"--ci",
"--authorized-org=oven-sh",
];
for (const feature of features || []) {
command.push(`--feature=${feature}`);
}
return {
key: `${getImageKey(platform)}-build-image`,

View File

@@ -22,15 +22,14 @@ error() {
if ! [ "$$" = "$pid" ]; then
kill -s TERM "$pid"
fi
exit 1
# Kill the shell. This used to be 'exit 1' but if the command is running inside a
# subshell, then only the subshell dies and the script keeps running uninterrupted.
kill $$
}
execute() {
local opts=$-
set -x
"$@"
{ local status=$?; set +x "$opts"; } 2> /dev/null
if [ "$status" -ne 0 ]; then
print "$ $@" >&2
if ! "$@"; then
error "Command failed: $@"
fi
}
@@ -380,37 +379,6 @@ check_operating_system() {
esac
}
check_inside_docker() {
if ! [ "$os" = "linux" ]; then
return
fi
print "Checking if inside Docker..."
if [ -f "/.dockerenv" ]; then
docker=1
else
if [ -f "/proc/1/cgroup" ]; then
case "$(cat /proc/1/cgroup)" in
*/docker/*)
docker=1
;;
esac
fi
if [ -f "/proc/self/mountinfo" ]; then
case "$(cat /proc/self/mountinfo)" in
*/docker/*)
docker=1
;;
esac
fi
fi
if [ "$docker" = "1" ]; then
print "Docker: enabled"
fi
}
check_package_manager() {
print "Checking package manager..."
@@ -739,7 +707,7 @@ install_common_software() {
git \
unzip \
wget \
dnf-plugins-core
dnf-plugins-core \
;;
apk)
# https://pkgs.alpinelinux.org/packages
@@ -1079,7 +1047,9 @@ install_build_essentials() {
;;
dnf | yum)
install_packages \
gcc-c++ \
gcc14-c++ \
gcc14-libstdc++-static \
gcc14-libatomic-static \
xz \
pkg-config \
golang
@@ -1175,11 +1145,18 @@ install_llvm() {
;;
esac
case "$os" in
freebsd)
case "$distro" in
freebsd)
# TODO: use llvm_version_exact
install_packages "devel/llvm$(llvm_version)"
;;
amzn)
install_packages \
"llvm$(llvm_version)" \
"clang$(llvm_version)" \
"lld$(llvm_version)" \
"compiler-rt$(llvm_version)" \
;;
esac
}
@@ -1304,8 +1281,15 @@ install_rust() {
;;
freebsd)
install_packages lang/rust
create_directory "$HOME/.cargo/bin"
append_to_path "$HOME/.cargo/bin"
create_directory "$home/.cargo/bin"
append_to_path "$home/.cargo/bin"
;;
amzn)
sh="$(require sh)"
rustup_script=$(download_file "https://sh.rustup.rs")
execute_as_user "$rustup_script -y --no-modify-path"
create_directory "$home/.cargo/bin"
append_to_path "$home/.cargo/bin"
;;
*)
rust_home="/opt/rust"
@@ -1342,13 +1326,17 @@ install_docker() {
sysutils/docker-compose \
;;
*)
case "$distro-$release" in
amzn-2 | amzn-1)
execute_sudo amazon-linux-extras install docker
;;
amzn-* | alpine-*)
case "$distro" in
alpine)
install_packages docker docker-cli-compose
;;
amzn)
install_packages docker
local compose_bin=$(download_file https://github.com/docker/compose/releases/latest/download/docker-compose-linux-$(uname -m))
execute_sudo chmod +x "$compose_bin"
execute_sudo mv "$compose_bin" "$(dirname $compose_bin)/docker-compose"
move_to_bin "$(dirname $compose_bin)/docker-compose"
;;
*)
sh="$(require sh)"
script=$(download_file "https://get.docker.com")
@@ -1420,10 +1408,6 @@ install_osxcross() {
}
install_tailscale() {
if [ "$docker" = "1" ]; then
return
fi
case "$os" in
linux)
sh="$(require sh)"
@@ -1445,6 +1429,9 @@ install_fuse_python() {
if ! [ "$os" = "linux" ]; then
return
fi
if [ "$distro" = "amzn" ]; then
return
fi
# only linux needs this
case "$pm" in
@@ -1776,7 +1763,6 @@ ensure_no_tmpfs() {
main() {
check_features "$@"
check_operating_system
check_inside_docker
check_user
check_ulimit
check_package_manager

View File

@@ -1,300 +0,0 @@
import { inspect } from "node:util";
import { $, isCI, spawn, spawnSafe, which } from "./utils.mjs";
export const docker = {
get name() {
return "docker";
},
/**
* @typedef {"linux" | "darwin" | "windows"} DockerOs
* @typedef {"amd64" | "arm64"} DockerArch
* @typedef {`${DockerOs}/${DockerArch}`} DockerPlatform
*/
/**
* @param {Platform} platform
* @returns {DockerPlatform}
*/
getPlatform(platform) {
const { os, arch } = platform;
if (arch === "aarch64") {
return `${os}/arm64`;
} else if (arch === "x64") {
return `${os}/amd64`;
}
throw new Error(`Unsupported platform: ${inspect(platform)}`);
},
/**
* @typedef DockerSpawnOptions
* @property {DockerPlatform} [platform]
* @property {boolean} [json]
*/
/**
* @param {string[]} args
* @param {DockerSpawnOptions & import("./utils.mjs").SpawnOptions} [options]
* @returns {Promise<unknown>}
*/
async spawn(args, options = {}) {
const docker = which("docker", { required: true });
let env = { ...process.env };
if (isCI) {
env["BUILDKIT_PROGRESS"] = "plain";
}
const { json, platform } = options;
if (json) {
args.push("--format=json");
}
if (platform) {
args.push(`--platform=${platform}`);
}
const { error, stdout } = await spawnSafe($`${docker} ${args}`, { env, ...options });
if (error) {
return;
}
if (!json) {
return stdout;
}
try {
return JSON.parse(stdout);
} catch {
return;
}
},
/**
* @typedef {Object} DockerImage
* @property {string} Id
* @property {string[]} RepoTags
* @property {string[]} RepoDigests
* @property {string} Created
* @property {DockerOs} Os
* @property {DockerArch} Architecture
* @property {number} Size
*/
/**
* @param {string} url
* @param {DockerPlatform} [platform]
* @returns {Promise<boolean>}
*/
async pullImage(url, platform) {
const done = await this.spawn($`pull ${url}`, {
platform,
throwOnError: error => !/No such image|manifest unknown/i.test(inspect(error)),
});
return !!done;
},
/**
* @param {string} url
* @param {DockerPlatform} [platform]
* @returns {Promise<DockerImage | undefined>}
*/
async inspectImage(url, platform) {
/** @type {DockerImage[]} */
const images = await this.spawn($`image inspect ${url}`, {
json: true,
throwOnError: error => !/No such image/i.test(inspect(error)),
});
if (!images) {
const pulled = await this.pullImage(url, platform);
if (pulled) {
return this.inspectImage(url, platform);
}
}
const { os, arch } = platform || {};
return images
?.filter(({ Os, Architecture }) => !os || !arch || (Os === os && Architecture === arch))
?.find((a, b) => (a.Created < b.Created ? 1 : -1));
},
/**
* @typedef {Object} DockerContainer
* @property {string} Id
* @property {string} Name
* @property {string} Image
* @property {string} Created
* @property {DockerContainerState} State
* @property {DockerContainerNetworkSettings} NetworkSettings
*/
/**
* @typedef {Object} DockerContainerState
* @property {"exited" | "running"} Status
* @property {number} [Pid]
* @property {number} ExitCode
* @property {string} [Error]
* @property {string} StartedAt
* @property {string} FinishedAt
*/
/**
* @typedef {Object} DockerContainerNetworkSettings
* @property {string} [IPAddress]
*/
/**
* @param {string} containerId
* @returns {Promise<DockerContainer | undefined>}
*/
async inspectContainer(containerId) {
const containers = await this.spawn($`container inspect ${containerId}`, { json: true });
return containers?.find(a => a.Id === containerId);
},
/**
* @returns {Promise<DockerContainer[]>}
*/
async listContainers() {
const containers = await this.spawn($`container ls --all`, { json: true });
return containers || [];
},
/**
* @typedef {Object} DockerRunOptions
* @property {string[]} [command]
* @property {DockerPlatform} [platform]
* @property {string} [name]
* @property {boolean} [detach]
* @property {"always" | "never"} [pull]
* @property {boolean} [rm]
* @property {"no" | "on-failure" | "always"} [restart]
*/
/**
* @param {string} url
* @param {DockerRunOptions} [options]
* @returns {Promise<DockerContainer>}
*/
async runContainer(url, options = {}) {
const { detach, command = [], ...containerOptions } = options;
const args = Object.entries(containerOptions)
.filter(([_, value]) => typeof value !== "undefined")
.map(([key, value]) => (typeof value === "boolean" ? `--${key}` : `--${key}=${value}`));
if (detach) {
args.push("--detach");
} else {
args.push("--tty", "--interactive");
}
const stdio = detach ? "pipe" : "inherit";
const result = await this.spawn($`run ${args} ${url} ${command}`, { stdio });
if (!detach) {
return;
}
const containerId = result.trim();
const container = await this.inspectContainer(containerId);
if (!container) {
throw new Error(`Failed to run container: ${inspect(result)}`);
}
return container;
},
/**
* @param {Platform} platform
* @returns {Promise<DockerImage>}
*/
async getBaseImage(platform) {
const { os, distro, release } = platform;
const dockerPlatform = this.getPlatform(platform);
let url;
if (os === "linux") {
if (distro === "debian" || distro === "ubuntu" || distro === "alpine") {
url = `docker.io/library/${distro}:${release}`;
} else if (distro === "amazonlinux") {
url = `public.ecr.aws/amazonlinux/amazonlinux:${release}`;
}
}
if (url) {
const image = await this.inspectImage(url, dockerPlatform);
if (image) {
return image;
}
}
throw new Error(`Unsupported platform: ${inspect(platform)}`);
},
/**
* @param {DockerContainer} container
* @param {MachineOptions} [options]
* @returns {Machine}
*/
toMachine(container, options = {}) {
const { Id: containerId } = container;
const exec = (command, options) => {
return spawn(["docker", "exec", containerId, ...command], options);
};
const execSafe = (command, options) => {
return spawnSafe(["docker", "exec", containerId, ...command], options);
};
const upload = async (source, destination) => {
await spawn(["docker", "cp", source, `${containerId}:${destination}`]);
};
const attach = async () => {
const { exitCode, error } = await spawn(["docker", "exec", "-it", containerId, "sh"], {
stdio: "inherit",
});
if (exitCode === 0 || exitCode === 130) {
return;
}
throw error;
};
const snapshot = async name => {
await spawn(["docker", "commit", containerId]);
};
const kill = async () => {
await spawn(["docker", "kill", containerId]);
};
return {
cloud: "docker",
id: containerId,
spawn: exec,
spawnSafe: execSafe,
upload,
attach,
snapshot,
close: kill,
[Symbol.asyncDispose]: kill,
};
},
/**
* @param {MachineOptions} options
* @returns {Promise<Machine>}
*/
async createMachine(options) {
const { Id: imageId, Os, Architecture } = await docker.getBaseImage(options);
const container = await docker.runContainer(imageId, {
platform: `${Os}/${Architecture}`,
command: ["sleep", "1d"],
detach: true,
rm: true,
restart: "no",
});
return this.toMachine(container, options);
},
};

View File

@@ -4,7 +4,6 @@ import { existsSync, mkdtempSync, readdirSync } from "node:fs";
import { basename, extname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { inspect, parseArgs } from "node:util";
import { docker } from "./docker.mjs";
import { tart } from "./tart.mjs";
import {
$,
@@ -1060,8 +1059,6 @@ function getRdpFile(hostname, username) {
*/
function getCloud(name) {
switch (name) {
case "docker":
return docker;
case "aws":
return aws;
case "tart":
@@ -1175,7 +1172,6 @@ async function main() {
"no-bootstrap": { type: "boolean" },
"buildkite-token": { type: "string" },
"tailscale-authkey": { type: "string" },
"docker": { type: "boolean" },
},
});
@@ -1221,13 +1217,12 @@ async function main() {
detached: !!args["detached"],
bootstrap: args["no-bootstrap"] !== true,
ci: !!args["ci"],
features: args["feature"],
rdp: !!args["rdp"] || !!args["vnc"],
sshKeys,
userData: args["user-data"] ? readFile(args["user-data"]) : undefined,
};
let { detached, bootstrap, ci, os, arch, distro, release, features } = options;
let { detached, bootstrap, ci, os, arch, distro, release } = options;
let name = `${os}-${arch}-${(release || "").replace(/\./g, "")}`;
@@ -1239,20 +1234,9 @@ async function main() {
name += `-musl`;
}
if (features?.length) {
name += `-with-${features.join("-")}`;
}
let bootstrapPath, agentPath, dockerfilePath;
let bootstrapPath, agentPath;
if (bootstrap) {
bootstrapPath = resolve(
import.meta.dirname,
os === "windows"
? "bootstrap.ps1"
: features?.includes("docker")
? "../.buildkite/Dockerfile-bootstrap.sh"
: "bootstrap.sh",
);
bootstrapPath = resolve(import.meta.dirname, os === "windows" ? "bootstrap.ps1" : "bootstrap.sh");
if (!existsSync(bootstrapPath)) {
throw new Error(`Script not found: ${bootstrapPath}`);
}
@@ -1266,14 +1250,6 @@ async function main() {
agentPath = join(tmpPath, "agent.mjs");
await spawnSafe($`${npx} esbuild ${entryPath} --bundle --platform=node --format=esm --outfile=${agentPath}`);
}
if (features?.includes("docker")) {
dockerfilePath = resolve(import.meta.dirname, "../.buildkite/Dockerfile");
if (!existsSync(dockerfilePath)) {
throw new Error(`Dockerfile not found: ${dockerfilePath}`);
}
}
}
/** @type {Machine} */
@@ -1363,31 +1339,12 @@ async function main() {
await machine.spawnSafe(["powershell", remotePath, ...args], { stdio: "inherit" });
});
} else {
if (!features?.includes("docker")) {
const remotePath = "/tmp/bootstrap.sh";
const args = ci ? ["--ci"] : [];
for (const feature of features || []) {
args.push(`--${feature}`);
}
await startGroup("Running bootstrap...", async () => {
await machine.upload(bootstrapPath, remotePath);
await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" });
});
} else if (dockerfilePath) {
const remotePath = "/tmp/bootstrap.sh";
await startGroup("Running Docker bootstrap...", async () => {
await machine.upload(bootstrapPath, remotePath);
console.log("Uploaded bootstrap.sh");
await machine.upload(dockerfilePath, "/tmp/Dockerfile");
console.log("Uploaded Dockerfile");
await machine.upload(agentPath, "/tmp/agent.mjs");
console.log("Uploaded agent.mjs");
agentPath = "";
bootstrapPath = "";
await machine.spawnSafe(["sudo", "bash", remotePath], { stdio: "inherit", cwd: "/tmp" });
});
}
const remotePath = "/tmp/bootstrap.sh";
const args = ci ? ["--ci"] : [];
await startGroup("Running bootstrap...", async () => {
await machine.upload(bootstrapPath, remotePath);
await machine.spawnSafe(["sh", remotePath, ...args], { stdio: "inherit" });
});
}
}
@@ -1396,9 +1353,6 @@ async function main() {
const remotePath = "C:\\buildkite-agent\\agent.mjs";
await startGroup("Installing agent...", async () => {
await machine.upload(agentPath, remotePath);
if (cloud.name === "docker" || features?.includes("docker")) {
return;
}
await machine.spawnSafe(["node", remotePath, "install"], { stdio: "inherit" });
});
} else {
@@ -1414,9 +1368,6 @@ async function main() {
}
}
await machine.spawnSafe([...command, "cp", tmpPath, remotePath]);
if (cloud.name === "docker") {
return;
}
{
const { stdout } = await machine.spawn(["node", "-v"]);
const version = parseInt(stdout.trim().replace(/^v/, ""));

View File

@@ -1,4 +1,4 @@
import { spawn, spawnSync } from "bun";
import { spawn, spawnSync, which } from "bun";
import { bunExe, bunEnv, isCI, isMusl } from "../../harness";
// Tests that intentionally abort and should not generate core dumps when they abort
@@ -20,8 +20,8 @@ export async function build(dir: string) {
// so we make it use clang instead
...(process.platform == "linux" && isCI
? {
CC: !isMusl ? "/usr/lib/llvm-19/bin/clang" : "/usr/lib/llvm19/bin/clang",
CXX: !isMusl ? "/usr/lib/llvm-19/bin/clang++" : "/usr/lib/llvm19/bin/clang++",
CC: which("clang-19") ?? (!isMusl ? "/usr/lib/llvm-19/bin/clang" : "/usr/lib/llvm19/bin/clang"),
CXX: which("clang++-19") ?? (!isMusl ? "/usr/lib/llvm-19/bin/clang++" : "/usr/lib/llvm19/bin/clang++"),
}
: {}),
},