From 78ca50e356930506e7e349cec8a00aafb9ff71f0 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sat, 17 Jan 2026 08:28:30 +0000 Subject: [PATCH] feat(ci): bake build dependencies into CI images This change pre-downloads all build dependencies (Zig, WebKit, BoringSSL, etc.) into the CI build images, significantly reducing build startup time by avoiding repeated downloads. Changes: - Add scripts/bake-dependencies.sh to clone Bun and run cmake configure to download all dependencies during image creation - Update bootstrap.sh to call bake-dependencies.sh when in CI mode - Add buildkite checkout hook to use git fetch instead of full clone when the repository already exists (preserves build/ and vendor/zig/) - Add buildkite pre-command hook to remove node_modules before each build, ensuring clean dependency installation - Update Dockerfile with the same hooks and pre-baked dependencies During CI builds: 1. git fetch origin (instead of full clone) 2. git checkout 3. rm -rf node_modules && bun install 4. Build uses pre-downloaded dependencies from build/ and vendor/ Co-Authored-By: Claude Opus 4.5 --- .buildkite/Dockerfile | 59 ++++++++++++- scripts/bake-dependencies.sh | 158 +++++++++++++++++++++++++++++++++++ scripts/bootstrap.sh | 77 ++++++++++++++++- 3 files changed, 292 insertions(+), 2 deletions(-) create mode 100755 scripts/bake-dependencies.sh diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile index 2b5f944834..5757cf5928 100644 --- a/.buildkite/Dockerfile +++ b/.buildkite/Dockerfile @@ -136,7 +136,64 @@ set -efu export BUILDKITE_BUILD_CHECKOUT_PATH=/var/lib/buildkite-agent/build EOF -RUN chmod 744 /var/lib/buildkite-agent/hooks/environment +# Checkout hook - uses git fetch instead of clone when repo exists +RUN cat <<'EOF' > /var/lib/buildkite-agent/hooks/checkout +#!/bin/sh +set -efu + +CHECKOUT_PATH="${BUILDKITE_BUILD_CHECKOUT_PATH}" +COMMIT="${BUILDKITE_COMMIT}" +REPO="${BUILDKITE_REPO}" + +echo "--- :git: Checkout" + +if [ -d "${CHECKOUT_PATH}/.git" ]; then + echo "Repository exists, using git fetch..." + cd "${CHECKOUT_PATH}" + git fetch --depth=1 origin "${COMMIT}" || git fetch origin "${COMMIT}" + git reset --hard + git clean -fdx -e 'build/' -e 'vendor/zig/' + git checkout -f "${COMMIT}" +else + echo "Repository does not exist, cloning..." + git clone --depth=1 "${REPO}" "${CHECKOUT_PATH}" + cd "${CHECKOUT_PATH}" + git fetch --depth=1 origin "${COMMIT}" || git fetch origin "${COMMIT}" + git checkout -f "${COMMIT}" +fi + +echo "Checked out ${COMMIT}" +EOF + +# Pre-command hook - removes node_modules before each build +RUN cat <<'EOF' > /var/lib/buildkite-agent/hooks/pre-command +#!/bin/sh +set -eu + +CHECKOUT_PATH="${BUILDKITE_BUILD_CHECKOUT_PATH}" + +if [ -d "${CHECKOUT_PATH}/node_modules" ]; then + echo "Removing existing node_modules..." + rm -rf "${CHECKOUT_PATH}/node_modules" +fi +EOF + +RUN chmod 755 /var/lib/buildkite-agent/hooks/environment \ + && chmod 755 /var/lib/buildkite-agent/hooks/checkout \ + && chmod 755 /var/lib/buildkite-agent/hooks/pre-command + +# Clone Bun repository and pre-download dependencies +# This avoids downloading dependencies during each CI build +RUN git clone --depth=1 https://github.com/oven-sh/bun.git /var/lib/buildkite-agent/build + +WORKDIR /var/lib/buildkite-agent/build + +# Install dependencies and run cmake configure to download build dependencies +RUN bun install --frozen-lockfile && \ + mkdir -p build/release && \ + cmake -S . -B build/release -G Ninja -DCMAKE_BUILD_TYPE=Release -DCI=ON && \ + rm -rf build/release/CMakeFiles build/release/CMakeCache.txt build/release/cmake_install.cmake && \ + rm -rf node_modules COPY ../*/agent.mjs /var/bun/scripts/ diff --git a/scripts/bake-dependencies.sh b/scripts/bake-dependencies.sh new file mode 100755 index 0000000000..9d22eefdb1 --- /dev/null +++ b/scripts/bake-dependencies.sh @@ -0,0 +1,158 @@ +#!/bin/sh +# Version: 1 +# +# This script pre-downloads all build dependencies into the image. +# It should be run during AMI baking to avoid downloading dependencies +# during each CI build. +# +# Dependencies downloaded: +# - Zig compiler +# - WebKit (JavaScriptCore) +# - BoringSSL +# - And other cmake-managed dependencies + +set -eu + +print() { + echo "$@" +} + +error() { + print "error: $@" >&2 + exit 1 +} + +# Detect architecture +detect_arch() { + arch="$(uname -m)" + case "$arch" in + x86_64 | x64 | amd64) + echo "x64" + ;; + aarch64 | arm64) + echo "aarch64" + ;; + *) + error "Unsupported architecture: $arch" + ;; + esac +} + +# Detect ABI (glibc vs musl) +detect_abi() { + if [ -f "/etc/alpine-release" ]; then + echo "musl" + else + ldd_output="$(ldd --version 2>&1 || true)" + case "$ldd_output" in + *musl*) + echo "musl" + ;; + *) + echo "" + ;; + esac + fi +} + +ARCH="$(detect_arch)" +ABI="$(detect_abi)" + +# Default paths - these should match what the buildkite agent uses +BUN_REPO_PATH="${BUN_REPO_PATH:-/var/lib/buildkite-agent/build}" +BUILD_TYPE="${BUILD_TYPE:-release}" + +print "=== Bun Dependency Baking Script ===" +print "Architecture: $ARCH" +print "ABI: ${ABI:-glibc}" +print "Repository path: $BUN_REPO_PATH" +print "Build type: $BUILD_TYPE" + +# Clone the Bun repository if it doesn't exist +if [ ! -d "$BUN_REPO_PATH/.git" ]; then + print "Cloning Bun repository..." + git clone --depth=1 https://github.com/oven-sh/bun.git "$BUN_REPO_PATH" +else + print "Bun repository already exists, updating..." + cd "$BUN_REPO_PATH" + git fetch --depth=1 origin main + git checkout FETCH_HEAD +fi + +cd "$BUN_REPO_PATH" + +# Install npm dependencies (will be cleaned later, but needed for codegen) +print "Installing npm dependencies..." +bun install --frozen-lockfile + +# Set up build directory +BUILD_PATH="$BUN_REPO_PATH/build/$BUILD_TYPE" +CACHE_PATH="$BUILD_PATH/cache" +mkdir -p "$BUILD_PATH" "$CACHE_PATH" + +print "Build path: $BUILD_PATH" +print "Cache path: $CACHE_PATH" + +# Run cmake configure to download all dependencies +# This will download: Zig, WebKit, BoringSSL, and all other dependencies +print "Running CMake configure to download dependencies..." + +CMAKE_ARGS="-S $BUN_REPO_PATH -B $BUILD_PATH" +CMAKE_ARGS="$CMAKE_ARGS -G Ninja" +CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release" +CMAKE_ARGS="$CMAKE_ARGS -DCI=ON" + +if [ -n "$ABI" ]; then + CMAKE_ARGS="$CMAKE_ARGS -DABI=$ABI" +fi + +# Run cmake configure - this downloads all dependencies +cmake $CMAKE_ARGS + +# Also download debug WebKit variant for debug builds +print "Downloading debug WebKit variant..." +cmake $CMAKE_ARGS -DCMAKE_BUILD_TYPE=Debug -B "$BUN_REPO_PATH/build/debug" || true + +# Clean up build artifacts but keep downloaded dependencies +print "Cleaning up build artifacts..." +rm -rf "$BUILD_PATH/CMakeFiles" "$BUILD_PATH/CMakeCache.txt" "$BUILD_PATH/cmake_install.cmake" +rm -rf "$BUN_REPO_PATH/build/debug/CMakeFiles" "$BUN_REPO_PATH/build/debug/CMakeCache.txt" 2>/dev/null || true + +# Remove node_modules - will be reinstalled during actual builds +print "Removing node_modules..." +rm -rf "$BUN_REPO_PATH/node_modules" + +# List what was downloaded +print "" +print "=== Downloaded Dependencies ===" +if [ -d "$BUN_REPO_PATH/vendor/zig" ]; then + print "✓ Zig compiler: $(ls -la "$BUN_REPO_PATH/vendor/zig/zig" 2>/dev/null | awk '{print $5}' || echo 'present')" +fi +if [ -d "$CACHE_PATH/webkit-"* ] 2>/dev/null; then + print "✓ WebKit: $(du -sh "$CACHE_PATH"/webkit-* 2>/dev/null | head -1 || echo 'present')" +fi +if [ -d "$BUILD_PATH/boringssl" ]; then + print "✓ BoringSSL: present" +fi +if [ -d "$BUILD_PATH/mimalloc" ]; then + print "✓ mimalloc: present" +fi +if [ -d "$BUILD_PATH/zstd" ]; then + print "✓ zstd: present" +fi +if [ -d "$BUILD_PATH/lol-html" ]; then + print "✓ lol-html: present" +fi + +# Calculate total size +TOTAL_SIZE="$(du -sh "$BUN_REPO_PATH" 2>/dev/null | cut -f1 || echo 'unknown')" +print "" +print "Total repository size: $TOTAL_SIZE" + +print "" +print "=== Dependency baking complete ===" +print "The following will happen during CI builds:" +print " 1. git fetch origin (instead of full clone)" +print " 2. git checkout " +print " 3. rm -rf node_modules && bun install" +print " 4. Build will use pre-downloaded dependencies" diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 5deb20a7c7..16175b26cf 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Version: 26 +# Version: 27 # A script that installs the dependencies needed to build and test Bun. # This should work on macOS and Linux with a POSIX shell. @@ -1455,13 +1455,69 @@ create_buildkite_user() { # reasons. local hook_dir=${home}/hooks mkdir -p -m 755 "${hook_dir}" + + # Environment hook - sets checkout path cat << EOF > "${hook_dir}/environment" #!/bin/sh set -efu export BUILDKITE_BUILD_CHECKOUT_PATH=${home}/build EOF + + # Checkout hook - uses git fetch instead of clone when repo exists + # This works with the pre-baked repository from bake-dependencies.sh + cat << 'CHECKOUT_EOF' > "${hook_dir}/checkout" +#!/bin/sh +set -efu + +CHECKOUT_PATH="${BUILDKITE_BUILD_CHECKOUT_PATH}" +COMMIT="${BUILDKITE_COMMIT}" +REPO="${BUILDKITE_REPO}" + +echo "--- :git: Checkout" + +if [ -d "${CHECKOUT_PATH}/.git" ]; then + echo "Repository exists, using git fetch..." + cd "${CHECKOUT_PATH}" + + # Fetch the specific commit + git fetch --depth=1 origin "${COMMIT}" || git fetch origin "${COMMIT}" + + # Clean up any local changes + git reset --hard + git clean -fdx -e 'build/' -e 'vendor/zig/' + + # Checkout the commit + git checkout -f "${COMMIT}" +else + echo "Repository does not exist, cloning..." + git clone --depth=1 "${REPO}" "${CHECKOUT_PATH}" + cd "${CHECKOUT_PATH}" + git fetch --depth=1 origin "${COMMIT}" || git fetch origin "${COMMIT}" + git checkout -f "${COMMIT}" +fi + +echo "Checked out ${COMMIT}" +CHECKOUT_EOF + + # Pre-command hook - removes node_modules before bun install + cat << 'PRECOMMAND_EOF' > "${hook_dir}/pre-command" +#!/bin/sh +set -eu + +CHECKOUT_PATH="${BUILDKITE_BUILD_CHECKOUT_PATH}" + +# Remove node_modules before each build to ensure clean state +# Dependencies are re-installed via bun install +if [ -d "${CHECKOUT_PATH}/node_modules" ]; then + echo "Removing existing node_modules..." + rm -rf "${CHECKOUT_PATH}/node_modules" +fi +PRECOMMAND_EOF + execute_sudo chmod +x "${hook_dir}/environment" + execute_sudo chmod +x "${hook_dir}/checkout" + execute_sudo chmod +x "${hook_dir}/pre-command" execute_sudo chown -R "$user:$group" "$hook_dir" set +ef -"$opts" @@ -1669,6 +1725,24 @@ ensure_no_tmpfs() { execute_sudo systemctl mask tmp.mount } +bake_bun_dependencies() { + if ! [ "$ci" = "1" ]; then + return + fi + + print "Baking Bun dependencies into image..." + + # The bake script will clone Bun and download all build dependencies + # This includes: Zig, WebKit, BoringSSL, mimalloc, zstd, etc. + bake_script="$(dirname "$0")/bake-dependencies.sh" + if [ -f "$bake_script" ]; then + # Run as buildkite-agent user so files have correct ownership + execute_as_user "BUN_REPO_PATH=/var/lib/buildkite-agent/build sh $bake_script" + else + print "Warning: bake-dependencies.sh not found, skipping dependency baking" + fi +} + main() { check_features "$@" check_operating_system @@ -1685,6 +1759,7 @@ main() { if [ "${BUN_NO_CORE_DUMP:-0}" != "1" ]; then configure_core_dumps fi + bake_bun_dependencies clean_system ensure_no_tmpfs }