mirror of
https://github.com/oven-sh/bun
synced 2026-02-02 15:08:46 +00:00
build(ENG-21330): Replace ccache with sccache (#24200)
This commit is contained in:
Marko Vejnovic
@@ -48,6 +48,14 @@ RUN --mount=type=tmpfs,target=/tmp \
|
||||
wget -O /tmp/cmake.sh "$cmake_url" && \
|
||||
sh /tmp/cmake.sh --skip-license --prefix=/usr
|
||||
|
||||
RUN --mount=type=tmpfs,target=/tmp \
|
||||
sccache_version="0.12.0" && \
|
||||
arch=$(uname -m) && \
|
||||
sccache_url="https://github.com/mozilla/sccache/releases/download/v${sccache_version}/sccache-v${sccache_version}-${arch}-unknown-linux-musl.tar.gz" && \
|
||||
wget -O /tmp/sccache.tar.gz "$sccache_url" && \
|
||||
tar -xzf /tmp/sccache.tar.gz -C /tmp && \
|
||||
install -m755 /tmp/sccache-v${sccache_version}-${arch}-unknown-linux-musl/sccache /usr/local/bin
|
||||
|
||||
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \
|
||||
--slave /usr/bin/g++ g++ /usr/bin/g++-13 \
|
||||
--slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-13 \
|
||||
@@ -110,14 +118,14 @@ ARG BUILDKITE_AGENT_TAGS
|
||||
|
||||
|
||||
# Install Rust nightly
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
|
||||
&& export PATH=$HOME/.cargo/bin:$PATH \
|
||||
&& rustup install nightly \
|
||||
&& rustup default nightly
|
||||
|
||||
|
||||
RUN ARCH=$(if [ "$TARGETARCH" = "arm64" ]; then echo "arm64"; else echo "amd64"; fi) && \
|
||||
echo "Downloading buildkite" && \
|
||||
echo "Downloading buildkite" && \
|
||||
curl -fsSL "https://github.com/buildkite/agent/releases/download/v3.87.0/buildkite-agent-linux-${ARCH}-3.87.0.tar.gz" -o /tmp/buildkite-agent.tar.gz && \
|
||||
mkdir -p /tmp/buildkite-agent && \
|
||||
tar -xzf /tmp/buildkite-agent.tar.gz -C /tmp/buildkite-agent && \
|
||||
@@ -147,7 +155,7 @@ COPY . /workspace/bun
|
||||
|
||||
|
||||
# Install Rust nightly
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \
|
||||
&& export PATH=$HOME/.cargo/bin:$PATH \
|
||||
&& rustup install nightly \
|
||||
&& rustup default nightly
|
||||
@@ -161,4 +169,4 @@ RUN --mount=type=tmpfs,target=/workspace/bun/build \
|
||||
ls -la \
|
||||
&& bun run build:release \
|
||||
&& mkdir -p /target \
|
||||
&& cp -r /workspace/bun/build/release/bun /target/bun
|
||||
&& cp -r /workspace/bun/build/release/bun /target/bun
|
||||
|
||||
@@ -24,7 +24,7 @@ if(CMAKE_HOST_APPLE)
|
||||
include(SetupMacSDK)
|
||||
endif()
|
||||
include(SetupLLVM)
|
||||
include(SetupCcache)
|
||||
include(SetupSccache)
|
||||
|
||||
# --- Project ---
|
||||
|
||||
|
||||
@@ -23,19 +23,19 @@ Using your system's package manager, install Bun's dependencies:
|
||||
{% codetabs group="os" %}
|
||||
|
||||
```bash#macOS (Homebrew)
|
||||
$ brew install automake ccache cmake coreutils gnu-sed go icu4c libiconv libtool ninja pkg-config rust ruby
|
||||
$ brew install automake cmake coreutils gnu-sed go icu4c libiconv libtool ninja pkg-config rust ruby sccache
|
||||
```
|
||||
|
||||
```bash#Ubuntu/Debian
|
||||
$ sudo apt install curl wget lsb-release software-properties-common cargo ccache cmake git golang libtool ninja-build pkg-config rustc ruby-full xz-utils
|
||||
$ sudo apt install curl wget lsb-release software-properties-common cargo cmake git golang libtool ninja-build pkg-config rustc ruby-full xz-utils
|
||||
```
|
||||
|
||||
```bash#Arch
|
||||
$ sudo pacman -S base-devel ccache cmake git go libiconv libtool make ninja pkg-config python rust sed unzip ruby
|
||||
$ sudo pacman -S base-devel cmake git go libiconv libtool make ninja pkg-config python rust sed unzip ruby
|
||||
```
|
||||
|
||||
```bash#Fedora
|
||||
$ sudo dnf install cargo clang19 llvm19 lld19 ccache cmake git golang libtool ninja-build pkg-config rustc ruby libatomic-static libstdc++-static sed unzip which libicu-devel 'perl(Math::BigInt)'
|
||||
$ sudo dnf install cargo clang19 llvm19 lld19 cmake git golang libtool ninja-build pkg-config rustc ruby libatomic-static libstdc++-static sed unzip which libicu-devel 'perl(Math::BigInt)'
|
||||
```
|
||||
|
||||
```bash#openSUSE Tumbleweed
|
||||
@@ -65,6 +65,44 @@ $ brew install bun
|
||||
|
||||
{% /codetabs %}
|
||||
|
||||
### Optional: Install `sccache`
|
||||
|
||||
sccache is used to cache compilation artifacts, significantly speeding up builds. It must be installed with S3 support:
|
||||
|
||||
```bash
|
||||
# For macOS
|
||||
$ brew install sccache
|
||||
|
||||
# For Linux. Note that the version in your package manager may not have S3 support.
|
||||
$ cargo install sccache --features=s3
|
||||
```
|
||||
|
||||
This will install `sccache` with S3 support. Our build scripts will automatically detect and use `sccache` with our shared S3 cache. **Note**: Not all versions of `sccache` are compiled with S3 support, hence we recommend installing it via `cargo`.
|
||||
|
||||
#### Registering AWS Credentials for `sccache` (Core Developers Only)
|
||||
|
||||
Core developers have write access to the shared S3 cache. To enable write access, you must log in with AWS credentials. The easiest way to do this is to use the [`aws` CLI](https://aws.amazon.com/cli/) and invoke [`aws configure` to provide your AWS security info](https://docs.aws.amazon.com/cli/latest/reference/configure/).
|
||||
|
||||
The `cmake` scripts should automatically detect your AWS credentials from the environment or the `~/.aws/credentials` file.
|
||||
|
||||
<details>
|
||||
<summary>Logging in to the `aws` CLI</summary>
|
||||
|
||||
1. Install the AWS CLI by following [the official guide](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
|
||||
2. Log in to your AWS account console. A team member should provide you with your credentials.
|
||||
3. Click your name in the top right > Security credentials.
|
||||
4. Scroll to "Access keys" and create a new access key.
|
||||
5. Run `aws configure` in your terminal and provide the access key ID and secret access key when prompted.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Common Issues You May Encounter</summary>
|
||||
|
||||
- To confirm that the cache is being used, you can use the `sccache --show-stats` command right after a build. This will expose very useful statistics, including cache hits/misses.
|
||||
- If you have multiple AWS profiles configured, ensure that the correct profile is set in the `AWS_PROFILE` environment variable.
|
||||
- `sccache` follows a server-client model. If you run into weird issues where `sccache` refuses to use S3, even though you have AWS credentials configured, try killing any running `sccache` servers with `sccache --stop-server` and then re-running the build.
|
||||
</details>
|
||||
|
||||
## Install LLVM
|
||||
|
||||
Bun requires LLVM 19 (`clang` is part of LLVM). This version requirement is to match WebKit (precompiled), as mismatching versions will cause memory allocation failures at runtime. In most cases, you can install LLVM through your system package manager:
|
||||
@@ -331,15 +369,6 @@ $ bun run build -DUSE_STATIC_LIBATOMIC=OFF
|
||||
|
||||
The built version of Bun may not work on other systems if compiled this way.
|
||||
|
||||
### ccache conflicts with building TinyCC on macOS
|
||||
|
||||
If you run into issues with `ccache` when building TinyCC, try reinstalling ccache
|
||||
|
||||
```bash
|
||||
brew uninstall ccache
|
||||
brew install ccache
|
||||
```
|
||||
|
||||
## Using bun-debug
|
||||
|
||||
- Disable logging: `BUN_DEBUG_QUIET_LOGS=1 bun-debug ...` (to disable all debug logging)
|
||||
|
||||
@@ -125,7 +125,7 @@ setx(CWD ${CMAKE_SOURCE_DIR})
|
||||
setx(BUILD_PATH ${CMAKE_BINARY_DIR})
|
||||
|
||||
optionx(CACHE_PATH FILEPATH "The path to the cache directory" DEFAULT ${BUILD_PATH}/cache)
|
||||
optionx(CACHE_STRATEGY "read-write|read-only|write-only|none" "The strategy to use for caching" DEFAULT "read-write")
|
||||
optionx(CACHE_STRATEGY "read-write|read-only|none" "The strategy to use for caching" DEFAULT "read-write")
|
||||
|
||||
optionx(CI BOOL "If CI is enabled" DEFAULT OFF)
|
||||
optionx(ENABLE_ANALYSIS BOOL "If static analysis targets should be enabled" DEFAULT OFF)
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
optionx(ENABLE_CCACHE BOOL "If ccache should be enabled" DEFAULT ON)
|
||||
|
||||
if(NOT ENABLE_CCACHE OR CACHE_STRATEGY STREQUAL "none")
|
||||
setenv(CCACHE_DISABLE 1)
|
||||
return()
|
||||
endif()
|
||||
|
||||
if (CI AND NOT APPLE)
|
||||
setenv(CCACHE_DISABLE 1)
|
||||
return()
|
||||
endif()
|
||||
|
||||
find_command(
|
||||
VARIABLE
|
||||
CCACHE_PROGRAM
|
||||
COMMAND
|
||||
ccache
|
||||
REQUIRED
|
||||
${CI}
|
||||
)
|
||||
|
||||
if(NOT CCACHE_PROGRAM)
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(CCACHE_ARGS CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER)
|
||||
foreach(arg ${CCACHE_ARGS})
|
||||
setx(${arg} ${CCACHE_PROGRAM})
|
||||
list(APPEND CMAKE_ARGS -D${arg}=${${arg}})
|
||||
endforeach()
|
||||
|
||||
setenv(CCACHE_DIR ${CACHE_PATH}/ccache)
|
||||
setenv(CCACHE_BASEDIR ${CWD})
|
||||
setenv(CCACHE_NOHASHDIR 1)
|
||||
|
||||
if(CACHE_STRATEGY STREQUAL "read-only")
|
||||
setenv(CCACHE_READONLY 1)
|
||||
elseif(CACHE_STRATEGY STREQUAL "write-only")
|
||||
setenv(CCACHE_RECACHE 1)
|
||||
endif()
|
||||
|
||||
setenv(CCACHE_FILECLONE 1)
|
||||
setenv(CCACHE_STATSLOG ${BUILD_PATH}/ccache.log)
|
||||
|
||||
if(CI)
|
||||
# FIXME: Does not work on Ubuntu 18.04
|
||||
# setenv(CCACHE_SLOPPINESS "pch_defines,time_macros,locale,clang_index_store,gcno_cwd,include_file_ctime,include_file_mtime")
|
||||
else()
|
||||
setenv(CCACHE_MAXSIZE 100G)
|
||||
setenv(CCACHE_SLOPPINESS "pch_defines,time_macros,locale,random_seed,clang_index_store,gcno_cwd")
|
||||
endif()
|
||||
90
cmake/tools/SetupSccache.cmake
Normal file
90
cmake/tools/SetupSccache.cmake
Normal file
@@ -0,0 +1,90 @@
|
||||
if(CACHE_STRATEGY STREQUAL "none")
|
||||
return()
|
||||
endif()
|
||||
|
||||
function(check_aws_credentials OUT_VAR)
|
||||
set(HAS_CREDENTIALS FALSE)
|
||||
|
||||
if(DEFINED ENV{AWS_ACCESS_KEY_ID} AND DEFINED ENV{AWS_SECRET_ACCESS_KEY})
|
||||
set(HAS_CREDENTIALS TRUE)
|
||||
message(NOTICE
|
||||
"sccache: Using AWS credentials found in environment variables")
|
||||
endif()
|
||||
|
||||
# Check for ~/.aws directory since sccache may use that.
|
||||
if(NOT HAS_CREDENTIALS)
|
||||
if(WIN32)
|
||||
set(AWS_CONFIG_DIR "$ENV{USERPROFILE}/.aws")
|
||||
else()
|
||||
set(AWS_CONFIG_DIR "$ENV{HOME}/.aws")
|
||||
endif()
|
||||
|
||||
if(EXISTS "${AWS_CONFIG_DIR}/credentials")
|
||||
set(HAS_CREDENTIALS TRUE)
|
||||
message(NOTICE
|
||||
"sccache: Using AWS credentials found in ${AWS_CONFIG_DIR}/credentials")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(${OUT_VAR} ${HAS_CREDENTIALS} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(check_running_in_ci OUT_VAR)
|
||||
set(IS_CI FALSE)
|
||||
|
||||
# Query EC2 instance metadata service to check if running on buildkite-agent
|
||||
# The IP address 169.254.169.254 is a well-known link-local address for querying EC2 instance
|
||||
# metdata:
|
||||
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
|
||||
execute_process(
|
||||
COMMAND curl -s -m 0.5 http://169.254.169.254/latest/meta-data/tags/instance/Service
|
||||
OUTPUT_VARIABLE METADATA_OUTPUT
|
||||
ERROR_VARIABLE METADATA_ERROR
|
||||
RESULT_VARIABLE METADATA_RESULT
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
ERROR_QUIET
|
||||
)
|
||||
|
||||
# Check if the request succeeded and returned exactly "buildkite-agent"
|
||||
if(METADATA_RESULT EQUAL 0 AND METADATA_OUTPUT STREQUAL "buildkite-agent")
|
||||
set(IS_CI TRUE)
|
||||
endif()
|
||||
|
||||
set(${OUT_VAR} ${IS_CI} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
check_running_in_ci(IS_IN_CI)
|
||||
find_command(VARIABLE SCCACHE_PROGRAM COMMAND sccache REQUIRED ${IS_IN_CI})
|
||||
if(NOT SCCACHE_PROGRAM)
|
||||
message(WARNING "sccache not found. Your builds will be slower.")
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(SCCACHE_ARGS CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER)
|
||||
foreach(arg ${SCCACHE_ARGS})
|
||||
setx(${arg} ${SCCACHE_PROGRAM})
|
||||
list(APPEND CMAKE_ARGS -D${arg}=${${arg}})
|
||||
endforeach()
|
||||
|
||||
# Configure S3 bucket for distributed caching
|
||||
setenv(SCCACHE_BUCKET "bun-build-sccache-store")
|
||||
setenv(SCCACHE_REGION "us-west-1")
|
||||
setenv(SCCACHE_DIR "${CACHE_PATH}/sccache")
|
||||
|
||||
# Handle credentials based on cache strategy
|
||||
if (CACHE_STRATEGY STREQUAL "read-only")
|
||||
setenv(SCCACHE_S3_NO_CREDENTIALS "1")
|
||||
message(STATUS "sccache configured in read-only mode.")
|
||||
else()
|
||||
# Check for AWS credentials and enable anonymous access if needed
|
||||
check_aws_credentials(HAS_AWS_CREDENTIALS)
|
||||
if(NOT IS_IN_CI AND NOT HAS_AWS_CREDENTIALS)
|
||||
setenv(SCCACHE_S3_NO_CREDENTIALS "1")
|
||||
message(NOTICE "sccache: No AWS credentials found, enabling anonymous S3 "
|
||||
"access. Writing to the cache will be disabled.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
setenv(SCCACHE_LOG "info")
|
||||
|
||||
message(STATUS "sccache configured for bun-build-sccache-store (us-west-1).")
|
||||
@@ -42,7 +42,6 @@ After Visual Studio, you need the following:
|
||||
- Perl
|
||||
- Ruby
|
||||
- Node.js
|
||||
- Ccache
|
||||
|
||||
<Note>The Zig compiler is automatically downloaded, installed, and updated by the building process.</Note>
|
||||
|
||||
@@ -50,7 +49,7 @@ After Visual Studio, you need the following:
|
||||
|
||||
```ps1 Scoop
|
||||
> irm https://get.scoop.sh | iex
|
||||
> scoop install nodejs-lts go rust nasm ruby perl ccache
|
||||
> scoop install nodejs-lts go rust nasm ruby perl sccache
|
||||
# scoop seems to be buggy if you install llvm and the rest at the same time
|
||||
> scoop install llvm@19.1.7
|
||||
```
|
||||
|
||||
@@ -40,7 +40,7 @@
|
||||
pkgs.cmake # Expected: 3.30+ on nixos-unstable as of 2025-10
|
||||
pkgs.ninja
|
||||
pkgs.pkg-config
|
||||
pkgs.ccache
|
||||
pkgs.sccache
|
||||
|
||||
# Compilers and toolchain - version pinned to LLVM 19
|
||||
clang
|
||||
|
||||
@@ -120,18 +120,18 @@ function Refresh-Path {
|
||||
function Add-To-Path {
|
||||
$absolutePath = Resolve-Path $args[0]
|
||||
$currentPath = [Environment]::GetEnvironmentVariable("Path", "Machine")
|
||||
if ($currentPath -like "*$absolutePath*") {
|
||||
if ($currentPath -like "*$absolutePath*") {
|
||||
return
|
||||
}
|
||||
|
||||
$newPath = $currentPath.TrimEnd(";") + ";" + $absolutePath
|
||||
if ($newPath.Length -ge 2048) {
|
||||
Write-Warning "PATH is too long, removing duplicate and old entries..."
|
||||
|
||||
$paths = $currentPath.Split(';', [StringSplitOptions]::RemoveEmptyEntries) |
|
||||
|
||||
$paths = $currentPath.Split(';', [StringSplitOptions]::RemoveEmptyEntries) |
|
||||
Where-Object { $_ -and (Test-Path $_) } |
|
||||
Select-Object -Unique
|
||||
|
||||
|
||||
$paths += $absolutePath
|
||||
$newPath = $paths -join ';'
|
||||
while ($newPath.Length -ge 2048 -and $paths.Count -gt 1) {
|
||||
@@ -274,7 +274,6 @@ function Install-Build-Essentials {
|
||||
cmake `
|
||||
make `
|
||||
ninja `
|
||||
ccache `
|
||||
python `
|
||||
golang `
|
||||
nasm `
|
||||
@@ -282,6 +281,7 @@ function Install-Build-Essentials {
|
||||
strawberryperl `
|
||||
mingw
|
||||
Install-Rust
|
||||
Install-Sccache
|
||||
# Needed to remap stack traces
|
||||
Install-PdbAddr2line
|
||||
Install-Llvm
|
||||
@@ -344,6 +344,10 @@ function Install-Rust {
|
||||
Add-To-Path "$rustPath\cargo\bin"
|
||||
}
|
||||
|
||||
function Install-Sccache {
|
||||
Install-Package sccache -Version "0.12.0"
|
||||
}
|
||||
|
||||
function Install-PdbAddr2line {
|
||||
Execute-Command cargo install --examples "pdb-addr2line@0.11.2"
|
||||
}
|
||||
|
||||
@@ -26,8 +26,11 @@ error() {
|
||||
}
|
||||
|
||||
execute() {
|
||||
print "$ $@" >&2
|
||||
if ! "$@"; then
|
||||
local opts=$-
|
||||
set -x
|
||||
"$@"
|
||||
{ local status=$?; set +x "$opts"; } 2> /dev/null
|
||||
if [ "$status" -ne 0 ]; then
|
||||
error "Command failed: $@"
|
||||
fi
|
||||
}
|
||||
@@ -1035,7 +1038,7 @@ install_build_essentials() {
|
||||
install_llvm
|
||||
install_osxcross
|
||||
install_gcc
|
||||
install_ccache
|
||||
install_sccache
|
||||
install_rust
|
||||
install_docker
|
||||
}
|
||||
@@ -1146,12 +1149,31 @@ install_gcc() {
|
||||
execute_sudo ln -sf $(which llvm-symbolizer-$llvm_v) /usr/bin/llvm-symbolizer
|
||||
}
|
||||
|
||||
install_ccache() {
|
||||
case "$pm" in
|
||||
apt | apk | brew)
|
||||
install_packages ccache
|
||||
;;
|
||||
esac
|
||||
install_sccache() {
|
||||
# Alright, look, this function is cobbled together but it's only as cobbled
|
||||
# together as this whole script is.
|
||||
#
|
||||
# For some reason, move_to_bin doesn't work here due to permissions so I'm
|
||||
# avoiding that function. It's also wrong with permissions and so on.
|
||||
#
|
||||
# Unfortunately, we cannot use install_packages since many package managers
|
||||
# don't compile `sccache` with S3 support.
|
||||
local opts=$-
|
||||
set -ef
|
||||
|
||||
local sccache_http
|
||||
sccache_http="https://github.com/mozilla/sccache/releases/download/v0.12.0/sccache-v0.12.0-$(uname -m)-unknown-linux-musl.tar.gz"
|
||||
|
||||
local file
|
||||
file=$(download_file "$sccache_http")
|
||||
|
||||
local tmpdir
|
||||
tmpdir=$(mktemp -d)
|
||||
|
||||
execute tar -xzf "$file" -C "$tmpdir"
|
||||
execute_sudo install -m755 "$tmpdir/sccache-v0.12.0-$(uname -m)-unknown-linux-musl/sccache" "/usr/local/bin"
|
||||
|
||||
set +ef -"$opts"
|
||||
}
|
||||
|
||||
install_rust() {
|
||||
@@ -1402,10 +1424,10 @@ install_chromium() {
|
||||
apk)
|
||||
install_packages \
|
||||
chromium \
|
||||
nss \
|
||||
freetype \
|
||||
harfbuzz \
|
||||
ttf-freefont
|
||||
nss \
|
||||
freetype \
|
||||
harfbuzz \
|
||||
ttf-freefont
|
||||
;;
|
||||
apt)
|
||||
install_packages \
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { spawn as nodeSpawn } from "node:child_process";
|
||||
import { chmodSync, cpSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { basename, join, relative, resolve } from "node:path";
|
||||
import {
|
||||
formatAnnotationToHtml,
|
||||
@@ -70,43 +70,8 @@ async function build(args) {
|
||||
generateOptions["-S"] = process.cwd();
|
||||
}
|
||||
|
||||
const cacheRead = isCacheReadEnabled();
|
||||
const cacheWrite = isCacheWriteEnabled();
|
||||
if (cacheRead || cacheWrite) {
|
||||
const cachePath = getCachePath();
|
||||
if (cacheRead && !existsSync(cachePath)) {
|
||||
const mainCachePath = getCachePath(getDefaultBranch());
|
||||
if (existsSync(mainCachePath)) {
|
||||
mkdirSync(cachePath, { recursive: true });
|
||||
try {
|
||||
cpSync(mainCachePath, cachePath, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
const { code } = error;
|
||||
switch (code) {
|
||||
case "EPERM":
|
||||
case "EACCES":
|
||||
try {
|
||||
chmodSync(mainCachePath, 0o777);
|
||||
cpSync(mainCachePath, cachePath, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
console.warn("Failed to copy cache with permissions fix", error);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
console.warn("Failed to copy cache", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
generateOptions["-DCACHE_PATH"] = cmakePath(cachePath);
|
||||
generateOptions["--fresh"] = undefined;
|
||||
if (cacheRead && cacheWrite) {
|
||||
generateOptions["-DCACHE_STRATEGY"] = "read-write";
|
||||
} else if (cacheRead) {
|
||||
generateOptions["-DCACHE_STRATEGY"] = "read-only";
|
||||
} else if (cacheWrite) {
|
||||
generateOptions["-DCACHE_STRATEGY"] = "write-only";
|
||||
}
|
||||
if (!generateOptions["-DCACHE_STRATEGY"]) {
|
||||
generateOptions["-DCACHE_STRATEGY"] = "read-write";
|
||||
}
|
||||
|
||||
const toolchain = generateOptions["--toolchain"];
|
||||
@@ -136,58 +101,17 @@ async function build(args) {
|
||||
|
||||
await startGroup("CMake Build", () => spawn("cmake", buildArgs, { env }));
|
||||
|
||||
await startGroup("sccache stats", () => {
|
||||
spawn("sccache", ["--show-stats"], { env });
|
||||
});
|
||||
|
||||
printDuration("total", Date.now() - startTime);
|
||||
}
|
||||
|
||||
function cmakePath(path) {
|
||||
return path.replace(/\\/g, "/");
|
||||
}
|
||||
|
||||
/** @param {string} str */
|
||||
const toAlphaNumeric = str => str.replace(/[^a-z0-9]/gi, "-");
|
||||
function getCachePath(branch) {
|
||||
const {
|
||||
BUILDKITE_BUILD_PATH: buildPath,
|
||||
BUILDKITE_REPO: repository,
|
||||
BUILDKITE_PULL_REQUEST_REPO: fork,
|
||||
BUILDKITE_BRANCH,
|
||||
BUILDKITE_STEP_KEY,
|
||||
} = process.env;
|
||||
|
||||
// NOTE: settings that could be long should be truncated to avoid hitting max
|
||||
// path length limit on windows (4096)
|
||||
const repositoryKey = toAlphaNumeric(
|
||||
// remove domain name, only leaving 'org/repo'
|
||||
(fork || repository).replace(/^https?:\/\/github\.com\/?/, ""),
|
||||
);
|
||||
const branchName = toAlphaNumeric(branch || BUILDKITE_BRANCH);
|
||||
const branchKey = branchName.startsWith("gh-readonly-queue-")
|
||||
? branchName.slice(18, branchName.indexOf("-pr-"))
|
||||
: branchName.slice(0, 32);
|
||||
const stepKey = toAlphaNumeric(BUILDKITE_STEP_KEY);
|
||||
return resolve(buildPath, "..", "cache", repositoryKey, branchKey, stepKey);
|
||||
}
|
||||
|
||||
function isCacheReadEnabled() {
|
||||
return (
|
||||
isBuildkite() &&
|
||||
process.env.BUILDKITE_CLEAN_CHECKOUT !== "true" &&
|
||||
process.env.BUILDKITE_BRANCH !== getDefaultBranch()
|
||||
);
|
||||
}
|
||||
|
||||
function isCacheWriteEnabled() {
|
||||
return isBuildkite();
|
||||
}
|
||||
|
||||
function isBuildkite() {
|
||||
return process.env.BUILDKITE === "true";
|
||||
}
|
||||
|
||||
function getDefaultBranch() {
|
||||
return process.env.BUILDKITE_PIPELINE_DEFAULT_BRANCH || "main";
|
||||
}
|
||||
|
||||
function parseOptions(args, flags = []) {
|
||||
const options = {};
|
||||
|
||||
|
||||
@@ -485,6 +485,12 @@ const aws = {
|
||||
});
|
||||
}
|
||||
|
||||
// Attach IAM instance profile for CI builds to enable S3 build cache access
|
||||
let iamInstanceProfile;
|
||||
if (options.ci) {
|
||||
iamInstanceProfile = JSON.stringify({ Name: "buildkite-build-agent" });
|
||||
}
|
||||
|
||||
const [instance] = await aws.runInstances({
|
||||
["image-id"]: ImageId,
|
||||
["instance-type"]: instanceType || (arch === "aarch64" ? "t4g.large" : "t3.large"),
|
||||
@@ -499,6 +505,7 @@ const aws = {
|
||||
["tag-specifications"]: JSON.stringify(tagSpecification),
|
||||
["key-name"]: keyName,
|
||||
["instance-market-options"]: marketOptions,
|
||||
["iam-instance-profile"]: iamInstanceProfile,
|
||||
});
|
||||
|
||||
const machine = aws.toMachine(instance, { ...options, username, keyPath });
|
||||
@@ -1191,6 +1198,9 @@ async function main() {
|
||||
const tags = {
|
||||
"robobun": "true",
|
||||
"robobun2": "true",
|
||||
// This tag controls the IAM role required to be able to write to the shared S3 build cache.
|
||||
// Don't want accidental polution from non-CI runs.
|
||||
"Service": args["ci"] ? "buildkite-agent" : undefined,
|
||||
"buildkite:token": args["buildkite-token"],
|
||||
"tailscale:authkey": args["tailscale-authkey"],
|
||||
...Object.fromEntries(args["tag"]?.map(tag => tag.split("=")) ?? []),
|
||||
|
||||
Reference in New Issue
Block a user