Compare commits

...

12 Commits

Author SHA1 Message Date
Ashcon Partovi
3e8ffbb1b2 [build images] 2025-01-07 16:02:30 -08:00
Ashcon Partovi
1a7eeb3577 [build images] 2025-01-07 15:57:04 -08:00
Ashcon Partovi
5098cb660d [build images] 2025-01-07 15:45:30 -08:00
Ashcon Partovi
ddf8cc08dd [build images] 2025-01-07 15:44:43 -08:00
Ashcon Partovi
5a7f65f951 [build images] 2025-01-07 15:39:35 -08:00
Ashcon Partovi
851e4032cc [build images] 2025-01-07 15:36:17 -08:00
Ashcon Partovi
d1a5fb2d7f [build images] 2025-01-07 15:04:31 -08:00
Ashcon Partovi
50a39a8c42 [build images] 2025-01-07 14:25:20 -08:00
Ashcon Partovi
526b8f4fbe [build images] 2025-01-07 13:53:17 -08:00
Ashcon Partovi
4883af39f5 Fix 2025-01-07 13:41:43 -08:00
Ashcon Partovi
6665f0e87b Fix 2025-01-07 13:33:29 -08:00
Ashcon Partovi
088dc974a7 gVisor 2025-01-07 13:27:55 -08:00
5 changed files with 231 additions and 139 deletions

View File

@@ -36,6 +36,7 @@ import {
* @typedef {"debian" | "ubuntu" | "alpine" | "amazonlinux"} Distro
* @typedef {"latest" | "previous" | "oldest" | "eol"} Tier
* @typedef {"release" | "assert" | "debug"} Profile
* @typedef {"docker" | "gvisor"} Feature
*/
/**
@@ -45,6 +46,7 @@ import {
* @property {Abi} [abi]
* @property {boolean} [baseline]
* @property {Profile} [profile]
* @property {Feature[]} [features]
*/
/**
@@ -95,50 +97,55 @@ function getTargetLabel(target) {
* @property {Distro} [distro]
* @property {string} release
* @property {Tier} [tier]
* @property {string[]} [features]
* @property {Feature[]} [features]
*/
/**
* @type {Platform[]}
*/
const buildPlatforms = [
/* macOS */
{ os: "darwin", arch: "aarch64", release: "14" },
{ os: "darwin", arch: "x64", release: "14" },
/* Linux GNU */
{ os: "linux", arch: "aarch64", distro: "amazonlinux", release: "2023", features: ["docker"] },
{ os: "linux", arch: "x64", distro: "amazonlinux", release: "2023", features: ["docker"] },
{ os: "linux", arch: "x64", baseline: true, distro: "amazonlinux", release: "2023", features: ["docker"] },
{ os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" },
{ os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20" },
{ os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20" },
{ os: "windows", arch: "x64", release: "2019" },
{ os: "windows", arch: "x64", baseline: true, release: "2019" },
/* Linux MUSL */
{ os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.21" },
{ os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.21" },
{ os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.21" },
/* Windows */
{ os: "windows", arch: "x64", release: "2025" },
{ os: "windows", arch: "x64", baseline: true, release: "2025" },
];
/**
* @type {Platform[]}
*/
const testPlatforms = [
{ os: "darwin", arch: "aarch64", release: "14", tier: "latest" },
{ os: "darwin", arch: "aarch64", release: "13", tier: "previous" },
{ os: "darwin", arch: "x64", release: "14", tier: "latest" },
{ os: "darwin", arch: "x64", release: "13", tier: "previous" },
{ os: "linux", arch: "aarch64", distro: "debian", release: "12", tier: "latest" },
{ os: "linux", arch: "x64", distro: "debian", release: "12", tier: "latest" },
{ os: "linux", arch: "x64", baseline: true, distro: "debian", release: "12", tier: "latest" },
{ os: "linux", arch: "aarch64", distro: "ubuntu", release: "24.04", tier: "latest" },
{ os: "linux", arch: "aarch64", distro: "ubuntu", release: "22.04", tier: "previous" },
{ os: "linux", arch: "aarch64", distro: "ubuntu", release: "20.04", tier: "oldest" },
{ os: "linux", arch: "x64", distro: "ubuntu", release: "24.04", tier: "latest" },
{ os: "linux", arch: "x64", distro: "ubuntu", release: "22.04", tier: "previous" },
{ os: "linux", arch: "x64", distro: "ubuntu", release: "20.04", tier: "oldest" },
{ os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "24.04", tier: "latest" },
{ os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "22.04", tier: "previous" },
{ os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "20.04", tier: "oldest" },
{ os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20", tier: "latest" },
{ os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20", tier: "latest" },
{ os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20", tier: "latest" },
{ os: "windows", arch: "x64", release: "2019", tier: "oldest" },
{ os: "windows", arch: "x64", release: "2019", baseline: true, tier: "oldest" },
/* macOS */
{ os: "darwin", arch: "aarch64", release: "14" },
{ os: "darwin", arch: "aarch64", release: "13" },
{ os: "darwin", arch: "x64", release: "14" },
{ os: "darwin", arch: "x64", release: "13" },
/* Linux GNU */
{ os: "linux", arch: "aarch64", distro: "debian", release: "12" },
{ os: "linux", arch: "x64", distro: "debian", release: "12" },
{ os: "linux", arch: "x64", baseline: true, distro: "debian", release: "12" },
{ os: "linux", arch: "aarch64", distro: "ubuntu", release: "24.04" },
{ os: "linux", arch: "x64", distro: "ubuntu", release: "24.04" },
{ os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "24.04" },
/* Linux MUSL */
{ os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.21" },
{ os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.21" },
{ os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.21" },
/* Linux GVisor */
{ os: "linux", arch: "x64", features: ["gvisor"], distro: "ubuntu", release: "24.04" },
{ os: "linux", arch: "aarch64", abi: "musl", features: ["gvisor"], distro: "alpine", release: "3.21" },
/* Windows */
{ os: "windows", arch: "x64", release: "2025" },
{ os: "windows", arch: "x64", release: "2025", baseline: true },
];
/**
@@ -146,13 +153,17 @@ const testPlatforms = [
* @returns {string}
*/
function getPlatformKey(platform) {
const { distro, release } = platform;
const { distro, release, features } = platform;
const target = getTargetKey(platform);
const version = release.replace(/\./g, "");
let key = `${target}-${version}`;
if (distro) {
return `${target}-${distro}-${version}`;
key += `-${distro}`;
}
return `${target}-${version}`;
if (features?.length) {
key += `-with-${features.join("-")}`;
}
return key;
}
/**
@@ -160,8 +171,12 @@ function getPlatformKey(platform) {
* @returns {string}
*/
function getPlatformLabel(platform) {
const { os, arch, baseline, profile, distro, release } = platform;
let label = `${getBuildkiteEmoji(distro || os)} ${release} ${arch}`;
const { os, arch, baseline, profile, distro, release, features } = platform;
let emoji = getBuildkiteEmoji(distro || os);
if (features?.includes("gvisor")) {
emoji += ` ${getBuildkiteEmoji("gvisor")}`;
}
let label = `${emoji} ${release} ${arch}`;
if (baseline) {
label += "-baseline";
}
@@ -182,14 +197,13 @@ function getImageKey(platform) {
if (distro) {
key += `-${distro}`;
}
if (features?.length) {
key += `-with-${features.join("-")}`;
}
if (abi) {
key += `-${abi}`;
}
const buildFeatures = features?.filter(feature => feature !== "gvisor");
if (buildFeatures?.length) {
key += `-with-${buildFeatures.join("-")}`;
}
return key;
}
@@ -522,7 +536,7 @@ function getBuildBunStep(platform, options) {
* @returns {Step}
*/
function getTestBunStep(platform, options, testOptions = {}) {
const { os } = platform;
const { os, features } = platform;
const { buildId, unifiedTests, testFiles } = testOptions;
const args = [`--step=${getTargetKey(platform)}-build-bun`];
@@ -532,6 +546,9 @@ function getTestBunStep(platform, options, testOptions = {}) {
if (testFiles) {
args.push(...testFiles.map(testFile => `--include=${testFile}`));
}
if (features?.includes("gvisor")) {
args.push("--gvisor");
}
const depends = [];
if (!buildId) {
@@ -836,8 +853,12 @@ function getOptionsStep() {
default: [],
options: [...new Map(testPlatforms.map(platform => [getImageKey(platform), platform])).entries()].map(
([key, platform]) => {
const { os, arch, abi, distro, release } = platform;
let label = `${getEmoji(os)} ${arch}`;
const { os, arch, abi, distro, release, features } = platform;
let emoji = getEmoji(os);
if (features?.includes("gvisor")) {
emoji += ` ${getEmoji("gvisor")}`;
}
let label = `${emoji} ${arch}`;
if (abi) {
label += `-${abi}`;
}
@@ -955,10 +976,10 @@ async function getPipelineOptions() {
unifiedTests: parseBoolean(options["unified-tests"]),
buildProfiles: parseArray(options["build-profiles"]),
buildPlatforms: buildPlatformKeys?.length
? buildPlatformKeys.map(key => buildPlatformsMap.get(key))
? buildPlatformKeys.map(key => buildPlatformsMap.get(key)).filter(Boolean)
: Array.from(buildPlatformsMap.values()),
testPlatforms: testPlatformKeys?.length
? testPlatformKeys.map(key => testPlatformsMap.get(key))
? testPlatformKeys.map(key => testPlatformsMap.get(key)).filter(Boolean)
: Array.from(testPlatformsMap.values()),
dryRun: parseBoolean(options["dry-run"]),
};

View File

@@ -1,5 +1,5 @@
#!/bin/sh
# Version: 9
# Version: 10
# A script that installs the dependencies needed to build and test Bun.
# This should work on macOS and Linux with a POSIX shell.
@@ -19,9 +19,7 @@ print() {
error() {
print "error: $@" >&2
if ! [ "$$" = "$pid" ]; then
kill -s TERM "$pid"
fi
kill -s TERM "$pid"
exit 1
}
@@ -59,14 +57,14 @@ execute_as_user() {
}
grant_to_user() {
path="$1"
if ! [ -f "$path" ] && ! [ -d "$path" ]; then
error "Could not find file or directory: \"$path\""
file_path="$1"
if ! [ -f "$file_path" ] && ! [ -d "$file_path" ]; then
error "Could not find file or directory: \"$file_path\""
fi
chown="$(require chown)"
execute_sudo "$chown" -R "$user:$group" "$path"
execute_sudo chmod -R 777 "$path"
execute_sudo "$chown" -R "$user:$group" "$file_path"
execute_sudo chmod -R 777 "$file_path"
}
which() {
@@ -74,11 +72,11 @@ which() {
}
require() {
path="$(which "$1")"
if ! [ -f "$path" ]; then
bin_path="$(which "$1")"
if ! [ -f "$bin_path" ]; then
error "Command \"$1\" is required, but is not installed."
fi
print "$path"
print "$bin_path"
}
fetch() {
@@ -106,79 +104,79 @@ compare_version() {
}
create_directory() {
path="$1"
path_dir="$path"
while ! [ -d "$path_dir" ]; do
path_dir="$(dirname "$path_dir")"
dir_path="$1"
dir_parent_path="$dir_path"
while ! [ -d "$dir_parent_path" ]; do
dir_parent_path="$(dirname "$dir_parent_path")"
done
path_needs_sudo="0"
if ! [ -r "$path_dir" ] || ! [ -w "$path_dir" ]; then
path_needs_sudo="1"
dir_needs_sudo="0"
if ! [ -r "$dir_parent_path" ] || ! [ -w "$dir_parent_path" ]; then
dir_needs_sudo="1"
fi
mkdir="$(require mkdir)"
if [ "$path_needs_sudo" = "1" ]; then
execute_sudo "$mkdir" -p "$path"
if [ "$dir_needs_sudo" = "1" ]; then
execute_sudo "$mkdir" -p "$dir_path"
else
execute "$mkdir" -p "$path"
execute "$mkdir" -p "$dir_path"
fi
grant_to_user "$path"
grant_to_user "$dir_path"
}
create_tmp_directory() {
mktemp="$(require mktemp)"
path="$(execute "$mktemp" -d)"
grant_to_user "$path"
print "$path"
tmp_dir_path="$(execute "$mktemp" -d)"
grant_to_user "$tmp_dir_path"
print "$tmp_dir_path"
}
create_file() {
path="$1"
path_dir="$(dirname "$path")"
if ! [ -d "$path_dir" ]; then
create_directory "$path_dir"
file_path="$1"
file_parent_path="$(dirname "$file_path")"
if ! [ -d "$file_parent_path" ]; then
create_directory "$file_parent_path"
fi
path_needs_sudo="0"
if ! [ -r "$path" ] || ! [ -w "$path" ]; then
path_needs_sudo="1"
file_needs_sudo="0"
if ! [ -r "$file_parent_path" ] || ! [ -w "$file_parent_path" ]; then
file_needs_sudo="1"
fi
if [ "$path_needs_sudo" = "1" ]; then
execute_sudo touch "$path"
if [ "$file_needs_sudo" = "1" ]; then
execute_sudo touch "$file_path"
else
execute touch "$path"
execute touch "$file_path"
fi
content="$2"
if [ -n "$content" ]; then
append_file "$path" "$content"
file_content="$2"
if [ -n "$file_content" ]; then
append_file "$file_path" "$file_content"
fi
grant_to_user "$path"
grant_to_user "$file_path"
}
append_file() {
path="$1"
if ! [ -f "$path" ]; then
create_file "$path"
file_path="$1"
if ! [ -f "$file_path" ]; then
create_file "$file_path"
fi
path_needs_sudo="0"
if ! [ -r "$path" ] || ! [ -w "$path" ]; then
path_needs_sudo="1"
file_needs_sudo="0"
if ! [ -r "$file_path" ] || ! [ -w "$file_path" ]; then
file_needs_sudo="1"
fi
content="$2"
print "$content" | while read -r line; do
if ! grep -q "$line" "$path"; then
file_content="$2"
print "$file_content" | while read -r line; do
if ! grep -q "$line" "$file_path"; then
sh="$(require sh)"
if [ "$path_needs_sudo" = "1" ]; then
execute_sudo "$sh" -c "echo '$line' >> '$path'"
if [ "$file_needs_sudo" = "1" ]; then
execute_sudo "$sh" -c "echo '$line' >> '$file_path'"
else
execute "$sh" -c "echo '$line' >> '$path'"
execute "$sh" -c "echo '$line' >> '$file_path'"
fi
fi
done
@@ -196,25 +194,26 @@ download_file() {
}
append_to_profile() {
content="$1"
profile_content="$1"
profiles=".profile .zprofile .bash_profile .bashrc .zshrc"
for profile in $profiles; do
for profile_path in "$current_home/$profile" "$home/$profile"; do
for profile_name in $profiles; do
for profile_path in "$current_home/$profile_name" "$home/$profile_name"; do
if [ "$ci" = "1" ] || [ -f "$profile_path" ]; then
append_file "$profile_path" "$content"
append_file "$profile_path" "$profile_content"
fi
done
done
}
append_to_path() {
path="$1"
if ! [ -d "$path" ]; then
error "Could not find directory: \"$path\""
dir_path="$1"
if ! [ -d "$dir_path" ]; then
error "Could not find directory: \"$dir_path\""
fi
append_to_profile "export PATH=\"$path:\$PATH\""
export PATH="$path:$PATH"
append_to_profile "export PATH=\"$dir_path:\$PATH\""
export PATH="$dir_path:$PATH"
}
move_to_bin() {
@@ -240,11 +239,11 @@ check_features() {
for arg in "$@"; do
case "$arg" in
*--ci*)
ci=1
ci="1"
print "CI: enabled"
;;
*--osxcross*)
osxcross=1
osxcross="1"
print "Cross-compiling to macOS: enabled"
;;
*--gcc-13*)
@@ -365,12 +364,12 @@ check_inside_docker() {
print "Checking if inside Docker..."
if [ -f "/.dockerenv" ]; then
docker=1
docker="1"
else
if [ -f "/proc/1/cgroup" ]; then
case "$(cat /proc/1/cgroup)" in
*/docker/*)
docker=1
docker="1"
;;
esac
fi
@@ -378,7 +377,7 @@ check_inside_docker() {
if [ -f "/proc/self/mountinfo" ]; then
case "$(cat /proc/self/mountinfo)" in
*/docker/*)
docker=1
docker="1"
;;
esac
fi
@@ -444,8 +443,8 @@ check_user() {
print "Group: $group"
home="$(execute_as_user echo '~')"
if [ -z "$home" ] || [ "$home" = "~" ]; then
error "Could not determine home directory for user: $user"
if [ "$home" = "~" ] || ! [ -d "$home" ]; then
error "Could not determine home directory for $user: $home"
fi
print "Home: $home"
@@ -463,6 +462,21 @@ check_user() {
current_home="$home"
}
check_locale() {
if ! [ "$ci" = "1" ]; then
return
fi
print "Checking locale..."
case "$os" in
linux)
if [ -f "$(which sudo)" ]; then
execute sudo touch "/var/lib/cloud/instance/locale-check.skip"
fi
;;
esac
}
check_ulimit() {
if ! [ "$ci" = "1" ]; then
return
@@ -526,24 +540,6 @@ check_ulimit() {
if [ -f "$systemctl" ]; then
execute_sudo "$systemctl" daemon-reload
fi
# Configure dpkg and apt for faster operation in CI environments
if [ "$ci" = "1" ] && [ "$pm" = "apt" ]; then
dpkg_conf="/etc/dpkg/dpkg.cfg.d/01-ci-options"
execute_sudo create_directory "$(dirname "$dpkg_conf")"
append_file "$dpkg_conf" "force-unsafe-io"
append_file "$dpkg_conf" "no-debsig"
apt_conf="/etc/apt/apt.conf.d/99-ci-options"
execute_sudo create_directory "$(dirname "$apt_conf")"
append_file "$apt_conf" 'Acquire::Languages "none";'
append_file "$apt_conf" 'Acquire::GzipIndexes "true";'
append_file "$apt_conf" 'Acquire::CompressionTypes::Order:: "gz";'
append_file "$apt_conf" 'APT::Get::Install-Recommends "false";'
append_file "$apt_conf" 'APT::Get::Install-Suggests "false";'
append_file "$apt_conf" 'Dpkg::Options { "--force-confdef"; "--force-confold"; }'
fi
}
package_manager() {
@@ -894,6 +890,7 @@ install_build_essentials() {
install_ccache
install_rust
install_docker
install_gvisor
}
llvm_version_exact() {
@@ -1073,6 +1070,25 @@ install_docker() {
fi
}
install_gvisor() {
if ! [ "$os" = "linux" ]; then
return
fi
runsc_arch="$(execute uname -m)"
runsc_url="https://storage.googleapis.com/gvisor/releases/release/latest/$runsc_arch/runsc"
runsc_bin="$(download_file "$runsc_url")"
move_to_bin "$runsc_bin"
runsc_exe="$(require runsc)"
execute_sudo "$runsc_exe" install
systemctl="$(which systemctl)"
if [ -f "$systemctl" ] && ! [ "$docker" = "1" ]; then
execute_sudo "$systemctl" reload docker
fi
}
macos_sdk_version() {
# https://github.com/alexey-lysiuk/macos-sdk/releases
print "13.3"
@@ -1177,8 +1193,8 @@ create_buildkite_user() {
fi
buildkite_paths="$home /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /var/run/buildkite-agent/buildkite-agent.sock"
for path in $buildkite_paths; do
create_directory "$path"
for buildkite_path in $buildkite_paths; do
create_directory "$buildkite_path"
done
buildkite_files="/var/run/buildkite-agent/buildkite-agent.pid"
@@ -1316,6 +1332,7 @@ main() {
check_features "$@"
check_operating_system
check_inside_docker
check_locale
check_user
check_ulimit
check_package_manager

View File

@@ -1215,8 +1215,9 @@ async function main() {
name += `-musl`;
}
if (features?.length) {
name += `-with-${features.join("-")}`;
const buildFeatures = features?.filter(feature => feature !== "gvisor");
if (buildFeatures?.length) {
name += `-with-${buildFeatures.join("-")}`;
}
let bootstrapPath, agentPath, dockerfilePath;

View File

@@ -39,6 +39,7 @@ import {
startGroup,
tmpdir,
unzip,
which,
} from "./utils.mjs";
import { userInfo } from "node:os";
let isQuiet = false;
@@ -106,6 +107,18 @@ const { values: options, positionals: filters } = parseArgs({
type: "string",
default: isCI ? "4" : "0", // N retries = N+1 attempts
},
["gvisor"]: {
type: "boolean",
default: false,
},
["strace"]: {
type: "boolean",
default: false,
},
["strace-filter"]: {
type: "string",
multiple: true,
},
},
});
@@ -569,10 +582,28 @@ async function spawnBun(execPath, { args, cwd, timeout, env, stdout, stderr }) {
}
bunEnv["TEMP"] = tmpdirPath;
}
let command = execPath;
let commandArgs = args || [];
if (bunEnv["gvisor"]) {
const runsc = which("runsc", { required: true });
const runscArgs = ["--rootless", "--ignore-cgroups", "--network=host"];
if (bunEnv["strace"]) {
runscArgs.push("--strace");
}
if (bunEnv["strace-filter"]) {
const syscalls = bunEnv["strace-filter"]
.split(",")
.map(syscall => syscall.trim())
.join(",");
runscArgs.push("--strace-syscalls", syscalls);
}
commandArgs = [...runscArgs, "do", command, ...commandArgs];
command = runsc;
}
try {
return await spawnSafe({
command: execPath,
args,
command,
args: commandArgs,
cwd,
timeout,
env: bunEnv,
@@ -580,11 +611,11 @@ async function spawnBun(execPath, { args, cwd, timeout, env, stdout, stderr }) {
stderr,
});
} finally {
// try {
// rmSync(tmpdirPath, { recursive: true, force: true });
// } catch (error) {
// console.warn(error);
// }
try {
rmSync(tmpdirPath, { recursive: true, force: true });
} catch (error) {
console.warn(error);
}
}
}

View File

@@ -1997,6 +1997,27 @@ export function getDistroVersion() {
}
}
/** @type {boolean | undefined} */
let gvisorDetected;
/**
* @returns {boolean | undefined}
*/
export function isGVisor() {
if (typeof gvisorDetected === "boolean") {
return gvisorDetected;
}
if (isLinux) {
const { error, stdout } = spawnSync(["dmesg"]);
if (!error && stdout.includes("Starting gVisor")) {
return (gvisorDetected = true);
}
}
return (gvisorDetected = false);
}
/**
* @returns {string | undefined}
*/
@@ -2781,6 +2802,7 @@ const emojiMap = {
gear: ["⚙️", "gear"],
clipboard: ["📋", "clipboard"],
rocket: ["🚀", "rocket"],
gvisor: ["👾", "space_invader"],
};
/**