ci: Add bootstrap.ps1 and automate Windows build images (#15606)

This commit is contained in:
Ashcon Partovi
2024-12-05 14:16:37 -08:00
committed by GitHub
parent 4c8cbecb08
commit fa6ac405a4
18 changed files with 3850 additions and 1277 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -8,4 +8,4 @@ function run_command() {
{ set +x; } 2>/dev/null
}
run_command node ".buildkite/ci.mjs"
run_command node ".buildkite/ci.mjs" "$@"

View File

@@ -20,7 +20,7 @@ else()
setx(RELEASE OFF)
endif()
if(CMAKE_BUILD_TYPE MATCHES "Debug|RelWithDebInfo")
if(CMAKE_BUILD_TYPE MATCHES "Debug")
setx(DEBUG ON)
else()
setx(DEBUG OFF)

View File

@@ -49,6 +49,8 @@ register_command(
CARGO_TERM_VERBOSE=true
CARGO_TERM_DIAGNOSTIC=true
CARGO_ENCODED_RUSTFLAGS=${RUSTFLAGS}
CARGO_HOME=${CARGO_HOME}
RUSTUP_HOME=${RUSTUP_HOME}
)
target_link_libraries(${bun} PRIVATE ${LOLHTML_LIBRARY})

View File

@@ -1,15 +1,42 @@
if(DEFINED ENV{CARGO_HOME})
set(CARGO_HOME $ENV{CARGO_HOME})
elseif(CMAKE_HOST_WIN32)
set(CARGO_HOME $ENV{USERPROFILE}/.cargo)
if(NOT EXISTS ${CARGO_HOME})
set(CARGO_HOME $ENV{PROGRAMFILES}/Rust/cargo)
endif()
else()
set(CARGO_HOME $ENV{HOME}/.cargo)
endif()
if(DEFINED ENV{RUSTUP_HOME})
set(RUSTUP_HOME $ENV{RUSTUP_HOME})
elseif(CMAKE_HOST_WIN32)
set(RUSTUP_HOME $ENV{USERPROFILE}/.rustup)
if(NOT EXISTS ${RUSTUP_HOME})
set(RUSTUP_HOME $ENV{PROGRAMFILES}/Rust/rustup)
endif()
else()
set(RUSTUP_HOME $ENV{HOME}/.rustup)
endif()
find_command(
VARIABLE
CARGO_EXECUTABLE
COMMAND
cargo
PATHS
$ENV{HOME}/.cargo/bin
${CARGO_HOME}/bin
REQUIRED
OFF
)
if(EXISTS ${CARGO_EXECUTABLE})
if(CARGO_EXECUTABLE MATCHES "^${CARGO_HOME}")
setx(CARGO_HOME ${CARGO_HOME})
setx(RUSTUP_HOME ${RUSTUP_HOME})
endif()
return()
endif()

View File

@@ -20,6 +20,8 @@ import {
getEnv,
writeFile,
spawnSafe,
spawn,
mkdir,
} from "./utils.mjs";
import { parseArgs } from "node:util";
@@ -49,16 +51,19 @@ async function doBuildkiteAgent(action) {
const args = [realpathSync(process.argv[1]), "start"];
if (isWindows) {
const serviceCommand = [
"New-Service",
"-Name",
"buildkite-agent",
"-StartupType",
"Automatic",
"-BinaryPathName",
`${escape(command)} ${escape(args.map(escape).join(" "))}`,
mkdir(logsPath);
const nssm = which("nssm", { required: true });
const nssmCommands = [
[nssm, "install", "buildkite-agent", command, ...args],
[nssm, "set", "buildkite-agent", "Start", "SERVICE_AUTO_START"],
[nssm, "set", "buildkite-agent", "AppDirectory", homePath],
[nssm, "set", "buildkite-agent", "AppStdout", agentLogPath],
[nssm, "set", "buildkite-agent", "AppStderr", agentLogPath],
];
await spawnSafe(["powershell", "-Command", serviceCommand.join(" ")], { stdio: "inherit" });
for (const command of nssmCommands) {
await spawnSafe(command, { stdio: "inherit" });
}
}
if (isOpenRc()) {
@@ -124,13 +129,21 @@ async function doBuildkiteAgent(action) {
token = await getCloudMetadataTag("buildkite:token");
}
if (!token) {
throw new Error(
"Buildkite token not found: either set BUILDKITE_AGENT_TOKEN or add a buildkite:token label to the instance",
);
}
let shell;
if (isWindows) {
const pwsh = which(["pwsh", "powershell"], { required: true });
shell = `${pwsh} -Command`;
// Command Prompt has a faster startup time than PowerShell.
// Also, it propogates the exit code of the command, which PowerShell does not.
const cmd = which("cmd", { required: true });
shell = `"${cmd}" /S /C`;
} else {
const sh = which(["bash", "sh"], { required: true });
shell = `${sh} -c`;
const sh = which("sh", { required: true });
shell = `${sh} -e -c`;
}
const flags = ["enable-job-log-tmpfile", "no-feature-reporting"];

View File

@@ -1,6 +1,6 @@
# Version: 4
# A powershell script that installs the dependencies needed to build and test Bun.
# This should work on Windows 10 or newer.
# Version: 7
# A script that installs the dependencies needed to build and test Bun.
# This should work on Windows 10 or newer with PowerShell.
# If this script does not work on your machine, please open an issue:
# https://github.com/oven-sh/bun/issues
@@ -16,6 +16,9 @@ param (
[switch]$Optimize = $CI
)
$ErrorActionPreference = "Stop"
Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
function Execute-Command {
$command = $args -join ' '
Write-Output "$ $command"
@@ -43,6 +46,47 @@ function Which {
}
}
function Execute-Script {
param (
[Parameter(Mandatory = $true, Position = 0)]
[string]$Path
)
$pwsh = Which pwsh powershell -Required
Execute-Command $pwsh $Path
}
function Download-File {
param (
[Parameter(Mandatory = $true, Position = 0)]
[string]$Url,
[Parameter(Mandatory = $false)]
[string]$Name,
[Parameter(Mandatory = $false)]
[string]$Path
)
if (-not $Name) {
$Name = [System.IO.Path]::ChangeExtension([System.IO.Path]::GetRandomFileName(), [System.IO.Path]::GetExtension($Url))
}
if (-not $Path) {
$Path = "$env:TEMP\$Name"
}
$client = New-Object System.Net.WebClient
for ($i = 0; $i -lt 10 -and -not (Test-Path $Path); $i++) {
try {
$client.DownloadFile($Url, $Path)
} catch {
Write-Warning "Failed to download $Url, retry $i..."
Start-Sleep -s $i
}
}
return $Path
}
function Install-Chocolatey {
if (Which choco) {
return
@@ -50,7 +94,8 @@ function Install-Chocolatey {
Write-Output "Installing Chocolatey..."
[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072
iex -Command ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
$installScript = Download-File "https://community.chocolatey.org/install.ps1"
Execute-Script $installScript
Refresh-Path
}
@@ -96,10 +141,23 @@ function Add-To-Path {
}
Write-Output "Adding $absolutePath to PATH..."
[Environment]::SetEnvironmentVariable("Path", $newPath, "Machine")
[Environment]::SetEnvironmentVariable("Path", "$newPath", "Machine")
Refresh-Path
}
function Set-Env {
param (
[Parameter(Mandatory = $true, Position = 0)]
[string]$Name,
[Parameter(Mandatory = $true, Position = 1)]
[string]$Value
)
Write-Output "Setting environment variable $Name=$Value..."
[System.Environment]::SetEnvironmentVariable("$Name", "$Value", "Machine")
[System.Environment]::SetEnvironmentVariable("$Name", "$Value", "Process")
}
function Install-Package {
param (
[Parameter(Mandatory = $true, Position = 0)]
@@ -137,7 +195,7 @@ function Install-Package {
function Install-Packages {
foreach ($package in $args) {
Install-Package -Name $package
Install-Package $package
}
}
@@ -145,12 +203,13 @@ function Install-Common-Software {
Install-Chocolatey
Install-Pwsh
Install-Git
Install-Packages curl 7zip
Install-Packages curl 7zip nssm
Install-NodeJs
Install-Bun
Install-Cygwin
if ($CI) {
Install-Tailscale
# FIXME: Installing tailscale causes the AWS metadata server to become unreachable
# Install-Tailscale
Install-Buildkite
}
}
@@ -204,12 +263,13 @@ function Install-Buildkite {
Write-Output "Installing Buildkite agent..."
$env:buildkiteAgentToken = "xxx"
iex ((New-Object System.Net.WebClient).DownloadString("https://raw.githubusercontent.com/buildkite/agent/main/install.ps1"))
$installScript = Download-File "https://raw.githubusercontent.com/buildkite/agent/main/install.ps1"
Execute-Script $installScript
Refresh-Path
}
function Install-Build-Essentials {
# Install-Visual-Studio
Install-Visual-Studio
Install-Packages `
cmake `
make `
@@ -219,41 +279,42 @@ function Install-Build-Essentials {
golang `
nasm `
ruby `
strawberryperl `
mingw
Install-Rust
Install-Llvm
}
function Install-Visual-Studio {
$components = @(
"Microsoft.VisualStudio.Workload.NativeDesktop",
"Microsoft.VisualStudio.Component.Windows10SDK.18362",
"Microsoft.VisualStudio.Component.Windows11SDK.22000",
"Microsoft.VisualStudio.Component.Windows11Sdk.WindowsPerformanceToolkit",
"Microsoft.VisualStudio.Component.VC.ASAN", # C++ AddressSanitizer
"Microsoft.VisualStudio.Component.VC.ATL", # C++ ATL for latest v143 build tools (x86 & x64)
"Microsoft.VisualStudio.Component.VC.DiagnosticTools", # C++ Diagnostic Tools
"Microsoft.VisualStudio.Component.VC.CLI.Support", # C++/CLI support for v143 build tools (Latest)
"Microsoft.VisualStudio.Component.VC.CoreIde", # C++ core features
"Microsoft.VisualStudio.Component.VC.Redist.14.Latest" # C++ 2022 Redistributable Update
param (
[Parameter(Mandatory = $false)]
[string]$Edition = "community"
)
$arch = (Get-WmiObject Win32_Processor).Architecture
if ($arch -eq 9) {
$components += @(
"Microsoft.VisualStudio.Component.VC.Tools.x86.x64", # MSVC v143 build tools (x86 & x64)
"Microsoft.VisualStudio.Component.VC.Modules.x86.x64" # MSVC v143 C++ Modules for latest v143 build tools (x86 & x64)
)
} elseif ($arch -eq 5) {
$components += @(
"Microsoft.VisualStudio.Component.VC.Tools.ARM64", # MSVC v143 build tools (ARM64)
"Microsoft.VisualStudio.Component.UWP.VC.ARM64" # C++ Universal Windows Platform support for v143 build tools (ARM64/ARM64EC)
Write-Output "Downloading Visual Studio installer..."
$vsInstaller = Download-File "https://aka.ms/vs/17/release/vs_$Edition.exe"
Write-Output "Installing Visual Studio..."
$vsInstallArgs = @(
"--passive",
"--norestart",
"--wait",
"--force",
"--locale en-US",
"--add Microsoft.VisualStudio.Workload.NativeDesktop",
"--includeRecommended"
)
$startInfo = New-Object System.Diagnostics.ProcessStartInfo
$startInfo.FileName = $vsInstaller
$startInfo.Arguments = $vsInstallArgs -join ' '
$startInfo.CreateNoWindow = $true
$process = New-Object System.Diagnostics.Process
$process.StartInfo = $startInfo
$process.Start()
$process.WaitForExit()
if ($process.ExitCode -ne 0) {
throw "Failed to install Visual Studio: code $($process.ExitCode)"
}
$packageParameters = $components | ForEach-Object { "--add $_" }
Install-Package visualstudio2022community `
-ExtraArgs "--package-parameters '--add Microsoft.VisualStudio.Workload.NativeDesktop --includeRecommended --includeOptional'"
}
function Install-Rust {
@@ -261,18 +322,31 @@ function Install-Rust {
return
}
Write-Output "Installing Rustup..."
$rustupInit = Download-File "https://win.rustup.rs/" -Name "rustup-init.exe"
Write-Output "Installing Rust..."
$rustupInit = "$env:TEMP\rustup-init.exe"
(New-Object System.Net.WebClient).DownloadFile("https://win.rustup.rs/", $rustupInit)
Execute-Command $rustupInit -y
Add-To-Path "$env:USERPROFILE\.cargo\bin"
Write-Output "Moving Rust to $env:ProgramFiles..."
$rustPath = Join-Path $env:ProgramFiles "Rust"
if (-not (Test-Path $rustPath)) {
New-Item -Path $rustPath -ItemType Directory
}
Move-Item "$env:UserProfile\.cargo" "$rustPath\cargo" -Force
Move-Item "$env:UserProfile\.rustup" "$rustPath\rustup" -Force
Write-Output "Setting environment variables for Rust..."
Set-Env "CARGO_HOME" "$rustPath\cargo"
Set-Env "RUSTUP_HOME" "$rustPath\rustup"
Add-To-Path "$rustPath\cargo\bin"
}
function Install-Llvm {
Install-Package llvm `
-Command clang-cl `
-Version "18.1.8"
Add-To-Path "C:\Program Files\LLVM\bin"
Add-To-Path "$env:ProgramFiles\LLVM\bin"
}
function Optimize-System {
@@ -280,6 +354,9 @@ function Optimize-System {
Disable-Windows-Threat-Protection
Disable-Windows-Services
Disable-Power-Management
}
function Optimize-System-Needs-Reboot {
Uninstall-Windows-Defender
}
@@ -319,7 +396,7 @@ function Disable-Windows-Services {
}
function Disable-Power-Management {
Write-Output "Disabling power management features..."
Write-Output "Disabling Power Management..."
powercfg /setactive 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c # High performance
powercfg /change monitor-timeout-ac 0
powercfg /change monitor-timeout-dc 0
@@ -329,7 +406,6 @@ function Disable-Power-Management {
powercfg /change hibernate-timeout-dc 0
}
Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force
if ($Optimize) {
Optimize-System
}
@@ -337,3 +413,6 @@ if ($Optimize) {
Install-Common-Software
Install-Build-Essentials
if ($Optimize) {
Optimize-System-Needs-Reboot
}

View File

@@ -1,5 +1,5 @@
#!/bin/sh
# Version: 5
# Version: 7
# A script that installs the dependencies needed to build and test Bun.
# This should work on macOS and Linux with a POSIX shell.
@@ -92,7 +92,7 @@ download_file() {
execute chmod 755 "$tmp"
path="$tmp/$filename"
fetch "$url" > "$path"
fetch "$url" >"$path"
execute chmod 644 "$path"
print "$path"
@@ -112,15 +112,24 @@ append_to_file() {
file="$1"
content="$2"
if ! [ -f "$file" ]; then
file_needs_sudo="0"
if [ -f "$file" ]; then
if ! [ -r "$file" ] || ! [ -w "$file" ]; then
file_needs_sudo="1"
fi
else
execute_as_user mkdir -p "$(dirname "$file")"
execute_as_user touch "$file"
fi
echo "$content" | while read -r line; do
if ! grep -q "$line" "$file"; then
if [ "$file_needs_sudo" = "1" ]; then
execute_sudo sh -c "echo '$line' >> '$file'"
else
echo "$line" >>"$file"
fi
fi
done
}
@@ -135,7 +144,7 @@ append_to_file_sudo() {
echo "$content" | while read -r line; do
if ! grep -q "$line" "$file"; then
echo "$line" | execute_sudo tee "$file" > /dev/null
echo "$line" | execute_sudo tee "$file" >/dev/null
fi
done
}
@@ -161,18 +170,21 @@ append_to_path() {
export PATH="$path:$PATH"
}
link_to_bin() {
path="$1"
if ! [ -d "$path" ]; then
error "Could not find directory: \"$path\""
move_to_bin() {
exe_path="$1"
if ! [ -f "$exe_path" ]; then
error "Could not find executable: \"$exe_path\""
fi
for file in "$path"/*; do
if [ -f "$file" ]; then
grant_to_user "$file"
execute_sudo ln -sf "$file" "/usr/bin/$(basename "$file")"
usr_paths="/usr/bin /usr/local/bin"
for usr_path in $usr_paths; do
if [ -d "$usr_path" ] && [ -w "$usr_path" ]; then
break
fi
done
grant_to_user "$exe_path"
execute_sudo mv -f "$exe_path" "$usr_path/$(basename "$exe_path")"
}
check_features() {
@@ -384,6 +396,74 @@ check_user() {
fi
}
check_ulimit() {
if ! [ "$ci" = "1" ]; then
return
fi
print "Checking ulimits..."
systemd_conf="/etc/systemd/system.conf"
if [ -f "$systemd_conf" ]; then
limits_conf="/etc/security/limits.d/99-unlimited.conf"
if ! [ -f "$limits_conf" ]; then
execute_sudo mkdir -p "$(dirname "$limits_conf")"
execute_sudo touch "$limits_conf"
fi
fi
limits="core data fsize memlock nofile rss stack cpu nproc as locks sigpending msgqueue"
for limit in $limits; do
limit_upper="$(echo "$limit" | tr '[:lower:]' '[:upper:]')"
limit_value="unlimited"
case "$limit" in
nofile | nproc)
limit_value="1048576"
;;
esac
if [ -f "$limits_conf" ]; then
limit_users="root *"
for limit_user in $limit_users; do
append_to_file "$limits_conf" "$limit_user soft $limit $limit_value"
append_to_file "$limits_conf" "$limit_user hard $limit $limit_value"
done
fi
if [ -f "$systemd_conf" ]; then
append_to_file "$systemd_conf" "DefaultLimit$limit_upper=$limit_value"
fi
done
rc_conf="/etc/rc.conf"
if [ -f "$rc_conf" ]; then
rc_ulimit=""
limit_flags="c d e f i l m n q r s t u v x"
for limit_flag in $limit_flags; do
limit_value="unlimited"
case "$limit_flag" in
n | u)
limit_value="1048576"
;;
esac
rc_ulimit="$rc_ulimit -$limit_flag $limit_value"
done
append_to_file "$rc_conf" "rc_ulimit=\"$rc_ulimit\""
fi
pam_confs="/etc/pam.d/common-session /etc/pam.d/common-session-noninteractive"
for pam_conf in $pam_confs; do
if [ -f "$pam_conf" ]; then
append_to_file "$pam_conf" "session optional pam_limits.so"
fi
done
systemctl="$(which systemctl)"
if [ -f "$systemctl" ]; then
execute_sudo "$systemctl" daemon-reload
fi
}
package_manager() {
case "$pm" in
apt)
@@ -602,6 +682,14 @@ install_nodejs_headers() {
}
install_bun() {
case "$pm" in
apk)
install_packages \
libgcc \
libstdc++
;;
esac
bash="$(require bash)"
script=$(download_file "https://bun.sh/install")
@@ -615,7 +703,10 @@ install_bun() {
;;
esac
link_to_bin "$home/.bun/bin"
move_to_bin "$home/.bun/bin/bun"
bun_path="$(which bun)"
bunx_path="$(dirname "$bun_path")/bunx"
execute_sudo ln -sf "$bun_path" "$bunx_path"
}
install_cmake() {
@@ -628,14 +719,14 @@ install_cmake() {
cmake_version="3.30.5"
case "$arch" in
x64)
url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-x86_64.sh"
cmake_url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-x86_64.sh"
;;
aarch64)
url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-aarch64.sh"
cmake_url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-aarch64.sh"
;;
esac
script=$(download_file "$url")
execute_sudo "$sh" "$script" \
cmake_script=$(download_file "$cmake_url")
execute_sudo "$sh" "$cmake_script" \
--skip-license \
--prefix=/usr
;;
@@ -732,13 +823,13 @@ install_llvm() {
case "$pm" in
apt)
bash="$(require bash)"
script="$(download_file "https://apt.llvm.org/llvm.sh")"
llvm_script="$(download_file "https://apt.llvm.org/llvm.sh")"
case "$distro-$release" in
ubuntu-24*)
execute_sudo "$bash" "$script" "$(llvm_version)" all -njammy
execute_sudo "$bash" "$llvm_script" "$(llvm_version)" all -njammy
;;
*)
execute_sudo "$bash" "$script" "$(llvm_version)" all
execute_sudo "$bash" "$llvm_script" "$(llvm_version)" all
;;
esac
;;
@@ -779,11 +870,6 @@ install_rust() {
execute_as_user "$sh" "$script" -y
;;
esac
# FIXME: This causes cargo to fail to build:
# > error: rustup could not choose a version of cargo to run,
# > because one wasn't specified explicitly, and no default is configured.
# link_to_bin "$home/.cargo/bin"
}
install_docker() {
@@ -796,7 +882,7 @@ install_docker() {
*)
case "$distro-$release" in
amzn-2 | amzn-1)
execute amazon-linux-extras install docker
execute_sudo amazon-linux-extras install docker
;;
amzn-* | alpine-*)
install_packages docker
@@ -832,8 +918,8 @@ install_tailscale() {
case "$os" in
linux)
sh="$(require sh)"
script=$(download_file "https://tailscale.com/install.sh")
execute "$sh" "$script"
tailscale_script=$(download_file "https://tailscale.com/install.sh")
execute "$sh" "$tailscale_script"
;;
darwin)
install_packages go
@@ -862,24 +948,39 @@ create_buildkite_user() {
esac
if [ -z "$(getent passwd "$user")" ]; then
case "$distro" in
alpine)
execute_sudo addgroup \
--system "$group"
execute_sudo adduser "$user" \
--system \
--ingroup "$group" \
--shell "$(require sh)" \
--home "$home" \
--disabled-password
;;
*)
execute_sudo useradd "$user" \
--system \
--shell "$(require sh)" \
--no-create-home \
--home-dir "$home"
;;
esac
fi
if [ -n "$(getent group docker)" ]; then
execute_sudo usermod -aG docker "$user"
fi
paths="$home /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /var/run/buildkite-agent/buildkite-agent.sock"
for path in $paths; do
buildkite_paths="$home /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /var/run/buildkite-agent/buildkite-agent.sock"
for path in $buildkite_paths; do
execute_sudo mkdir -p "$path"
execute_sudo chown -R "$user:$group" "$path"
done
files="/var/run/buildkite-agent/buildkite-agent.pid"
for file in $files; do
buildkite_files="/var/run/buildkite-agent/buildkite-agent.pid"
for file in $buildkite_files; do
execute_sudo touch "$file"
execute_sudo chown "$user:$group" "$file"
done
@@ -890,19 +991,42 @@ install_buildkite() {
return
fi
bash="$(require bash)"
script="$(download_file "https://raw.githubusercontent.com/buildkite/agent/main/install.sh")"
tmp_dir="$(execute dirname "$script")"
HOME="$tmp_dir" execute "$bash" "$script"
buildkite_version="3.87.0"
case "$os-$arch" in
linux-aarch64)
buildkite_filename="buildkite-agent-linux-arm64-$buildkite_version.tar.gz"
;;
linux-x64)
buildkite_filename="buildkite-agent-linux-amd64-$buildkite_version.tar.gz"
;;
darwin-aarch64)
buildkite_filename="buildkite-agent-darwin-arm64-$buildkite_version.tar.gz"
;;
darwin-x64)
buildkite_filename="buildkite-agent-darwin-amd64-$buildkite_version.tar.gz"
;;
esac
buildkite_url="https://github.com/buildkite/agent/releases/download/v$buildkite_version/$buildkite_filename"
buildkite_filepath="$(download_file "$buildkite_url" "$buildkite_filename")"
buildkite_tmpdir="$(dirname "$buildkite_filepath")"
out_dir="$tmp_dir/.buildkite-agent"
execute_sudo mv -f "$out_dir/bin/buildkite-agent" "/usr/bin/buildkite-agent"
execute tar -xzf "$buildkite_filepath" -C "$buildkite_tmpdir"
move_to_bin "$buildkite_tmpdir/buildkite-agent"
execute rm -rf "$buildkite_tmpdir"
}
install_chrome_dependencies() {
install_chromium() {
# https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#chrome-doesnt-launch-on-linux
# https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-in-the-cloud
case "$pm" in
apk)
install_packages \
chromium \
nss \
freetype \
harfbuzz \
ttf-freefont
;;
apt)
install_packages \
fonts-liberation \
@@ -979,22 +1103,17 @@ install_chrome_dependencies() {
esac
}
raise_file_descriptor_limit() {
append_to_file_sudo /etc/security/limits.conf '* soft nofile 262144'
append_to_file_sudo /etc/security/limits.conf '* hard nofile 262144'
}
main() {
check_features "$@"
check_operating_system
check_inside_docker
check_user
check_ulimit
check_package_manager
create_buildkite_user
install_common_software
install_build_essentials
install_chrome_dependencies
raise_file_descriptor_limit # XXX: temporary
install_chromium
}
main "$@"

View File

@@ -3,6 +3,7 @@
import { spawn as nodeSpawn } from "node:child_process";
import { existsSync, readFileSync, mkdirSync, cpSync, chmodSync } from "node:fs";
import { basename, join, resolve } from "node:path";
import { isCI, printEnvironment, startGroup } from "./utils.mjs";
// https://cmake.org/cmake/help/latest/manual/cmake.1.html#generate-a-project-buildsystem
const generateFlags = [
@@ -37,6 +38,10 @@ async function build(args) {
return spawn("pwsh", ["-NoProfile", "-NoLogo", "-File", shellPath, process.argv0, scriptPath, ...args]);
}
if (isCI) {
printEnvironment();
}
const env = {
...process.env,
FORCE_COLOR: "1",
@@ -102,7 +107,8 @@ async function build(args) {
const generateArgs = Object.entries(generateOptions).flatMap(([flag, value]) =>
flag.startsWith("-D") ? [`${flag}=${value}`] : [flag, value],
);
await spawn("cmake", generateArgs, { env }, "configuration");
await startGroup("CMake Configure", () => spawn("cmake", generateArgs, { env }));
const envPath = resolve(buildPath, ".env");
if (existsSync(envPath)) {
@@ -116,7 +122,8 @@ async function build(args) {
const buildArgs = Object.entries(buildOptions)
.sort(([a], [b]) => (a === "--build" ? -1 : a.localeCompare(b)))
.flatMap(([flag, value]) => [flag, value]);
await spawn("cmake", buildArgs, { env }, "compilation");
await startGroup("CMake Build", () => spawn("cmake", buildArgs, { env }));
printDuration("total", Date.now() - startTime);
}

File diff suppressed because it is too large Load Diff

View File

@@ -17,7 +17,6 @@ import {
accessSync,
appendFileSync,
readdirSync,
rmSync,
} from "node:fs";
import { spawn, spawnSync } from "node:child_process";
import { join, basename, dirname, relative, sep } from "node:path";
@@ -27,6 +26,8 @@ import {
getBuildUrl,
getEnv,
getFileUrl,
getLoggedInUserCount,
getShell,
getWindowsExitReason,
isArm64,
isBuildkite,
@@ -59,6 +60,10 @@ const { values: options, positionals: filters } = parseArgs({
type: "string",
default: undefined,
},
["build-id"]: {
type: "string",
default: undefined,
},
["bail"]: {
type: "boolean",
default: false,
@@ -99,32 +104,7 @@ const { values: options, positionals: filters } = parseArgs({
async function runTests() {
let execPath;
if (options["step"]) {
downloadLoop: for (let i = 0; i < 10; i++) {
execPath = await getExecPathFromBuildKite(options["step"]);
for (let j = 0; j < 10; j++) {
const { error } = spawnSync(execPath, ["--version"], {
encoding: "utf-8",
timeout: spawnTimeout,
env: {
PATH: process.env.PATH,
BUN_DEBUG_QUIET_LOGS: 1,
},
});
if (!error) {
break downloadLoop;
}
const { code } = error;
if (code === "EBUSY") {
console.log("Bun appears to be busy, retrying...");
continue;
}
if (code === "UNKNOWN") {
console.log("Bun appears to be corrupted, downloading again...");
rmSync(execPath, { force: true });
continue downloadLoop;
}
}
}
execPath = await getExecPathFromBuildKite(options["step"], options["build-id"]);
} else {
execPath = getExecPath(options["exec-path"]);
}
@@ -482,12 +462,14 @@ async function spawnBun(execPath, { args, cwd, timeout, env, stdout, stderr }) {
const path = addPath(dirname(execPath), process.env.PATH);
const tmpdirPath = mkdtempSync(join(tmpdir(), "buntmp-"));
const { username, homedir } = userInfo();
const shellPath = getShell();
const bunEnv = {
...process.env,
PATH: path,
TMPDIR: tmpdirPath,
USER: username,
HOME: homedir,
SHELL: shellPath,
FORCE_COLOR: "1",
BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING: "1",
BUN_DEBUG_QUIET_LOGS: "1",
@@ -604,7 +586,7 @@ async function spawnBunTest(execPath, testPath, options = { cwd }) {
* @returns {number}
*/
function getTestTimeout(testPath) {
if (/integration|3rd_party|docker/i.test(testPath)) {
if (/integration|3rd_party|docker|bun-install-registry|v8/i.test(testPath)) {
return integrationTimeout;
}
return testTimeout;
@@ -1080,9 +1062,10 @@ function getExecPath(bunExe) {
/**
* @param {string} target
* @param {string} [buildId]
* @returns {Promise<string>}
*/
async function getExecPathFromBuildKite(target) {
async function getExecPathFromBuildKite(target, buildId) {
if (existsSync(target) || target.includes("/")) {
return getExecPath(target);
}
@@ -1090,8 +1073,9 @@ async function getExecPathFromBuildKite(target) {
const releasePath = join(cwd, "release");
mkdirSync(releasePath, { recursive: true });
let zipPath;
downloadLoop: for (let i = 0; i < 10; i++) {
const args = ["artifact", "download", "**", releasePath, "--step", target];
const buildId = process.env["BUILDKITE_ARTIFACT_BUILD_ID"];
if (buildId) {
args.push("--build", buildId);
}
@@ -1101,27 +1085,32 @@ async function getExecPathFromBuildKite(target) {
args,
});
let zipPath;
for (const entry of readdirSync(releasePath, { recursive: true, encoding: "utf-8" })) {
if (/^bun.*\.zip$/i.test(entry) && !entry.includes("-profile.zip")) {
zipPath = join(releasePath, entry);
break;
break downloadLoop;
}
}
console.warn(`Waiting for ${target}.zip to be available...`);
await new Promise(resolve => setTimeout(resolve, i * 1000));
}
if (!zipPath) {
throw new Error(`Could not find ${target}.zip from Buildkite: ${releasePath}`);
}
await unzip(zipPath, releasePath);
for (const entry of readdirSync(releasePath, { recursive: true, encoding: "utf-8" })) {
const releaseFiles = readdirSync(releasePath, { recursive: true, encoding: "utf-8" });
for (const entry of releaseFiles) {
const execPath = join(releasePath, entry);
if (/bun(?:\.exe)?$/i.test(entry) && isExecutable(execPath)) {
if (/bun(?:\.exe)?$/i.test(entry) && statSync(execPath).isFile()) {
return execPath;
}
}
console.warn(`Found ${releaseFiles.length} files in ${releasePath}:`);
throw new Error(`Could not find executable from BuildKite: ${releasePath}`);
}
@@ -1466,8 +1455,39 @@ export async function main() {
}
printEnvironment();
// FIXME: Some DNS tests hang unless we set the DNS server to 8.8.8.8
// It also appears to hang on 1.1.1.1, which could explain this issue:
// https://github.com/oven-sh/bun/issues/11136
if (isWindows && isCI) {
await spawn("pwsh", [
"-Command",
"Set-DnsClientServerAddress -InterfaceAlias 'Ethernet 4' -ServerAddresses ('8.8.8.8','8.8.4.4')",
]);
}
const results = await runTests();
const ok = results.every(({ ok }) => ok);
let waitForUser = false;
while (isCI) {
const userCount = getLoggedInUserCount();
if (!userCount) {
if (waitForUser) {
console.log("No users logged in, exiting runner...");
}
break;
}
if (!waitForUser) {
startGroup("Summary");
console.warn(`Found ${userCount} users logged in, keeping the runner alive until logout...`);
waitForUser = true;
}
await new Promise(resolve => setTimeout(resolve, 60_000));
}
process.exit(getExitCode(ok ? "pass" : "fail"));
}

File diff suppressed because it is too large Load Diff

View File

@@ -74,7 +74,7 @@ JSC::JSValue generateModule(JSC::JSGlobalObject* globalObject, JSC::VM& vm, cons
return result;
}
#if BUN_DEBUG
#if BUN_DYNAMIC_JS_LOAD_PATH
JSValue initializeInternalModuleFromDisk(
JSGlobalObject* globalObject,
VM& vm,

Binary file not shown.

View File

@@ -5,7 +5,7 @@ import { readFile, readlink, writeFile } from "fs/promises";
import fs, { closeSync, openSync } from "node:fs";
import os from "node:os";
import { dirname, isAbsolute, join } from "path";
import detect_libc from "detect-libc";
import detectLibc from "detect-libc";
type Awaitable<T> = T | Promise<T>;
@@ -19,7 +19,8 @@ export const isIntelMacOS = isMacOS && process.arch === "x64";
export const isDebug = Bun.version.includes("debug");
export const isCI = process.env.CI !== undefined;
export const isBuildKite = process.env.BUILDKITE === "true";
export const libc_family = detect_libc.familySync();
export const libcFamily = detectLibc.familySync() as "glibc" | "musl";
export const isVerbose = process.env.DEBUG === "1";
// Use these to mark a test as flaky or broken.
// This will help us keep track of these tests.
@@ -1372,7 +1373,7 @@ export function waitForFileToExist(path: string, interval: number) {
export function libcPathForDlopen() {
switch (process.platform) {
case "linux":
switch (libc_family) {
switch (libcFamily) {
case "glibc":
return "libc.so.6";
case "musl":

View File

@@ -1,8 +1,9 @@
import assert from "assert";
import { copyFileSync } from "fs";
import { join } from "path";
import { ConsoleMessage, Page, launch } from "puppeteer";
import type { ConsoleMessage, Page } from "puppeteer";
import { launch } from "puppeteer";
import { which } from "bun";
const root = join(import.meta.dir, "../");
copyFileSync(join(root, "src/Counter1.txt"), join(root, "src/Counter.tsx"));
@@ -12,28 +13,38 @@ if (process.argv.length > 2) {
url = process.argv[2];
}
const browserPath = which("chromium-browser") || which("chromium") || which("chrome") || undefined;
if (!browserPath) {
console.warn("Since a Chromium browser was not found, it will be downloaded by Puppeteer.");
}
const b = await launch({
// While puppeteer is migrating to their new headless: `true` mode,
// this causes strange issues on macOS in the cloud (AWS and MacStadium).
//
// There is a GitHub issue, but the discussion is unhelpful:
// https://github.com/puppeteer/puppeteer/issues/10153
//
// Fixes: 'TargetCloseError: Protocol error (Target.setAutoAttach): Target closed'
headless: "shell",
// On macOS, there are issues using the new headless mode.
// "TargetCloseError: Protocol error (Target.setAutoAttach): Target closed"
headless: process.platform === "darwin" ? "shell" : true,
// Inherit the stdout and stderr of the browser process.
dumpio: true,
// Prefer to use a pipe to connect to the browser, instead of a WebSocket.
pipe: true,
// Disable timeouts.
timeout: 0,
protocolTimeout: 0,
// Specify that chrome should be used, for consistent test results.
// If a browser path is not found, it will be downloaded.
browser: "chrome",
executablePath: browserPath,
args: [
// Fixes: 'dock_plist is not an NSDictionary'
// On Linux, there are issues with the sandbox, so disable it.
// On macOS, this fixes: "dock_plist is not an NSDictionary"
"--no-sandbox",
"--single-process",
"--disable-setuid-sandbox",
// On Docker, the default /dev/shm is too small for Chrome, which causes
// crashes when rendering large pages, so disable it.
"--disable-dev-shm-usage",
// Fixes: 'Navigating frame was detached'
// Fixes: "Navigating frame was detached"
"--disable-features=site-per-process",
// Uncomment if you want debug logs from Chromium:
// "--enable-logging=stderr",
// "--v=1",
],
});

View File

@@ -1,8 +1,14 @@
import { isBroken, isWindows } from "harness";
import assert from "node:assert";
import cluster from "node:cluster";
import http from "node:http";
import { availableParallelism } from "node:os";
if (isWindows && isBroken) {
console.log("Skipping on Windows because it does not work when there are more than 1 CPU");
process.exit(0);
}
const numCPUs = availableParallelism();
let workers = 0;

View File

@@ -4,7 +4,9 @@
"@types/react": "18.0.28",
"@types/react-dom": "18.0.11",
"@types/supertest": "2.0.12",
"@types/utf-8-validate": "5.0.0"
"@types/utf-8-validate": "5.0.0",
"@types/ws": "8.5.10",
"@types/puppeteer": "7.0.4"
},
"dependencies": {
"@azure/service-bus": "7.9.4",
@@ -16,7 +18,6 @@
"@remix-run/serve": "2.10.3",
"@resvg/resvg-js": "2.4.1",
"@swc/core": "1.3.38",
"@types/ws": "8.5.10",
"aws-cdk-lib": "2.148.0",
"axios": "1.6.8",
"body-parser": "1.20.2",