diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index bf346e6c69..d64ecd178b 100755 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -31,7 +31,7 @@ import { } from "../scripts/utils.mjs"; /** - * @typedef {"linux" | "darwin" | "windows"} Os + * @typedef {"linux" | "darwin" | "windows" | "freebsd"} Os * @typedef {"aarch64" | "x64"} Arch * @typedef {"musl"} Abi * @typedef {"debian" | "ubuntu" | "alpine" | "amazonlinux"} Distro @@ -114,6 +114,7 @@ const buildPlatforms = [ { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.22" }, { os: "windows", arch: "x64", release: "2019" }, { os: "windows", arch: "x64", baseline: true, release: "2019" }, + { os: "freebsd", arch: "x64", release: "14.3" }, ]; /** @@ -1072,7 +1073,7 @@ async function getPipeline(options = {}) { const imagePlatforms = new Map( buildImages || publishImages ? [...buildPlatforms, ...testPlatforms] - .filter(({ os }) => os === "linux" || os === "windows") + .filter(({ os }) => os !== "darwin") .map(platform => [getImageKey(platform), platform]) : [], ); @@ -1106,10 +1107,13 @@ async function getPipeline(options = {}) { const includeASAN = !isMainBranch(); if (!buildId) { - const relevantBuildPlatforms = includeASAN + let relevantBuildPlatforms = includeASAN ? buildPlatforms : buildPlatforms.filter(({ profile }) => profile !== "asan"); + // run build-image but no build-bun yet + relevantBuildPlatforms = relevantBuildPlatforms.filter(({ os }) => os !== "freebsd"); + steps.push( ...relevantBuildPlatforms.map(target => { const imageKey = getImageKey(target); diff --git a/package.json b/package.json index 549bdf8405..a8caf066ad 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,7 @@ "machine:linux:alpine": "./scripts/machine.mjs ssh --cloud=aws --arch=x64 --instance-type c7i.2xlarge --os=linux --distro=alpine --release=3.22", "machine:linux:amazonlinux": "./scripts/machine.mjs ssh --cloud=aws --arch=x64 --instance-type c7i.2xlarge --os=linux --distro=amazonlinux --release=2023", "machine:windows:2019": "./scripts/machine.mjs ssh --cloud=aws --arch=x64 --instance-type c7i.2xlarge --os=windows --release=2019", - "machine:freebsd": "./scripts/machine.mjs ssh --cloud=aws --arch=x64 --instance-type c7i.large --os=freebsd --release=14.3", + "machine:freebsd": "./scripts/machine.mjs ssh --cloud=aws --arch=x64 --instance-type c7i.2xlarge --os=freebsd --release=14.3", "sync-webkit-source": "bun ./scripts/sync-webkit-source.ts" } } diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 7bc2a53ad6..912e213760 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -192,7 +192,7 @@ download_file() { file_tmp_dir="$(create_tmp_directory)" file_tmp_path="$file_tmp_dir/$(basename "$file_url")" - fetch "$file_url" >"$file_tmp_path" + fetch "$file_url" > "$file_tmp_path" grant_to_user "$file_tmp_path" print "$file_tmp_path" @@ -204,14 +204,14 @@ download_and_verify_file() { hash="$2" path=$(download_file "$file_url") - execute sh -c 'echo "'"$hash $path"'" | sha256sum -c' >/dev/null 2>&1 + execute sh -c 'echo "'"$hash $path"'" | sha256sum -c -' >/dev/null 2>&1 print "$path" } append_to_profile() { content="$1" - profiles=".profile .zprofile .bash_profile .bashrc .zshrc" + profiles=".profile .zprofile .bash_profile .bashrc .zshrc .cshrc" for profile in $profiles; do for profile_path in "$current_home/$profile" "$home/$profile"; do if [ "$ci" = "1" ] || [ -f "$profile_path" ]; then @@ -275,12 +275,15 @@ check_operating_system() { os="$("$uname" -s)" case "$os" in - Linux*) + Linux) os="linux" ;; - Darwin*) + Darwin) os="darwin" ;; + FreeBSD) + os="freebsd" + ;; *) error "Unsupported operating system: $os" ;; @@ -343,6 +346,11 @@ check_operating_system() { ;; esac ;; + freebsd) + . /etc/os-release + distro="$ID" + release="$VERSION_ID" + ;; esac if [ -n "$distro" ]; then @@ -426,6 +434,9 @@ check_package_manager() { error "No package manager found. (apt, dnf, yum, apk)" fi ;; + freebsd) + pm="pkg" + ;; esac print "Package manager: $pm" @@ -438,6 +449,13 @@ check_package_manager() { apk) package_manager update ;; + pkg) + # may need to switch betwen 'latest' and 'quarterly' depending on which repo www/chromium is in. check https://www.freshports.org/www/chromium/. + # mkdir -p /usr/local/etc/pkg/repos + # echo 'FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/${ABI}/quarterly" }' > /usr/local/etc/pkg/repos/FreeBSD.conf + export ASSUME_ALWAYS_YES=yes + package_manager update -f + ;; esac } @@ -478,6 +496,9 @@ check_user() { } check_ulimit() { + if ! [ "$os" = "linux" ]; then + return + fi if ! [ "$ci" = "1" ]; then return fi @@ -588,6 +609,9 @@ package_manager() { brew) execute_as_user brew "$@" ;; + pkg) + execute_sudo pkg "$@" + ;; *) error "Unsupported package manager: $pm" ;; @@ -645,6 +669,9 @@ install_packages() { --no-progress \ "$@" ;; + pkg) + package_manager install "$@" + ;; *) error "Unsupported package manager: $pm" ;; @@ -688,13 +715,57 @@ install_common_software() { apt-transport-https \ software-properties-common fi + # https://packages.debian.org + # https://packages.ubuntu.com install_packages \ + bash \ + ca-certificates \ + curl \ + htop \ + gnupg \ + git \ + unzip \ + wget \ libc6-dbg ;; dnf) + # https://packages.fedoraproject.org install_packages \ + bash \ + ca-certificates \ + curl \ + htop \ + gnupg \ + git \ + unzip \ + wget \ dnf-plugins-core ;; + apk) + # https://pkgs.alpinelinux.org/packages + install_packages \ + bash \ + ca-certificates \ + curl \ + htop \ + gnupg \ + git \ + unzip \ + wget \ + ;; + pkg) + # https://www.freshports.org + install_packages \ + shells/bash \ + ftp/curl \ + sysutils/htop \ + security/gnupg \ + devel/git \ + archivers/unzip \ + ftp/wget \ + editors/vim \ + sysutils/neofetch \ + ;; esac case "$distro" in @@ -718,16 +789,6 @@ install_common_software() { execute "$crb" enable fi - install_packages \ - bash \ - ca-certificates \ - curl \ - htop \ - gnupg \ - git \ - unzip \ - wget - install_rosetta install_nodejs install_bun @@ -755,6 +816,9 @@ install_nodejs() { linux) nodejs_platform="linux" ;; + freebsd) + nodejs_platform="freebsd" + ;; *) error "Unsupported OS for Node.js download: $os" ;; @@ -773,6 +837,12 @@ install_nodejs() { ;; esac + if [ "$os" = "freebsd" ]; then + # TODO: use nodejs_version_exact + install_packages "www/node$(nodejs_version)" "www/npm-node$(nodejs_version)" + return + fi + case "$abi" in musl) nodejs_mirror="https://bun-nodejs-release.s3.us-west-1.amazonaws.com" @@ -862,6 +932,10 @@ install_nodejs() { } install_nodejs_headers() { + if [ "$os" = "freebsd" ]; then + return + fi + nodejs_version="$(nodejs_version_exact)" nodejs_headers_tar="$(download_file "https://nodejs.org/download/release/v$nodejs_version/node-v$nodejs_version-headers.tar.gz")" nodejs_headers_dir="$(dirname "$nodejs_headers_tar")" @@ -876,6 +950,10 @@ install_nodejs_headers() { } setup_node_gyp_cache() { + if [ "$os" = "freebsd" ]; then + return + fi + nodejs_version="$1" headers_source="$2" @@ -914,6 +992,11 @@ bun_version_exact() { } install_bun() { + if [ "$os" = "freebsd" ]; then + # TODO: need to complete bun bootstrap for for this work + return + fi + install_packages unzip case "$pm" in @@ -965,6 +1048,9 @@ install_cmake() { --skip-license \ --prefix=/usr ;; + freebsd-pkg) + install_packages devel/cmake + ;; esac } @@ -1027,19 +1113,34 @@ install_build_essentials() { ;; esac - install_packages \ - make \ - python3 \ - libtool \ - ruby \ - perl + case "$os" in + linux) + install_packages \ + make \ + python3 \ + libtool \ + ruby \ + perl \ + ;; + freebsd) + install_packages \ + devel/ninja \ + devel/pkgconf \ + lang/go \ + devel/gmake \ + lang/python3 \ + devel/libtool \ + lang/ruby33 \ + perl5 \ + ;; + esac install_cmake install_llvm install_osxcross install_gcc - install_sccache install_rust + install_sccache install_docker } @@ -1073,11 +1174,17 @@ install_llvm() { "llvm$(llvm_version)-dev" # Ensures llvm-symbolizer is installed ;; esac + + case "$os" in + freebsd) + # TODO: use llvm_version_exact + install_packages "devel/llvm$(llvm_version)" + ;; + esac } install_gcc() { - if ! [ "$os" = "linux" ] || ! [ "$distro" = "ubuntu" ] || [ -z "$gcc_version" ] - then + if ! [ "$os" = "linux" ] || ! [ "$distro" = "ubuntu" ] || [ -z "$gcc_version" ]; then return fi @@ -1150,6 +1257,18 @@ install_gcc() { } install_sccache() { + case "$os" in + linux) + ;; + freebsd) + cargo install sccache --locked --version 0.12.0 + return + ;; + *) + error "Unsupported platform: $os" + ;; + esac + # Alright, look, this function is cobbled together but it's only as cobbled # together as this whole script is. # @@ -1177,12 +1296,17 @@ install_sccache() { } install_rust() { - case "$pm" in - apk) + case "$distro" in + alpine) install_packages \ rust \ cargo ;; + freebsd) + install_packages lang/rust + create_directory "$HOME/.cargo/bin" + append_to_path "$HOME/.cargo/bin" + ;; *) rust_home="/opt/rust" create_directory "$rust_home" @@ -1212,6 +1336,11 @@ install_docker() { package_manager install docker --cask fi ;; + pkg) + install_packages \ + sysutils/docker \ + sysutils/docker-compose \ + ;; *) case "$distro-$release" in amzn-2 | amzn-1) @@ -1306,10 +1435,17 @@ install_tailscale() { execute_as_user go install tailscale.com/cmd/tailscale{,d}@latest append_to_path "$home/go/bin" ;; + freebsd) + install_packages security/tailscale + ;; esac } install_fuse_python() { + if ! [ "$os" = "linux" ]; then + return + fi + # only linux needs this case "$pm" in apk) @@ -1339,7 +1475,7 @@ install_fuse_python() { } create_buildkite_user() { - if ! [ "$ci" = "1" ] || ! [ "$os" = "linux" ]; then + if ! [ "$ci" = "1" ]; then return fi @@ -1368,6 +1504,14 @@ create_buildkite_user() { --home "$home" \ --disabled-password ;; + freebsd) + execute_sudo pw group add -n "$group" + execute_sudo pw user add \ + -n "$user" \ + -g "$group" \ + -s "$(require sh)" \ + -d "$home" \ + ;; *) execute_sudo useradd "$user" \ --system \ @@ -1518,6 +1662,10 @@ install_chromium() { xorg-x11-fonts-Type1 \ xorg-x11-utils ;; + pkg) + install_packages \ + www/chromium \ + ;; esac case "$distro" in @@ -1529,28 +1677,46 @@ install_chromium() { } install_age() { - # we only use this to encrypt core dumps, which we only have on Linux + age_version="1.2.1" case "$os" in linux) - age_tarball="" case "$arch" in x64) - age_tarball="$(download_and_verify_file https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-linux-amd64.tar.gz 7df45a6cc87d4da11cc03a539a7470c15b1041ab2b396af088fe9990f7c79d50)" + age_arch="amd64" + age_hash="7df45a6cc87d4da11cc03a539a7470c15b1041ab2b396af088fe9990f7c79d50" ;; aarch64) - age_tarball="$(download_and_verify_file https://github.com/FiloSottile/age/releases/download/v1.2.1/age-v1.2.1-linux-arm64.tar.gz 57fd79a7ece5fe501f351b9dd51a82fbee1ea8db65a8839db17f5c080245e99f)" + age_arch="arm64" + age_hash="57fd79a7ece5fe501f351b9dd51a82fbee1ea8db65a8839db17f5c080245e99f" + ;; + *) + error "Unsupported platform: $os-$arch" ;; esac - - age_extract_dir="$(create_tmp_directory)" - execute tar -C "$age_extract_dir" -zxf "$age_tarball" age/age - move_to_bin "$age_extract_dir/age/age" + ;; + freebsd) + case "$arch" in + x64) + age_arch="amd64" + age_hash="943a7510a9973a1e589b913a70228aa1361a63cde39e3ed581435a4d4802df29" + ;; + *) + error "Unsupported platform: $os-$arch" + ;; + esac + ;; + *) + error "Unsupported platform: $os-$arch" ;; esac + + age_tarball="$(download_and_verify_file https://github.com/FiloSottile/age/releases/download/v$age_version/age-v$age_version-$os-$age_arch.tar.gz "$age_hash")" + age_extract_dir="$(create_tmp_directory)" + execute tar -C "$age_extract_dir" -zxf "$age_tarball" age/age + move_to_bin "$age_extract_dir/age/age" } configure_core_dumps() { - # we only have core dumps on Linux case "$os" in linux) # set up a directory that the test runner will look in after running tests diff --git a/scripts/machine.mjs b/scripts/machine.mjs index c1212d8fee..e63f4f2a46 100755 --- a/scripts/machine.mjs +++ b/scripts/machine.mjs @@ -712,7 +712,7 @@ write_files: HostKey /etc/ssh/ssh_host_ed25519_key SyslogFacility AUTHPRIV PermitRootLogin yes - AuthorizedKeysFile .ssh/authorized_keys + AuthorizedKeysFile %h/.ssh/authorized_keys PasswordAuthentication no ChallengeResponseAuthentication no GSSAPIAuthentication yes @@ -883,8 +883,7 @@ function getSshKeys() { const sshFiles = readdirSync(sshPath, { withFileTypes: true, encoding: "utf-8" }); const publicPaths = sshFiles .filter(entry => entry.isFile() && entry.name.endsWith(".pub")) - .map(({ name }) => join(sshPath, name)) - .filter(path => !readFile(path, { cache: true }).startsWith("ssh-ed25519")); + .map(({ name }) => join(sshPath, name)); sshKeys.push( ...publicPaths.map(publicPath => ({ @@ -960,10 +959,11 @@ async function getGithubOrgSshKeys(organization) { * @returns {Promise} */ async function spawnScp(options) { - const { hostname, port, username, identityPaths, password, source, destination, retries = 10 } = options; + const { hostname, port, username, identityPaths, password, source, destination, retries = 3 } = options; await waitForPort({ hostname, port: port || 22 }); const command = ["scp", "-o", "StrictHostKeyChecking=no"]; + command.push("-O"); // use SCP instead of SFTP if (!password) { command.push("-o", "BatchMode=yes"); } @@ -1228,7 +1228,6 @@ async function main() { }; let { detached, bootstrap, ci, os, arch, distro, release, features } = options; - if (os === "freebsd") bootstrap = false; let name = `${os}-${arch}-${(release || "").replace(/\./g, "")}`; diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 609db9f8f4..b2cec8d209 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -2986,6 +2986,9 @@ const emojiMap = { gear: ["⚙️", "gear"], clipboard: ["📋", "clipboard"], rocket: ["🚀", "rocket"], + freebsd: ["😈", "freebsd"], + openbsd: ["🐡", "openbsd"], + netbsd: ["🚩", "netbsd"], }; /**