From fa6ac405a4aeccab2dd12eba74ec6dabf601aeab Mon Sep 17 00:00:00 2001 From: Ashcon Partovi Date: Thu, 5 Dec 2024 14:16:37 -0800 Subject: [PATCH] ci: Add bootstrap.ps1 and automate Windows build images (#15606) --- .buildkite/ci.mjs | 1754 +++++++++------ .buildkite/scripts/prepare-build.sh | 2 +- cmake/Options.cmake | 2 +- cmake/targets/BuildLolHtml.cmake | 2 + cmake/tools/SetupRust.cmake | 29 +- scripts/agent.mjs | 39 +- scripts/bootstrap.ps1 | 163 +- scripts/bootstrap.sh | 221 +- scripts/build.mjs | 11 +- scripts/machine.mjs | 1985 ++++++++++++++--- scripts/runner.node.mjs | 112 +- scripts/utils.mjs | 744 +++++- .../bindings/InternalModuleRegistry.cpp | 2 +- test/bun.lockb | Bin 380850 -> 400730 bytes test/harness.ts | 7 +- .../next-pages/test/dev-server-puppeteer.ts | 43 +- test/js/node/cluster/test-docs-http-server.ts | 6 + test/package.json | 5 +- 18 files changed, 3850 insertions(+), 1277 deletions(-) diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index 70d4e44c1d..e3adb70fb1 100755 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -5,838 +5,1126 @@ * @link https://buildkite.com/docs/pipelines/defining-steps */ -import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { getBootstrapVersion, + getBuildkiteEmoji, + getBuildMetadata, getBuildNumber, - getCanaryRevision, - getChangedFiles, - getCommit, getCommitMessage, + getEmoji, getEnv, getLastSuccessfulBuild, - getMainBranch, - getTargetBranch, isBuildkite, + isBuildManual, isFork, isMainBranch, isMergeQueue, - printEnvironment, + parseBoolean, spawnSafe, + startGroup, toYaml, uploadArtifact, + writeFile, } from "../scripts/utils.mjs"; /** - * @typedef PipelineOptions - * @property {string} [buildId] - * @property {boolean} [buildImages] - * @property {boolean} [publishImages] - * @property {boolean} [skipTests] + * @typedef {"linux" | "darwin" | "windows"} Os + * @typedef {"aarch64" | "x64"} Arch + * @typedef {"musl"} Abi + * @typedef {"debian" | "ubuntu" | "alpine" | "amazonlinux"} Distro + * @typedef {"latest" | "previous" | "oldest" | "eol"} Tier + * @typedef {"release" | "assert" | "debug"} Profile */ /** - * @param {PipelineOptions} options + * @typedef Target + * @property {Os} os + * @property {Arch} arch + * @property {Abi} [abi] + * @property {boolean} [baseline] + * @property {boolean} [canary] + * @property {Profile} [profile] */ -function getPipeline(options) { - const { buildId, buildImages, publishImages, skipTests } = options; - /** - * Helpers - */ +/** + * @param {Target} target + * @returns {string} + */ +function getTargetKey(target) { + const { os, arch, abi, baseline, profile } = target; + let key = `${os}-${arch}`; + if (abi) { + key += `-${abi}`; + } + if (baseline) { + key += "-baseline"; + } + if (profile && profile !== "release") { + key += `-${profile}`; + } + return key; +} - /** - * @param {string} text - * @returns {string} - * @link https://github.com/buildkite/emojis#emoji-reference - */ - const getEmoji = string => { - if (string === "amazonlinux") { - return ":aws:"; - } - return `:${string}:`; +/** + * @param {Target} target + * @returns {string} + */ +function getTargetLabel(target) { + const { os, arch, abi, baseline, profile } = target; + let label = `${getBuildkiteEmoji(os)} ${arch}`; + if (abi) { + label += `-${abi}`; + } + if (baseline) { + label += "-baseline"; + } + if (profile && profile !== "release") { + label += `-${profile}`; + } + return label; +} + +/** + * @typedef Platform + * @property {Os} os + * @property {Arch} arch + * @property {Abi} [abi] + * @property {boolean} [baseline] + * @property {boolean} [canary] + * @property {Profile} [profile] + * @property {Distro} [distro] + * @property {string} release + * @property {Tier} [tier] + */ + +/** + * @type {Platform[]} + */ +const buildPlatforms = [ + { os: "darwin", arch: "aarch64", release: "14" }, + { os: "darwin", arch: "x64", release: "14" }, + { os: "linux", arch: "aarch64", distro: "debian", release: "11" }, + { os: "linux", arch: "x64", distro: "debian", release: "11" }, + { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" }, + { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" }, + { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20" }, + { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20" }, + { os: "windows", arch: "x64", release: "2019" }, + { os: "windows", arch: "x64", baseline: true, release: "2019" }, +]; + +/** + * @type {Platform[]} + */ +const testPlatforms = [ + { os: "darwin", arch: "aarch64", release: "14", tier: "latest" }, + { os: "darwin", arch: "aarch64", release: "13", tier: "previous" }, + { os: "darwin", arch: "x64", release: "14", tier: "latest" }, + { os: "darwin", arch: "x64", release: "13", tier: "previous" }, + { os: "linux", arch: "aarch64", distro: "debian", release: "12", tier: "latest" }, + { os: "linux", arch: "aarch64", distro: "debian", release: "11", tier: "previous" }, + { os: "linux", arch: "x64", distro: "debian", release: "12", tier: "latest" }, + { os: "linux", arch: "x64", distro: "debian", release: "11", tier: "previous" }, + { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "12", tier: "latest" }, + { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11", tier: "previous" }, + { os: "linux", arch: "aarch64", distro: "ubuntu", release: "24.04", tier: "latest" }, + { os: "linux", arch: "aarch64", distro: "ubuntu", release: "22.04", tier: "previous" }, + { os: "linux", arch: "aarch64", distro: "ubuntu", release: "20.04", tier: "oldest" }, + { os: "linux", arch: "x64", distro: "ubuntu", release: "24.04", tier: "latest" }, + { os: "linux", arch: "x64", distro: "ubuntu", release: "22.04", tier: "previous" }, + { os: "linux", arch: "x64", distro: "ubuntu", release: "20.04", tier: "oldest" }, + { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "24.04", tier: "latest" }, + { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "22.04", tier: "previous" }, + { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "20.04", tier: "oldest" }, + { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20", tier: "latest" }, + { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20", tier: "latest" }, + { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20", tier: "latest" }, + { os: "windows", arch: "x64", release: "2025", tier: "latest" }, + { os: "windows", arch: "x64", release: "2022", tier: "previous" }, + { os: "windows", arch: "x64", release: "2019", tier: "oldest" }, + { os: "windows", arch: "x64", release: "2025", baseline: true, tier: "latest" }, + { os: "windows", arch: "x64", release: "2022", baseline: true, tier: "previous" }, + { os: "windows", arch: "x64", release: "2019", baseline: true, tier: "oldest" }, +]; + +/** + * @param {Platform} platform + * @returns {string} + */ +function getPlatformKey(platform) { + const { distro, release } = platform; + const target = getTargetKey(platform); + const version = release.replace(/\./g, ""); + if (distro) { + return `${target}-${distro}-${version}`; + } + return `${target}-${version}`; +} + +/** + * @param {Platform} platform + * @returns {string} + */ +function getPlatformLabel(platform) { + const { os, arch, baseline, profile, distro, release } = platform; + let label = `${getBuildkiteEmoji(distro || os)} ${release} ${arch}`; + if (baseline) { + label += "-baseline"; + } + if (profile && profile !== "release") { + label += `-${profile}`; + } + return label; +} + +/** + * @param {Platform} platform + * @returns {string} + */ +function getImageKey(platform) { + const { os, arch, distro, release } = platform; + const version = release.replace(/\./g, ""); + if (distro) { + return `${os}-${arch}-${distro}-${version}`; + } + return `${os}-${arch}-${version}`; +} + +/** + * @param {Platform} platform + * @returns {string} + */ +function getImageLabel(platform) { + const { os, arch, distro, release } = platform; + return `${getBuildkiteEmoji(distro || os)} ${release} ${arch}`; +} + +/** + * @param {Platform} platform + * @param {boolean} [dryRun] + * @returns {string} + */ +function getImageName(platform, dryRun) { + const { os, arch, distro, release } = platform; + const name = distro ? `${os}-${arch}-${distro}-${release}` : `${os}-${arch}-${release}`; + if (dryRun) { + return `${name}-build-${getBuildNumber()}`; + } + return `${name}-v${getBootstrapVersion(os)}`; +} + +/** + * @param {number} [limit] + * @link https://buildkite.com/docs/pipelines/command-step#retry-attributes + */ +function getRetry(limit = 0) { + return { + manual: { + permit_on_passed: true, + }, + automatic: [ + { exit_status: 1, limit }, + { exit_status: -1, limit: 3 }, + { exit_status: 255, limit: 3 }, + { signal_reason: "cancel", limit: 3 }, + { signal_reason: "agent_stop", limit: 3 }, + ], }; +} - /** - * @typedef {"linux" | "darwin" | "windows"} Os - * @typedef {"aarch64" | "x64"} Arch - * @typedef {"musl"} Abi - */ +/** + * @returns {number} + * @link https://buildkite.com/docs/pipelines/managing-priorities + */ +function getPriority() { + if (isFork()) { + return -1; + } + if (isMainBranch()) { + return 2; + } + if (isMergeQueue()) { + return 1; + } + return 0; +} - /** - * @typedef Target - * @property {Os} os - * @property {Arch} arch - * @property {Abi} [abi] - * @property {boolean} [baseline] - */ +/** + * Agents + */ - /** - * @param {Target} target - * @returns {string} - */ - const getTargetKey = target => { - const { os, arch, abi, baseline } = target; - let key = `${os}-${arch}`; - if (abi) { - key += `-${abi}`; - } - if (baseline) { - key += "-baseline"; - } - return key; +/** + * @typedef {Object} Ec2Options + * @property {string} instanceType + * @property {number} cpuCount + * @property {number} threadsPerCore + */ + +/** + * @param {Platform} platform + * @param {Ec2Options} options + * @returns {Agent} + */ +function getEc2Agent(platform, options) { + const { os, arch, abi, distro, release } = platform; + const { instanceType, cpuCount, threadsPerCore } = options; + return { + os, + arch, + abi, + distro, + release, + // The agent is created by robobun, see more details here: + // https://github.com/oven-sh/robobun/blob/d46c07e0ac5ac0f9ffe1012f0e98b59e1a0d387a/src/robobun.ts#L1707 + robobun: true, + robobun2: true, + "image-name": getImageName(platform), + "instance-type": instanceType, + "cpu-count": cpuCount, + "threads-per-core": threadsPerCore, + "preemptible": false, }; +} - /** - * @param {Target} target - * @returns {string} - */ - const getTargetLabel = target => { - const { os, arch, abi, baseline } = target; - let label = `${getEmoji(os)} ${arch}`; - if (abi) { - label += `-${abi}`; - } - if (baseline) { - label += "-baseline"; - } - return label; - }; +/** + * @param {Platform} platform + * @returns {string} + */ +function getCppAgent(platform) { + const { os, arch } = platform; - /** - * @typedef Platform - * @property {Os} os - * @property {Arch} arch - * @property {Abi} [abi] - * @property {boolean} [baseline] - * @property {string} [distro] - * @property {string} release - */ - - /** - * @param {Platform} platform - * @returns {string} - */ - const getPlatformKey = platform => { - const { os, arch, abi, baseline, distro, release } = platform; - const target = getTargetKey({ os, arch, abi, baseline }); - if (distro) { - return `${target}-${distro}-${release.replace(/\./g, "")}`; - } - return `${target}-${release.replace(/\./g, "")}`; - }; - - /** - * @param {Platform} platform - * @returns {string} - */ - const getPlatformLabel = platform => { - const { os, arch, baseline, distro, release } = platform; - let label = `${getEmoji(distro || os)} ${release} ${arch}`; - if (baseline) { - label += "-baseline"; - } - return label; - }; - - /** - * @param {Platform} platform - * @returns {string} - */ - const getImageKey = platform => { - const { os, arch, distro, release } = platform; - if (distro) { - return `${os}-${arch}-${distro}-${release.replace(/\./g, "")}`; - } - return `${os}-${arch}-${release.replace(/\./g, "")}`; - }; - - /** - * @param {Platform} platform - * @returns {string} - */ - const getImageLabel = platform => { - const { os, arch, distro, release } = platform; - return `${getEmoji(distro || os)} ${release} ${arch}`; - }; - - /** - * @param {number} [limit] - * @link https://buildkite.com/docs/pipelines/command-step#retry-attributes - */ - const getRetry = (limit = 0) => { - return { - automatic: [ - { exit_status: 1, limit }, - { exit_status: -1, limit: 3 }, - { exit_status: 255, limit: 3 }, - { signal_reason: "agent_stop", limit: 3 }, - ], - }; - }; - - /** - * @returns {number} - * @link https://buildkite.com/docs/pipelines/managing-priorities - */ - const getPriority = () => { - if (isFork()) { - return -1; - } - if (isMainBranch()) { - return 2; - } - if (isMergeQueue()) { - return 1; - } - return 0; - }; - - /** - * @param {Target} target - * @returns {Record} - */ - const getBuildEnv = target => { - const { baseline, abi } = target; - return { - ENABLE_BASELINE: baseline ? "ON" : "OFF", - ABI: abi === "musl" ? "musl" : undefined, - }; - }; - - /** - * @param {Target} target - * @returns {string} - */ - const getBuildToolchain = target => { - const { os, arch, abi, baseline } = target; - let key = `${os}-${arch}`; - if (abi) { - key += `-${abi}`; - } - if (baseline) { - key += "-baseline"; - } - return key; - }; - - /** - * Agents - */ - - /** - * @typedef {Record} Agent - */ - - /** - * @param {Platform} platform - * @returns {boolean} - */ - const isUsingNewAgent = platform => { - const { os } = platform; - if (os === "linux") { - return true; - } - return false; - }; - - /** - * @param {"v1" | "v2"} version - * @param {Platform} platform - * @param {string} [instanceType] - * @returns {Agent} - */ - const getEmphemeralAgent = (version, platform, instanceType) => { - const { os, arch, abi, distro, release } = platform; - if (version === "v1") { - return { - robobun: true, - os, - arch, - distro, - release, - }; - } - let image; - if (distro) { - image = `${os}-${arch}-${distro}-${release}`; - } else { - image = `${os}-${arch}-${release}`; - } - if (buildImages && !publishImages) { - image += `-build-${getBuildNumber()}`; - } else { - image += `-v${getBootstrapVersion()}`; - } - return { - robobun: true, - robobun2: true, - os, - arch, - abi, - distro, - release, - "image-name": image, - "instance-type": instanceType, - }; - }; - - /** - * @param {Target} target - * @returns {Agent} - */ - const getBuildAgent = target => { - const { os, arch, abi } = target; - if (isUsingNewAgent(target)) { - const instanceType = arch === "aarch64" ? "c8g.8xlarge" : "c7i.8xlarge"; - return getEmphemeralAgent("v2", target, instanceType); - } + if (os === "darwin") { return { queue: `build-${os}`, os, arch, - abi, }; + } + + return getEc2Agent(platform, { + instanceType: arch === "aarch64" ? "c8g.16xlarge" : "c7i.16xlarge", + cpuCount: 64, + threadsPerCore: 1, + }); +} + +/** + * @param {Platform} platform + * @returns {Agent} + */ +function getZigAgent(platform) { + const { arch } = platform; + + return { + queue: "build-zig", }; - /** - * @param {Target} target - * @returns {Agent} - */ - const getZigAgent = platform => { - const { arch } = platform; - const instanceType = arch === "aarch64" ? "c8g.2xlarge" : "c7i.2xlarge"; + // return getEc2Agent( + // { + // os: "linux", + // arch, + // distro: "debian", + // release: "11", + // }, + // { + // instanceType: arch === "aarch64" ? "c8g.2xlarge" : "c7i.2xlarge", + // cpuCount: 8, + // threadsPerCore: 1, + // }, + // ); +} + +/** + * @param {Platform} platform + * @returns {Agent} + */ +function getTestAgent(platform) { + const { os, arch } = platform; + + if (os === "darwin") { return { - robobun: true, - robobun2: true, - os: "linux", + queue: `test-${os}`, + os, arch, - distro: "debian", - release: "11", - "image-name": `linux-${arch}-debian-11-v5`, // v5 is not on main yet - "instance-type": instanceType, }; - // TODO: Temporarily disable due to configuration - // return { - // queue: "build-zig", - // }; + } + + // TODO: `dev-server-ssr-110.test.ts` and `next-build.test.ts` run out of memory + // at 8GB of memory, so use 16GB instead. + if (os === "windows") { + return getEc2Agent(platform, { + instanceType: "c7i.2xlarge", + cpuCount: 1, + threadsPerCore: 1, + }); + } + + if (arch === "aarch64") { + return getEc2Agent(platform, { + instanceType: "c8g.xlarge", + cpuCount: 1, + threadsPerCore: 1, + }); + } + + return getEc2Agent(platform, { + instanceType: "c7i.xlarge", + cpuCount: 1, + threadsPerCore: 1, + }); +} + +/** + * Steps + */ + +/** + * @param {Target} target + * @returns {Record} + */ +function getBuildEnv(target) { + const { profile, baseline, canary, abi } = target; + const release = !profile || profile === "release"; + + return { + CMAKE_BUILD_TYPE: release ? "Release" : profile === "debug" ? "Debug" : "RelWithDebInfo", + ENABLE_BASELINE: baseline ? "ON" : "OFF", + ENABLE_CANARY: canary ? "ON" : "OFF", + ENABLE_ASSERTIONS: release ? "OFF" : "ON", + ENABLE_LOGS: release ? "OFF" : "ON", + ABI: abi === "musl" ? "musl" : undefined, }; +} - /** - * @param {Platform} platform - * @returns {Agent} - */ - const getTestAgent = platform => { - const { os, arch, release } = platform; - if (isUsingNewAgent(platform)) { - const instanceType = arch === "aarch64" ? "t4g.large" : "t3.large"; - return getEmphemeralAgent("v2", platform, instanceType); - } - if (os === "darwin") { - return { - os, - arch, - release, - queue: "test-darwin", - }; - } - return getEmphemeralAgent("v1", platform); +/** + * @param {Platform} platform + * @returns {Step} + */ +function getBuildVendorStep(platform) { + return { + key: `${getTargetKey(platform)}-build-vendor`, + label: `${getTargetLabel(platform)} - build-vendor`, + agents: getCppAgent(platform), + retry: getRetry(), + cancel_on_build_failing: isMergeQueue(), + env: getBuildEnv(platform), + command: "bun run build:ci --target dependencies", }; +} - /** - * Steps - */ - - /** - * @typedef Step - * @property {string} key - * @property {string} [label] - * @property {Record} [agents] - * @property {Record} [env] - * @property {string} command - * @property {string[]} [depends_on] - * @property {Record} [retry] - * @property {boolean} [cancel_on_build_failing] - * @property {boolean} [soft_fail] - * @property {number} [parallelism] - * @property {number} [concurrency] - * @property {string} [concurrency_group] - * @property {number} [priority] - * @property {number} [timeout_in_minutes] - * @link https://buildkite.com/docs/pipelines/command-step - */ - - /** - * @param {Platform} platform - * @param {string} [step] - * @returns {string[]} - */ - const getDependsOn = (platform, step) => { - if (imagePlatforms.has(getImageKey(platform))) { - const key = `${getImageKey(platform)}-build-image`; - if (key !== step) { - return [key]; - } - } - return []; +/** + * @param {Platform} platform + * @returns {Step} + */ +function getBuildCppStep(platform) { + return { + key: `${getTargetKey(platform)}-build-cpp`, + label: `${getTargetLabel(platform)} - build-cpp`, + agents: getCppAgent(platform), + retry: getRetry(), + cancel_on_build_failing: isMergeQueue(), + env: { + BUN_CPP_ONLY: "ON", + ...getBuildEnv(platform), + }, + command: "bun run build:ci --target bun", }; +} - /** - * @param {Platform} platform - * @returns {Step} - */ - const getBuildImageStep = platform => { - const { os, arch, distro, release } = platform; - const action = publishImages ? "publish-image" : "create-image"; - return { - key: `${getImageKey(platform)}-build-image`, - label: `${getImageLabel(platform)} - build-image`, - agents: { - queue: "build-image", - }, - env: { - DEBUG: "1", - }, - retry: getRetry(), - command: `node ./scripts/machine.mjs ${action} --ci --cloud=aws --os=${os} --arch=${arch} --distro=${distro} --distro-version=${release}`, - }; +/** + * @param {Target} target + * @returns {string} + */ +function getBuildToolchain(target) { + const { os, arch, abi, baseline } = target; + let key = `${os}-${arch}`; + if (abi) { + key += `-${abi}`; + } + if (baseline) { + key += "-baseline"; + } + return key; +} + +/** + * @param {Platform} platform + * @returns {Step} + */ +function getBuildZigStep(platform) { + const toolchain = getBuildToolchain(platform); + return { + key: `${getTargetKey(platform)}-build-zig`, + label: `${getTargetLabel(platform)} - build-zig`, + agents: getZigAgent(platform), + retry: getRetry(), + cancel_on_build_failing: isMergeQueue(), + env: getBuildEnv(platform), + command: `bun run build:ci --target bun-zig --toolchain ${toolchain}`, }; +} - /** - * @param {Platform} platform - * @returns {Step} - */ - const getBuildVendorStep = platform => { - return { - key: `${getTargetKey(platform)}-build-vendor`, - label: `${getTargetLabel(platform)} - build-vendor`, - depends_on: getDependsOn(platform), - agents: getBuildAgent(platform), - retry: getRetry(), - cancel_on_build_failing: isMergeQueue(), - env: getBuildEnv(platform), - command: "bun run build:ci --target dependencies", - }; +/** + * @param {Platform} platform + * @returns {Step} + */ +function getLinkBunStep(platform) { + return { + key: `${getTargetKey(platform)}-build-bun`, + label: `${getTargetLabel(platform)} - build-bun`, + depends_on: [ + `${getTargetKey(platform)}-build-vendor`, + `${getTargetKey(platform)}-build-cpp`, + `${getTargetKey(platform)}-build-zig`, + ], + agents: getCppAgent(platform), + retry: getRetry(), + cancel_on_build_failing: isMergeQueue(), + env: { + BUN_LINK_ONLY: "ON", + ...getBuildEnv(platform), + }, + command: "bun run build:ci --target bun", }; +} - /** - * @param {Platform} platform - * @returns {Step} - */ - const getBuildCppStep = platform => { - return { - key: `${getTargetKey(platform)}-build-cpp`, - label: `${getTargetLabel(platform)} - build-cpp`, - depends_on: getDependsOn(platform), - agents: getBuildAgent(platform), - retry: getRetry(), - cancel_on_build_failing: isMergeQueue(), - env: { - BUN_CPP_ONLY: "ON", - ...getBuildEnv(platform), - }, - command: "bun run build:ci --target bun", - }; +/** + * @param {Platform} platform + * @returns {Step} + */ +function getBuildBunStep(platform) { + return { + key: `${getTargetKey(platform)}-build-bun`, + label: `${getTargetLabel(platform)} - build-bun`, + agents: getCppAgent(platform), + retry: getRetry(), + cancel_on_build_failing: isMergeQueue(), + env: getBuildEnv(platform), + command: "bun run build:ci", }; +} - /** - * @param {Platform} platform - * @returns {Step} - */ - const getBuildZigStep = platform => { - const toolchain = getBuildToolchain(platform); - return { - key: `${getTargetKey(platform)}-build-zig`, - label: `${getTargetLabel(platform)} - build-zig`, - depends_on: getDependsOn(platform), - agents: getZigAgent(platform), - retry: getRetry(), - cancel_on_build_failing: isMergeQueue(), - env: getBuildEnv(platform), - command: `bun run build:ci --target bun-zig --toolchain ${toolchain}`, - }; +/** + * @typedef {Object} TestOptions + * @property {string} [buildId] + * @property {boolean} [unifiedTests] + * @property {string[]} [testFiles] + */ + +/** + * @param {Platform} platform + * @param {TestOptions} [options] + * @returns {Step} + */ +function getTestBunStep(platform, options = {}) { + const { os } = platform; + const { buildId, unifiedTests, testFiles } = options; + + const args = [`--step=${getTargetKey(platform)}-build-bun`]; + if (buildId) { + args.push(`--build-id=${buildId}`); + } + if (testFiles) { + args.push(...testFiles.map(testFile => `--include=${testFile}`)); + } + + const depends = []; + if (!buildId) { + depends.push(`${getTargetKey(platform)}-build-bun`); + } + + return { + key: `${getPlatformKey(platform)}-test-bun`, + label: `${getPlatformLabel(platform)} - test-bun`, + depends_on: depends, + agents: getTestAgent(platform), + cancel_on_build_failing: isMergeQueue(), + retry: getRetry(), + soft_fail: isMainBranch() ? true : [{ exit_status: 2 }], + parallelism: unifiedTests ? undefined : os === "darwin" ? 2 : 10, + command: + os === "windows" + ? `node .\\scripts\\runner.node.mjs ${args.join(" ")}` + : `./scripts/runner.node.mjs ${args.join(" ")}`, }; +} - /** - * @param {Platform} platform - * @returns {Step} - */ - const getBuildBunStep = platform => { - return { - key: `${getTargetKey(platform)}-build-bun`, - label: `${getTargetLabel(platform)} - build-bun`, - depends_on: [ - `${getTargetKey(platform)}-build-vendor`, - `${getTargetKey(platform)}-build-cpp`, - `${getTargetKey(platform)}-build-zig`, - ], - agents: getBuildAgent(platform), - retry: getRetry(), - cancel_on_build_failing: isMergeQueue(), - env: { - BUN_LINK_ONLY: "ON", - ...getBuildEnv(platform), - }, - command: "bun run build:ci --target bun", - }; +/** + * @param {Platform} platform + * @param {boolean} [dryRun] + * @returns {Step} + */ +function getBuildImageStep(platform, dryRun) { + const { os, arch, distro, release } = platform; + const action = dryRun ? "create-image" : "publish-image"; + const command = [ + "node", + "./scripts/machine.mjs", + action, + `--os=${os}`, + `--arch=${arch}`, + distro && `--distro=${distro}`, + `--release=${release}`, + "--cloud=aws", + "--ci", + "--authorized-org=oven-sh", + ]; + return { + key: `${getImageKey(platform)}-build-image`, + label: `${getImageLabel(platform)} - build-image`, + agents: { + queue: "build-image", + }, + env: { + DEBUG: "1", + }, + retry: getRetry(), + command: command.filter(Boolean).join(" "), + timeout_in_minutes: 3 * 60, }; +} - /** - * @param {Platform} platform - * @returns {Step} - */ - const getTestBunStep = platform => { - const { os } = platform; - let command; - if (os === "windows") { - command = `node .\\scripts\\runner.node.mjs --step ${getTargetKey(platform)}-build-bun`; - } else { - command = `./scripts/runner.node.mjs --step ${getTargetKey(platform)}-build-bun`; - } - let parallelism; - if (os === "darwin") { - parallelism = 2; - } else { - parallelism = 10; - } - let env; - let depends = []; - if (buildId) { - env = { - BUILDKITE_ARTIFACT_BUILD_ID: buildId, - }; - } else { - depends = [`${getTargetKey(platform)}-build-bun`]; - } - let retry; - if (os !== "windows") { - // When the runner fails on Windows, Buildkite only detects an exit code of 1. - // Because of this, we don't know if the run was fatal, or soft-failed. - retry = getRetry(1); - } - let soft_fail; - if (isMainBranch()) { - soft_fail = true; - } else { - soft_fail = [{ exit_status: 2 }]; - } - return { - key: `${getPlatformKey(platform)}-test-bun`, - label: `${getPlatformLabel(platform)} - test-bun`, - depends_on: [...depends, ...getDependsOn(platform)], - agents: getTestAgent(platform), - retry, - cancel_on_build_failing: isMergeQueue(), - soft_fail, - parallelism, - command, - env, - }; +/** + * @param {Platform[]} [buildPlatforms] + * @returns {Step} + */ +function getReleaseStep(buildPlatforms) { + return { + key: "release", + label: getBuildkiteEmoji("rocket"), + agents: { + queue: "test-darwin", + }, + depends_on: buildPlatforms.map(platform => `${getTargetKey(platform)}-build-bun`), + command: ".buildkite/scripts/upload-release.sh", }; +} - /** - * Config - */ +/** + * @typedef {Object} Pipeline + * @property {Step[]} [steps] + * @property {number} [priority] + */ - /** - * @type {Platform[]} - */ - const buildPlatforms = [ - { os: "darwin", arch: "aarch64", release: "14" }, - { os: "darwin", arch: "x64", release: "14" }, - { os: "linux", arch: "aarch64", distro: "debian", release: "11" }, - { os: "linux", arch: "x64", distro: "debian", release: "11" }, - { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" }, - { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" }, - { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20" }, - { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20" }, - { os: "windows", arch: "x64", release: "2019" }, - { os: "windows", arch: "x64", baseline: true, release: "2019" }, +/** + * @typedef {Record} Agent + */ + +/** + * @typedef {GroupStep | CommandStep | BlockStep} Step + */ + +/** + * @typedef {Object} GroupStep + * @property {string} key + * @property {string} group + * @property {Step[]} steps + * @property {string[]} [depends_on] + */ + +/** + * @typedef {Object} CommandStep + * @property {string} key + * @property {string} [label] + * @property {Record} [agents] + * @property {Record} [env] + * @property {string} command + * @property {string[]} [depends_on] + * @property {Record} [retry] + * @property {boolean} [cancel_on_build_failing] + * @property {boolean} [soft_fail] + * @property {number} [parallelism] + * @property {number} [concurrency] + * @property {string} [concurrency_group] + * @property {number} [priority] + * @property {number} [timeout_in_minutes] + * @link https://buildkite.com/docs/pipelines/command-step + */ + +/** + * @typedef {Object} BlockStep + * @property {string} key + * @property {string} block + * @property {string} [prompt] + * @property {"passed" | "failed" | "running"} [blocked_state] + * @property {(SelectInput | TextInput)[]} [fields] + */ + +/** + * @typedef {Object} TextInput + * @property {string} key + * @property {string} text + * @property {string} [default] + * @property {boolean} [required] + * @property {string} [hint] + */ + +/** + * @typedef {Object} SelectInput + * @property {string} key + * @property {string} select + * @property {string | string[]} [default] + * @property {boolean} [required] + * @property {boolean} [multiple] + * @property {string} [hint] + * @property {SelectOption[]} [options] + */ + +/** + * @typedef {Object} SelectOption + * @property {string} label + * @property {string} value + */ + +/** + * @typedef {Object} PipelineOptions + * @property {string | boolean} [skipEverything] + * @property {string | boolean} [skipBuilds] + * @property {string | boolean} [skipTests] + * @property {string | boolean} [forceBuilds] + * @property {string | boolean} [forceTests] + * @property {string | boolean} [buildImages] + * @property {string | boolean} [publishImages] + * @property {boolean} [canary] + * @property {Profile[]} [buildProfiles] + * @property {Platform[]} [buildPlatforms] + * @property {Platform[]} [testPlatforms] + * @property {string[]} [testFiles] + * @property {boolean} [unifiedBuilds] + * @property {boolean} [unifiedTests] + */ + +/** + * @param {Step} step + * @param {(string | undefined)[]} dependsOn + * @returns {Step} + */ +function getStepWithDependsOn(step, ...dependsOn) { + const { depends_on: existingDependsOn = [] } = step; + return { + ...step, + depends_on: [...existingDependsOn, ...dependsOn.filter(Boolean)], + }; +} + +/** + * @returns {BlockStep} + */ +function getOptionsStep() { + const booleanOptions = [ + { + label: `${getEmoji("true")} Yes`, + value: "true", + }, + { + label: `${getEmoji("false")} No`, + value: "false", + }, ]; - /** - * @type {Platform[]} - */ - const testPlatforms = [ - { os: "darwin", arch: "aarch64", release: "14" }, - { os: "darwin", arch: "aarch64", release: "13" }, - { os: "darwin", arch: "x64", release: "14" }, - { os: "darwin", arch: "x64", release: "13" }, - { os: "linux", arch: "aarch64", distro: "debian", release: "12" }, - { os: "linux", arch: "aarch64", distro: "debian", release: "11" }, - { os: "linux", arch: "x64", distro: "debian", release: "12" }, - { os: "linux", arch: "x64", distro: "debian", release: "11" }, - { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "12" }, - { os: "linux", arch: "x64", baseline: true, distro: "debian", release: "11" }, - { os: "linux", arch: "aarch64", distro: "ubuntu", release: "22.04" }, - { os: "linux", arch: "aarch64", distro: "ubuntu", release: "20.04" }, - { os: "linux", arch: "x64", distro: "ubuntu", release: "22.04" }, - { os: "linux", arch: "x64", distro: "ubuntu", release: "20.04" }, - { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "22.04" }, - { os: "linux", arch: "x64", baseline: true, distro: "ubuntu", release: "20.04" }, - { os: "linux", arch: "aarch64", abi: "musl", distro: "alpine", release: "3.20" }, - { os: "linux", arch: "x64", abi: "musl", distro: "alpine", release: "3.20" }, - { os: "linux", arch: "x64", abi: "musl", baseline: true, distro: "alpine", release: "3.20" }, - { os: "windows", arch: "x64", release: "2019" }, - { os: "windows", arch: "x64", baseline: true, release: "2019" }, - ]; + return { + key: "options", + block: getBuildkiteEmoji("clipboard"), + blocked_state: "running", + fields: [ + { + key: "canary", + select: "If building, is this a canary build?", + hint: "If you are building for a release, this should be false", + required: false, + default: "true", + options: booleanOptions, + }, + { + key: "skip-builds", + select: "Do you want to skip the build?", + hint: "If true, artifacts will be downloaded from the last successful build", + required: false, + default: "false", + options: booleanOptions, + }, + { + key: "skip-tests", + select: "Do you want to skip the tests?", + required: false, + default: "false", + options: booleanOptions, + }, + { + key: "force-builds", + select: "Do you want to force run the build?", + hint: "If true, the build will run even if no source files have changed", + required: false, + default: "false", + options: booleanOptions, + }, + { + key: "force-tests", + select: "Do you want to force run the tests?", + hint: "If true, the tests will run even if no test files have changed", + required: false, + default: "false", + options: booleanOptions, + }, + { + key: "build-profiles", + select: "If building, which profiles do you want to build?", + required: false, + multiple: true, + default: ["release"], + options: [ + { + label: `${getEmoji("release")} Release`, + value: "release", + }, + { + label: `${getEmoji("assert")} Release with Assertions`, + value: "assert", + }, + { + label: `${getEmoji("debug")} Debug`, + value: "debug", + }, + ], + }, + { + key: "build-platforms", + select: "If building, which platforms do you want to build?", + hint: "If this is left blank, all platforms are built", + required: false, + multiple: true, + default: [], + options: buildPlatforms.map(platform => { + const { os, arch, abi, baseline } = platform; + let label = `${getEmoji(os)} ${arch}`; + if (abi) { + label += `-${abi}`; + } + if (baseline) { + label += `-baseline`; + } + return { + label, + value: getTargetKey(platform), + }; + }), + }, + { + key: "test-platforms", + select: "If testing, which platforms do you want to test?", + hint: "If this is left blank, all platforms are tested", + required: false, + multiple: true, + default: [], + options: [...new Map(testPlatforms.map(platform => [getImageKey(platform), platform])).entries()].map( + ([key, platform]) => { + const { os, arch, abi, distro, release } = platform; + let label = `${getEmoji(os)} ${arch}`; + if (abi) { + label += `-${abi}`; + } + if (distro) { + label += ` ${distro}`; + } + if (release) { + label += ` ${release}`; + } + return { + label, + value: key, + }; + }, + ), + }, + { + key: "test-files", + text: "If testing, which files do you want to test?", + hint: "If specified, only run test paths that include the list of strings (e.g. 'test/js', 'test/cli/hot/watch.ts')", + required: false, + }, + { + key: "build-images", + select: "Do you want to re-build the base images?", + hint: "This can take 2-3 hours to complete, only do so if you've tested locally", + required: false, + default: "false", + options: booleanOptions, + }, + { + key: "publish-images", + select: "Do you want to re-build and publish the base images?", + hint: "This can take 2-3 hours to complete, only do so if you've tested locally", + required: false, + default: "false", + options: booleanOptions, + }, + { + key: "unified-builds", + select: "Do you want to build each platform in a single step?", + hint: "If true, builds will not be split into seperate steps (this will likely slow down the build)", + required: false, + default: "false", + options: booleanOptions, + }, + { + key: "unified-tests", + select: "Do you want to run tests in a single step?", + hint: "If true, tests will not be split into seperate steps (this will be very slow)", + required: false, + default: "false", + options: booleanOptions, + }, + ], + }; +} +/** + * @returns {Step} + */ +function getOptionsApplyStep() { + const command = getEnv("BUILDKITE_COMMAND"); + return { + key: "options-apply", + label: getBuildkiteEmoji("gear"), + command: `${command} --apply`, + depends_on: ["options"], + agents: { + queue: getEnv("BUILDKITE_AGENT_META_DATA_QUEUE", false), + }, + }; +} + +/** + * @returns {Promise} + */ +async function getPipelineOptions() { + const isManual = isBuildManual(); + if (isManual && !process.argv.includes("--apply")) { + return; + } + + const buildPlatformsMap = new Map(buildPlatforms.map(platform => [getTargetKey(platform), platform])); + const testPlatformsMap = new Map(testPlatforms.map(platform => [getPlatformKey(platform), platform])); + + if (isManual) { + const { fields } = getOptionsStep(); + const keys = fields?.map(({ key }) => key) ?? []; + const values = await Promise.all(keys.map(getBuildMetadata)); + const options = Object.fromEntries(keys.map((key, index) => [key, values[index]])); + + /** + * @param {string} value + * @returns {string[] | undefined} + */ + const parseArray = value => + value + ?.split("\n") + ?.map(item => item.trim()) + ?.filter(Boolean); + + const buildPlatformKeys = parseArray(options["build-platforms"]); + const testPlatformKeys = parseArray(options["test-platforms"]); + return { + canary: parseBoolean(options["canary"]), + skipBuilds: parseBoolean(options["skip-builds"]), + forceBuilds: parseBoolean(options["force-builds"]), + skipTests: parseBoolean(options["skip-tests"]), + testFiles: parseArray(options["test-files"]), + buildImages: parseBoolean(options["build-images"]), + publishImages: parseBoolean(options["publish-images"]), + unifiedBuilds: parseBoolean(options["unified-builds"]), + unifiedTests: parseBoolean(options["unified-tests"]), + buildProfiles: parseArray(options["build-profiles"]), + buildPlatforms: buildPlatformKeys?.length + ? buildPlatformKeys.map(key => buildPlatformsMap.get(key)) + : Array.from(buildPlatformsMap.values()), + testPlatforms: testPlatformKeys?.length + ? testPlatformKeys.map(key => testPlatformsMap.get(key)) + : Array.from(testPlatformsMap.values()), + }; + } + + const commitMessage = getCommitMessage(); + + /** + * @param {RegExp} pattern + * @returns {string | boolean} + */ + const parseOption = pattern => { + const match = pattern.exec(commitMessage); + if (match) { + const [, value] = match; + return value; + } + return false; + }; + + return { + canary: + !parseBoolean(getEnv("RELEASE", false) || "false") && + !/\[(release|build release|release build)\]/i.test(commitMessage), + skipEverything: parseOption(/\[(skip ci|no ci)\]/i), + skipBuilds: parseOption(/\[(skip builds?|no builds?|only tests?)\]/i), + forceBuilds: parseOption(/\[(force builds?)\]/i), + skipTests: parseOption(/\[(skip tests?|no tests?|only builds?)\]/i), + buildPlatforms: Array.from(buildPlatformsMap.values()), + testPlatforms: Array.from(testPlatformsMap.values()), + buildProfiles: ["release"], + }; +} + +/** + * @param {PipelineOptions} [options] + * @returns {Promise} + */ +async function getPipeline(options = {}) { + const priority = getPriority(); + + if (isBuildManual() && !Object.keys(options).length) { + return { + priority, + steps: [getOptionsStep(), getOptionsApplyStep()], + }; + } + + const { skipEverything } = options; + if (skipEverything) { + return; + } + + const { buildProfiles = [], buildPlatforms = [], testPlatforms = [], buildImages, publishImages } = options; const imagePlatforms = new Map( - [...buildPlatforms, ...testPlatforms] - .filter(platform => buildImages && isUsingNewAgent(platform)) - .map(platform => [getImageKey(platform), platform]), + buildImages || publishImages + ? [...buildPlatforms, ...testPlatforms] + .filter(({ os }) => os === "linux" || os === "windows") + .map(platform => [getImageKey(platform), platform]) + : [], ); - /** - * @type {Step[]} - */ + /** @type {Step[]} */ const steps = []; if (imagePlatforms.size) { steps.push({ - group: ":docker:", - steps: [...imagePlatforms.values()].map(platform => getBuildImageStep(platform)), + key: "build-images", + group: getBuildkiteEmoji("aws"), + steps: [...imagePlatforms.values()].map(platform => getBuildImageStep(platform, !publishImages)), }); } - for (const platform of buildPlatforms) { - const { os, arch, abi, baseline } = platform; + const { skipBuilds, forceBuilds, unifiedBuilds } = options; - /** @type {Step[]} */ - const platformSteps = []; - - if (buildImages || !buildId) { - platformSteps.push( - getBuildVendorStep(platform), - getBuildCppStep(platform), - getBuildZigStep(platform), - getBuildBunStep(platform), - ); + /** @type {string | undefined} */ + let buildId; + if (skipBuilds && !forceBuilds) { + const lastBuild = await getLastSuccessfulBuild(); + if (lastBuild) { + const { id } = lastBuild; + buildId = id; + } else { + console.warn("No last successful build found, must force builds..."); } + } - if (!skipTests) { - platformSteps.push( - ...testPlatforms - .filter( - testPlatform => - testPlatform.os === os && - testPlatform.arch === arch && - testPlatform.abi === abi && - testPlatform.baseline === baseline, - ) - .map(testPlatform => getTestBunStep(testPlatform)), - ); - } + if (!buildId) { + steps.push( + ...buildPlatforms + .flatMap(platform => buildProfiles.map(profile => ({ ...platform, profile }))) + .map(target => { + const imageKey = getImageKey(target); + const imagePlatform = imagePlatforms.get(imageKey); - if (!platformSteps.length) { + return getStepWithDependsOn( + { + key: getTargetKey(target), + group: getTargetLabel(target), + steps: unifiedBuilds + ? [getBuildBunStep(target)] + : [ + getBuildVendorStep(target), + getBuildCppStep(target), + getBuildZigStep(target), + getLinkBunStep(target), + ], + }, + imagePlatform ? `${imageKey}-build-image` : undefined, + ); + }), + ); + } + + const { skipTests, forceTests, unifiedTests, testFiles } = options; + if (!skipTests || forceTests) { + steps.push( + ...testPlatforms + .flatMap(platform => buildProfiles.map(profile => ({ ...platform, profile }))) + .map(target => ({ + key: getTargetKey(target), + group: getTargetLabel(target), + steps: [getTestBunStep(target, { unifiedTests, testFiles, buildId })], + })), + ); + } + + if (isMainBranch()) { + steps.push(getReleaseStep(buildPlatforms)); + } + + /** @type {Map} */ + const stepsByGroup = new Map(); + + for (let i = 0; i < steps.length; i++) { + const step = steps[i]; + if (!("group" in step)) { continue; } - steps.push({ - key: getTargetKey(platform), - group: getTargetLabel(platform), - steps: platformSteps, - }); - } + const { group, steps: groupSteps } = step; + if (stepsByGroup.has(group)) { + stepsByGroup.get(group).steps.push(...groupSteps); + } else { + stepsByGroup.set(group, step); + } - if (isMainBranch() && !isFork()) { - steps.push({ - label: ":github:", - agents: { - queue: "test-darwin", - }, - depends_on: buildPlatforms.map(platform => `${getTargetKey(platform)}-build-bun`), - command: ".buildkite/scripts/upload-release.sh", - }); + steps[i] = undefined; } return { - priority: getPriority(), - steps, + priority, + steps: [...steps.filter(step => typeof step !== "undefined"), ...Array.from(stepsByGroup.values())], }; } async function main() { - printEnvironment(); - - console.log("Checking last successful build..."); - const lastBuild = await getLastSuccessfulBuild(); - if (lastBuild) { - const { id, path, commit_id: commit } = lastBuild; - console.log(" - Build ID:", id); - console.log(" - Build URL:", new URL(path, "https://buildkite.com/").toString()); - console.log(" - Commit:", commit); - } else { - console.log(" - No build found"); + startGroup("Generating options..."); + const options = await getPipelineOptions(); + if (options) { + console.log("Generated options:", options); } - let changedFiles; - let changedFilesBranch; - if (!isFork() && !isMainBranch()) { - console.log("Checking changed files..."); - const targetRef = getTargetBranch(); - console.log(" - Target Ref:", targetRef); - const baseRef = lastBuild?.commit_id || targetRef || getMainBranch(); - console.log(" - Base Ref:", baseRef); - const headRef = getCommit(); - console.log(" - Head Ref:", headRef); - - changedFiles = await getChangedFiles(undefined, baseRef, headRef); - changedFilesBranch = await getChangedFiles(undefined, targetRef, headRef); - if (changedFiles) { - if (changedFiles.length) { - changedFiles.forEach(filename => console.log(` - ${filename}`)); - } else { - console.log(" - No changed files"); - } - } + startGroup("Generating pipeline..."); + const pipeline = await getPipeline(options); + if (!pipeline) { + console.log("Generated pipeline is empty, skipping..."); + return; } - const isDocumentationFile = filename => /^(\.vscode|\.github|bench|docs|examples)|\.(md)$/i.test(filename); - const isTestFile = filename => /^test/i.test(filename) || /runner\.node\.mjs$/i.test(filename); - - console.log("Checking if CI should be forced..."); - let forceBuild; - let ciFileChanged; - { - const message = getCommitMessage(); - const match = /\[(force ci|ci force|ci force build)\]/i.exec(message); - if (match) { - const [, reason] = match; - console.log(" - Yes, because commit message contains:", reason); - forceBuild = true; - } - for (const coref of [".buildkite/ci.mjs", "scripts/utils.mjs", "scripts/bootstrap.sh", "scripts/machine.mjs"]) { - if (changedFilesBranch && changedFilesBranch.includes(coref)) { - console.log(" - Yes, because the list of changed files contains:", coref); - forceBuild = true; - ciFileChanged = true; - } - } - } - - console.log("Checking if CI should be skipped..."); - if (!forceBuild) { - const message = getCommitMessage(); - const match = /\[(skip ci|no ci|ci skip|ci no)\]/i.exec(message); - if (match) { - const [, reason] = match; - console.log(" - Yes, because commit message contains:", reason); - return; - } - if (changedFiles && changedFiles.every(filename => isDocumentationFile(filename))) { - console.log(" - Yes, because all changed files are documentation"); - return; - } - } - - console.log("Checking if CI should re-build images..."); - let buildImages; - { - const message = getCommitMessage(); - const match = /\[(build images?|images? build)\]/i.exec(message); - if (match) { - const [, reason] = match; - console.log(" - Yes, because commit message contains:", reason); - buildImages = true; - } - if (ciFileChanged) { - console.log(" - Yes, because a core CI file changed"); - buildImages = true; - } - } - - console.log("Checking if CI should publish images..."); - let publishImages; - { - const message = getCommitMessage(); - const match = /\[(publish images?|images? publish)\]/i.exec(message); - if (match) { - const [, reason] = match; - console.log(" - Yes, because commit message contains:", reason); - publishImages = true; - buildImages = true; - } - if (ciFileChanged && isMainBranch()) { - console.log(" - Yes, because a core CI file changed and this is main branch"); - publishImages = true; - buildImages = true; - } - } - - console.log("Checking if build should be skipped..."); - let skipBuild; - if (!forceBuild) { - const message = getCommitMessage(); - const match = /\[(only tests?|tests? only|skip build|no build|build skip|build no)\]/i.exec(message); - if (match) { - const [, reason] = match; - console.log(" - Yes, because commit message contains:", reason); - skipBuild = true; - } - if (changedFiles && changedFiles.every(filename => isTestFile(filename) || isDocumentationFile(filename))) { - console.log(" - Yes, because all changed files are tests or documentation"); - skipBuild = true; - } - } - - console.log("Checking if tests should be skipped..."); - let skipTests; - { - const message = getCommitMessage(); - const match = /\[(skip tests?|tests? skip|no tests?|tests? no)\]/i.exec(message); - if (match) { - console.log(" - Yes, because commit message contains:", match[1]); - skipTests = true; - } - if (isMainBranch()) { - console.log(" - Yes, because we're on main branch"); - skipTests = true; - } - } - - console.log("Checking if build is a named release..."); - let buildRelease; - if (/^(1|true|on|yes)$/i.test(getEnv("RELEASE", false))) { - console.log(" - Yes, because RELEASE environment variable is set"); - buildRelease = true; - } else { - const message = getCommitMessage(); - const match = /\[(release|release build|build release)\]/i.exec(message); - if (match) { - const [, reason] = match; - console.log(" - Yes, because commit message contains:", reason); - buildRelease = true; - } - } - - console.log("Generating pipeline..."); - const pipeline = getPipeline({ - buildId: lastBuild && skipBuild && !forceBuild ? lastBuild.id : undefined, - buildImages, - publishImages, - skipTests, - }); - const content = toYaml(pipeline); const contentPath = join(process.cwd(), ".buildkite", "ci.yml"); - writeFileSync(contentPath, content); + writeFile(contentPath, content); console.log("Generated pipeline:"); console.log(" - Path:", contentPath); console.log(" - Size:", (content.length / 1024).toFixed(), "KB"); - if (isBuildkite) { - await uploadArtifact(contentPath); - } if (isBuildkite) { - console.log("Setting canary revision..."); - const canaryRevision = buildRelease ? 0 : await getCanaryRevision(); - await spawnSafe(["buildkite-agent", "meta-data", "set", "canary", `${canaryRevision}`], { stdio: "inherit" }); - - console.log("Uploading pipeline..."); - await spawnSafe(["buildkite-agent", "pipeline", "upload", contentPath], { stdio: "inherit" }); + startGroup("Uploading pipeline..."); + try { + await spawnSafe(["buildkite-agent", "pipeline", "upload", contentPath], { stdio: "inherit" }); + } finally { + await uploadArtifact(contentPath); + } } } diff --git a/.buildkite/scripts/prepare-build.sh b/.buildkite/scripts/prepare-build.sh index a76370fd7c..b0b3f9f37e 100755 --- a/.buildkite/scripts/prepare-build.sh +++ b/.buildkite/scripts/prepare-build.sh @@ -8,4 +8,4 @@ function run_command() { { set +x; } 2>/dev/null } -run_command node ".buildkite/ci.mjs" +run_command node ".buildkite/ci.mjs" "$@" diff --git a/cmake/Options.cmake b/cmake/Options.cmake index d6cc8582ea..201bf8c8e1 100644 --- a/cmake/Options.cmake +++ b/cmake/Options.cmake @@ -20,7 +20,7 @@ else() setx(RELEASE OFF) endif() -if(CMAKE_BUILD_TYPE MATCHES "Debug|RelWithDebInfo") +if(CMAKE_BUILD_TYPE MATCHES "Debug") setx(DEBUG ON) else() setx(DEBUG OFF) diff --git a/cmake/targets/BuildLolHtml.cmake b/cmake/targets/BuildLolHtml.cmake index 934f8d0be9..3b0d80a723 100644 --- a/cmake/targets/BuildLolHtml.cmake +++ b/cmake/targets/BuildLolHtml.cmake @@ -49,6 +49,8 @@ register_command( CARGO_TERM_VERBOSE=true CARGO_TERM_DIAGNOSTIC=true CARGO_ENCODED_RUSTFLAGS=${RUSTFLAGS} + CARGO_HOME=${CARGO_HOME} + RUSTUP_HOME=${RUSTUP_HOME} ) target_link_libraries(${bun} PRIVATE ${LOLHTML_LIBRARY}) diff --git a/cmake/tools/SetupRust.cmake b/cmake/tools/SetupRust.cmake index a83b28bc5f..8a45d243eb 100644 --- a/cmake/tools/SetupRust.cmake +++ b/cmake/tools/SetupRust.cmake @@ -1,15 +1,42 @@ +if(DEFINED ENV{CARGO_HOME}) + set(CARGO_HOME $ENV{CARGO_HOME}) +elseif(CMAKE_HOST_WIN32) + set(CARGO_HOME $ENV{USERPROFILE}/.cargo) + if(NOT EXISTS ${CARGO_HOME}) + set(CARGO_HOME $ENV{PROGRAMFILES}/Rust/cargo) + endif() +else() + set(CARGO_HOME $ENV{HOME}/.cargo) +endif() + +if(DEFINED ENV{RUSTUP_HOME}) + set(RUSTUP_HOME $ENV{RUSTUP_HOME}) +elseif(CMAKE_HOST_WIN32) + set(RUSTUP_HOME $ENV{USERPROFILE}/.rustup) + if(NOT EXISTS ${RUSTUP_HOME}) + set(RUSTUP_HOME $ENV{PROGRAMFILES}/Rust/rustup) + endif() +else() + set(RUSTUP_HOME $ENV{HOME}/.rustup) +endif() + find_command( VARIABLE CARGO_EXECUTABLE COMMAND cargo PATHS - $ENV{HOME}/.cargo/bin + ${CARGO_HOME}/bin REQUIRED OFF ) if(EXISTS ${CARGO_EXECUTABLE}) + if(CARGO_EXECUTABLE MATCHES "^${CARGO_HOME}") + setx(CARGO_HOME ${CARGO_HOME}) + setx(RUSTUP_HOME ${RUSTUP_HOME}) + endif() + return() endif() diff --git a/scripts/agent.mjs b/scripts/agent.mjs index e40b694f6e..e94f0658d0 100755 --- a/scripts/agent.mjs +++ b/scripts/agent.mjs @@ -20,6 +20,8 @@ import { getEnv, writeFile, spawnSafe, + spawn, + mkdir, } from "./utils.mjs"; import { parseArgs } from "node:util"; @@ -49,16 +51,19 @@ async function doBuildkiteAgent(action) { const args = [realpathSync(process.argv[1]), "start"]; if (isWindows) { - const serviceCommand = [ - "New-Service", - "-Name", - "buildkite-agent", - "-StartupType", - "Automatic", - "-BinaryPathName", - `${escape(command)} ${escape(args.map(escape).join(" "))}`, + mkdir(logsPath); + + const nssm = which("nssm", { required: true }); + const nssmCommands = [ + [nssm, "install", "buildkite-agent", command, ...args], + [nssm, "set", "buildkite-agent", "Start", "SERVICE_AUTO_START"], + [nssm, "set", "buildkite-agent", "AppDirectory", homePath], + [nssm, "set", "buildkite-agent", "AppStdout", agentLogPath], + [nssm, "set", "buildkite-agent", "AppStderr", agentLogPath], ]; - await spawnSafe(["powershell", "-Command", serviceCommand.join(" ")], { stdio: "inherit" }); + for (const command of nssmCommands) { + await spawnSafe(command, { stdio: "inherit" }); + } } if (isOpenRc()) { @@ -124,13 +129,21 @@ async function doBuildkiteAgent(action) { token = await getCloudMetadataTag("buildkite:token"); } + if (!token) { + throw new Error( + "Buildkite token not found: either set BUILDKITE_AGENT_TOKEN or add a buildkite:token label to the instance", + ); + } + let shell; if (isWindows) { - const pwsh = which(["pwsh", "powershell"], { required: true }); - shell = `${pwsh} -Command`; + // Command Prompt has a faster startup time than PowerShell. + // Also, it propogates the exit code of the command, which PowerShell does not. + const cmd = which("cmd", { required: true }); + shell = `"${cmd}" /S /C`; } else { - const sh = which(["bash", "sh"], { required: true }); - shell = `${sh} -c`; + const sh = which("sh", { required: true }); + shell = `${sh} -e -c`; } const flags = ["enable-job-log-tmpfile", "no-feature-reporting"]; diff --git a/scripts/bootstrap.ps1 b/scripts/bootstrap.ps1 index eda27d917a..e9a698c941 100755 --- a/scripts/bootstrap.ps1 +++ b/scripts/bootstrap.ps1 @@ -1,6 +1,6 @@ -# Version: 4 -# A powershell script that installs the dependencies needed to build and test Bun. -# This should work on Windows 10 or newer. +# Version: 7 +# A script that installs the dependencies needed to build and test Bun. +# This should work on Windows 10 or newer with PowerShell. # If this script does not work on your machine, please open an issue: # https://github.com/oven-sh/bun/issues @@ -16,6 +16,9 @@ param ( [switch]$Optimize = $CI ) +$ErrorActionPreference = "Stop" +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force + function Execute-Command { $command = $args -join ' ' Write-Output "$ $command" @@ -43,6 +46,47 @@ function Which { } } +function Execute-Script { + param ( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Path + ) + + $pwsh = Which pwsh powershell -Required + Execute-Command $pwsh $Path +} + +function Download-File { + param ( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Url, + [Parameter(Mandatory = $false)] + [string]$Name, + [Parameter(Mandatory = $false)] + [string]$Path + ) + + if (-not $Name) { + $Name = [System.IO.Path]::ChangeExtension([System.IO.Path]::GetRandomFileName(), [System.IO.Path]::GetExtension($Url)) + } + + if (-not $Path) { + $Path = "$env:TEMP\$Name" + } + + $client = New-Object System.Net.WebClient + for ($i = 0; $i -lt 10 -and -not (Test-Path $Path); $i++) { + try { + $client.DownloadFile($Url, $Path) + } catch { + Write-Warning "Failed to download $Url, retry $i..." + Start-Sleep -s $i + } + } + + return $Path +} + function Install-Chocolatey { if (Which choco) { return @@ -50,7 +94,8 @@ function Install-Chocolatey { Write-Output "Installing Chocolatey..." [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072 - iex -Command ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + $installScript = Download-File "https://community.chocolatey.org/install.ps1" + Execute-Script $installScript Refresh-Path } @@ -96,10 +141,23 @@ function Add-To-Path { } Write-Output "Adding $absolutePath to PATH..." - [Environment]::SetEnvironmentVariable("Path", $newPath, "Machine") + [Environment]::SetEnvironmentVariable("Path", "$newPath", "Machine") Refresh-Path } +function Set-Env { + param ( + [Parameter(Mandatory = $true, Position = 0)] + [string]$Name, + [Parameter(Mandatory = $true, Position = 1)] + [string]$Value + ) + + Write-Output "Setting environment variable $Name=$Value..." + [System.Environment]::SetEnvironmentVariable("$Name", "$Value", "Machine") + [System.Environment]::SetEnvironmentVariable("$Name", "$Value", "Process") +} + function Install-Package { param ( [Parameter(Mandatory = $true, Position = 0)] @@ -137,7 +195,7 @@ function Install-Package { function Install-Packages { foreach ($package in $args) { - Install-Package -Name $package + Install-Package $package } } @@ -145,12 +203,13 @@ function Install-Common-Software { Install-Chocolatey Install-Pwsh Install-Git - Install-Packages curl 7zip + Install-Packages curl 7zip nssm Install-NodeJs Install-Bun Install-Cygwin if ($CI) { - Install-Tailscale + # FIXME: Installing tailscale causes the AWS metadata server to become unreachable + # Install-Tailscale Install-Buildkite } } @@ -204,12 +263,13 @@ function Install-Buildkite { Write-Output "Installing Buildkite agent..." $env:buildkiteAgentToken = "xxx" - iex ((New-Object System.Net.WebClient).DownloadString("https://raw.githubusercontent.com/buildkite/agent/main/install.ps1")) + $installScript = Download-File "https://raw.githubusercontent.com/buildkite/agent/main/install.ps1" + Execute-Script $installScript Refresh-Path } function Install-Build-Essentials { - # Install-Visual-Studio + Install-Visual-Studio Install-Packages ` cmake ` make ` @@ -219,41 +279,42 @@ function Install-Build-Essentials { golang ` nasm ` ruby ` + strawberryperl ` mingw Install-Rust Install-Llvm } function Install-Visual-Studio { - $components = @( - "Microsoft.VisualStudio.Workload.NativeDesktop", - "Microsoft.VisualStudio.Component.Windows10SDK.18362", - "Microsoft.VisualStudio.Component.Windows11SDK.22000", - "Microsoft.VisualStudio.Component.Windows11Sdk.WindowsPerformanceToolkit", - "Microsoft.VisualStudio.Component.VC.ASAN", # C++ AddressSanitizer - "Microsoft.VisualStudio.Component.VC.ATL", # C++ ATL for latest v143 build tools (x86 & x64) - "Microsoft.VisualStudio.Component.VC.DiagnosticTools", # C++ Diagnostic Tools - "Microsoft.VisualStudio.Component.VC.CLI.Support", # C++/CLI support for v143 build tools (Latest) - "Microsoft.VisualStudio.Component.VC.CoreIde", # C++ core features - "Microsoft.VisualStudio.Component.VC.Redist.14.Latest" # C++ 2022 Redistributable Update + param ( + [Parameter(Mandatory = $false)] + [string]$Edition = "community" ) - $arch = (Get-WmiObject Win32_Processor).Architecture - if ($arch -eq 9) { - $components += @( - "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", # MSVC v143 build tools (x86 & x64) - "Microsoft.VisualStudio.Component.VC.Modules.x86.x64" # MSVC v143 C++ Modules for latest v143 build tools (x86 & x64) - ) - } elseif ($arch -eq 5) { - $components += @( - "Microsoft.VisualStudio.Component.VC.Tools.ARM64", # MSVC v143 build tools (ARM64) - "Microsoft.VisualStudio.Component.UWP.VC.ARM64" # C++ Universal Windows Platform support for v143 build tools (ARM64/ARM64EC) - ) - } + Write-Output "Downloading Visual Studio installer..." + $vsInstaller = Download-File "https://aka.ms/vs/17/release/vs_$Edition.exe" - $packageParameters = $components | ForEach-Object { "--add $_" } - Install-Package visualstudio2022community ` - -ExtraArgs "--package-parameters '--add Microsoft.VisualStudio.Workload.NativeDesktop --includeRecommended --includeOptional'" + Write-Output "Installing Visual Studio..." + $vsInstallArgs = @( + "--passive", + "--norestart", + "--wait", + "--force", + "--locale en-US", + "--add Microsoft.VisualStudio.Workload.NativeDesktop", + "--includeRecommended" + ) + $startInfo = New-Object System.Diagnostics.ProcessStartInfo + $startInfo.FileName = $vsInstaller + $startInfo.Arguments = $vsInstallArgs -join ' ' + $startInfo.CreateNoWindow = $true + $process = New-Object System.Diagnostics.Process + $process.StartInfo = $startInfo + $process.Start() + $process.WaitForExit() + if ($process.ExitCode -ne 0) { + throw "Failed to install Visual Studio: code $($process.ExitCode)" + } } function Install-Rust { @@ -261,18 +322,31 @@ function Install-Rust { return } + Write-Output "Installing Rustup..." + $rustupInit = Download-File "https://win.rustup.rs/" -Name "rustup-init.exe" + Write-Output "Installing Rust..." - $rustupInit = "$env:TEMP\rustup-init.exe" - (New-Object System.Net.WebClient).DownloadFile("https://win.rustup.rs/", $rustupInit) Execute-Command $rustupInit -y - Add-To-Path "$env:USERPROFILE\.cargo\bin" + + Write-Output "Moving Rust to $env:ProgramFiles..." + $rustPath = Join-Path $env:ProgramFiles "Rust" + if (-not (Test-Path $rustPath)) { + New-Item -Path $rustPath -ItemType Directory + } + Move-Item "$env:UserProfile\.cargo" "$rustPath\cargo" -Force + Move-Item "$env:UserProfile\.rustup" "$rustPath\rustup" -Force + + Write-Output "Setting environment variables for Rust..." + Set-Env "CARGO_HOME" "$rustPath\cargo" + Set-Env "RUSTUP_HOME" "$rustPath\rustup" + Add-To-Path "$rustPath\cargo\bin" } function Install-Llvm { Install-Package llvm ` -Command clang-cl ` -Version "18.1.8" - Add-To-Path "C:\Program Files\LLVM\bin" + Add-To-Path "$env:ProgramFiles\LLVM\bin" } function Optimize-System { @@ -280,6 +354,9 @@ function Optimize-System { Disable-Windows-Threat-Protection Disable-Windows-Services Disable-Power-Management +} + +function Optimize-System-Needs-Reboot { Uninstall-Windows-Defender } @@ -319,7 +396,7 @@ function Disable-Windows-Services { } function Disable-Power-Management { - Write-Output "Disabling power management features..." + Write-Output "Disabling Power Management..." powercfg /setactive 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c # High performance powercfg /change monitor-timeout-ac 0 powercfg /change monitor-timeout-dc 0 @@ -329,7 +406,6 @@ function Disable-Power-Management { powercfg /change hibernate-timeout-dc 0 } -Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force if ($Optimize) { Optimize-System } @@ -337,3 +413,6 @@ if ($Optimize) { Install-Common-Software Install-Build-Essentials +if ($Optimize) { + Optimize-System-Needs-Reboot +} diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 871340f1fc..18defcfe88 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Version: 5 +# Version: 7 # A script that installs the dependencies needed to build and test Bun. # This should work on macOS and Linux with a POSIX shell. @@ -92,7 +92,7 @@ download_file() { execute chmod 755 "$tmp" path="$tmp/$filename" - fetch "$url" > "$path" + fetch "$url" >"$path" execute chmod 644 "$path" print "$path" @@ -112,14 +112,23 @@ append_to_file() { file="$1" content="$2" - if ! [ -f "$file" ]; then + file_needs_sudo="0" + if [ -f "$file" ]; then + if ! [ -r "$file" ] || ! [ -w "$file" ]; then + file_needs_sudo="1" + fi + else execute_as_user mkdir -p "$(dirname "$file")" execute_as_user touch "$file" fi echo "$content" | while read -r line; do if ! grep -q "$line" "$file"; then - echo "$line" >>"$file" + if [ "$file_needs_sudo" = "1" ]; then + execute_sudo sh -c "echo '$line' >> '$file'" + else + echo "$line" >>"$file" + fi fi done } @@ -135,7 +144,7 @@ append_to_file_sudo() { echo "$content" | while read -r line; do if ! grep -q "$line" "$file"; then - echo "$line" | execute_sudo tee "$file" > /dev/null + echo "$line" | execute_sudo tee "$file" >/dev/null fi done } @@ -161,18 +170,21 @@ append_to_path() { export PATH="$path:$PATH" } -link_to_bin() { - path="$1" - if ! [ -d "$path" ]; then - error "Could not find directory: \"$path\"" +move_to_bin() { + exe_path="$1" + if ! [ -f "$exe_path" ]; then + error "Could not find executable: \"$exe_path\"" fi - for file in "$path"/*; do - if [ -f "$file" ]; then - grant_to_user "$file" - execute_sudo ln -sf "$file" "/usr/bin/$(basename "$file")" + usr_paths="/usr/bin /usr/local/bin" + for usr_path in $usr_paths; do + if [ -d "$usr_path" ] && [ -w "$usr_path" ]; then + break fi done + + grant_to_user "$exe_path" + execute_sudo mv -f "$exe_path" "$usr_path/$(basename "$exe_path")" } check_features() { @@ -384,6 +396,74 @@ check_user() { fi } +check_ulimit() { + if ! [ "$ci" = "1" ]; then + return + fi + + print "Checking ulimits..." + systemd_conf="/etc/systemd/system.conf" + if [ -f "$systemd_conf" ]; then + limits_conf="/etc/security/limits.d/99-unlimited.conf" + if ! [ -f "$limits_conf" ]; then + execute_sudo mkdir -p "$(dirname "$limits_conf")" + execute_sudo touch "$limits_conf" + fi + fi + + limits="core data fsize memlock nofile rss stack cpu nproc as locks sigpending msgqueue" + for limit in $limits; do + limit_upper="$(echo "$limit" | tr '[:lower:]' '[:upper:]')" + + limit_value="unlimited" + case "$limit" in + nofile | nproc) + limit_value="1048576" + ;; + esac + + if [ -f "$limits_conf" ]; then + limit_users="root *" + for limit_user in $limit_users; do + append_to_file "$limits_conf" "$limit_user soft $limit $limit_value" + append_to_file "$limits_conf" "$limit_user hard $limit $limit_value" + done + fi + + if [ -f "$systemd_conf" ]; then + append_to_file "$systemd_conf" "DefaultLimit$limit_upper=$limit_value" + fi + done + + rc_conf="/etc/rc.conf" + if [ -f "$rc_conf" ]; then + rc_ulimit="" + limit_flags="c d e f i l m n q r s t u v x" + for limit_flag in $limit_flags; do + limit_value="unlimited" + case "$limit_flag" in + n | u) + limit_value="1048576" + ;; + esac + rc_ulimit="$rc_ulimit -$limit_flag $limit_value" + done + append_to_file "$rc_conf" "rc_ulimit=\"$rc_ulimit\"" + fi + + pam_confs="/etc/pam.d/common-session /etc/pam.d/common-session-noninteractive" + for pam_conf in $pam_confs; do + if [ -f "$pam_conf" ]; then + append_to_file "$pam_conf" "session optional pam_limits.so" + fi + done + + systemctl="$(which systemctl)" + if [ -f "$systemctl" ]; then + execute_sudo "$systemctl" daemon-reload + fi +} + package_manager() { case "$pm" in apt) @@ -602,6 +682,14 @@ install_nodejs_headers() { } install_bun() { + case "$pm" in + apk) + install_packages \ + libgcc \ + libstdc++ + ;; + esac + bash="$(require bash)" script=$(download_file "https://bun.sh/install") @@ -615,7 +703,10 @@ install_bun() { ;; esac - link_to_bin "$home/.bun/bin" + move_to_bin "$home/.bun/bin/bun" + bun_path="$(which bun)" + bunx_path="$(dirname "$bun_path")/bunx" + execute_sudo ln -sf "$bun_path" "$bunx_path" } install_cmake() { @@ -628,14 +719,14 @@ install_cmake() { cmake_version="3.30.5" case "$arch" in x64) - url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-x86_64.sh" + cmake_url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-x86_64.sh" ;; aarch64) - url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-aarch64.sh" + cmake_url="https://github.com/Kitware/CMake/releases/download/v$cmake_version/cmake-$cmake_version-linux-aarch64.sh" ;; esac - script=$(download_file "$url") - execute_sudo "$sh" "$script" \ + cmake_script=$(download_file "$cmake_url") + execute_sudo "$sh" "$cmake_script" \ --skip-license \ --prefix=/usr ;; @@ -732,13 +823,13 @@ install_llvm() { case "$pm" in apt) bash="$(require bash)" - script="$(download_file "https://apt.llvm.org/llvm.sh")" + llvm_script="$(download_file "https://apt.llvm.org/llvm.sh")" case "$distro-$release" in ubuntu-24*) - execute_sudo "$bash" "$script" "$(llvm_version)" all -njammy + execute_sudo "$bash" "$llvm_script" "$(llvm_version)" all -njammy ;; *) - execute_sudo "$bash" "$script" "$(llvm_version)" all + execute_sudo "$bash" "$llvm_script" "$(llvm_version)" all ;; esac ;; @@ -779,11 +870,6 @@ install_rust() { execute_as_user "$sh" "$script" -y ;; esac - - # FIXME: This causes cargo to fail to build: - # > error: rustup could not choose a version of cargo to run, - # > because one wasn't specified explicitly, and no default is configured. - # link_to_bin "$home/.cargo/bin" } install_docker() { @@ -796,7 +882,7 @@ install_docker() { *) case "$distro-$release" in amzn-2 | amzn-1) - execute amazon-linux-extras install docker + execute_sudo amazon-linux-extras install docker ;; amzn-* | alpine-*) install_packages docker @@ -832,8 +918,8 @@ install_tailscale() { case "$os" in linux) sh="$(require sh)" - script=$(download_file "https://tailscale.com/install.sh") - execute "$sh" "$script" + tailscale_script=$(download_file "https://tailscale.com/install.sh") + execute "$sh" "$tailscale_script" ;; darwin) install_packages go @@ -862,24 +948,39 @@ create_buildkite_user() { esac if [ -z "$(getent passwd "$user")" ]; then - execute_sudo useradd "$user" \ - --system \ - --no-create-home \ - --home-dir "$home" + case "$distro" in + alpine) + execute_sudo addgroup \ + --system "$group" + execute_sudo adduser "$user" \ + --system \ + --ingroup "$group" \ + --shell "$(require sh)" \ + --home "$home" \ + --disabled-password + ;; + *) + execute_sudo useradd "$user" \ + --system \ + --shell "$(require sh)" \ + --no-create-home \ + --home-dir "$home" + ;; + esac fi if [ -n "$(getent group docker)" ]; then execute_sudo usermod -aG docker "$user" fi - paths="$home /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /var/run/buildkite-agent/buildkite-agent.sock" - for path in $paths; do + buildkite_paths="$home /var/cache/buildkite-agent /var/log/buildkite-agent /var/run/buildkite-agent /var/run/buildkite-agent/buildkite-agent.sock" + for path in $buildkite_paths; do execute_sudo mkdir -p "$path" execute_sudo chown -R "$user:$group" "$path" done - files="/var/run/buildkite-agent/buildkite-agent.pid" - for file in $files; do + buildkite_files="/var/run/buildkite-agent/buildkite-agent.pid" + for file in $buildkite_files; do execute_sudo touch "$file" execute_sudo chown "$user:$group" "$file" done @@ -890,19 +991,42 @@ install_buildkite() { return fi - bash="$(require bash)" - script="$(download_file "https://raw.githubusercontent.com/buildkite/agent/main/install.sh")" - tmp_dir="$(execute dirname "$script")" - HOME="$tmp_dir" execute "$bash" "$script" + buildkite_version="3.87.0" + case "$os-$arch" in + linux-aarch64) + buildkite_filename="buildkite-agent-linux-arm64-$buildkite_version.tar.gz" + ;; + linux-x64) + buildkite_filename="buildkite-agent-linux-amd64-$buildkite_version.tar.gz" + ;; + darwin-aarch64) + buildkite_filename="buildkite-agent-darwin-arm64-$buildkite_version.tar.gz" + ;; + darwin-x64) + buildkite_filename="buildkite-agent-darwin-amd64-$buildkite_version.tar.gz" + ;; + esac + buildkite_url="https://github.com/buildkite/agent/releases/download/v$buildkite_version/$buildkite_filename" + buildkite_filepath="$(download_file "$buildkite_url" "$buildkite_filename")" + buildkite_tmpdir="$(dirname "$buildkite_filepath")" - out_dir="$tmp_dir/.buildkite-agent" - execute_sudo mv -f "$out_dir/bin/buildkite-agent" "/usr/bin/buildkite-agent" + execute tar -xzf "$buildkite_filepath" -C "$buildkite_tmpdir" + move_to_bin "$buildkite_tmpdir/buildkite-agent" + execute rm -rf "$buildkite_tmpdir" } -install_chrome_dependencies() { +install_chromium() { # https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#chrome-doesnt-launch-on-linux # https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-in-the-cloud case "$pm" in + apk) + install_packages \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ttf-freefont + ;; apt) install_packages \ fonts-liberation \ @@ -979,22 +1103,17 @@ install_chrome_dependencies() { esac } -raise_file_descriptor_limit() { - append_to_file_sudo /etc/security/limits.conf '* soft nofile 262144' - append_to_file_sudo /etc/security/limits.conf '* hard nofile 262144' -} - main() { check_features "$@" check_operating_system check_inside_docker check_user + check_ulimit check_package_manager create_buildkite_user install_common_software install_build_essentials - install_chrome_dependencies - raise_file_descriptor_limit # XXX: temporary + install_chromium } main "$@" diff --git a/scripts/build.mjs b/scripts/build.mjs index a35c21eac3..2fab14a959 100755 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -3,6 +3,7 @@ import { spawn as nodeSpawn } from "node:child_process"; import { existsSync, readFileSync, mkdirSync, cpSync, chmodSync } from "node:fs"; import { basename, join, resolve } from "node:path"; +import { isCI, printEnvironment, startGroup } from "./utils.mjs"; // https://cmake.org/cmake/help/latest/manual/cmake.1.html#generate-a-project-buildsystem const generateFlags = [ @@ -37,6 +38,10 @@ async function build(args) { return spawn("pwsh", ["-NoProfile", "-NoLogo", "-File", shellPath, process.argv0, scriptPath, ...args]); } + if (isCI) { + printEnvironment(); + } + const env = { ...process.env, FORCE_COLOR: "1", @@ -102,7 +107,8 @@ async function build(args) { const generateArgs = Object.entries(generateOptions).flatMap(([flag, value]) => flag.startsWith("-D") ? [`${flag}=${value}`] : [flag, value], ); - await spawn("cmake", generateArgs, { env }, "configuration"); + + await startGroup("CMake Configure", () => spawn("cmake", generateArgs, { env })); const envPath = resolve(buildPath, ".env"); if (existsSync(envPath)) { @@ -116,7 +122,8 @@ async function build(args) { const buildArgs = Object.entries(buildOptions) .sort(([a], [b]) => (a === "--build" ? -1 : a.localeCompare(b))) .flatMap(([flag, value]) => [flag, value]); - await spawn("cmake", buildArgs, { env }, "compilation"); + + await startGroup("CMake Build", () => spawn("cmake", buildArgs, { env })); printDuration("total", Date.now() - startTime); } diff --git a/scripts/machine.mjs b/scripts/machine.mjs index 3ddfd6ac3a..479dbb4cfd 100755 --- a/scripts/machine.mjs +++ b/scripts/machine.mjs @@ -18,45 +18,740 @@ import { waitForPort, which, escapePowershell, + getGithubUrl, + getGithubApiUrl, + curlSafe, + mkdtemp, + writeFile, + copyFile, + isMacOS, + mkdir, + rm, + homedir, + isWindows, + sha256, + isPrivileged, } from "./utils.mjs"; -import { join, relative, resolve } from "node:path"; -import { homedir } from "node:os"; -import { existsSync, mkdirSync, mkdtempSync, readdirSync } from "node:fs"; +import { basename, extname, join, relative, resolve } from "node:path"; +import { existsSync, mkdtempSync, readdirSync } from "node:fs"; import { fileURLToPath } from "node:url"; +/** + * @link https://tart.run/ + * @link https://github.com/cirruslabs/tart + */ +const tart = { + get name() { + return "tart"; + }, + + /** + * @param {string[]} args + * @param {import("./utils.mjs").SpawnOptions} options + * @returns {Promise} + */ + async spawn(args, options) { + const tart = which("tart", { required: true }); + const { json } = options || {}; + const command = json ? [tart, ...args, "--format=json"] : [tart, ...args]; + + const { stdout } = await spawnSafe(command, options); + if (!json) { + return stdout; + } + + try { + return JSON.parse(stdout); + } catch { + return; + } + }, + + /** + * @typedef {"sequoia" | "sonoma" | "ventura" | "monterey"} TartDistro + * @typedef {`ghcr.io/cirruslabs/macos-${TartDistro}-xcode`} TartImage + * @link https://github.com/orgs/cirruslabs/packages?repo_name=macos-image-templates + */ + + /** + * @param {Platform} platform + * @returns {TartImage} + */ + getImage(platform) { + const { os, arch, release } = platform; + if (os !== "darwin" || arch !== "aarch64") { + throw new Error(`Unsupported platform: ${inspect(platform)}`); + } + const distros = { + "15": "sequoia", + "14": "sonoma", + "13": "ventura", + "12": "monterey", + }; + const distro = distros[release]; + if (!distro) { + throw new Error(`Unsupported macOS release: ${distro}`); + } + return `ghcr.io/cirruslabs/macos-${distro}-xcode`; + }, + + /** + * @typedef {Object} TartVm + * @property {string} Name + * @property {"running" | "stopped"} State + * @property {"local"} Source + * @property {number} Size + * @property {number} Disk + * @property {number} [CPU] + * @property {number} [Memory] + */ + + /** + * @returns {Promise} + */ + async listVms() { + return this.spawn(["list"], { json: true }); + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async getVm(name) { + const result = await this.spawn(["get", name], { + json: true, + throwOnError: error => !/does not exist/i.test(inspect(error)), + }); + return { + Name: name, + ...result, + }; + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async stopVm(name) { + await this.spawn(["stop", name, "--timeout=0"], { + throwOnError: error => !/does not exist|is not running/i.test(inspect(error)), + }); + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async deleteVm(name) { + await this.stopVm(name); + await this.spawn(["delete", name], { + throwOnError: error => !/does not exist/i.test(inspect(error)), + }); + }, + + /** + * @param {string} name + * @param {TartImage} image + * @returns {Promise} + */ + async cloneVm(name, image) { + const localName = image.split("/").pop(); + const localVm = await this.getVm(localName); + if (localVm) { + const { Name } = localVm; + await this.spawn(["clone", Name, name]); + return; + } + + console.log(`Cloning macOS image: ${image} (this will take a long time)`); + await this.spawn(["clone", image, localName]); + await this.spawn(["clone", localName, name]); + }, + + /** + * @typedef {Object} TartMount + * @property {boolean} [readOnly] + * @property {string} source + * @property {string} destination + */ + + /** + * @typedef {Object} TartVmOptions + * @property {number} [cpuCount] + * @property {number} [memoryGb] + * @property {number} [diskSizeGb] + * @property {boolean} [no-graphics] + * @property {boolean} [no-audio] + * @property {boolean} [no-clipboard] + * @property {boolean} [recovery] + * @property {boolean} [vnc] + * @property {boolean} [vnc-experimental] + * @property {boolean} [net-softnet] + * @property {TartMount[]} [dir] + */ + + /** + * @param {string} name + * @param {TartVmOptions} options + * @returns {Promise} + */ + async runVm(name, options = {}) { + const { cpuCount, memoryGb, diskSizeGb, dir, ...vmOptions } = options; + + const setArgs = ["--random-mac", "--random-serial"]; + if (cpuCount) { + setArgs.push(`--cpu=${cpuCount}`); + } + if (memoryGb) { + setArgs.push(`--memory=${memoryGb}`); + } + if (diskSizeGb) { + setArgs.push(`--disk-size=${diskSizeGb}`); + } + await this.spawn(["set", name, ...setArgs]); + + const args = Object.entries(vmOptions) + .filter(([, value]) => value !== undefined) + .flatMap(([key, value]) => (typeof value === "boolean" ? (value ? [`--${key}`] : []) : [`--${key}=${value}`])); + if (dir?.length) { + args.push( + ...dir.map(({ source, destination, readOnly }) => `--dir=${source}:${destination}${readOnly ? ":ro" : ""}`), + ); + } + + // This command is blocking, so it needs to be detached and not awaited + this.spawn(["run", name, ...args], { detached: true }); + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async getVmIp(name) { + const stdout = await this.spawn(["ip", name], { + retryOnError: error => /no IP address found/i.test(inspect(error)), + throwOnError: error => !/does not exist/i.test(inspect(error)), + }); + return stdout?.trim(); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { name, imageName, cpuCount, memoryGb, diskSizeGb, rdp } = options; + + const image = imageName || this.getImage(options); + const machineId = name || `i-${Math.random().toString(36).slice(2, 11)}`; + await this.cloneVm(machineId, image); + + await this.runVm(machineId, { + cpuCount, + memoryGb, + diskSizeGb, + "net-softnet": isPrivileged(), + "no-audio": true, + "no-clipboard": true, + "no-graphics": true, + "vnc-experimental": rdp, + }); + + return this.toMachine(machineId); + }, + + /** + * @param {string} name + * @returns {Machine} + */ + toMachine(name) { + const connect = async () => { + const hostname = await this.getVmIp(name); + return { + hostname, + // hardcoded by base images + username: "admin", + password: "admin", + }; + }; + + const exec = async (command, options) => { + const connectOptions = await connect(); + return spawnSsh({ ...connectOptions, command }, options); + }; + + const execSafe = async (command, options) => { + const connectOptions = await connect(); + return spawnSshSafe({ ...connectOptions, command }, options); + }; + + const attach = async () => { + const connectOptions = await connect(); + await spawnSshSafe({ ...connectOptions }); + }; + + const upload = async (source, destination) => { + const connectOptions = await connect(); + await spawnScp({ ...connectOptions, source, destination }); + }; + + const rdp = async () => { + const connectOptions = await connect(); + await spawnRdp({ ...connectOptions }); + }; + + const close = async () => { + await this.deleteVm(name); + }; + + return { + cloud: "tart", + id: name, + spawn: exec, + spawnSafe: execSafe, + attach, + upload, + close, + [Symbol.asyncDispose]: close, + }; + }, +}; + +/** + * @link https://docs.orbstack.dev/ + */ +const orbstack = { + get name() { + return "orbstack"; + }, + + /** + * @typedef {Object} OrbstackImage + * @property {string} distro + * @property {string} version + * @property {string} arch + */ + + /** + * @param {Platform} platform + * @returns {OrbstackImage} + */ + getImage(platform) { + const { os, arch, distro, release } = platform; + if (os !== "linux" || !/^debian|ubuntu|alpine|fedora|centos$/.test(distro)) { + throw new Error(`Unsupported platform: ${inspect(platform)}`); + } + + return { + distro, + version: release, + arch: arch === "aarch64" ? "arm64" : "amd64", + }; + }, + + /** + * @typedef {Object} OrbstackVm + * @property {string} id + * @property {string} name + * @property {"running"} state + * @property {OrbstackImage} image + * @property {OrbstackConfig} config + */ + + /** + * @typedef {Object} OrbstackConfig + * @property {string} default_username + * @property {boolean} isolated + */ + + /** + * @typedef {Object} OrbstackVmOptions + * @property {string} [name] + * @property {OrbstackImage} image + * @property {string} [username] + * @property {string} [password] + * @property {string} [userData] + */ + + /** + * @param {OrbstackVmOptions} options + * @returns {Promise} + */ + async createVm(options) { + const { name, image, username, password, userData } = options; + const { distro, version, arch } = image; + const uniqueId = name || `linux-${distro}-${version}-${arch}-${Math.random().toString(36).slice(2, 11)}`; + + const args = [`--arch=${arch}`, `${distro}:${version}`, uniqueId]; + if (username) { + args.push(`--user=${username}`); + } + if (password) { + args.push(`--set-password=${password}`); + } + + let userDataPath; + if (userData) { + userDataPath = mkdtemp("orbstack-user-data-", "user-data.txt"); + writeFile(userDataPath, userData); + args.push(`--user-data=${userDataPath}`); + } + + try { + await spawnSafe($`orbctl create ${args}`); + } finally { + if (userDataPath) { + rm(userDataPath); + } + } + + return this.inspectVm(uniqueId); + }, + + /** + * @param {string} name + */ + async deleteVm(name) { + await spawnSafe($`orbctl delete ${name}`, { + throwOnError: error => !/machine not found/i.test(inspect(error)), + }); + }, + + /** + * @param {string} name + * @returns {Promise} + */ + async inspectVm(name) { + const { exitCode, stdout } = await spawnSafe($`orbctl info ${name} --format=json`, { + throwOnError: error => !/machine not found/i.test(inspect(error)), + }); + if (exitCode === 0) { + return JSON.parse(stdout); + } + }, + + /** + * @returns {Promise} + */ + async listVms() { + const { stdout } = await spawnSafe($`orbctl list --format=json`); + return JSON.parse(stdout); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { distro } = options; + const username = getUsername(distro); + const userData = getUserData({ ...options, username }); + + const image = this.getImage(options); + const vm = await this.createVm({ + image, + username, + userData, + }); + + return this.toMachine(vm, options); + }, + + /** + * @param {OrbstackVm} vm + * @returns {Machine} + */ + toMachine(vm) { + const { id, name, config } = vm; + + const { default_username: username } = config; + const connectOptions = { + username, + hostname: `${name}@orb`, + }; + + const exec = async (command, options) => { + return spawnSsh({ ...connectOptions, command }, options); + }; + + const execSafe = async (command, options) => { + return spawnSshSafe({ ...connectOptions, command }, options); + }; + + const attach = async () => { + await spawnSshSafe({ ...connectOptions }); + }; + + const upload = async (source, destination) => { + await spawnSafe(["orbctl", "push", `--machine=${name}`, source, destination]); + }; + + const close = async () => { + await this.deleteVm(name); + }; + + return { + cloud: "orbstack", + id, + name, + spawn: exec, + spawnSafe: execSafe, + upload, + attach, + close, + [Symbol.asyncDispose]: close, + }; + }, +}; + const docker = { + get name() { + return "docker"; + }, + + /** + * @typedef {"linux" | "darwin" | "windows"} DockerOs + * @typedef {"amd64" | "arm64"} DockerArch + * @typedef {`${DockerOs}/${DockerArch}`} DockerPlatform + */ + + /** + * @param {Platform} platform + * @returns {DockerPlatform} + */ getPlatform(platform) { const { os, arch } = platform; + if (arch === "aarch64") { + return `${os}/arm64`; + } else if (arch === "x64") { + return `${os}/amd64`; + } + throw new Error(`Unsupported platform: ${inspect(platform)}`); + }, - if (os === "linux" || os === "windows") { - if (arch === "aarch64") { - return `${os}/arm64`; - } else if (arch === "x64") { - return `${os}/amd64`; + /** + * @typedef DockerSpawnOptions + * @property {DockerPlatform} [platform] + * @property {boolean} [json] + */ + + /** + * @param {string[]} args + * @param {DockerSpawnOptions & import("./utils.mjs").SpawnOptions} [options] + * @returns {Promise} + */ + async spawn(args, options = {}) { + const docker = which("docker", { required: true }); + + let env = { ...process.env }; + if (isCI) { + env["BUILDKIT_PROGRESS"] = "plain"; + } + + const { json, platform } = options; + if (json) { + args.push("--format=json"); + } + if (platform) { + args.push(`--platform=${platform}`); + } + + const { error, stdout } = await spawnSafe($`${docker} ${args}`, { env, ...options }); + if (error) { + return; + } + if (!json) { + return stdout; + } + + try { + return JSON.parse(stdout); + } catch { + return; + } + }, + + /** + * @typedef {Object} DockerImage + * @property {string} Id + * @property {string[]} RepoTags + * @property {string[]} RepoDigests + * @property {string} Created + * @property {DockerOs} Os + * @property {DockerArch} Architecture + * @property {number} Size + */ + + /** + * @param {string} url + * @param {DockerPlatform} [platform] + * @returns {Promise} + */ + async pullImage(url, platform) { + const done = await this.spawn($`pull ${url}`, { + platform, + throwOnError: error => !/No such image|manifest unknown/i.test(inspect(error)), + }); + return !!done; + }, + + /** + * @param {string} url + * @param {DockerPlatform} [platform] + * @returns {Promise} + */ + async inspectImage(url, platform) { + /** @type {DockerImage[]} */ + const images = await this.spawn($`image inspect ${url}`, { + json: true, + throwOnError: error => !/No such image/i.test(inspect(error)), + }); + + if (!images) { + const pulled = await this.pullImage(url, platform); + if (pulled) { + return this.inspectImage(url, platform); + } + } + + const { os, arch } = platform || {}; + return images + ?.filter(({ Os, Architecture }) => !os || !arch || (Os === os && Architecture === arch)) + ?.find((a, b) => (a.Created < b.Created ? 1 : -1)); + }, + + /** + * @typedef {Object} DockerContainer + * @property {string} Id + * @property {string} Name + * @property {string} Image + * @property {string} Created + * @property {DockerContainerState} State + * @property {DockerContainerNetworkSettings} NetworkSettings + */ + + /** + * @typedef {Object} DockerContainerState + * @property {"exited" | "running"} Status + * @property {number} [Pid] + * @property {number} ExitCode + * @property {string} [Error] + * @property {string} StartedAt + * @property {string} FinishedAt + */ + + /** + * @typedef {Object} DockerContainerNetworkSettings + * @property {string} [IPAddress] + */ + + /** + * @param {string} containerId + * @returns {Promise} + */ + async inspectContainer(containerId) { + const containers = await this.spawn($`container inspect ${containerId}`, { json: true }); + return containers?.find(a => a.Id === containerId); + }, + + /** + * @returns {Promise} + */ + async listContainers() { + const containers = await this.spawn($`container ls --all`, { json: true }); + return containers || []; + }, + + /** + * @typedef {Object} DockerRunOptions + * @property {string[]} [command] + * @property {DockerPlatform} [platform] + * @property {string} [name] + * @property {boolean} [detach] + * @property {"always" | "never"} [pull] + * @property {boolean} [rm] + * @property {"no" | "on-failure" | "always"} [restart] + */ + + /** + * @param {string} url + * @param {DockerRunOptions} [options] + * @returns {Promise} + */ + async runContainer(url, options = {}) { + const { detach, command = [], ...containerOptions } = options; + const args = Object.entries(containerOptions) + .filter(([_, value]) => typeof value !== "undefined") + .map(([key, value]) => (typeof value === "boolean" ? `--${key}` : `--${key}=${value}`)); + if (detach) { + args.push("--detach"); + } else { + args.push("--tty", "--interactive"); + } + + const stdio = detach ? "pipe" : "inherit"; + const result = await this.spawn($`run ${args} ${url} ${command}`, { stdio }); + if (!detach) { + return; + } + + const containerId = result.trim(); + const container = await this.inspectContainer(containerId); + if (!container) { + throw new Error(`Failed to run container: ${inspect(result)}`); + } + return container; + }, + + /** + * @param {Platform} platform + * @returns {Promise} + */ + async getBaseImage(platform) { + const { os, distro, release } = platform; + const dockerPlatform = this.getPlatform(platform); + + let url; + if (os === "linux") { + if (distro === "debian" || distro === "ubuntu" || distro === "alpine") { + url = `docker.io/library/${distro}:${release}`; + } else if (distro === "amazonlinux") { + url = `public.ecr.aws/amazonlinux/amazonlinux:${release}`; + } + } + + if (url) { + const image = await this.inspectImage(url, dockerPlatform); + if (image) { + return image; } } throw new Error(`Unsupported platform: ${inspect(platform)}`); }, - async createMachine(platform) { - const { id } = await docker.getImage(platform); - const platformString = docker.getPlatform(platform); + /** + * @param {DockerContainer} container + * @param {MachineOptions} [options] + * @returns {Machine} + */ + toMachine(container, options = {}) { + const { Id: containerId } = container; - const command = ["sleep", "1d"]; - const { stdout } = await spawnSafe(["docker", "run", "--rm", "--platform", platformString, "-d", id, ...command]); - const containerId = stdout.trim(); - - const spawn = async command => { - return spawn(["docker", "exec", containerId, ...command]); + const exec = (command, options) => { + return spawn(["docker", "exec", containerId, ...command], options); }; - const spawnSafe = async command => { - return spawnSafe(["docker", "exec", containerId, ...command]); + const execSafe = (command, options) => { + return spawnSafe(["docker", "exec", containerId, ...command], options); + }; + + const upload = async (source, destination) => { + await spawn(["docker", "cp", source, `${containerId}:${destination}`]); }; const attach = async () => { - const { exitCode, spawnError } = await spawn(["docker", "exec", "-it", containerId, "bash"], { + const { exitCode, error } = await spawn(["docker", "exec", "-it", containerId, "sh"], { stdio: "inherit", }); @@ -64,69 +759,60 @@ const docker = { return; } - throw spawnError; + throw error; + }; + + const snapshot = async name => { + await spawn(["docker", "commit", containerId]); }; const kill = async () => { - await spawnSafe(["docker", "kill", containerId]); + await spawn(["docker", "kill", containerId]); }; return { - spawn, - spawnSafe, + cloud: "docker", + id: containerId, + spawn: exec, + spawnSafe: execSafe, + upload, attach, close: kill, [Symbol.asyncDispose]: kill, }; }, - async getImage(platform) { - const os = platform["os"]; - const distro = platform["distro"]; - const release = platform["release"] || "latest"; + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { Id: imageId, Os, Architecture } = await docker.getBaseImage(options); - let url; - if (os === "linux") { - if (distro === "debian") { - url = `docker.io/library/debian:${release}`; - } else if (distro === "ubuntu") { - url = `docker.io/library/ubuntu:${release}`; - } else if (distro === "amazonlinux") { - url = `public.ecr.aws/amazonlinux/amazonlinux:${release}`; - } else if (distro === "alpine") { - url = `docker.io/library/alpine:${release}`; - } - } + const container = await docker.runContainer(imageId, { + platform: `${Os}/${Architecture}`, + command: ["sleep", "1d"], + detach: true, + rm: true, + restart: "no", + }); - if (url) { - await spawnSafe(["docker", "pull", "--platform", docker.getPlatform(platform), url]); - const { stdout } = await spawnSafe(["docker", "image", "inspect", url, "--format", "json"]); - const [{ Id }] = JSON.parse(stdout); - return { - id: Id, - name: url, - username: "root", - }; - } - - throw new Error(`Unsupported platform: ${inspect(platform)}`); + return this.toMachine(container, options); }, }; -export const aws = { +const aws = { get name() { return "aws"; }, /** * @param {string[]} args + * @param {import("./utils.mjs").SpawnOptions} [options] * @returns {Promise} */ - async spawn(args) { - const aws = which("aws"); - if (!aws) { - throw new Error("AWS CLI is not installed, please install it"); - } + async spawn(args, options = {}) { + const aws = which("aws", { required: true }); let env; if (isCI) { @@ -137,14 +823,7 @@ export const aws = { }; } - const { error, stdout } = await spawn($`${aws} ${args} --output json`, { env }); - if (error) { - if (/max attempts exceeded/i.test(inspect(error))) { - return this.spawn(args); - } - throw error; - } - + const { stdout } = await spawnSafe($`${aws} ${args} --output json`, { env, ...options }); try { return JSON.parse(stdout); } catch { @@ -202,9 +881,28 @@ export const aws = { * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/run-instances.html */ async runInstances(options) { - const flags = aws.getFlags(options); - const { Instances } = await aws.spawn($`ec2 run-instances ${flags}`); - return Instances.sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1)); + for (let i = 0; i < 3; i++) { + const flags = aws.getFlags(options); + const result = await aws.spawn($`ec2 run-instances ${flags}`, { + throwOnError: error => { + if (options["instance-market-options"] && /InsufficientInstanceCapacity/i.test(inspect(error))) { + delete options["instance-market-options"]; + const instanceType = options["instance-type"] || "default"; + console.warn(`There is not enough capacity for ${instanceType} spot instances, retrying with on-demand...`); + return false; + } + return true; + }, + }); + if (result) { + const { Instances } = result; + if (Instances.length) { + return Instances.sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1)); + } + } + await new Promise(resolve => setTimeout(resolve, i * Math.random() * 15_000)); + } + throw new Error(`Failed to run instances: ${inspect(instanceOptions)}`); }, /** @@ -220,7 +918,9 @@ export const aws = { * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/terminate-instances.html */ async terminateInstances(...instanceIds) { - await aws.spawn($`ec2 terminate-instances --instance-ids ${instanceIds}`); + await aws.spawn($`ec2 terminate-instances --instance-ids ${instanceIds}`, { + throwOnError: error => !/InvalidInstanceID\.NotFound/i.test(inspect(error)), + }); }, /** @@ -229,7 +929,29 @@ export const aws = { * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait.html */ async waitInstances(action, ...instanceIds) { - await aws.spawn($`ec2 wait ${action} --instance-ids ${instanceIds}`); + await aws.spawn($`ec2 wait ${action} --instance-ids ${instanceIds}`, { + retryOnError: error => /max attempts exceeded/i.test(inspect(error)), + }); + }, + + /** + * @param {string} instanceId + * @param {string} privateKeyPath + * @param {object} [passwordOptions] + * @param {boolean} [passwordOptions.wait] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/get-password-data.html + */ + async getPasswordData(instanceId, privateKeyPath, passwordOptions = {}) { + const attempts = passwordOptions.wait ? 15 : 1; + for (let i = 0; i < attempts; i++) { + const { PasswordData } = await aws.spawn($`ec2 get-password-data --instance-id ${instanceId}`); + if (PasswordData) { + return decryptPassword(PasswordData, privateKeyPath); + } + await new Promise(resolve => setTimeout(resolve, 60000 * i)); + } + throw new Error(`Failed to get password data for instance: ${instanceId}`); }, /** @@ -262,19 +984,31 @@ export const aws = { */ async createImage(options) { const flags = aws.getFlags(options); - try { - const { ImageId } = await aws.spawn($`ec2 create-image ${flags}`); - return ImageId; - } catch (error) { - const match = /already in use by AMI (ami-[a-z0-9]+)/i.exec(inspect(error)); - if (!match) { - throw error; - } - const [, existingImageId] = match; - await aws.spawn($`ec2 deregister-image --image-id ${existingImageId}`); - const { ImageId } = await aws.spawn($`ec2 create-image ${flags}`); + + /** @type {string | undefined} */ + let existingImageId; + + /** @type {AwsImage | undefined} */ + const image = await aws.spawn($`ec2 create-image ${flags}`, { + throwOnError: error => { + const match = /already in use by AMI (ami-[a-z0-9]+)/i.exec(inspect(error)); + if (!match) { + return true; + } + const [, imageId] = match; + existingImageId = imageId; + return false; + }, + }); + + if (!existingImageId) { + const { ImageId } = image; return ImageId; } + + await aws.spawn($`ec2 deregister-image --image-id ${existingImageId}`); + const { ImageId } = await aws.spawn($`ec2 create-image ${flags}`); + return ImageId; }, /** @@ -294,7 +1028,60 @@ export const aws = { * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/wait/image-available.html */ async waitImage(action, ...imageIds) { - await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`); + await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`, { + retryOnError: error => /max attempts exceeded/i.test(inspect(error)), + }); + }, + + /** + * @typedef {Object} AwsKeyPair + * @property {string} KeyPairId + * @property {string} KeyName + * @property {string} KeyFingerprint + * @property {string} [PublicKeyMaterial] + */ + + /** + * @param {string[]} [names] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-key-pairs.html + */ + async describeKeyPairs(names) { + const command = names + ? $`ec2 describe-key-pairs --include-public-key --key-names ${names}` + : $`ec2 describe-key-pairs --include-public-key`; + const { KeyPairs } = await aws.spawn(command); + return KeyPairs; + }, + + /** + * @param {string | Buffer} publicKey + * @param {string} [name] + * @returns {Promise} + * @link https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/import-key-pair.html + */ + async importKeyPair(publicKey, name) { + const keyName = name || `key-pair-${sha256(publicKey)}`; + const publicKeyBase64 = Buffer.from(publicKey).toString("base64"); + + /** @type {AwsKeyPair | undefined} */ + const keyPair = await aws.spawn( + $`ec2 import-key-pair --key-name ${keyName} --public-key-material ${publicKeyBase64}`, + { + throwOnError: error => !/InvalidKeyPair\.Duplicate/i.test(inspect(error)), + }, + ); + + if (keyPair) { + return keyPair; + } + + const keyPairs = await aws.describeKeyPairs(keyName); + if (keyPairs.length) { + return keyPairs[0]; + } + + throw new Error(`Failed to import key pair: ${keyName}`); }, /** @@ -329,36 +1116,36 @@ export const aws = { * @returns {Promise} */ async getBaseImage(options) { - const { os, arch, distro, distroVersion } = options; + const { os, arch, distro, release } = options; let name, owner; if (os === "linux") { if (!distro || distro === "debian") { owner = "amazon"; - name = `debian-${distroVersion || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-*`; + name = `debian-${release || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-*`; } else if (distro === "ubuntu") { owner = "099720109477"; - name = `ubuntu/images/hvm-ssd*/ubuntu-*-${distroVersion || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-server-*`; + name = `ubuntu/images/hvm-ssd*/ubuntu-*-${release || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-server-*`; } else if (distro === "amazonlinux") { owner = "amazon"; - if (distroVersion === "1") { - // EOL - } else if (distroVersion === "2") { + if (release === "1" && arch === "x64") { + name = `amzn-ami-2018.03.*`; + } else if (release === "2") { name = `amzn2-ami-hvm-*-${arch === "aarch64" ? "arm64" : "x86_64"}-gp2`; } else { - name = `al${distroVersion || "*"}-ami-*-${arch === "aarch64" ? "arm64" : "x86_64"}`; + name = `al${release || "*"}-ami-*-${arch === "aarch64" ? "arm64" : "x86_64"}`; } } else if (distro === "alpine") { owner = "538276064493"; - name = `alpine-${distroVersion || "*"}.*-${arch === "aarch64" ? "aarch64" : "x86_64"}-uefi-cloudinit-*`; + name = `alpine-${release || "*"}.*-${arch === "aarch64" ? "aarch64" : "x86_64"}-uefi-cloudinit-*`; } else if (distro === "centos") { owner = "aws-marketplace"; - name = `CentOS-Stream-ec2-${distroVersion || "*"}-*.${arch === "aarch64" ? "aarch64" : "x86_64"}-*`; + name = `CentOS-Stream-ec2-${release || "*"}-*.${arch === "aarch64" ? "aarch64" : "x86_64"}-*`; } } else if (os === "windows") { if (!distro || distro === "server") { owner = "amazon"; - name = `Windows_Server-${distroVersion || "*"}-English-Full-Base-*`; + name = `Windows_Server-${release || "*"}-English-Full-Base-*`; } } @@ -385,7 +1172,7 @@ export const aws = { * @returns {Promise} */ async createMachine(options) { - const { os, arch, imageId, instanceType, tags } = options; + const { os, arch, imageId, instanceType, tags, sshKeys, preemptible } = options; /** @type {AwsImage} */ let image; @@ -413,7 +1200,7 @@ export const aws = { let userData = getUserData({ ...options, username }); if (os === "windows") { - userData = `${userData}-ExecutionPolicy Unrestricted -NoProfile -NonInteractivefalse`; + userData = `${userData}-ExecutionPolicy Unrestricted -NoProfile -NonInteractivetrue`; } let tagSpecification = []; @@ -426,6 +1213,29 @@ export const aws = { }); } + /** @type {string | undefined} */ + let keyName, keyPath; + if (os === "windows") { + const sshKey = sshKeys.find(({ privatePath }) => existsSync(privatePath)); + if (sshKey) { + const { publicKey, privatePath } = sshKey; + const { KeyName } = await aws.importKeyPair(publicKey); + keyName = KeyName; + keyPath = privatePath; + } + } + + let marketOptions; + if (preemptible) { + marketOptions = JSON.stringify({ + MarketType: "spot", + SpotOptions: { + InstanceInterruptionBehavior: "terminate", + SpotInstanceType: "one-time", + }, + }); + } + const [instance] = await aws.runInstances({ ["image-id"]: ImageId, ["instance-type"]: instanceType || (arch === "aarch64" ? "t4g.large" : "t3.large"), @@ -438,10 +1248,11 @@ export const aws = { "InstanceMetadataTags": "enabled", }), ["tag-specifications"]: JSON.stringify(tagSpecification), - ["key-name"]: "ashcon-bun", + ["key-name"]: keyName, + ["instance-market-options"]: marketOptions, }); - return aws.toMachine(instance, { ...options, username }); + return aws.toMachine(instance, { ...options, username, keyPath }); }, /** @@ -479,6 +1290,13 @@ export const aws = { return spawnSshSafe({ ...connectOptions, command }, options); }; + const rdp = async () => { + const { keyPath } = options; + const { hostname, username } = await connect(); + const password = await aws.getPasswordData(InstanceId, keyPath, { wait: true }); + return { hostname, username, password }; + }; + const attach = async () => { const connectOptions = await connect(); await spawnSshSafe({ ...connectOptions }); @@ -517,6 +1335,7 @@ export const aws = { spawnSafe, upload, attach, + rdp, snapshot, close: terminate, [Symbol.asyncDispose]: terminate, @@ -525,70 +1344,478 @@ export const aws = { }; const google = { - async createMachine(platform) { - const image = await google.getImage(platform); - const { id: imageId, username } = image; + get cloud() { + return "google"; + }, - const authorizedKeys = await getAuthorizedKeys(); - const sshKeys = authorizedKeys?.map(key => `${username}:${key}`).join("\n") ?? ""; + /** + * @param {string[]} args + * @param {import("./utils.mjs").SpawnOptions} [options] + * @returns {Promise} + */ + async spawn(args, options = {}) { + const gcloud = which("gcloud", { required: true }); - const { os, ["instance-type"]: type } = platform; - const instanceType = type || "e2-standard-4"; + let env = { ...process.env }; + // if (isCI) { + // env; // TODO: Add Google Cloud credentials + // } else { + // env["TERM"] = "dumb"; + // } - let metadata = `ssh-keys=${sshKeys}`; - if (os === "windows") { - metadata += `,sysprep-specialize-script-cmd=googet -noconfirm=true install google-compute-engine-ssh,enable-windows-ssh=TRUE`; - } - - const [{ id, networkInterfaces }] = await google.createInstances({ - ["zone"]: "us-central1-a", - ["image"]: imageId, - ["machine-type"]: instanceType, - ["boot-disk-auto-delete"]: true, - // ["boot-disk-size"]: "10GB", - // ["boot-disk-type"]: "pd-standard", - ["metadata"]: metadata, + const { stdout } = await spawnSafe($`${gcloud} ${args} --format json`, { + env, + ...options, }); + try { + return JSON.parse(stdout); + } catch { + return; + } + }, - const publicIp = () => { - for (const { accessConfigs } of networkInterfaces) { - for (const { natIP } of accessConfigs) { - return natIP; + /** + * @param {Record} [options] + * @returns {string[]} + */ + getFilters(options = {}) { + const filter = Object.entries(options) + .filter(([, value]) => value !== undefined) + .map(([key, value]) => [value.includes("*") ? `${key}~${value}` : `${key}=${value}`]) + .join(" AND "); + return filter ? ["--filter", filter] : []; + }, + + /** + * @param {Record} options + * @returns {string[]} + */ + getFlags(options) { + return Object.entries(options) + .filter(([, value]) => value !== undefined) + .flatMap(([key, value]) => { + if (typeof value === "boolean") { + return value ? [`--${key}`] : []; + } + return [`--${key}=${value}`]; + }); + }, + + /** + * @param {Record} options + * @returns {string} + * @link https://cloud.google.com/sdk/gcloud/reference/topic/escaping + */ + getMetadata(options) { + const delimiter = Math.random().toString(36).substring(2, 15); + const entries = Object.entries(options) + .map(([key, value]) => `${key}=${value}`) + .join(delimiter); + return `^${delimiter}^${entries}`; + }, + + /** + * @param {string} name + * @returns {string} + */ + getLabel(name) { + return name.replace(/[^a-z0-9_-]/g, "-").toLowerCase(); + }, + + /** + * @typedef {Object} GoogleImage + * @property {string} id + * @property {string} name + * @property {string} family + * @property {"X86_64" | "ARM64"} architecture + * @property {string} diskSizeGb + * @property {string} selfLink + * @property {"READY"} status + * @property {string} creationTimestamp + */ + + /** + * @param {Partial} [options] + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/images/list + */ + async listImages(options) { + const filters = google.getFilters(options); + const images = await google.spawn($`compute images list ${filters} --preview-images --show-deprecated`); + return images.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + }, + + /** + * @param {Record} options + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/images/create + */ + async createImage(options) { + const { name, ...otherOptions } = options; + const flags = this.getFlags(otherOptions); + const imageId = name || "i-" + Math.random().toString(36).substring(2, 15); + return this.spawn($`compute images create ${imageId} ${flags}`); + }, + + /** + * @typedef {Object} GoogleInstance + * @property {string} id + * @property {string} name + * @property {"RUNNING"} status + * @property {string} machineType + * @property {string} zone + * @property {GoogleDisk[]} disks + * @property {GoogleNetworkInterface[]} networkInterfaces + * @property {object} [scheduling] + * @property {"STANDARD" | "SPOT"} [scheduling.provisioningModel] + * @property {boolean} [scheduling.preemptible] + * @property {Record} [labels] + * @property {string} selfLink + * @property {string} creationTimestamp + */ + + /** + * @typedef {Object} GoogleDisk + * @property {string} deviceName + * @property {boolean} boot + * @property {"X86_64" | "ARM64"} architecture + * @property {string[]} [licenses] + * @property {number} diskSizeGb + */ + + /** + * @typedef {Object} GoogleNetworkInterface + * @property {"IPV4_ONLY" | "IPV4_IPV6" | "IPV6_ONLY"} stackType + * @property {string} name + * @property {string} network + * @property {string} networkIP + * @property {string} subnetwork + * @property {GoogleAccessConfig[]} accessConfigs + */ + + /** + * @typedef {Object} GoogleAccessConfig + * @property {string} name + * @property {"ONE_TO_ONE_NAT" | "INTERNAL_NAT"} type + * @property {string} [natIP] + */ + + /** + * @param {Record} options + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/create + */ + async createInstance(options) { + const { name, ...otherOptions } = options || {}; + const flags = this.getFlags(otherOptions); + const instanceId = name || "i-" + Math.random().toString(36).substring(2, 15); + const [instance] = await this.spawn($`compute instances create ${instanceId} ${flags}`); + return instance; + }, + + /** + * @param {string} instanceId + * @param {string} zoneId + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/stop + */ + async stopInstance(instanceId, zoneId) { + await this.spawn($`compute instances stop ${instanceId} --zone=${zoneId}`); + }, + + /** + * @param {string} instanceId + * @param {string} zoneId + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/instances/delete + */ + async deleteInstance(instanceId, zoneId) { + await this.spawn($`compute instances delete ${instanceId} --delete-disks=all --zone=${zoneId}`, { + throwOnError: error => !/not found/i.test(inspect(error)), + }); + }, + + /** + * @param {string} instanceId + * @param {string} username + * @param {string} zoneId + * @param {object} [options] + * @param {boolean} [options.wait] + * @returns {Promise} + * @link https://cloud.google.com/sdk/gcloud/reference/compute/reset-windows-password + */ + async resetWindowsPassword(instanceId, username, zoneId, options = {}) { + const attempts = options.wait ? 15 : 1; + for (let i = 0; i < attempts; i++) { + const result = await this.spawn( + $`compute reset-windows-password ${instanceId} --user=${username} --zone=${zoneId}`, + { + throwOnError: error => !/instance may not be ready for use/i.test(inspect(error)), + }, + ); + if (result) { + const { password } = result; + if (password) { + return password; } } - throw new Error(`Failed to find public IP for instance: ${id}`); + await new Promise(resolve => setTimeout(resolve, 60000 * i)); + } + }, + + /** + * @param {Partial} options + * @returns {Promise} + */ + async listInstances(options) { + const filters = this.getFilters(options); + const instances = await this.spawn($`compute instances list ${filters}`); + return instances.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async getMachineImage(options) { + const { os, arch, distro, release } = options; + const architecture = arch === "aarch64" ? "ARM64" : "X86_64"; + + /** @type {string | undefined} */ + let family; + if (os === "linux") { + if (!distro || distro === "debian") { + family = `debian-${release || "*"}`; + } else if (distro === "ubuntu") { + family = `ubuntu-${release?.replace(/\./g, "") || "*"}`; + } else if (distro === "fedora") { + family = `fedora-coreos-${release || "*"}`; + } else if (distro === "rhel") { + family = `rhel-${release || "*"}`; + } + } else if (os === "windows" && arch === "x64") { + if (!distro || distro === "server") { + family = `windows-${release || "*"}`; + } + } + + if (family) { + const images = await this.listImages({ family, architecture }); + if (images.length) { + const [image] = images; + return image; + } + } + + throw new Error(`Unsupported platform: ${inspect(options)}`); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async createMachine(options) { + const { name, os, arch, distro, instanceType, tags, preemptible, detached } = options; + const image = await google.getMachineImage(options); + const { selfLink: imageUrl } = image; + + const username = getUsername(distro || os); + const userData = getUserData({ ...options, username }); + + /** @type {Record} */ + let metadata; + if (os === "windows") { + metadata = { + "enable-windows-ssh": "TRUE", + "sysprep-specialize-script-ps1": userData, + }; + } else { + metadata = { + "user-data": userData, + }; + } + + const instance = await google.createInstance({ + "name": name, + "zone": "us-central1-a", + "image": imageUrl, + "machine-type": instanceType || (arch === "aarch64" ? "t2a-standard-2" : "t2d-standard-2"), + "boot-disk-auto-delete": true, + "boot-disk-size": `${getDiskSize(options)}GB`, + "metadata": this.getMetadata(metadata), + "labels": Object.entries(tags || {}) + .filter(([, value]) => value !== undefined) + .map(([key, value]) => `${this.getLabel(key)}=${value}`) + .join(","), + "provisioning-model": preemptible ? "SPOT" : "STANDARD", + "instance-termination-action": preemptible || !detached ? "DELETE" : undefined, + "no-restart-on-failure": true, + "threads-per-core": 1, + "max-run-duration": detached ? undefined : "6h", + }); + + return this.toMachine(instance, options); + }, + + /** + * @param {GoogleInstance} instance + * @param {MachineOptions} [options] + * @returns {Machine} + */ + toMachine(instance, options = {}) { + const { id: instanceId, name, zone: zoneUrl, machineType: machineTypeUrl, labels } = instance; + const machineType = machineTypeUrl.split("/").pop(); + const zoneId = zoneUrl.split("/").pop(); + + let os, arch, distro, release; + const { disks = [] } = instance; + for (const { boot, architecture, licenses = [] } of disks) { + if (!boot) { + continue; + } + + if (architecture === "X86_64") { + arch = "x64"; + } else if (architecture === "ARM64") { + arch = "aarch64"; + } + + for (const license of licenses) { + const linuxMatch = /(debian|ubuntu|fedora|rhel)-(\d+)/i.exec(license); + if (linuxMatch) { + os = "linux"; + [, distro, release] = linuxMatch; + } else { + const windowsMatch = /windows-server-(\d+)-dc-core/i.exec(license); + if (windowsMatch) { + os = "windows"; + distro = "windowsserver"; + [, release] = windowsMatch; + } + } + } + } + + let publicIp; + const { networkInterfaces = [] } = instance; + for (const { accessConfigs = [] } of networkInterfaces) { + for (const { type, natIP } of accessConfigs) { + if (type === "ONE_TO_ONE_NAT" && natIP) { + publicIp = natIP; + } + } + } + + let preemptible; + const { scheduling } = instance; + if (scheduling) { + const { provisioningModel, preemptible: isPreemptible } = scheduling; + preemptible = provisioningModel === "SPOT" || isPreemptible; + } + + /** + * @returns {SshOptions} + */ + const connect = () => { + if (!publicIp) { + throw new Error(`Failed to find public IP for instance: ${name}`); + } + + /** @type {string | undefined} */ + let username; + + const { os, distro } = options; + if (os || distro) { + username = getUsername(distro || os); + } + + return { hostname: publicIp, username }; }; - const spawn = command => { - const hostname = publicIp(); - return spawnSsh({ hostname, username, command }); + const spawn = async (command, options) => { + const connectOptions = connect(); + return spawnSsh({ ...connectOptions, command }, options); }; - const spawnSafe = command => { - const hostname = publicIp(); - return spawnSshSafe({ hostname, username, command }); + const spawnSafe = async (command, options) => { + const connectOptions = connect(); + return spawnSshSafe({ ...connectOptions, command }, options); + }; + + const rdp = async () => { + const { hostname, username } = connect(); + const rdpUsername = `${username}-rdp`; + const password = await google.resetWindowsPassword(instanceId, rdpUsername, zoneId, { wait: true }); + return { hostname, username: rdpUsername, password }; }; const attach = async () => { - const hostname = publicIp(); - await spawnSshSafe({ hostname, username }); + const connectOptions = connect(); + await spawnSshSafe({ ...connectOptions }); + }; + + const upload = async (source, destination) => { + const connectOptions = connect(); + await spawnScp({ ...connectOptions, source, destination }); + }; + + const snapshot = async name => { + const stopResult = await this.stopInstance(instanceId, zoneId); + console.log(stopResult); + const image = await this.createImage({ + ["source-disk"]: instanceId, + ["zone"]: zoneId, + ["name"]: name || `${instanceId}-snapshot-${Date.now()}`, + }); + console.log(image); + return; }; const terminate = async () => { - await google.deleteInstance(id); + await google.deleteInstance(instanceId, zoneId); }; return { + cloud: "google", + os, + arch, + distro, + release, + id: instanceId, + imageId: undefined, + name, + instanceType: machineType, + region: zoneId, + publicIp, + preemptible, + labels, spawn, spawnSafe, + rdp, attach, + upload, + snapshot, close: terminate, [Symbol.asyncDispose]: terminate, }; }, - async getImage(platform) { - const { os, arch, distro, release } = platform; + /** + * @param {Record} [labels] + * @returns {Promise} + */ + async getMachines(labels) { + const filters = labels ? this.getFilters({ labels }) : {}; + const instances = await google.listInstances(filters); + return instances.map(instance => this.toMachine(instance)); + }, + + /** + * @param {MachineOptions} options + * @returns {Promise} + */ + async getImage(options) { + const { os, arch, distro, release } = options; const architecture = arch === "aarch64" ? "ARM64" : "X86_64"; let name; @@ -623,49 +1850,6 @@ const google = { throw new Error(`Unsupported platform: ${inspect(platform)}`); }, - - async listImages(options = {}) { - const filter = Object.entries(options) - .map(([key, value]) => [value.includes("*") ? `${key}~${value}` : `${key}=${value}`]) - .join(" AND "); - const filters = filter ? ["--filter", filter] : []; - const { stdout } = await spawnSafe(["gcloud", "compute", "images", "list", ...filters, "--format", "json"]); - const images = JSON.parse(stdout); - return images.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); - }, - - async listInstances(options = {}) { - const filter = Object.entries(options) - .map(([key, value]) => [value.includes("*") ? `${key}~${value}` : `${key}=${value}`]) - .join(" AND "); - const filters = filter ? ["--filter", filter] : []; - const { stdout } = await spawnSafe(["gcloud", "compute", "instances", "list", ...filters, "--format", "json"]); - const instances = JSON.parse(stdout); - return instances.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); - }, - - async createInstances(options = {}) { - const flags = Object.entries(options).flatMap(([key, value]) => - typeof value === "boolean" ? `--${key}` : `--${key}=${value}`, - ); - const randomId = "i-" + Math.random().toString(36).substring(2, 15); - const { stdout } = await spawnSafe([ - "gcloud", - "compute", - "instances", - "create", - randomId, - ...flags, - "--format", - "json", - ]); - const instances = JSON.parse(stdout); - return instances.sort((a, b) => (a.creationTimestamp < b.creationTimestamp ? 1 : -1)); - }, - - async deleteInstance(instanceId) { - await spawnSafe(["gcloud", "compute", "instances", "delete", instanceId, "--zone", "us-central1-a", "--quiet"]); - }, }; /** @@ -676,6 +1860,10 @@ const google = { * @property {string} [password] */ +/** + * @param {CloudInit} cloudInit + * @returns {string} + */ function getUserData(cloudInit) { const { os } = cloudInit; if (os === "windows") { @@ -705,24 +1893,28 @@ function getCloudInit(cloudInit) { break; } + let users; + if (username === "root") { + users = [`root:${password}`]; + } else { + users = [`root:${password}`, `${username}:${password}`]; + } + // https://cloudinit.readthedocs.io/en/stable/ return `#cloud-config - write_files: - path: /etc/ssh/sshd_config content: | PermitRootLogin yes - PasswordAuthentication yes + PasswordAuthentication no + PubkeyAuthentication yes + UsePAM yes + UseLogin yes Subsystem sftp ${sftpPath} - chpasswd: expire: false - list: | - root:${password} - ${username}:${password} - + list: ${JSON.stringify(users)} disable_root: false - ssh_pwauth: true ssh_authorized_keys: ${authorizedKeys} `; @@ -734,39 +1926,47 @@ function getCloudInit(cloudInit) { */ function getWindowsStartupScript(cloudInit) { const { sshKeys } = cloudInit; - const authorizedKeys = sshKeys.filter(({ publicKey }) => publicKey).map(({ publicKey }) => publicKey); + const authorizedKeys = sshKeys.map(({ publicKey }) => publicKey); return ` $ErrorActionPreference = "Stop" Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force function Install-Ssh { - $sshService = Get-WindowsCapability -Online | Where-Object Name -like 'OpenSSH.Server*' - if ($sshService.State -ne "Installed") { - Write-Output "Installing OpenSSH server..." - Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 + $sshdService = Get-Service -Name sshd -ErrorAction SilentlyContinue + if (-not $sshdService) { + $buildNumber = Get-WmiObject Win32_OperatingSystem | Select-Object -ExpandProperty BuildNumber + if ($buildNumber -lt 17763) { + Write-Output "Installing OpenSSH server through Github..." + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri "https://github.com/PowerShell/Win32-OpenSSH/releases/download/v9.8.0.0p1-Preview/OpenSSH-Win64.zip" -OutFile "$env:TEMP\\OpenSSH.zip" + Expand-Archive -Path "$env:TEMP\\OpenSSH.zip" -DestinationPath "$env:TEMP\\OpenSSH" -Force + Get-ChildItem -Path "$env:TEMP\\OpenSSH\\OpenSSH-Win64" -Recurse | Move-Item -Destination "$env:ProgramFiles\\OpenSSH" -Force + & "$env:ProgramFiles\\OpenSSH\\install-sshd.ps1" + } else { + Write-Output "Installing OpenSSH server through Windows Update..." + Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 + } } + Write-Output "Enabling OpenSSH server..." + Set-Service -Name sshd -StartupType Automatic + Start-Service sshd + $pwshPath = Get-Command pwsh -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path if (-not $pwshPath) { $pwshPath = Get-Command powershell -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path } - if (-not (Get-Service -Name sshd -ErrorAction SilentlyContinue)) { - Write-Output "Enabling OpenSSH server..." - Set-Service -Name sshd -StartupType Automatic - Start-Service sshd - } - if ($pwshPath) { Write-Output "Setting default shell to $pwshPath..." New-ItemProperty -Path "HKLM:\\SOFTWARE\\OpenSSH" -Name DefaultShell -Value $pwshPath -PropertyType String -Force } - $firewallRule = Get-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -ErrorAction SilentlyContinue + $firewallRule = Get-NetFirewallRule -Name "OpenSSH-Server" -ErrorAction SilentlyContinue if (-not $firewallRule) { Write-Output "Configuring firewall..." - New-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 + New-NetFirewallRule -Profile Any -Name 'OpenSSH-Server' -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 } $sshPath = "C:\\ProgramData\\ssh" @@ -841,13 +2041,19 @@ function getDiskSize(options) { return diskSizeGb; } - return os === "windows" ? 50 : 30; + // After Visual Studio and dependencies are installed, + // there is ~50GB of used disk space. + if (os === "windows") { + return 60; + } + + return 30; } /** * @typedef SshKey - * @property {string} privatePath - * @property {string} publicPath + * @property {string} [privatePath] + * @property {string} [publicPath] * @property {string} publicKey */ @@ -855,24 +2061,27 @@ function getDiskSize(options) { * @returns {SshKey} */ function createSshKey() { + const sshKeyGen = which("ssh-keygen", { required: true }); + const sshAdd = which("ssh-add", { required: true }); + const sshPath = join(homedir(), ".ssh"); - if (!existsSync(sshPath)) { - mkdirSync(sshPath, { recursive: true }); - } - - const name = `id_rsa_${crypto.randomUUID()}`; - const privatePath = join(sshPath, name); - const publicPath = join(sshPath, `${name}.pub`); - spawnSyncSafe(["ssh-keygen", "-t", "rsa", "-b", "4096", "-f", privatePath, "-N", ""], { stdio: "inherit" }); + mkdir(sshPath); + const filename = `id_rsa_${crypto.randomUUID()}`; + const privatePath = join(sshPath, filename); + const publicPath = join(sshPath, `${filename}.pub`); + spawnSyncSafe([sshKeyGen, "-t", "rsa", "-b", "4096", "-f", privatePath, "-N", ""], { stdio: "inherit" }); if (!existsSync(privatePath) || !existsSync(publicPath)) { throw new Error(`Failed to generate SSH key: ${privatePath} / ${publicPath}`); } - const sshAgent = which("ssh-agent"); - const sshAdd = which("ssh-add"); - if (sshAgent && sshAdd) { - spawnSyncSafe(["sh", "-c", `eval $(${sshAgent} -s) && ${sshAdd} ${privatePath}`], { stdio: "inherit" }); + if (isWindows) { + spawnSyncSafe([sshAdd, privatePath], { stdio: "inherit" }); + } else { + const sshAgent = which("ssh-agent"); + if (sshAgent) { + spawnSyncSafe(["sh", "-c", `eval $(${sshAgent} -s) && ${sshAdd} ${privatePath}`], { stdio: "inherit" }); + } } return { @@ -894,7 +2103,7 @@ function getSshKeys() { /** @type {SshKey[]} */ const sshKeys = []; if (existsSync(sshPath)) { - const sshFiles = readdirSync(sshPath, { withFileTypes: true }); + const sshFiles = readdirSync(sshPath, { withFileTypes: true, encoding: "utf-8" }); const publicPaths = sshFiles .filter(entry => entry.isFile() && entry.name.endsWith(".pub")) .map(({ name }) => join(sshPath, name)); @@ -917,11 +2126,41 @@ function getSshKeys() { return sshKeys; } +/** + * @param {string} username + * @returns {Promise} + */ +async function getGithubUserSshKeys(username) { + const url = new URL(`${username}.keys`, getGithubUrl()); + const publicKeys = await curlSafe(url); + return publicKeys + .split("\n") + .filter(key => key.length) + .map(key => ({ publicKey: `${key} github@${username}` })); +} + +/** + * @param {string} organization + * @returns {Promise} + */ +async function getGithubOrgSshKeys(organization) { + const url = new URL(`orgs/${encodeURIComponent(organization)}/members`, getGithubApiUrl()); + const members = await curlSafe(url, { json: true }); + + /** @type {SshKey[][]} */ + const sshKeys = await Promise.all( + members.filter(({ type, login }) => type === "User" && login).map(({ login }) => getGithubUserSshKeys(login)), + ); + + return sshKeys.flat(); +} + /** * @typedef SshOptions * @property {string} hostname * @property {number} [port] * @property {string} [username] + * @property {string} [password] * @property {string[]} [command] * @property {string[]} [identityPaths] * @property {number} [retries] @@ -929,56 +2168,76 @@ function getSshKeys() { /** * @param {SshOptions} options - * @param {object} [spawnOptions] + * @param {import("./utils.mjs").SpawnOptions} [spawnOptions] * @returns {Promise} */ async function spawnSsh(options, spawnOptions = {}) { - const { hostname, port, username, identityPaths, command } = options; - await waitForPort({ hostname, port: port || 22 }); + const { hostname, port, username, identityPaths, password, retries = 10, command: spawnCommand } = options; - const ssh = ["ssh", hostname, "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"]; + if (!hostname.includes("@")) { + await waitForPort({ + hostname, + port: port || 22, + }); + } + + const logPath = mkdtemp("ssh-", "ssh.log"); + const command = ["ssh", hostname, "-v", "-C", "-E", logPath, "-o", "StrictHostKeyChecking=no"]; + if (!password) { + command.push("-o", "BatchMode=yes"); + } if (port) { - ssh.push("-p", port); + command.push("-p", port); } if (username) { - ssh.push("-l", username); + command.push("-l", username); } - if (identityPaths) { - ssh.push(...identityPaths.flatMap(path => ["-i", path])); + if (password) { + const sshPass = which("sshpass", { required: true }); + command.unshift(sshPass, "-p", password); + } else if (identityPaths) { + command.push(...identityPaths.flatMap(path => ["-i", path])); } - const stdio = command ? "pipe" : "inherit"; - if (command) { - ssh.push(...command); + const stdio = spawnCommand ? "pipe" : "inherit"; + if (spawnCommand) { + command.push(...spawnCommand); } - return spawn(ssh, { stdio, ...spawnOptions }); + /** @type {import("./utils.mjs").SpawnResult} */ + let result; + for (let i = 0; i < retries; i++) { + result = await spawn(command, { stdio, ...spawnOptions, throwOnError: undefined }); + + const { exitCode } = result; + if (exitCode !== 255) { + break; + } + + const sshLogs = readFile(logPath, { encoding: "utf-8" }); + if (sshLogs.includes("Authenticated")) { + break; + } + + await new Promise(resolve => setTimeout(resolve, (i + 1) * 15000)); + } + + if (spawnOptions?.throwOnError) { + const { error } = result; + if (error) { + throw error; + } + } + + return result; } /** * @param {SshOptions} options - * @param {object} [spawnOptions] + * @param {import("./utils.mjs").SpawnOptions} [spawnOptions] * @returns {Promise} */ async function spawnSshSafe(options, spawnOptions = {}) { - const { hostname, port, username, identityPaths, command } = options; - await waitForPort({ hostname, port: port || 22 }); - - const ssh = ["ssh", hostname, "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"]; - if (port) { - ssh.push("-p", port); - } - if (username) { - ssh.push("-l", username); - } - if (identityPaths) { - ssh.push(...identityPaths.flatMap(path => ["-i", path])); - } - const stdio = command ? "pipe" : "inherit"; - if (command) { - ssh.push(...command); - } - - return spawnSafe(ssh, { stdio, ...spawnOptions }); + return spawnSsh(options, { throwOnError: true, ...spawnOptions }); } /** @@ -997,14 +2256,20 @@ async function spawnSshSafe(options, spawnOptions = {}) { * @returns {Promise} */ async function spawnScp(options) { - const { hostname, port, username, identityPaths, source, destination, retries = 10 } = options; + const { hostname, port, username, identityPaths, password, source, destination, retries = 10 } = options; await waitForPort({ hostname, port: port || 22 }); - const command = ["scp", "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"]; + const command = ["scp", "-o", "StrictHostKeyChecking=no"]; + if (!password) { + command.push("-o", "BatchMode=yes"); + } if (port) { command.push("-P", port); } - if (identityPaths) { + if (password) { + const sshPass = which("sshpass", { required: true }); + command.unshift(sshPass, "-p", password); + } else if (identityPaths) { command.push(...identityPaths.flatMap(path => ["-i", path])); } command.push(resolve(source)); @@ -1032,6 +2297,53 @@ async function spawnScp(options) { throw new Error(`SCP failed: ${source} -> ${username}@${hostname}:${destination}`, { cause }); } +/** + * @param {string} passwordData + * @param {string} privateKeyPath + * @returns {string} + */ +function decryptPassword(passwordData, privateKeyPath) { + const name = basename(privateKeyPath, extname(privateKeyPath)); + const tmpPemPath = mkdtemp("pem-", `${name}.pem`); + try { + copyFile(privateKeyPath, tmpPemPath, { mode: 0o600 }); + spawnSyncSafe(["ssh-keygen", "-p", "-m", "PEM", "-f", tmpPemPath, "-N", ""]); + const { stdout } = spawnSyncSafe( + ["openssl", "pkeyutl", "-decrypt", "-inkey", tmpPemPath, "-pkeyopt", "rsa_padding_mode:pkcs1"], + { + stdin: Buffer.from(passwordData, "base64"), + }, + ); + return stdout.trim(); + } finally { + rm(tmpPemPath); + } +} + +/** + * @typedef RdpCredentials + * @property {string} hostname + * @property {string} username + * @property {string} password + */ + +/** + * @param {string} hostname + * @param {string} [username] + * @param {string} [password] + * @returns {string} + */ +function getRdpFile(hostname, username) { + const options = [ + "auto connect:i:1", // start the connection automatically + `full address:s:${hostname}`, + ]; + if (username) { + options.push(`username:s:${username}`); + } + return options.join("\n"); +} + /** * @typedef Cloud * @property {string} name @@ -1044,52 +2356,80 @@ async function spawnScp(options) { */ function getCloud(name) { switch (name) { + case "docker": + return docker; + case "orbstack": + return orbstack; + case "tart": + return tart; case "aws": return aws; + case "google": + return google; } throw new Error(`Unsupported cloud: ${name}`); } /** - * @typedef Machine + * @typedef {"linux" | "darwin" | "windows"} Os + * @typedef {"aarch64" | "x64"} Arch + * @typedef {"macos" | "windowsserver" | "debian" | "ubuntu" | "alpine" | "amazonlinux"} Distro + */ + +/** + * @typedef {Object} Platform + * @property {Os} os + * @property {Arch} arch + * @property {Distro} distro + * @property {string} release + * @property {string} [eol] + */ + +/** + * @typedef {Object} Machine * @property {string} cloud + * @property {Os} [os] + * @property {Arch} [arch] + * @property {Distro} [distro] + * @property {string} [release] * @property {string} [name] * @property {string} id * @property {string} imageId * @property {string} instanceType * @property {string} region * @property {string} [publicIp] - * @property {(command: string[]) => Promise} spawn - * @property {(command: string[]) => Promise} spawnSafe + * @property {boolean} [preemptible] + * @property {Record} tags + * @property {(command: string[], options?: import("./utils.mjs").SpawnOptions) => Promise} spawn + * @property {(command: string[], options?: import("./utils.mjs").SpawnOptions) => Promise} spawnSafe * @property {(source: string, destination: string) => Promise} upload + * @property {() => Promise} [rdp] * @property {() => Promise} attach * @property {() => Promise} snapshot * @property {() => Promise} close */ -/** - * @typedef {"linux" | "darwin" | "windows"} Os - * @typedef {"aarch64" | "x64"} Arch - */ - /** * @typedef MachineOptions * @property {Cloud} cloud * @property {Os} os * @property {Arch} arch - * @property {string} distro - * @property {string} [distroVersion] + * @property {Distro} distro + * @property {string} [release] + * @property {string} [name] + * @property {string} [instanceType] * @property {string} [imageId] * @property {string} [imageName] * @property {number} [cpuCount] * @property {number} [memoryGb] * @property {number} [diskSizeGb] - * @property {boolean} [persistent] + * @property {boolean} [preemptible] * @property {boolean} [detached] * @property {Record} [tags] * @property {boolean} [bootstrap] * @property {boolean} [ci] - * @property {SshKey[]} [sshKeys] + * @property {boolean} [rdp] + * @property {SshKey[]} sshKeys */ async function main() { @@ -1111,52 +2451,74 @@ async function main() { "os": { type: "string", default: "linux" }, "arch": { type: "string", default: "x64" }, "distro": { type: "string" }, - "distro-version": { type: "string" }, + "release": { type: "string" }, + "name": { type: "string" }, "instance-type": { type: "string" }, "image-id": { type: "string" }, "image-name": { type: "string" }, "cpu-count": { type: "string" }, "memory-gb": { type: "string" }, "disk-size-gb": { type: "string" }, - "persistent": { type: "boolean" }, + "preemptible": { type: "boolean" }, + "spot": { type: "boolean" }, "detached": { type: "boolean" }, "tag": { type: "string", multiple: true }, "ci": { type: "boolean" }, + "rdp": { type: "boolean" }, + "vnc": { type: "boolean" }, + "authorized-user": { type: "string", multiple: true }, + "authorized-org": { type: "string", multiple: true }, "no-bootstrap": { type: "boolean" }, "buildkite-token": { type: "string" }, "tailscale-authkey": { type: "string" }, }, }); + const sshKeys = getSshKeys(); + if (args["authorized-user"]) { + const userSshKeys = await Promise.all(args["authorized-user"].map(getGithubUserSshKeys)); + sshKeys.push(...userSshKeys.flat()); + } + if (args["authorized-org"]) { + const orgSshKeys = await Promise.all(args["authorized-org"].map(getGithubOrgSshKeys)); + sshKeys.push(...orgSshKeys.flat()); + } + + const tags = { + "robobun": "true", + "robobun2": "true", + "buildkite:token": args["buildkite-token"], + "tailscale:authkey": args["tailscale-authkey"], + ...Object.fromEntries(args["tag"]?.map(tag => tag.split("=")) ?? []), + }; + + const cloud = getCloud(args["cloud"]); + /** @type {MachineOptions} */ const options = { - cloud: getCloud(args["cloud"]), + cloud: args["cloud"], os: parseOs(args["os"]), arch: parseArch(args["arch"]), distro: args["distro"], - distroVersion: args["distro-version"], + release: args["release"], + name: args["name"], instanceType: args["instance-type"], imageId: args["image-id"], imageName: args["image-name"], - tags: { - "robobun": "true", - "robobun2": "true", - "buildkite:token": args["buildkite-token"], - "tailscale:authkey": args["tailscale-authkey"], - ...Object.fromEntries(args["tag"]?.map(tag => tag.split("=")) ?? []), - }, + tags, cpuCount: parseInt(args["cpu-count"]) || undefined, memoryGb: parseInt(args["memory-gb"]) || undefined, diskSizeGb: parseInt(args["disk-size-gb"]) || undefined, - persistent: !!args["persistent"], + preemptible: !!args["preemptible"] || !!args["spot"], detached: !!args["detached"], bootstrap: args["no-bootstrap"] !== true, ci: !!args["ci"], - sshKeys: getSshKeys(), + rdp: !!args["rdp"] || !!args["vnc"], + sshKeys, }; - const { cloud, detached, bootstrap, ci, os, arch, distro, distroVersion } = options; - const name = `${os}-${arch}-${distro}-${distroVersion}`; + const { detached, bootstrap, ci, os, arch, distro, release } = options; + const name = distro ? `${os}-${arch}-${distro}-${release}` : `${os}-${arch}-${release}`; let bootstrapPath, agentPath; if (bootstrap) { @@ -1178,9 +2540,26 @@ async function main() { /** @type {Machine} */ const machine = await startGroup("Creating machine...", async () => { - console.log("Creating machine:", JSON.parse(JSON.stringify(options))); + console.log("Creating machine:"); + console.table({ + "Operating System": os, + "Architecture": arch, + "Distribution": distro ? `${distro} ${release}` : release, + "CI": ci ? "Yes" : "No", + }); + const result = await cloud.createMachine(options); - console.log("Created machine:", result); + const { id, name, imageId, instanceType, region, publicIp } = result; + console.log("Created machine:"); + console.table({ + "ID": id, + "Name": name || "N/A", + "Image ID": imageId, + "Instance Type": instanceType, + "Region": region, + "IP Address": publicIp || "TBD", + }); + return result; }); @@ -1201,7 +2580,38 @@ async function main() { } try { - await startGroup("Connecting...", async () => { + if (options.rdp) { + await startGroup("Connecting with RDP...", async () => { + const { hostname, username, password } = await machine.rdp(); + + console.log("You can now connect with RDP using these credentials:"); + console.table({ + Hostname: hostname, + Username: username, + Password: password, + }); + + const { cloud, id } = machine; + const rdpPath = mkdtemp("rdp-", `${cloud}-${id}.rdp`); + + /** @type {string[]} */ + let command; + if (isMacOS) { + command = [ + "osascript", + "-e", + `'tell application "Microsoft Remote Desktop" to open POSIX file ${JSON.stringify(rdpPath)}'`, + ]; + } + + if (command) { + writeFile(rdpPath, getRdpFile(hostname, username)); + await spawn(command, { detached: true }); + } + }); + } + + await startGroup("Connecting with SSH...", async () => { const command = os === "windows" ? ["cmd", "/c", "ver"] : ["uname", "-a"]; await machine.spawnSafe(command, { stdio: "inherit" }); }); @@ -1226,12 +2636,14 @@ async function main() { if (agentPath) { if (os === "windows") { - // TODO - // const remotePath = "C:\\Windows\\Temp\\agent.mjs"; - // await startGroup("Installing agent...", async () => { - // await machine.upload(agentPath, remotePath); - // await machine.spawnSafe(["node", remotePath, "install"], { stdio: "inherit" }); - // }); + const remotePath = "C:\\buildkite-agent\\agent.mjs"; + await startGroup("Installing agent...", async () => { + await machine.upload(agentPath, remotePath); + if (cloud.name === "docker") { + return; + } + await machine.spawnSafe(["node", remotePath, "install"], { stdio: "inherit" }); + }); } else { const tmpPath = "/tmp/agent.mjs"; const remotePath = "/var/lib/buildkite-agent/agent.mjs"; @@ -1245,6 +2657,9 @@ async function main() { } } await machine.spawnSafe([...command, "cp", tmpPath, remotePath]); + if (cloud.name === "docker") { + return; + } { const { stdout } = await machine.spawn(["node", "-v"]); const version = parseInt(stdout.trim().replace(/^v/, "")); @@ -1262,7 +2677,7 @@ async function main() { if (command === "create-image" || command === "publish-image") { let suffix; if (command === "publish-image") { - suffix = `v${getBootstrapVersion()}`; + suffix = `v${getBootstrapVersion(os)}`; } else if (isCI) { suffix = `build-${getBuildNumber()}`; } else { diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs index 483f918a27..3fa3eaa66a 100755 --- a/scripts/runner.node.mjs +++ b/scripts/runner.node.mjs @@ -17,7 +17,6 @@ import { accessSync, appendFileSync, readdirSync, - rmSync, } from "node:fs"; import { spawn, spawnSync } from "node:child_process"; import { join, basename, dirname, relative, sep } from "node:path"; @@ -27,6 +26,8 @@ import { getBuildUrl, getEnv, getFileUrl, + getLoggedInUserCount, + getShell, getWindowsExitReason, isArm64, isBuildkite, @@ -59,6 +60,10 @@ const { values: options, positionals: filters } = parseArgs({ type: "string", default: undefined, }, + ["build-id"]: { + type: "string", + default: undefined, + }, ["bail"]: { type: "boolean", default: false, @@ -99,32 +104,7 @@ const { values: options, positionals: filters } = parseArgs({ async function runTests() { let execPath; if (options["step"]) { - downloadLoop: for (let i = 0; i < 10; i++) { - execPath = await getExecPathFromBuildKite(options["step"]); - for (let j = 0; j < 10; j++) { - const { error } = spawnSync(execPath, ["--version"], { - encoding: "utf-8", - timeout: spawnTimeout, - env: { - PATH: process.env.PATH, - BUN_DEBUG_QUIET_LOGS: 1, - }, - }); - if (!error) { - break downloadLoop; - } - const { code } = error; - if (code === "EBUSY") { - console.log("Bun appears to be busy, retrying..."); - continue; - } - if (code === "UNKNOWN") { - console.log("Bun appears to be corrupted, downloading again..."); - rmSync(execPath, { force: true }); - continue downloadLoop; - } - } - } + execPath = await getExecPathFromBuildKite(options["step"], options["build-id"]); } else { execPath = getExecPath(options["exec-path"]); } @@ -482,12 +462,14 @@ async function spawnBun(execPath, { args, cwd, timeout, env, stdout, stderr }) { const path = addPath(dirname(execPath), process.env.PATH); const tmpdirPath = mkdtempSync(join(tmpdir(), "buntmp-")); const { username, homedir } = userInfo(); + const shellPath = getShell(); const bunEnv = { ...process.env, PATH: path, TMPDIR: tmpdirPath, USER: username, HOME: homedir, + SHELL: shellPath, FORCE_COLOR: "1", BUN_FEATURE_FLAG_INTERNAL_FOR_TESTING: "1", BUN_DEBUG_QUIET_LOGS: "1", @@ -604,7 +586,7 @@ async function spawnBunTest(execPath, testPath, options = { cwd }) { * @returns {number} */ function getTestTimeout(testPath) { - if (/integration|3rd_party|docker/i.test(testPath)) { + if (/integration|3rd_party|docker|bun-install-registry|v8/i.test(testPath)) { return integrationTimeout; } return testTimeout; @@ -1080,9 +1062,10 @@ function getExecPath(bunExe) { /** * @param {string} target + * @param {string} [buildId] * @returns {Promise} */ -async function getExecPathFromBuildKite(target) { +async function getExecPathFromBuildKite(target, buildId) { if (existsSync(target) || target.includes("/")) { return getExecPath(target); } @@ -1090,23 +1073,27 @@ async function getExecPathFromBuildKite(target) { const releasePath = join(cwd, "release"); mkdirSync(releasePath, { recursive: true }); - const args = ["artifact", "download", "**", releasePath, "--step", target]; - const buildId = process.env["BUILDKITE_ARTIFACT_BUILD_ID"]; - if (buildId) { - args.push("--build", buildId); - } - - await spawnSafe({ - command: "buildkite-agent", - args, - }); - let zipPath; - for (const entry of readdirSync(releasePath, { recursive: true, encoding: "utf-8" })) { - if (/^bun.*\.zip$/i.test(entry) && !entry.includes("-profile.zip")) { - zipPath = join(releasePath, entry); - break; + downloadLoop: for (let i = 0; i < 10; i++) { + const args = ["artifact", "download", "**", releasePath, "--step", target]; + if (buildId) { + args.push("--build", buildId); } + + await spawnSafe({ + command: "buildkite-agent", + args, + }); + + for (const entry of readdirSync(releasePath, { recursive: true, encoding: "utf-8" })) { + if (/^bun.*\.zip$/i.test(entry) && !entry.includes("-profile.zip")) { + zipPath = join(releasePath, entry); + break downloadLoop; + } + } + + console.warn(`Waiting for ${target}.zip to be available...`); + await new Promise(resolve => setTimeout(resolve, i * 1000)); } if (!zipPath) { @@ -1115,13 +1102,15 @@ async function getExecPathFromBuildKite(target) { await unzip(zipPath, releasePath); - for (const entry of readdirSync(releasePath, { recursive: true, encoding: "utf-8" })) { + const releaseFiles = readdirSync(releasePath, { recursive: true, encoding: "utf-8" }); + for (const entry of releaseFiles) { const execPath = join(releasePath, entry); - if (/bun(?:\.exe)?$/i.test(entry) && isExecutable(execPath)) { + if (/bun(?:\.exe)?$/i.test(entry) && statSync(execPath).isFile()) { return execPath; } } + console.warn(`Found ${releaseFiles.length} files in ${releasePath}:`); throw new Error(`Could not find executable from BuildKite: ${releasePath}`); } @@ -1466,8 +1455,39 @@ export async function main() { } printEnvironment(); + + // FIXME: Some DNS tests hang unless we set the DNS server to 8.8.8.8 + // It also appears to hang on 1.1.1.1, which could explain this issue: + // https://github.com/oven-sh/bun/issues/11136 + if (isWindows && isCI) { + await spawn("pwsh", [ + "-Command", + "Set-DnsClientServerAddress -InterfaceAlias 'Ethernet 4' -ServerAddresses ('8.8.8.8','8.8.4.4')", + ]); + } + const results = await runTests(); const ok = results.every(({ ok }) => ok); + + let waitForUser = false; + while (isCI) { + const userCount = getLoggedInUserCount(); + if (!userCount) { + if (waitForUser) { + console.log("No users logged in, exiting runner..."); + } + break; + } + + if (!waitForUser) { + startGroup("Summary"); + console.warn(`Found ${userCount} users logged in, keeping the runner alive until logout...`); + waitForUser = true; + } + + await new Promise(resolve => setTimeout(resolve, 60_000)); + } + process.exit(getExitCode(ok ? "pass" : "fail")); } diff --git a/scripts/utils.mjs b/scripts/utils.mjs index eb22222296..198712a34c 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -6,6 +6,7 @@ import { createHash } from "node:crypto"; import { appendFileSync, chmodSync, + copyFileSync, existsSync, mkdirSync, mkdtempSync, @@ -14,7 +15,7 @@ import { writeFileSync, } from "node:fs"; import { connect } from "node:net"; -import { hostname, tmpdir as nodeTmpdir, userInfo, release } from "node:os"; +import { hostname, tmpdir as nodeTmpdir, homedir as nodeHomedir, userInfo, release } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { normalize as normalizeWindows } from "node:path/win32"; @@ -118,6 +119,8 @@ export function setEnv(name, value) { * @property {string} [cwd] * @property {number} [timeout] * @property {Record} [env] + * @property {boolean | ((error: Error) => boolean)} [throwOnError] + * @property {(error: Error) => boolean} [retryOnError] * @property {string} [stdin] * @property {boolean} [privileged] */ @@ -156,9 +159,6 @@ export function $(strings, ...values) { return result; } -/** @type {string[] | undefined} */ -let priviledgedCommand; - /** * @param {string[]} command * @param {SpawnOptions} options @@ -170,6 +170,9 @@ function parseCommand(command, options) { return command; } +/** @type {string[] | undefined} */ +let priviledgedCommand; + /** * @returns {string[]} */ @@ -203,6 +206,28 @@ function getPrivilegedCommand() { return (priviledgedCommand = []); } +/** @type {boolean | undefined} */ +let privileged; + +/** + * @returns {boolean} + */ +export function isPrivileged() { + if (typeof privileged !== "undefined") { + return privileged; + } + + const command = getPrivilegedCommand(); + if (command.length) { + const { error } = spawnSync(command); + privileged = !error; + } else { + privileged = false; + } + + return privileged; +} + /** * @param {string[]} command * @param {SpawnOptions} options @@ -279,6 +304,24 @@ export async function spawn(command, options = {}) { } } + if (error) { + const retryOnError = options["retryOnError"]; + if (typeof retryOnError === "function") { + if (retryOnError(error)) { + return spawn(command, options); + } + } + + const throwOnError = options["throwOnError"]; + if (typeof throwOnError === "function") { + if (throwOnError(error)) { + throw error; + } + } else if (throwOnError) { + throw error; + } + } + return { exitCode, signalCode, @@ -293,15 +336,8 @@ export async function spawn(command, options = {}) { * @param {SpawnOptions} options * @returns {Promise} */ -export async function spawnSafe(command, options) { - const result = await spawn(command, options); - - const { error } = result; - if (error) { - throw error; - } - - return result; +export async function spawnSafe(command, options = {}) { + return spawn(command, { throwOnError: true, ...options }); } /** @@ -313,11 +349,13 @@ export function spawnSync(command, options = {}) { const [cmd, ...args] = parseCommand(command, options); debugLog("$", cmd, ...args); + const stdin = options["stdin"]; const spawnOptions = { cwd: options["cwd"] ?? process.cwd(), timeout: options["timeout"] ?? undefined, env: options["env"] ?? undefined, - stdio: ["ignore", "pipe", "pipe"], + stdio: [typeof stdin === "undefined" ? "ignore" : "pipe", "pipe", "pipe"], + input: stdin, ...options, }; @@ -362,6 +400,24 @@ export function spawnSync(command, options = {}) { } } + if (error) { + const retryOnError = options["retryOnError"]; + if (typeof retryOnError === "function") { + if (retryOnError(error)) { + return spawn(command, options); + } + } + + const throwOnError = options["throwOnError"]; + if (typeof throwOnError === "function") { + if (throwOnError(error)) { + throw error; + } + } else if (throwOnError) { + throw error; + } + } + return { exitCode, signalCode, @@ -376,15 +432,8 @@ export function spawnSync(command, options = {}) { * @param {SpawnOptions} options * @returns {SpawnResult} */ -export function spawnSyncSafe(command, options) { - const result = spawnSync(command, options); - - const { error } = result; - if (error) { - throw error; - } - - return result; +export function spawnSyncSafe(command, options = {}) { + return spawnSync(command, { throwOnError: true, ...options }); } /** @@ -403,8 +452,8 @@ export function getWindowsExitReason(exitCode) { } /** - * @param {string} url - * @returns {URL} + * @param {string | URL} url + * @returns {URL | undefined} */ export function parseGitUrl(url) { const string = typeof url === "string" ? url : url.toString(); @@ -416,8 +465,20 @@ export function parseGitUrl(url) { if (/^https:\/\/github\.com\//.test(string)) { return new URL(string.slice(19).replace(/\.git$/, ""), githubUrl); } +} - throw new Error(`Unsupported git url: ${string}`); +/** + * @param {string | URL} url + * @returns {string | undefined} + */ +export function parseGitRepository(url) { + const parsed = parseGitUrl(url); + if (parsed) { + const { hostname, pathname } = parsed; + if (hostname == "github.com") { + return pathname.slice(1); + } + } } /** @@ -427,7 +488,7 @@ export function parseGitUrl(url) { export function getRepositoryUrl(cwd) { if (!cwd) { if (isBuildkite) { - const repository = getEnv("BUILDKITE_PULL_REQUEST_REPO", false) || getEnv("BUILDKITE_REPO", false); + const repository = getEnv("BUILDKITE_REPO", false); if (repository) { return parseGitUrl(repository); } @@ -464,9 +525,18 @@ export function getRepository(cwd) { const url = getRepositoryUrl(cwd); if (url) { - const { hostname, pathname } = new URL(url); - if (hostname == "github.com") { - return pathname.slice(1); + return parseGitRepository(url); + } +} + +/** + * @returns {string | undefined} + */ +export function getPullRequestRepository() { + if (isBuildkite) { + const repository = getEnv("BUILDKITE_PULL_REQUEST_REPO", false); + if (repository) { + return parseGitRepository(repository); } } } @@ -561,7 +631,7 @@ export function getBranch(cwd) { /** * @param {string} [cwd] - * @returns {string} + * @returns {string | undefined} */ export function getMainBranch(cwd) { if (!cwd) { @@ -684,7 +754,7 @@ export function isMergeQueue(cwd) { export function getGithubToken() { const cachedToken = getSecret("GITHUB_TOKEN", { required: false }); - if (typeof cachedToken === "string") { + if (typeof cachedToken === "string" || !which("gh")) { return cachedToken || undefined; } @@ -701,6 +771,7 @@ export function getGithubToken() { * @property {string} [body] * @property {Record} [headers] * @property {number} [timeout] + * @property {boolean} [cache] * @property {number} [retries] * @property {boolean} [json] * @property {boolean} [arrayBuffer] @@ -715,6 +786,9 @@ export function getGithubToken() { * @property {any} body */ +/** @type {Record} */ +let cachedResults; + /** * @param {string} url * @param {CurlOptions} [options] @@ -730,6 +804,15 @@ export async function curl(url, options = {}) { let arrayBuffer = options["arrayBuffer"]; let filename = options["filename"]; + let cacheKey; + let cache = options["cache"]; + if (cache) { + cacheKey = `${method} ${href}`; + if (cachedResults?.[cacheKey]) { + return cachedResults[cacheKey]; + } + } + if (typeof headers["Authorization"] === "undefined") { if (hostname === "api.github.com" || hostname === "uploads.github.com") { const githubToken = getGithubToken(); @@ -789,6 +872,11 @@ export async function curl(url, options = {}) { } } + if (cacheKey) { + cachedResults ||= {}; + cachedResults[cacheKey] = { status, statusText, error, body }; + } + return { status, statusText, @@ -813,6 +901,7 @@ export async function curlSafe(url, options) { return body; } +/** @type {Record | undefined} */ let cachedFiles; /** @@ -829,14 +918,13 @@ export function readFile(filename, options = {}) { } } - const relativePath = relative(process.cwd(), absolutePath); - debugLog("$", "cat", relativePath); + debugLog("$", "cat", absolutePath); let content; try { content = readFileSync(absolutePath, "utf-8"); } catch (cause) { - throw new Error(`Read failed: ${relativePath}`, { cause }); + throw new Error(`Read failed: ${absolutePath}`, { cause }); } if (options["cache"]) { @@ -847,22 +935,80 @@ export function readFile(filename, options = {}) { return content; } +/** + * @param {string} path + * @param {number} mode + */ +export function chmod(path, mode) { + debugLog("$", "chmod", path, mode); + chmodSync(path, mode); +} + /** * @param {string} filename * @param {string | Buffer} content * @param {object} [options] * @param {number} [options.mode] */ -export function writeFile(filename, content, options = {}) { - const parent = dirname(filename); - if (!existsSync(parent)) { - mkdirSync(parent, { recursive: true }); - } +export function writeFile(filename, content, options) { + mkdir(dirname(filename)); + debugLog("$", "touch", filename); writeFileSync(filename, content); - if (options["mode"]) { - chmodSync(filename, options["mode"]); + if (options?.mode) { + chmod(filename, options.mode); + } +} + +/** + * @param {string} source + * @param {string} destination + * @param {object} [options] + * @param {number} [options.mode] + */ +export function copyFile(source, destination, options) { + mkdir(dirname(destination)); + + debugLog("$", "cp", source, destination); + copyFileSync(source, destination); + + if (options?.mode) { + chmod(destination, options.mode); + } +} + +/** + * @param {string} path + * @param {object} [options] + * @param {number} [options.mode] + */ +export function mkdir(path, options = {}) { + if (existsSync(path)) { + return; + } + + debugLog("$", "mkdir", path); + mkdirSync(path, { ...options, recursive: true }); +} + +/** + * @param {string} path + */ +export function rm(path) { + let stats; + try { + stats = statSync(path); + } catch { + return; + } + + if (stats?.isDirectory()) { + debugLog("$", "rm", "-rf", path); + rmSync(path, { recursive: true, force: true }); + } else { + debugLog("$", "rm", "-f", path); + rmSync(path, { force: true }); } } @@ -894,10 +1040,16 @@ export function which(command, options = {}) { } } +/** + * @typedef {object} GitRef + * @property {string} [repository] + * @property {string} [commit] + */ + /** * @param {string} [cwd] - * @param {string} [base] - * @param {string} [head] + * @param {string | GitRef} [base] + * @param {string | GitRef} [head] * @returns {Promise} */ export async function getChangedFiles(cwd, base, head) { @@ -905,7 +1057,7 @@ export async function getChangedFiles(cwd, base, head) { head ||= getCommit(cwd); base ||= `${head}^1`; - const url = `https://api.github.com/repos/${repository}/compare/${base}...${head}`; + const url = new URL(`repos/${repository}/compare/${base}...${head}`, getGithubApiUrl()); const { error, body } = await curl(url, { json: true }); if (error) { @@ -997,17 +1149,32 @@ export function getBuildLabel() { } /** + * @returns {boolean | undefined} + */ +export function isBuildManual() { + if (isBuildkite) { + const buildSource = getEnv("BUILDKITE_SOURCE", false); + if (buildSource) { + const buildId = getEnv("BUILDKITE_REBUILT_FROM_BUILD_ID", false); + return buildSource === "ui" && !buildId; + } + } +} + +/** + * @param {string} [os] * @returns {number} */ -export function getBootstrapVersion() { - if (isWindows) { - return 0; // TODO - } - const scriptPath = join(import.meta.dirname, "bootstrap.sh"); +export function getBootstrapVersion(os) { + const scriptPath = join( + import.meta.dirname, + os === "windows" || (!os && isWindows) ? "bootstrap.ps1" : "bootstrap.sh", + ); const scriptContent = readFile(scriptPath, { cache: true }); const match = /# Version: (\d+)/.exec(scriptContent); if (match) { - return parseInt(match[1]); + const [, version] = match; + return parseInt(version); } return 0; } @@ -1046,9 +1213,8 @@ export async function getBuildkiteBuildNumber() { return; } - const { status, error, body } = await curl(`https://api.github.com/repos/${repository}/commits/${commit}/statuses`, { - json: true, - }); + const url = new URL(`repos/${repository}/commits/${commit}/statuses`, getGithubApiUrl()); + const { status, error, body } = await curl(url, { json: true }); if (status === 404) { return; } @@ -1147,7 +1313,7 @@ export async function getLastSuccessfulBuild() { } while (url) { - const { error, body } = await curl(`${url}.json`, { json: true }); + const { error, body } = await curl(`${url}.json`, { json: true, cache: true }); if (error) { return; } @@ -1190,7 +1356,7 @@ export async function uploadArtifact(filename, cwd) { * @returns {string} */ export function stripAnsi(string) { - return string.replace(/\u001b\[\d+m/g, ""); + return string.replace(/\u001b\[[0-9;]*[a-zA-Z]/g, ""); } /** @@ -1250,6 +1416,13 @@ export function escapePowershell(string) { return string.replace(/'/g, "''").replace(/`/g, "``"); } +/** + * @returns {string} + */ +export function homedir() { + return nodeHomedir(); +} + /** * @returns {string} */ @@ -1281,6 +1454,16 @@ export function tmpdir() { return nodeTmpdir(); } +/** + * @param {string} [prefix] + * @param {string} [filename] + * @returns {string} + */ +export function mkdtemp(prefix, filename) { + const tmpPath = mkdtempSync(join(tmpdir(), prefix || "bun-")); + return filename ? join(tmpPath, filename) : tmpPath; +} + /** * @param {string} filename * @param {string} [output] @@ -1297,6 +1480,30 @@ export async function unzip(filename, output) { return destination; } +/** + * @param {string} value + * @returns {boolean | undefined} + */ +export function parseBoolean(value) { + if (/^(true|yes|1|on)$/i.test(value)) { + return true; + } + if (/^(false|no|0|off)$/i.test(value)) { + return false; + } +} + +/** + * @param {string} value + * @returns {number | undefined} + */ +export function parseNumber(value) { + const number = Number(value); + if (!isNaN(number)) { + return number; + } +} + /** * @param {string} string * @returns {"darwin" | "linux" | "windows"} @@ -1343,9 +1550,13 @@ export function getArch() { } /** - * @returns {string} + * @returns {string | undefined} */ export function getKernel() { + if (isWindows) { + return; + } + const kernel = release(); const match = /(\d+)\.(\d+)(?:\.(\d+))?/.exec(kernel); @@ -1496,7 +1707,7 @@ export async function getTargetDownloadUrl(target, release) { return canaryUrl; } - const statusUrl = new URL(`https://api.github.com/repos/oven-sh/bun/commits/${release}/status`).toString(); + const statusUrl = new URL(`repos/oven-sh/bun/commits/${release}/status`, getGithubApiUrl()); const { error, body } = await curl(statusUrl, { json: true }); if (error) { throw new Error(`Failed to fetch commit status: ${release}`, { cause: error }); @@ -1690,9 +1901,10 @@ export function getDistro() { const releasePath = "/etc/os-release"; if (existsSync(releasePath)) { const releaseFile = readFile(releasePath, { cache: true }); - const match = releaseFile.match(/^ID=\"?(.*)\"?/m); + const match = releaseFile.match(/^ID=(.*)/m); if (match) { - return match[1]; + const [, id] = match; + return id.includes('"') ? JSON.parse(id) : id; } } @@ -1735,9 +1947,10 @@ export function getDistroVersion() { const releasePath = "/etc/os-release"; if (existsSync(releasePath)) { const releaseFile = readFile(releasePath, { cache: true }); - const match = releaseFile.match(/^VERSION_ID=\"?(.*)\"?/m); + const match = releaseFile.match(/^VERSION_ID=(.*)/m); if (match) { - return match[1]; + const [, release] = match; + return release.includes('"') ? JSON.parse(release) : release; } } @@ -1755,6 +1968,25 @@ export function getDistroVersion() { } } +/** + * @returns {string | undefined} + */ +export function getShell() { + if (isWindows) { + const pwsh = which(["pwsh", "powershell"]); + if (pwsh) { + return pwsh; + } + } + + const sh = which(["bash", "sh"]); + if (sh) { + return sh; + } + + return getEnv("SHELL", false); +} + /** * @typedef {"aws" | "google"} Cloud */ @@ -1808,11 +2040,6 @@ export async function isAws() { return stdout.includes("Amazon"); } } - - const instanceId = await getCloudMetadata("instance-id", "google"); - if (instanceId) { - return true; - } } if (await checkAws()) { @@ -1846,11 +2073,6 @@ export async function isGoogleCloud() { } } } - - const instanceId = await getCloudMetadata("id", "google"); - if (instanceId) { - return true; - } } if (await detectGoogleCloud()) { @@ -1902,8 +2124,9 @@ export async function getCloudMetadata(name, cloud) { throw new Error(`Unsupported cloud: ${inspect(cloud)}`); } - const { error, body } = await curl(url, { headers, retries: 0 }); + const { error, body } = await curl(url, { headers, retries: 10 }); if (error) { + console.warn("Failed to get cloud metadata:", error); return; } @@ -1918,6 +2141,7 @@ export async function getCloudMetadata(name, cloud) { export function getCloudMetadataTag(tag, cloud) { const metadata = { "aws": `tags/instance/${tag}`, + "google": `labels/${tag.replace(":", "-")}`, }; return getCloudMetadata(metadata, cloud); @@ -1952,6 +2176,7 @@ export async function getBuildMetadata(name) { */ export async function waitForPort(options) { const { hostname, port, retries = 10 } = options; + console.log("Connecting...", `${hostname}:${port}`); let cause; for (let i = 0; i < retries; i++) { @@ -1963,6 +2188,7 @@ export async function waitForPort(options) { const socket = connect({ host: hostname, port }); socket.on("connect", () => { socket.destroy(); + console.log("Connected:", `${hostname}:${port}`); resolve(); }); socket.on("error", error => { @@ -1978,12 +2204,17 @@ export async function waitForPort(options) { } } + console.error("Connection failed:", `${hostname}:${port}`); return cause; } /** * @returns {Promise} */ export async function getCanaryRevision() { + if (isPullRequest() || isFork()) { + return 1; + } + const repository = getRepository() || "oven-sh/bun"; const { error: releaseError, body: release } = await curl( new URL(`repos/${repository}/releases/latest`, getGithubApiUrl()), @@ -2025,6 +2256,269 @@ export function getGithubUrl() { return new URL(getEnv("GITHUB_SERVER_URL", false) || "https://github.com"); } +/** + * @param {string} string + * @returns {string} + */ +export function sha256(string) { + return createHash("sha256").update(Buffer.from(string)).digest("hex"); +} + +/** + * @param {string} [level] + * @returns {"info" | "warning" | "error"} + */ +function parseLevel(level) { + if (/error|fatal|fail/i.test(level)) { + return "error"; + } + if (/warn|caution/i.test(level)) { + return "warning"; + } + return "notice"; +} + +/** + * @typedef {Object} Annotation + * @property {string} title + * @property {string} [content] + * @property {string} [source] + * @property {"notice" | "warning" | "error"} [level] + * @property {string} [url] + * @property {string} [filename] + * @property {number} [line] + * @property {number} [column] + * @property {Record} [metadata] + */ + +/** + * @typedef {Object} AnnotationContext + * @property {string} [cwd] + * @property {string[]} [command] + */ + +/** + * @param {Record} options + * @param {AnnotationContext} [context] + * @returns {Annotation} + */ +export function parseAnnotation(options, context) { + const source = options["source"]; + const level = parseLevel(options["level"]); + const title = options["title"] || (source ? `${source} ${level}` : level); + const filename = options["filename"]; + const line = parseInt(options["line"]) || undefined; + const column = parseInt(options["column"]) || undefined; + const content = options["content"]; + const lines = Array.isArray(content) ? content : content?.split(/(\r?\n)/) || []; + const metadata = Object.fromEntries( + Object.entries(options["metadata"] || {}).filter(([, value]) => value !== undefined), + ); + + const relevantLines = []; + let lastLine; + for (const line of lines) { + if (!lastLine && !line.trim()) { + continue; + } + lastLine = line.trim(); + relevantLines.push(line); + } + + return { + source, + title, + level, + filename, + line, + column, + content: relevantLines.join("\n"), + metadata, + }; +} + +/** + * @typedef {Object} AnnotationResult + * @property {Annotation[]} annotations + * @property {string} content + * @property {string} preview + */ + +/** + * @param {string} content + * @param {AnnotationOptions} [options] + * @returns {AnnotationResult} + */ +export function parseAnnotations(content, options = {}) { + /** @type {Annotation[]} */ + const annotations = []; + + const originalLines = content.split(/(\r?\n)/); + const lines = []; + + for (let i = 0; i < originalLines.length; i++) { + const originalLine = originalLines[i]; + const line = stripAnsi(originalLine).trim(); + const bufferedLines = [originalLine]; + + /** + * @param {RegExp} pattern + * @param {number} [maxLength] + * @returns {{lines: string[], match: string[] | undefined}} + */ + const readUntil = (pattern, maxLength = 100) => { + let length = 0; + let match; + + while (i + length <= originalLines.length && length < maxLength) { + const originalLine = originalLines[i + length++]; + const line = stripAnsi(originalLine).trim(); + const patternMatch = pattern.exec(line); + if (patternMatch) { + match = patternMatch; + break; + } + } + + const lines = originalLines.slice(i + 1, (i += length)); + bufferedLines.push(...lines); + return { lines, match }; + }; + + // Github Actions + // https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions + const githubAnnotation = line.match(/^::(error|warning|notice|debug)(?: (.*))?::(.*)$/); + if (githubAnnotation) { + const [, level, attributes, content] = githubAnnotation; + const { file, line, col, title } = Object.fromEntries( + attributes?.split(",")?.map(entry => entry.split("=")) || {}, + ); + + const annotation = parseAnnotation({ + level, + filename: file, + line, + column: col, + content: unescapeGitHubAction(title) + unescapeGitHubAction(content), + }); + annotations.push(annotation); + continue; + } + + const githubCommand = line.match(/^::(group|endgroup|add-mask|stop-commands)::$/); + if (githubCommand) { + continue; + } + + // CMake error format + // e.g. CMake Error at /path/to/thing.cmake:123 (message): ... + const cmakeMessage = line.match(/CMake (Error|Warning|Deprecation Warning) at (.*):(\d+)/i); + if (cmakeMessage) { + let [, level, filename, line] = cmakeMessage; + + const { match: callStackMatch } = readUntil(/Call Stack \(most recent call first\)/i); + if (callStackMatch) { + const { match: callFrameMatch } = readUntil(/(CMakeLists\.txt|[^\s]+\.cmake):(\d+)/i, 5); + if (callFrameMatch) { + const [, frame, location] = callFrameMatch; + filename = frame; + line = location; + } + } + + const annotation = parseAnnotation({ + source: "cmake", + level, + filename, + line, + content: bufferedLines, + }); + annotations.push(annotation); + } + + // Zig compiler error + // e.g. /path/to/build.zig:8:19: error: ... + const zigMessage = line.match(/^(.+\.zig):(\d+):(\d+): (error|warning): (.+)$/); + if (zigMessage) { + const [, filename, line, column, level] = zigMessage; + + const { match: callStackMatch } = readUntil(/referenced by:/i); + if (callStackMatch) { + readUntil(/(.+\.zig):(\d+):(\d+)/i, 5); + } + + const annotation = parseAnnotation({ + source: "zig", + level, + filename, + line, + column, + content: bufferedLines, + }); + annotations.push(annotation); + } + + const nodeJsError = line.match(/^file:\/\/(.+\.(?:c|m)js):(\d+)/i); + if (nodeJsError) { + const [, filename, line] = nodeJsError; + + let metadata; + const { match: nodeJsVersionMatch } = readUntil(/^Node\.js v(\d+\.\d+\.\d+)/i); + if (nodeJsVersionMatch) { + const [, version] = nodeJsVersionMatch; + metadata = { + "node-version": version, + }; + } + + const annotation = parseAnnotation({ + source: "node", + level: "error", + filename, + line, + content: bufferedLines, + metadata, + }); + annotations.push(annotation); + } + + const clangError = line.match(/^(.+\.(?:cpp|c|m|h)):(\d+):(\d+): (error|warning): (.+)/i); + if (clangError) { + const [, filename, line, column, level] = clangError; + readUntil(/^\d+ (?:error|warning)s? generated/); + const annotation = parseAnnotation({ + source: "clang", + level, + filename, + line, + column, + content: bufferedLines, + }); + annotations.push(annotation); + } + + const shellMessage = line.match(/(.+\.sh): line (\d+): (.+)/i); + if (shellMessage) { + const [, filename, line] = shellMessage; + const annotation = parseAnnotation({ + source: "shell", + level: "error", + filename, + line, + content: bufferedLines, + }); + annotations.push(annotation); + } + + lines.push(originalLine); + } + + return { + annotations, + content: lines.join("\n"), + }; +} + /** * @param {object} obj * @param {number} indent @@ -2061,7 +2555,12 @@ export function toYaml(obj, indent = 0) { } if ( typeof value === "string" && - (value.includes(":") || value.includes("#") || value.includes("'") || value.includes('"') || value.includes("\n")) + (value.includes(":") || + value.includes("#") || + value.includes("'") || + value.includes('"') || + value.includes("\n") || + value.includes("*")) ) { result += `${spaces}${key}: "${value.replace(/"/g, '\\"')}"\n`; continue; @@ -2071,11 +2570,19 @@ export function toYaml(obj, indent = 0) { return result; } +/** @type {string | undefined} */ +let lastGroup; + /** * @param {string} title * @param {function} [fn] */ export function startGroup(title, fn) { + if (lastGroup && lastGroup !== title) { + lastGroup = title; + endGroup(); + } + if (isGithubAction) { console.log(`::group::${stripAnsi(title)}`); } else if (isBuildkite) { @@ -2099,6 +2606,10 @@ export function startGroup(title, fn) { } export function endGroup() { + if (lastGroup) { + lastGroup = undefined; + } + if (isGithubAction) { console.log("::endgroup::"); } else { @@ -2125,12 +2636,12 @@ export function printEnvironment() { console.log("Username:", getUsername()); console.log("Working Directory:", process.cwd()); console.log("Temporary Directory:", tmpdir()); + if (process.isBun) { + console.log("Bun Version:", Bun.version, Bun.revision); + } else { + console.log("Node Version:", process.version); + } }); - if (isPosix) { - startGroup("ulimit -a", () => { - spawnSync(["ulimit", "-a"], { stdio: ["ignore", "inherit", "inherit"] }); - }); - } if (isCI) { startGroup("Environment", () => { @@ -2138,6 +2649,15 @@ export function printEnvironment() { console.log(`${key}:`, value); } }); + + if (isPosix) { + startGroup("Limits", () => { + const shell = which(["sh", "bash"]); + if (shell) { + spawnSync([shell, "-c", "ulimit -a"], { stdio: "inherit" }); + } + }); + } } startGroup("Repository", () => { @@ -2159,7 +2679,71 @@ export function printEnvironment() { startGroup("CI", () => { console.log("Build ID:", getBuildId()); console.log("Build Label:", getBuildLabel()); - console.log("Build URL:", `${getBuildUrl()}`); + console.log("Build URL:", getBuildUrl()?.toString()); }); } } + +/** + * @returns {number | undefined} + */ +export function getLoggedInUserCount() { + if (isWindows) { + const pwsh = which(["pwsh", "powershell"]); + if (pwsh) { + const { error, stdout } = spawnSync([ + pwsh, + "-Command", + `Get-CimInstance -ClassName Win32_Process -Filter "Name = 'sshd.exe'" | Get-CimAssociatedInstance -Association Win32_SessionProcess | Get-CimAssociatedInstance -Association Win32_LoggedOnUser | Where-Object {$_.Name -ne 'SYSTEM'} | Measure-Object | Select-Object -ExpandProperty Count`, + ]); + if (!error) { + return parseInt(stdout) || undefined; + } + } + } + + const { error, stdout } = spawnSync(["who"]); + if (!error) { + return stdout.split("\n").filter(line => /tty|pts/i.test(line)).length; + } +} + +/** @typedef {keyof typeof emojiMap} Emoji */ + +const emojiMap = { + darwin: ["🍎", "darwin"], + linux: ["🐧", "linux"], + debian: ["🐧", "debian"], + ubuntu: ["🐧", "ubuntu"], + alpine: ["🐧", "alpine"], + aws: ["☁️", "aws"], + amazonlinux: ["🐧", "aws"], + windows: ["🪟", "windows"], + true: ["✅", "white_check_mark"], + false: ["❌", "x"], + debug: ["🐞", "bug"], + assert: ["🔍", "mag"], + release: ["🏆", "trophy"], + gear: ["⚙️", "gear"], + clipboard: ["📋", "clipboard"], + rocket: ["🚀", "rocket"], +}; + +/** + * @param {Emoji} emoji + * @returns {string} + */ +export function getEmoji(emoji) { + const [unicode] = emojiMap[emoji] || []; + return unicode || ""; +} + +/** + * @param {Emoji} emoji + * @returns {string} + * @link https://github.com/buildkite/emojis#emoji-reference + */ +export function getBuildkiteEmoji(emoji) { + const [, name] = emojiMap[emoji] || []; + return name ? `:${name}:` : ""; +} diff --git a/src/bun.js/bindings/InternalModuleRegistry.cpp b/src/bun.js/bindings/InternalModuleRegistry.cpp index fc3407c702..2f5d95c92f 100644 --- a/src/bun.js/bindings/InternalModuleRegistry.cpp +++ b/src/bun.js/bindings/InternalModuleRegistry.cpp @@ -74,7 +74,7 @@ JSC::JSValue generateModule(JSC::JSGlobalObject* globalObject, JSC::VM& vm, cons return result; } -#if BUN_DEBUG +#if BUN_DYNAMIC_JS_LOAD_PATH JSValue initializeInternalModuleFromDisk( JSGlobalObject* globalObject, VM& vm, diff --git a/test/bun.lockb b/test/bun.lockb index 699279fcb23528b9d24e1f7fbf0fc08fe1dd6b90..592e2bd028cbec1c45bc4a767a5aa41dd3122432 100755 GIT binary patch delta 67240 zcmeEvcU%<7+HUv2=(LJr1Wbt8Rm?gl!hjKsm_bDyV1QAQ45(ljjB8k9>$a$ zgUSRU$z(~QV2Uv@$^z`K5rjg5kd{)RC;)vN$Lc^A@N&R{z)r|l6zDhO zjEglyD+m(Ed>}>M13Cd4q7GzaXlzXV0Vr)9=N98oDofD72&Erl3QI7C4-_WDNICF9 zz|z3O$Xpz_0ay~)2Z_nvT>L2oybX?WtZ73aRDuu;bO*lG2|_WT2V%+KOkj0j2n@Rd zuM`%9GQfjCYSkK!alo?RlYkY0&5@qsjp6Z$aj38-I8`LclxQ&}3c_FLf?~ozA?-A} zjcV2drBaIy=4U-}3kj&E1AuP8t7r}td@l!Hxyz$Sv?MLsXz`4UF-8>wCoB1Z6z_q|#7m=D z)X(ct3XR)V=!81xWbZz7XJC>s!59`}vLc2VxCEqv3!?di$H1w;-9R#+0I7h57zFB` zpMkKM$0zgni{)82`D5@X-7#>rB0y?r7Le==M*0FmxR4fWjx~8kgbA?_$Y2CV(_m9v zl0^_4!O6f7b4*0IF);$IPa==|-Gdb{0i#Rt1I=*}p7D`F3OF^uY>5vWU5#~f8l1fWX5N20n z@ss30NB1Vjk&AlYG}IJsL>;X)7*7OJ9V3CN6=PDg=Ri|RZ*UsDAUV^~#jR9LR`W+d zO5GZ@~|dC2~HtQLGFXq%R3GKMF4nk|Enz}5&88o3Odx^e`Ns%>e& zcuah90>!@rR~wIbDqw}YQR`wJ-iWc7_z0sVS`d_mtk|hQ38_q$5;iIbPs+5J+RTk>f&+Q{<5R?*3m|vaSk`kBbb8A1Z_! z!>K2PhG;BxZB35URSEI&F_xqVPk4{;9#fyYT!NXV$288z@;m|tfo!z{l4n+v zA4%@k5`CCD$U>bb2$R66CU1aL!-qhsp&ZK6O#)I&Z$Kx!$Z>vO*3zQ=*wu3qICXne zJR6qJ;3dE#P#Y?JW`8w*>9mO~kQQr7ijI%42*&VmQv#OfVRB{xcmGI?9%TzdrKwsi z;c(Q5MnD=o4TrE}WJ%VMwAe0Sk@b4xReARV*v6C(Kjh%@)t7RC9vR0i3Gm z%Zp7g4veS#4^chRElF5eV}}ZXP${1ekUBA}p^=ga4ZxAEe%haLte_mUkKA-`0-Hm1 z2C@EH3#6Tht$qG-N+EY^1-J}VuYV%zty|!f+maAtPO2{mOOT7Iu_lQXYrDQg7*lAY zCnT8T)at%XW;6a4kY?-&Aa&AiAlaMJiS_DwaB{^ZKs3XeHU|M}m#v_=s6GvTax$&4 zg3uCuNbU9FSSpoqCm>aPHINKu0Lj-AEa-3?HITjtcqt&ciP^$>QUa&+AF(qnjF>cw zB|2dsGISon37MLS z*xh=9r<#nWP^zZzd+LMS(ahRJj;6Tq_y}`clx;Hi8^hv%7|T2~DX}3gYl0BV?R?_8 z9-L~kXdJ8Y&p;ZZuoPU&1;JbJL;?~vjc3&z5}z1B{upbH!`0D}REfuL|B=P>DIn(0A zWIZtmNd4R&NLEr8F)K-y7&8`LApp87^qmU?p#rcrcxm7q@QT3F;55UBgHvBPg3~p> zFL-(2Gazw4&aVKG-kMe!0m_gcSPu9wgKfUg0jWZ}fwWL<1X6Ru<748nIth=LGdF6y zf;rFxAjMmZxJcvBtYm{09vwf>9D%jjWQ@ROJRvbYCI)+P!Fv^zhkH181gHWURS?o1 zWwDj=cVKzwSwL#xB%nKRAjb|ozBcE@fMn+{6ioMeE4Y`S#8|rexyfR2mnP@dvwls8 zwL0BtyF79H@gepN*fHao2_13nI+SXOu zdTU$dC!iD>orEZ|B?!8$%;6HE64dTWi1JKINibR7K&Mh4^7t!2nw`fP{s04>=C~v; z;qw+&ZE)HZ3NOHE9lgVGq+GN_HBph9lxSmpxSbhoxSLg?5|9ch2_)nBIS$yvIIT-m ztq;4H%|ugDBGwzhl$hv=CDQbi>reJFI~Hs+X~;+KV;(USNS#w)zlzqh0hX{h#0$0+ z*R}#*g#l+Iw9WUsFigFR5exT>iI2iw*ccvd659R3(lrAXLHr{ijamJJ43!<6Lsv(| zGuB9h=6;CM zfNg+9f%#A%=?{5DY|JgOaH_}tj=D&`fb3~wz{K&<< zMt9W53Cwzf;b0)O+jcwN;U?1+xt^kXzIK!*}HW<_$T# z=)c{OH)C!ZVH#|VNj3@L@v$-HI5;>4f;!ST)ErL>UDcPY!%G3Fr`|xLc*`r+zbVGp z7)U}AJ+%YY`jhF2z(Pn@4p;p%;{v2BX%OPPkU|FCeuoCdzCw2ut!f_>#uJDtA zlztG9D%h3lSS8f{9jOu3<-_KY@!;yZKGH!{4VZxH^mP=~^`#@n=0NK3&WNX)$0pJP zOhGuSWd`>EOG00rkHs$oryl5&pXFN#qj7y0% zCWS}G#wSJ@ssnq0;|^e1#4iNWKu?9e zV!)JAtO1cOR1wU~9%=wjRXH{QQh!whQco1+xDOerU*7|1;O+w{-FYAxHUp_;qfihn z)T6Mb(tz3uj^`DqR+c%CGtk9~1fR<=Lpy+EAdBM&AQ@5<0-u0WK{*_c@pS1N2LY+e z6QfN=lqF08ryhLi&JZ_BamlfQ5DlH?>HysTS}A2DP7*o;sfGSPs>oIk)&p)8nU6?7 z((Cc~DnKeQ95>k5Q3?%lRiXGwKrtibm`>X9cv8e-flq5U|nK%flSKx%P# zRn{YTo|{DRsqqm6!cxRjmrt+841^^|M$$W)Bx9myq(unVX1taa0qXh${^Zd1X#WseN;OTa!@Ttv zkSdCOF)af_ZFfN1xqiK_sIJSA;M5S?Yl-FcSdTWU&&p}#ROo!{w$$>|p4?ugPkZ)p zv#WbR>CMqiE<5O!{5~gT;efMd+0Q4)>bI<sz3+Xl_0~D~Tx!XZ#g-PR)46Wq5vSpv zjqf~Z)6#mtkblpIUv91M*|_2_l_#$kRvZp&`BU}O1Jj-zZEY^IDEjEksipl}Zd|bb z)0)vUj&BXH_WShcdhG5=+H*6PuIzI7eZRMF`fWb+bMNGddnW~?^xf8{;I%S?eLjs! zShgiQzUg(>9jgx-69=7=2HjYm^Y+}s5o3oOly|H(`a9m({;bMT`K8Xh;o0?xKj!b< zs=$LqvpoIwHPauMyzS=}w@x`F-)XbkA=@+MR@lLisEX}=>(+Nf))MRN`Kt=vtABZ6 zi_{9`BT5C1nKd(4DN zFS7Sp=X|>5->YAAqkhZl)OtEQ^{Tt`@o>+S+(-8(T&q2M&fB=3N0%tDyi(h(Uv6IS zbF=&Xpj3wr-P5zLZ(Cg?yvwphf4mFJ?N_|GuHkBr;Kd)teX0DUPzASwqpQ@Hr~3qG zoHX(kA73#>cJnnzYc+yU7lW;pJqCKo=}-b41fdy}e6q&`FC8aZ`IfI<_d8e*L@7mA zYU;|TeRPtCqagS~(aD)+z0?n^Etp33@YRc1vYVeld@hIh8Ki1jJY-bU!HyIL))Xv1 zJf#%5E(TngRvnUM+mZ%W`@PgVq(^-%`$K;VnkWZY>Sk`QWWp{#;H6uL&>jPh2}0 zp>1_?-D>(;$y=1p`1}Guz4AM2Mn`8&$)>f}AjcdH0 zoZG#b?AG2OZkI#a8>GrEcxi(ajuDyp)Q#FMG~Q2A16 z4;#CMx^)AiZgx;hm>|3LFi01mc|p^VZON%(-VAJ~mwdpeT(~ykW`i|nx!1t;dm1Ds zY)q*iP*#v$>Iz1d9MoD)mfeEYTHZ}3qLNvY`7#n#S)=6NrW~g=1?gr~5XLR+J3f5q{ zI;_%6u#PO1S+Bhb)|~uRs`vw|JL||WFxF9U1!*oA)d7PNqL(gnmR~mZ)Juk%wo;8h zdFk>9EpJP}Sl@%`-h$ECnxN|T$TZqvho|4n?*G0}T8nhj23qqir(b5}F$U?#l;(a+i%piGV-DWNKz*x-$ zqmdJ6K&9hgtS71BntFoJ3UP3QwtBIRoD*))TA}&N8DZYq69~1C%XRdYoYCEUY+$h? zSQF$bN*2W*WjB*SItGo#L|vb>9})CXZFn`noTiXCN_LAhh)3j*NQ0z>hfx=)PA0Vh zW7f%acYw85V`dX+7h z(t0pz3>+5~dCF6wB4Cx!YY-5ZiN^E>qm<|}R4W~fd@ClY^_ALUA*G@)M`H9?o!#OL+9S}Kleb8vu#(Uu zM;}D$#qM$rVq2juo#fm&Z|y9EI;f#rd7;_{n>H*jbQGa>YP#ZBO#{_XpS;krywJ0} zklx>xj_w0kj$?VDVrVhbdLz_EwYNAg^e8V>w}mZs2tuvYbi4CHB368+wYG)CxpI0E zt{>nuQ0U@dy`+QVWBPz)#_FZUVBBM{tS89n$p&$`oP)m?WVgWvO_A1eufe`jAcly# zRgg1{dTA6G`2pN9(@O_J7V@cU-mh@RK41maWwciy`dbbS@|LC|#O4Y4+i$YlP=oFh zG^$tLf+5vv%NBdgkCu8dTFx12KpWgr4AODLpd-=>(Dhp?+>Xta0*XgrT$91Dh@h02 z2q7U-(Q5sUGq!d&$K=AWl4oFk%>~2opciP~TmfTV1Eww3fwoDZ7n|u2U@}e6sbExH z)d1dTx(%nTNXT%56p2*_7HCQ9sMl@+!~SQux5lB9TzZ7BHn5XwZ@9NKA0etA+J}j7 zAB?;HTI^Lu8l>a%BRl@|S)|u@_)C-t21dJ+(bsHjP*>aJ!rYL2!D^8e#iO3+r2~Vbr8&vQ;d*H; z7+sQ`s`$~Pfvu27)rYCk`Iin5YGuds7L2A$=Z+gSq zlI3)0+o08hH9@`=rPn?H!_KU&w^XgCZMQMBF`P~D|v_5O2En?S-{9B^Om(> z%v(HKU~vPZRfLU!N3Xm%KfPoGBQIlqwGa%81(-2VFJ70^C!h~{v-Ik+C0W3zUGRwZ zdhKek`szM5S5BX3klZnq$eN(sa>pou;lhZScFU;O{tSkm8tpAPgxaoVMqIrEz-XqR z217C8V5~m0*j@vpRnZ*%HWHzyijnFP4@AVC`vbkes5}ts|z?2rnH7<_u<($8f#&7FYmv zq*z0Cn`Mwp(Ka734%SN zbLSbfgQ2Lax%Lu5!8CT-z(Io0OZB)z2w}yd34n%_PGogJ_qW$e31BVNI7}~bi=3We z(Eb4p7sQ_4+8P$LMP2CnA;cynI#HJnM(+3ZqWB65T{Lj(YW313v7Tf16bWE7W#EPI zjO}2o=ja;yPR?0q5F5yDiwt6v9J0tD6&j2ic%){Y-5;zA7`y{{H-XWNz(Wp9Pt6e9 zbqfBVYY#?U%c|tOR?#On6gtAzvvi%!kaM72 zhUNu}%!czw))*{azIu({NO^9uuQV7U_g)Oh7TIm3L3}EQtTagFtZW$BjX)$A_VF}T zdg!HjVAN+|*d$y6BO5RrjH_E3EzN+$VD=59)L|@%lTi0cN@p1&g-V2kg@HiAh zAu(7^Uq=sza@HB74^VmM;10a;c(&fkidb=*h|k z-0MJNcIZ~(02nnFU5@MfN3h0V>bi-0qmYdT+8J*&NLdqD47vq-k{e)pq{CCYje6P& zZZb##6IqQ=@=Cof6|62&e68Oq*=@5ydH|g~4hseLK{e#`%?54XN$S!t-dkFLkRM{% zqVgDwYKqO}dN18%Hi95nD))jh|AHH1eRJDt(E9#FPuen!-n!`s{o_h<844N3`0w^| zoIO)+O zaIIbnMpiM=F*&Y-kt?!st3HEGqateOgo06H)n}^GMlh;99xt@hYo5-a2NT*FGvU|j zjbREx21FO2Ij1`eRtqd|*rZQf!N6mSP+ey6xLjh@?*>NA#@!IAG#8AI33j8G!J5jU zo4lnmv+|q@jqMD^9S#;IgHh$Mg09m`zk<=Yq1!RW?Pl8;wsTq;tUGNGrH5S4CmYc% zU4CZUGPYe^07e$zB`x*Z`(U1O#z=3erou{9EsIv!Z9g7ha19Ipe!Wy?4x2ctgNr@n z9Hf{EjaCz8`v_P=wXU~j>7@@~^}*Q9Uz>F1Fqq#)y>2oX<@?%;2cghVVu*3;Sz@kj z&7zA}Fc?ib+y-Dhn+isD)eVVwS`ImAkc!M>J&R2nrfdhWb}YS*UYY~u4+hW1da@o3%A5%8z|=-!eeEw!4_JP14i{?*DNW6*<$T&3r6i#S7ULc>~_SUox6~B zapjJ9Ywsh3y(dBq7ty5#A#tMY_NzfU3QjFRsgZi^2e57wD|O5i@I@T85Ab+yBUrGi zON;SwraV0XOf5m$jZp7A-DL@REZK=ds9&CbBu_83)Lu^pV|ET8lxIgOyG#&5ZMmfs zFr$rWuYj>ORbNhP36++L5P3EF1Zy-%eL2I|Tl*FvY@eohOD$Kh!GsGx^wNQ_YZfgP zXXKpY2FYz@p6^BCIV~9Z2KpbPJvooz@+ut!qaKHsVv!AAg}qiD+XNQOSa&boYWt`N zur7HCw}JJ^W5w18LPQ=L4AwV~o&N_`f31DKsbF7I{sG1dN5xvMV_QG9n%cQwc!sma zTe^b~IWpEOxK^e0%=zFxnD+g^$SKsj327l1H5Ahnop2kBI*qXk8(0UyCE@JBUm^QG~2aIaY zo*PZfQv#-=~ zXI{hL8{;`+y}uWXx2|RJz~f=KhY2h%v6K!*D1MK>TI82F-VBUzswH}!NW6J)@QsPx5dMG!K(Gucvl#K{`7+?aUN?}A$ zsU=`7nH8`HJWh3Wsmd|dVVF+v$yhMT$CjnlU|*N4_h8H`C}o4+Y+Xe68uP*0BOj(= z2fg+oSX+7xtM&OE_T-E`&2$K`%yeX6{I22Hi|({z&ol z>TnK9D=66hVH&xfXL(TKJ1-pwbqtL6)N2=k8Su!d8Oc-%du~wa0^4|Da$yw*p?L>m zxTBxP8K&A?y?9X$d2NvVFWNkYyfFI`K6Hfi@K>;!mjxj)k7a-*O-(F#9^UA^?F}pPq^VZ=&fw9u86~Of<>%A zE^Adm_02&WFmh1!s-}~{$g{ro;Xx?WBj~kcy;SNt8+mm#&_;t{zhuN`0uZ8n*uLX2 z)MYR{vA`OT?}p7c;eXnOVCveS{RttwlZ^D1P9a3yT28$+FL0C9TV>i_U_O+JKK=30 zAUy(ag0#id``U`P*g}nGAZzqecQCeO(#%;1M#}?MJA6LkEf{+W-}pB3Gqx*90Hftu z?Lpl^upVGv+wOLUrN$8AK6(NerN*TvM6W$Taq1^3^54yy@aWxEV3dk2-V-FV}0!JhEBsjgWy^or{}AFm|z|7e{Bo^h$;n*M!GdE|pvd z(OcUS!S*b;4yaYcY*p{U|gJ6IXs)MaRD5Tmlehp@Q4Zi#uZ1OZO?WnK8 zV_$>wAXrV<3HgX z>O>of5G~c}220utMuV;14{9A>xE*>B@KnVBm81`5t#0lp1LLl!_Z&S4&h{>Vw z$i{D&Poo9!`~hGz@33NEHCY12ilQsb9WWYmR0!);nYXr04ehJjfdwEA*9pYQVBuhF zEA@`57~?8ely zUxQUY;zo+8LlF#O!J`PaSA$ZCPt1o=FWd+1|IBV48T0tto{jzob_c9CYXS24ei4L_ zJf?s}<*~oP`sJ}ue188c+YJ_)7w0C5b~cm1y643m1ItS()>G0;p{u5AM4`EoQA+d{ z&nj*%P$Yb~yqT&<^AYTU{OUDU`w9%LM^Y)lQB)U8^RD{f`rnfLM0bhciR@6^Hv+R4k15}$*JSMd2}gLW-M zJcYn5QyF}|y^m^^wmCzmPb%LoKSbLM#z&-@o*8536!eDCi!oi!5}Wb1RHi>C_U_7AU42IIr@#Mq6|rhj11zA{`Xe{hBK=cSARqaI?b&8j>`Bm5AI zc@te>Yr8S$WhJD7krhk^>=Ji_aSz3w`x97z`i8V6KC17dTBqB>*eSF&E3NeU zLG%?XDd|6;hy9csfHVsq-tWZhqRcyBe2!4f%c1v~wHJD6;8*6O*RBD>2ZZqA`2j*? zfz`8d`8>C#`?9YL7WRXYJ$MINE8^4q#85l*XLB%Wk-BG-MuX8#jcrReff>M10*1bD z1@w}fak7~X0XEmMnbJ-G!&V1RX@5b8u0Cu-^qQx_CUYsCBIBC??GUHl@QW5Dy$<@| z2t=A^c%q)H*H){DK2%#4fe@=Ntpm%zC>8EhXX0awN={wO?v|BAY-unN#8f4u9=dh6 zl3q{Ls~<}i@zHn6uHGLzMT3#UxCr=~5+-y_HH}jhQRoa6bF7WtsV^8gz(1y03e^)T z?%4>itu=RnS(>3P|_c5?J4U8Jk9v}$#;sUKVXf*u2EtsDgN6Y$b zFjj(bwwDfsOtQ^uKus0}r$WLkFtuVzrkm(h2+SKPsoU@(KNyXUmvpsQ2J8%i^x9A` zJi5h;wJd}vE0!BL*B!84N@zV?m3%zes)>n+jdU^?ZxA-D`@ovAi`RPu8z4V?y@%e( zzczCPDjGv=tSLLcl5`NM2zBCUcZ1bcAJzSV5L@Z!`c@rZgrE{6`PO?c9mv2_7mUU+KguqRy^%WBPC`8_n_P9nm93^c{gkAG z=m!xq2e;CjiK+XxKM`sUUEOy}zW53T)!RYoyvw116&^!gBanU6Mm_A2d<@rCLB!cn zjs5qK@?(k!gxzqG9qcz$#J&MF!GVCF!5K^hhdmYsM^ov$PQpl>bP(oKGU|)&RN2=g9Rb@y8X5%E>&UHdcw+kofyJ<#H4fk{50FaLV9duAW zHxTKYUxIKLC)qnf1P3A2^C%GW{tA|&ItaZ=`cIO(uQ ziocJO@BvOb?2#&idsH?4F-77ar1ZGoR4e-wNC!|^(nu_d{eU36P^EuFNy$Y;)#wjA z+8!O0Dcz_Yjm7F}JJdob9oeG;NI~rJ)pD^{R}Vs}rF!1Mdd_y zz67-3G9j644J4@z*9mEf>&Ur1lGF)*2!nV$Aq6{g>;lXO9?W?NMdAQj=^JuF1Q!UY z0#TgXV=3q(xc;9*HrD?S45PNZ!c^yx_(Ov>nm2%u8al2$7smY?SOpnZAw8|an|MA# zt>V)}#Qs}3(UUJtJD^g6og8-o$yxW1$WaUYh4X(03n1Tdo{x|Q_$-i`d66#8BwXVO z{uNS&n>^jOP*m0ii`9&j;x11{NZB58ZjbKJ^P`azrK86F4@h5 z&-W9@DJ0?`q=B1Z<-!agWt_#iJyOB5xlTyw<^ajSe6ABxx(v?0g%q_Ae<gQ<{}+${KR~NG9G_r-Z2Zj)e&Jk1DO7I_M+YFK(^BdG1*x5dp;J^*{2@Pf z1yX)D68{U58ucu+kW z@MH~nGD0%fnDcKTnb-4pLQ3C+qc@Niq*g#m*BVIU+ZIR%A*FBcz<;y=0rjv)O3)EH zMRmp>GSr2q`xdG@8y+7Fq+oB3eSws}ACT-CDN;qYFxm;MrY4>+iYFwb0gU0?9?4)F zbaH@XAn_qU>VZ)_Jt0+KH0OC}g+Li6@&uE30zxWq8rP?DosbO8;5d`(_DB_&1)ZW4 z9{aN3qW*&}jA-Sj% z;wf)oo{o^_QVFh?(GaTe41|=RD#vO-imJ(ZEh0Dw zsi4}N+apbl2GFVfjd?mRo{o_6HFa!(ziQwsQo-Ilfe%Lm&-g7Yq?9v>E)r$yNXZl* zP_^W0r|ZhNHU-JnxH_TuUK@N|R}AIf$5u?RW{DZL3u&5GtYkjL90TeL`^cE<7q z-$L4@B_o~+9L#YDPyg>A?II@f{F59}J{g?M4cQ~LUHlw#*5Zxa&cA_dnoW!%99LaNywAQi9|Nbv`F{J%oV zcbMlp0;GbD@_dA3?-=KV#DB9QK%;dENCwXF1ca2}Jm-W|kxQHtQUN)f6H@vsoFgc- z;3v(-wj|)NN3v(_fIpP+|G_I5Sp6U0z?Ay_3P$ZF_9})(k;D;T&Hj&Hq+p{w#uT8I7#et!k?{S^%D#Bk6nneVS) zs3*R^f}vi(E1U1HVCe3{6_;&VLchO)`Th##`zx65uVAno&?WOgM|^82-xT`zsj!BI5s_S1>`S@qhFc%#S&7Vs=iP=B|fQa)zdz*j_Qt(DW2LC_4x` zDn(}kIw@v?Amso-XT?nhbWsKobXATKbW=QL0lF(G1U-~91U;1+vjM@%D1s0rhoF~I z?`J@7Wgu34~Ef`Vt6p zmO%JS!WboRDTIKf5Y{e*Fiv?-!aEW|mqGYZ$yx?s)ezE^oYff7xk|k?fO*P9fHGkX3_M)}0~tz_wa^zR zGYA$cj|dhi{_6mlN;<(}>y$LCZs6087bB)=FJeIH$ylhaC_ODCs*O%-I3qGYJQkz?~2Rc0yRY z6T)HTJqhnf*fU+zE<0$KCYoNU|6Sy7?AUFeOruMt=`skAqMI$>t>gV&*@A;RlsIzI=l{1}wZ>6-AAI^n6 zo87atyXS}FjO@Z6KJK~juEf!@hxV79Hf+)k>n#7;`PP-)em}}|x7{7%uJyBPj@YVI zoC60|IZ&&N!{b>k2Y33MGb5p7f$VxqQV-%*!e;oKz2U$eRixZjrvuWxE8XthZ}7T} zIVTpx^;%uH{`AmJ z6J46sJ@Dq2h@4HSbKT=CE~fccf4%U0b>_qZb8d9_)oIf--wP-1>{;~3+v6Et20OzS z?G5)U(`mcsDYq=whc7-R#qIoiT7mrwf1f^LpYw{Xul02w9B-0)_1)jErxy$S($0I+ z*g@9DtA9F`R->L@*|^Qg-EU`&^tCg5$=>kx`57@$T@U&l>f#l>SX`evWMX&Imp7AF z&4`+I{Mb_MPo7ByR^H1!_V?ry)qDN@CAvrG#5o1_F3vtSGq8T8N0oPWv9o*mt6h8? z$Nfda$7jbpJG9li``flh{kz|6QeyY~)^i?Da9QP=HEO@}f#2gDGh9!$DRE+0+@2bh zSKoj1esz__7b||aqo2>rT3`2XJlYz+F5lD@MH90dEId5Y{c!D>V}ETh{o2gv%Eq?&?F?VBH{7S} z2ytl7nU~M68j>~2YwW%EeX5TyHTlE+Vg9A3JqteD^Xc~f13omHvFm-tE%UP@D~nvb zpS$$*n*Obe9NB%LtfAavJG)ou)4R6A@liv;@^@Uz(T5&gT6`#@d7IxZ9c%n&pK&MB zp0+qMpyKP&i!DCmdd`_-NNCePpy`rvLic3yG(a_3!#mc6TQ{O$1PQ+xk%T`~Rm zA>E$1I;RgFNFCR!?K69y$h9}@Jfwbw7eO~mjQ{!FflFCUeh{Rq8zyWRY`8oj~ROY^j$We|Ep_p+Ens^qn_lcC5Nr{rc7P24}OMt_fYQBP8b->h)?i~F=mN6YrHGknv&!l#leR*G4C zF>K4mAv2Os7CxWVEc*VwVYfPjD8(nduqMC0cyxEx_D&;eSH0P4Y30eeL#wX;a#Xu% z_I|792XnO=mp8VuGknY5aNEv4Db2!$zc{wmch0Cs(=Qmu&3>@{R6gH64sZIdtnwi9 zY2x-Z1^fRfWvpGj!hhL;w;e5m4)~WSUAFv^M+2K*tUcMzE`6BTb~uJEsIu_Y?3>o5 zfBOsx2&unh!#3Ylk4Hzh8?fqfzA9Z`ye?4ac|zj3p-Y~Ouar7-i@D&hf&H&O%)dXX zz@+HjJ6~nhOtLe4$KLR+wqc%~R_JaWPm4YiS7UMC{yx)B{r=kj){Xj~E;(nc*s(W! zK%0uk?#G^f=-gB|Wt~%gz{C@6R_**)XD{lF&q3>PVe8Tq3jcV=a7@WN){p6dG zLD@?>pW4^q~=@4{`WR``wS{Iz_DCP$#)5_ z$G^9J>S?X>WaQ0B@x6cQ6gH_9M#6o@clf)uzUC1P3y`1og(Bh z`L-=M-+ukn@b;@NJDN*&9l5Dfz@opcN9yLwwl4aRX?o>+rG?|9h`wVwt$o|N`-Daz zf2J<%9@uB5o#6-ehLcLSsCO;?^`zX@>$1;{nLIpj`rC^qT813%ct5Q6w0T7y|8*ns zz?@%(yB51Nvx`gLLZLUFe15mAnrT_lyRX_0+}nDSon7lgd&3<=%=O%#o<13~vdWEV zC&O+HPCMl`^Zo_DODhKK$tkqoyt&eYo}MW$t}V>&e)VIwHNG9)y4KrS;BlRS84IQ? zxcr_z)z<+)RM!0=yH8+x_hI#KKBn@zx*KZhuDrM z>lJ?m~2tTOss)tbLY74{7~Iwhe* z*{Ne{8vj_R34L4beztYN?U(lkFC6c*!p`tBd&4(fDxCjPZ(O@oiPiVDYdzu@kE3g~ zo$5Wl-=N2e-|x8GjT&Dh`$`e({f$}&*PT`Va+rT=+MT!}7po3hwP;3a?FymW?CjFF z6KscLLAmVX>?rHT#Vvk%dVZ;CpXB)KmWp%3y3IE=NxV6;_FtO@E%{KcfX8c(CI>?+ zc546l&<~wn4XTbt>2ywtd}!j644E?#C@VOn0-Rs*kCAse|t0@xajX z{d-5$4VhT{amDz7tuIZw*mmHX<7=zt&UJcX%}Kd3@!IF+k2215y;6Uh$>qX`4R(fK z+Sl0CtJ~XQ3ES6<554H$#qsE{Czmp!Uyq#It=Kt-I-M3LU+=PYeeu#W{@7%_zH`6N z+|5tkE@|;w+{{Dy+duU)q@{Mz1dx94rLzhvcB8Wyl@RpNHXllA7csuwq*$;thmo_bZ=H;ua*T(d^SF^{faUJ^)I^Msj?qmLz2WNb>YZczu z8&2pjV1xUGgO#hFSh(uKnT>6JIne)N-^_O{lUuY7IC8qkpbb}o8uYu^`}*w?4_jaR z6tZdRG{2Cj2Mhg5=u+3Fel)!LYFKz{Z`Wn;ntfAd?HJ-Q#Czo7)P=pfj-MUW_d`st zE7k!M&dzCaRLt>x85usf>EJC3KSeZhIQjD^qtoj(w>}TJS!3N|w*!~(YbJT?#9#J? z=NBF6(rdpdXx-lno=o!+-j9Db`>o5m^kD;+MONL~Y=6Ii^A*<{Ue|4QvCaHP39(IF zA6OSlFG3-L!lXiCB+uI#j zVbb3CM`JxUHf>!rpi(pIl?pEsZy1VAJX=hdc{rxUv%hx!G_=Bokf9@H2RXj(FseX- zN%yRoms_ogIUPE`%)}SF+2`5VmhFSR;pa6HXFqXk@#lrBmv1>Y8PfUP!$-3}ru{H+ z{fuGFRz0@lK7ab@>~8~;^UZhcdTf8^Va--Id%mjF9qZV2L6L=j%6@s*etOuxm88z+ zr)SzU99$=@(6^gxhvO-qrtN;WCM{{|(C5J39ogHS=tsA|aIN~7 zV-a(9k1c)pcD3fU2YTt^gSM=DoG-NZ;@TaH)H6>#UgoZRziidx%Nx(J@8Zw)hFhiw zlq<8Kd7)C1E-Z36m{D?6dcP679m@MlkEWFF_2bC&s8)uSZR<$WL*mUntYzBWs~y{; z>EL;v{^;F!m)!VmrB>bTYW%mo;iA{Kc3pPNk}$i~d&icu$DH~xaQ}*36Lu`GdgNs3 z=Lv0lIR*5NEf=bLRd|*r-^W#nJCxeJT{q|69+#syX9%Y|(yukxeC-Rjt3C^N+;iV` zT)w?Ux`{QLJRj%(GXB>qM|!>JGCXEz-7=a5P5$Umx7OvAIh$wxF!XZa;+5V{NL#&j z(3%68%{zsKhBR^Ua&BN}Sp8r)?_RZx5)M-8x!`fo>t{p&eKx$?Z-~!ZxqS4qnp#% zkkUatPd$8Bs9D(RekS=wqmt=it#2oFinzbi^tW^N-ZQ^yUoOA=*5x zHKx}qp7=03z3fIemHhc%~jmp&`LtCm^J9`#N$Q^YS>IeKXD zv;uF0(<{_roA}Z;}=boL&?cVkMD7$g~ym{53sOzsu!~}4r>%qD^*50%Bbs_VN|SCwJN@P zhonuj5B#d>CpuNB$%?N7bf#oCD4F8?n?|Q8R9mTiM^j&E58`yK_Sbq|$Tt3_i5In6 z{4Smo)pLbfPbZ%`?0+W@#ZMMFDfRAX>L|NUpz03z2_Cf$2Gw|>lNwzq9e(8rUvgC2 zZL)+Vn`0t`w#~8^ywL1rjXwPZIVZQoe=Sed%2sRt9z%mKCZ=U8droVrX@de$Cn~?7 zhN25tOtFKhU$l*sk=Hdd)hq)no+-xI7-4#L<~dEjeB!HY@rq`cqZpJu0*!U>@NC1{ z6+QlIeUhVM%#kSrEW4l;Qrm&wlv{gSV?~!_qf54lY&59o7Ob{)@1wTcXY&3yr^%lF zP7|ZnN=bX731$uXn%)WjS-BHI{Y_Wht1D4kIKhC$SEDGhQic3_}VTtiW;NzCc z)$sqwlj;1TTJ@5c1%ln2RKq3kTW58sTES{n)&I&puc|Avn-z4Zy-va3F#7XG4Uss! zM8W<1dhMLwQE@%u(2T$R`}+$!sE-F@F@)Qx-vqPgG-1(I>M?|;p>>nKP({a3u2E&x zpXC9jaE)HlpySjdm1}r+Bb4Ul3&XfbFRRP5bi#11(Hs9tTpPhP`nE3nvK@Z?f@@e+ z==bo{gPtT&BdEd+n56zT9%)oz{M4O#m`3r0^!+#d&Z2tIx5=q?^qpj1CJ7Uu;U76F z{c@ptOy+rMi2Rr&{KPftf#zJBqHO%tp}5af9#tH&H`&Kw`#w6BD4`kGY~M*Q3C)LV zGZgVCOw8maToBcQN6FA=Y+P}E=I=6R^K@>|zHselr8AjOcoh9|f&(`(hihe_P3Ote zxmFI^3}~RjTqP4Gtn(1265Vmm=E*a-RsovAwT0Y-2edG*E#m1aLTksh#XMalXq~vW zglkyMg=gIVmom*NRDt*bBBfmhkt$Oar|lb?D|qH=&@LbyHFG7`sv}Il@Is!J#kCp; z+rGZLifiOuNxb0IT*Ea((9+j+=~%YT5C&C_t$xYXD4c8B0 zJi>Hr08)@#I359V-OWI9*?Kq=DH6vP@@)#!mt-wm%mz|I^7SOHZRZ;KdKkjw_dB?T z3z-nk)6vfeP~OHkyKrq6*Ywc3a&0%)tX>eCa*6$}Zz%_c&L3vx?T*x(gmq5XmI5W9+$pPh4xE0PNT)fN^wuZKxYdKuQ z^;V$g33Ob6hJW7a)er1ii#l54NH z))QJZv|>Qqd9$Af3U70 z58!*Qg(5tFYah7Qm)hSHbGs@c>DG)4_QQFS&v&}5BCS8p<CUx$T#JHM zk!w02<&DN!iE9$q%+QK)tpL{sKr7BQTGZ$+T^NWn1=XjOsSr;XgK#7?T9|AYY7PZK%@1hDAxv2`}Gj%pqf%;5^;KQjcQ7o z1?Lk4Xb~y_jd~^t=Tokg;^~s1{lzsGt__A(klSWgOJK zRB#H;k7`2Jy%e2_^E6Lag{K<^?L0F018u+EF&yDj(3%73cRa}62%Pk#j26J^(6luF zM&ew+4SMo~q#s5Onr5}3(Q=!HlYUf>4*DSv3Xa0L76Dp>ns9A2!o8r;B1Fzz3OEL5 zf69O(2pSbP7N@l<7rXGx;~>&)dOKiOu8l{y5j0wG$h)aGf5eI3_Ei^`o;=+IX!sRh zRSV|nCPF)olaAgz-6Vu};KU+i6>PuxF&V;6oOICdf549L6V9b1;ONV>DG2|mD(rVb zDBV!V@k4sm~MAR*4E; zh_eRQY`>JU2pUaAnnHsSM+P%-(zGLOD9^hXVfyJXnpG)0-4cX9L!((WjHR=x<9m&) z;26#`FGF}S*GBNn%b}%mje3nLxB}-euBCBpB{Zra&7e^{Zx+I|R**J^r(1>aU-XuQ z#Iamljqom<Zg;2zS9rPBwvS>k%dgByA$sHXyv5 z`^O||2{m9N&eL4{i6`6yZ3)6Ou2Z*<&VBzs$Nl84hVgQweu za5%!0cP7`Y*&yWgbjU!;yd5VwAvp^D@CdG{+^qu!gzwK-hd z4Q)6y8m@F`w0`WtxyJUx6Z0Wbp1n9}DpHFU@XY%Vrf#K*F67#NgsEFeqgg-|Jb-f= zj8Fw=7EmF-;Czacj-@;u4P`d(>1B@2tYAOB>A&Wt?7i<$Q3*QY;Fztu;Gh%Tnj?J* z($lYR48b`R=OCO3*&Y`iaARNVl7p9%2^$OgA(HMmdnki0Ik-E~%}-~XU2t|)GA=pv zY}y9Fr>Mp=oX>H-#Q6&6pEzIRe2eohobPbH$N3TG-#EWGDzz^=RJ2OSp96jc=T)57 zaOUE?jPoMSOE}Nt)S&IhfWP7Fi7pAo8G^GH&fYlt;0(pt7v~|Q*@$x!&doTt;G|Ea zug19s=USZqtF`ZdiYf`(W#A5iE((Gu3`jDpVaNj}M9c{l*PIXuisWQQFs(UrSL08HB`@O`V}LM{Us!@yPXeQ18xAPfs?=~I&#Ix%i;tcd9A{0lexe=GP`Qj z3isT)UIhbQ@y7xQKqA0PEnZrM0HHMfs*$BdCp@+R+5+u>I<)DkkykYjJh}j5(Z?nN zlYyzgG=NWgW&*PSTnNlJ^0HrsM_x4WA|V^#g#n-Y^BF)13duKeF)fQnE1(=r&NtFj zw#Q>%^fC6}A_4Z$Vt`nH9Vs|T28nbc->9kudk;K=@Pxq=M0ZkOGja$F!(%6)GtdZV z3^W0n0_+#CAHc5pP+%C43b0$wA3fo3&LjhWfQG-fnM)n78TCyoj~Z`BZ`%d1ts8)F zAi&?m;qU151^NLOk;Y%5{1a$`JYEQQp+b8B8$7oIf`EHy-TS~z;3RMgI0zg9_5s_0 z9e@wu%O6Kujt4eJvw$hUG++SG0q6*H2D$)U0UN*;um>CfN5Bbi23&wjKn0*QU=EZ8 zECDls9ej4^*@6EI7$ELFuL^JBg|DmdM@iQKTky;a?sWhG69L{I;*S$L0|S9nARXur z1Oat{dO!of7ht!x62P1PjtDCP4uBnC53rkh8EN_Km9s0#A1U1j@a@1qfP=tc;0SOO zI0l>m*d09soCVmEOaKyren2D;4a5NKJ@O%NGoU%p0%!@e0{An}-GSbK@Cuvap%zdZ z;HzkL0YAVWz+W5?pAWB!@D#Lm8aM--1&#wJ0G>s7hWHg=Z}A|o3|Im1T*^+{RDkEk z83512BY=@?l1Gu>O(U0{|KR-r@DL~ft^ikoe4rh`UMl-(QvmkOrUC4ajRVF569D$b zP5`lh32+Nd{2G1tE=|8_q)Fq?J0%1B`KLG_9!LP90sg3}58w;b0%`+~ko^I08aNB= z2X+H{fL%Z>pf;E(tI z4s=9)c{^tYFcX*x1Oq*Rra&`*T?BR~uA^=6(`SQxfE_IUvh78H9V`BZ?;L;(Y27U& z>oj(v_>09GkY^LXcN@Y0cAU}?jsOCH20%lAXYX+Ud-Cj7i~&9a?|}EfW8e{R8@L19 z1+D?tfg8Ynw8H^_{h;~m1uVb=e@ux#z{GygS)c$Fe}q;z3;0900H6_25AXx(1CK#- z4(JW^0m6WGKx<$Jl);;)ErDmCeGU`?*MXbBE#MAtmp{RL4-c1tD?k=7510=u0A>N~ z(~LyTKSBB@z*FEBz@L~p1grw!&luzY832D9i@yfNUyQm9Va}rx7Xbe5b#tHv&=2uJ zK>IZO7YMvWyhAw`fKDi5I6~gqs=X8c@jhiA;8)ag955c31n{mOU)VAMc>jvGp(0Q~ z51<)Mzis53Ru^PJN<>gUjVOAJ^`>U z28Mtd(17+X;?xK)BD@4#1$cAdJ#Z2@hj(7v^B3fKJ--NX{55a>%tQxZKQAJ9t$q+# z1Mqr$9l)p3ybfn6c}dNGYrqZQrF1)hm(Mi-cCA$i+aPQUv;^1-o`hDL35*2B0L+IM zk@caWdcX&0l4%3ka&qDsM6LuffJ}hh;Ao%+&<$t|GzDA%51dLf2o-lf**j7z$M@o=ouUFyam__YzKA$8NmN*^X&kS zE%<8!zW0nerVYc(YDmdzxZ!wa$!h{yz!hNCfBk+EbiCl;1;c*e10d9ikQW@hzz78R zV{`j}rNDAv7H}E(T2t9<$eB0`Y77QK0d6C2paxK+I#!zz)Fvsay^Z%3z)D~oFbpW7 zEmAWJc>wqb6bZNn0;~u419co1DMiZ<+;Lcta1>fpiy}$+Q{z*CbYLVf3fK(>djQT; z#Qf{@=ZN2m^c{eSEtuk8yiTl=bi5tUem8I59|jHq2Z0R$pMYcm^8oh8c_@bho$@Rm z7;Pv=$3GZV!cVeZelT*=c1Ao8_i$h|FbX(;*5Y9v2Gm5E{LJT?;nd=zQCiwayz+5h zx{@4?khdZG0!-Ef*;S}s6yDjSvr88FRXSUd;Q%iP*zj?_(Z~~vFb0SRc%i_H1YRt# zBa(`67%-GA4x1k~N$ef5@5i3e0E9jW2O(rr#WWnpG#qlFTo^NEztIpV0gMGqfM0z5>;-sL$KH1y!1EMO>+Az<2DSkk0cODT zMRYUq&NDXCY*NyhhEep+{ED87()4_0HE*1-J}wI@7&G z_yV{C+yJfth&C|7f7kJR65%L-eV4Ah-Z-N*C2uhQ6MeTer5z!Tsx@Q7Ddyi{EaJVJz4QPOifKLfa6 z#@~2;4ZH%l0scbxC-4S9Erc)p8PBCa`xYT<=Oe=RfHmmfA^ZS*0=UETikbar;awQw zAyX%tvI7rhAC;Gj?EpM(f@3uiK>>X&eg*mV;(7GkOIjo3bAg6SBHxiUi5=UnGAV7S@Q)>w-R7ktRkK5 zR^)Fgsa5`zHW*R&D40{cX|O4(x{lW3O%qiDyGmeW<6)ula2UWZl5%5v9OAh8A!zfId%> ztR+uhNOuYmx}uwaadS_V89uHxfsMDThc}vtOk@cb%&W93aOTQC4ONg<^n)z2{|O@I zfPi~k{a%JU&KH{86g3hm+(AJgP#vMQ)}Uj1!JVK%X7mx8f1>{qbQmeH$CLp&pZj3M zRq0o_v9P^&ay1a3DAwT{q_EQ7EZ)^*QsXBPBC7}VW?*7U&wr9CXl&7lmY|ql+o$BG z)mJ}(!VBFMV^E7f{tF=*Qe6a^uAr*`y2Y;-HY{I%*g`$01d3yVi6F2CLEQ;UuX#GX z-3tOw3|2__hC6=3qX(WbW%J2Wgr=(Cb(VgFUGWY;Anag-P*n^}#ZL z{6Vg}T}sxKwYyEiyCzW+MDe1voS;H3cT1Pr7iR6i0*I}V$SRklnE`^IK~VCfPkzT< zWp?Wc6p7>_REn6fRHn=qF*OrPW2X$67vzgh+)s{~^&O@9mQH$uk2FE?Gz_unxb z%7c}4^>oLMRT@_!3*CeV%ZPIa?Tn3Fx$V$$+;&rFgM}2RDj-!^$xE{zV}=L)#nQE7 z%a&`^LbS0bciF$GZCTW^6HRXfey0`td-v%!+dYmS&xitHg0X=_i$GVbL^^{5bcPw_ zlY^yH-rn2bx|M7m3O=1!_p`UzT(I)MFokG>#70+K<-85X!K|R$q@U09e;iQop@)#x zgQYk_x%DM0WAWUL##+K`MALQ-9#Ekrkp*cMqw`=8geJbk#C$SQ`WIk{NDK$ozg zdQ+tes!M5yNt9LI};KKdBoMa>TQZsZcBCsoFkA$O1-3b&+aV zLmvS&-A=N$Mk(JKI&;jP6RjnuA}f`SKE&Ft)$-_b&jq%F6!~Cqk5Pd&`p{r%&h~uibX5ki*>-u0kk%X7WXpP#3dV>k5*sW_%U7h$a1f zHywIARWKXrGDRQat53(vN!Dp(ag1z@5Wsht-3&_x&?LP)qItq5*C2 zt&laK`_gT@(@ngpBIt<&f&|iVkV;y6@R|T#+(p*yUwR|U|LHpL!a(8L5`};OpK=(= zbQ?ft4!ZVBkLJycY7xg?o)@N2tQ;tZ=`PR_d$iy~`pD!j6%mbF9m@FaYT;0;FC*YR zc~pdKYIkZ~5i^oGnv1nPvevYb_bkl@>j~{B4TP#1G_NA~j;9m+){Y+Vn^K28q;o(= zDBh8QcQPEgU_3gYTWxlbyoM@O7BV}<()ZzBoK!EJZm z@2|C2gXfe00(>ZCq7yPJ&7(;4$_bv_1Tu35!91$!41#~~2kaq!va{r^TLyQSnKpmA zF{{Vpl?}j@dnaOxNr8Sv{+Z}_2`*Ayd63)W9dwZ-6?c*2m0;QG)4fV4N$x+It#xIE z)VPq9cqZz1wxw)_!D=!ERz}q_>dBtQgE`5smiWKfF0s##KIo3qxe*|Md$xe-E`W|_ zdiRF&gY%5x<$s}jNT)!cd#@0TlYZ+LY*A;Bo}jd!oYlmlevK`vS;zG$PGnyNymcU8 zqgLVFJCF89eZutwEhq>C_^eG7(%yNPMxW-XJ>%VKK538#iCAL z zD@w}Ig=;qrG!FOCr#z;%H6fu?o~ z*}rspG^yXwWz$m7V(h!2Tv6Kva1~;-Am2l3Ejp2S3xol|{Dp3ctraS8M>6~xEvBY) z@^cl&DZv%9q1>PP7vJW^x9tC>;S}xD6gMoS-%&8+^8VkHK)YNeTWahMt0;4^`~er5 z=#IskQv1pD9))Qhw37SO0;$@~iJY2QHkd-j;Oz`e=E%_$ z=Z#=0KG=m|0lge4S(XsZZLpGdB1yBMgFHPh7*{((J7yLsJWdW$SFPg6?n_LyS4tXDZ zrHU$fQ{~(0i3(ZUWB#k!U6%i?ne(P{4KiOCsx(Rr4X=eJRn#K3X1p|3*feH?)u-#q z5$o?yg&E-9&rLRf9O}TXNoQ@@IorJ`%~*!}G-S7_*;7i*`jiSNR? z5L2+h46^V8BcT#_?cN4Msfi!@vwr8QtSMY&;hf!}wV>6#>Z9x#sNZM!JW5#uiP@J% z2QdtlQ`VSG967e2vl~*_pYVpsr1!|GaRLF`yUa7&;?I=lBi~6ud zVwJ_6_!viSBC}KkN~9dntT9qq%5;-VsI@;lu}~V}4;r}(XnruRl*u86PJzzyd)t^w z@BK0CA5u^Q@Oed12>xSEE53x{?OXjwc|P1TdiCMY4-+1s_V9FAJ%RMB0hHdEoC08K z=Tlq&-kd0l-?a2HK-9ieLufFdzdS!|KYPDVcGAYNSSMmCVWZxb{F&g4vUGLIck5NP zmlF!XLQ*VO9Vn?GblZjU8-i^Ys@4dF%%`5b>~0bxkA>`&nQ_mnnqy`ZJszV)UYDyJ zXiXzzeodB*;UvF~mHX4Ei=C?6Noi~$$k9u18ciXMA*~+`<)8)4ZY%{_?TVLYtUHOD zJ{kx2gjW&VjOw+M9Gh(`Au}@lJUyd#a)FBND!L}hEndNH3GxcN^pLgACA#h$kNJ;! zHj(P;if5LEEbInpJ9jqgKD!P!a=ct?30tt1PBal}P&I|Ty-Cwl>iYvFLmR?&oS{Wb z=SX=?k*k27@Y`?nIUGS2IW&VR=cXu*($tlIo*a>Yf(quPRh-4 z`X$fX=9-~u0QU^=%OWnYNX!$I&>Zc5k#;vnOWqr#ELRS!J+=QMeMEP0clF`^^oq<{ zz$JPQ0(M9;b<@{wJoOWHfY9MEs?lzPsbLGrQ)4|?wqTlFW`58991hPAJx;9p2GCTn zP*tN%ChXeGv#)#+NTD?_TQVb3&+642m^W00Px68qpfY*s2xgALV&i zD_D)Hl-ml0HK%(BG(oAdP(K;|emSxGiuIx#(IU;st2O8Zxw82qvr)KdzMqW2aj{AkTn??KKFeR=H z`bseEYy)Xp(aSclo!{5>2HCg8c20HjZz~0+6)!^Zrm=Y8iuc(+jFw&T=JgWWdX{eH zqPWP=d3y0)`VTyAs@qPgYx#q6XnZ><4x4Co7oq{bx2tSc(UlD{NgE{9P(7orK~RFS z>3^DL2SH`O(-EYZt{W@2*B1I11btqmR_&2EgVNfAbGM1It8Wmo>%g;V2|N}Q=Q5w3 z{DS7sp%Y9VM=E|xBMk@RN%xBss3BzLfpp~Yo13pDEjf?O%4D2Hi$JGIz~%`P%-P&3 z%f$Bp4ySkr7tQOjfzB`^xsc+H6fiZ*8y=9Nb_YlyZ^Nqk(^!7-F|v;n+(oVcZQC(e+MX7)KyF>&*tAvV)}>w-+hi<%>nmy@ z27(2=2X+Ww97v5{c6w(W9cPx@;x>ehIKh7E%F@X0&yhLw1M>)Mf{*Yoqz_!F<`Fy9X zq_K}FwJ$`1%@vJiP>DV=T?29e9lEqfFoI#!ii5dyrZ<9Zl!iB-tKh-2@AGlvu9-AT z7^yGoIZ}B0X6VyCFMb-eYN0e#hPFo6^2M&u7=t20P$+`aBY_=HS2>w z86e+p|CP$7c73Ees=sMsAMh@_O78fZK0kd`()MU&35VV|o`7L>rQIOV1%jX=)MK1{ zHZ1z&YwY_K5kw)y0Vx|QOFi8mj|WLeDmIoypJT$o+M|I;WIwo83?9&u6!h1=~5I z8VEE?*2`Ks{GxDpgF{CKiheCll7&6&tGIah)kZ-WxX&lygsvtxxRbi?eO94TC}um@ zTB25M*Pa>p?2?U687@dsJWS!@yBdqdzs!{GhGQKnC;x|sD!ZJrJXVfkh4=&aQ#q}l zOe5fTD()w|U*TlRKIwl5{{y#|CoI?moNf2jYvB$>^@R#wY?S@eE;Xw^+aB{eiTx&d z43DD>CjUt1`eGh5*(B4g-u$6$PW>7mCH4`83s{co_LBm2$3eh8;(>AJPj>tBv<>X5 z;%;A6Qd*q0Gwm_-qU_;{G9J>-eyHJJbO{_Z$G6C>IltYRSKUs|DG9YJ$Nm?{KN6zI zM{g3Q%!hfCBPCyzit_&GFBz$Hcel$~H9=R~J$$?ZUv7bPVrKb%o`Nm)Vc+%{eT+gk zc!#XKOgQb=^-@%i2$cce~>dd7T!;AjqeHWeDof8Gd_0PdKR8lw&~AAOH81^qN$w(xAn>mZp>?=i zMKQ6M^h)lbrLnN~*C}{7sxPx7=QwF0_9wU$o##H8#g5j!h8g_c>4*?fOlq`Mp0S=h zqghdZM;m@Cyrv$q^5${RUM)*$52a4f6SSg$FSYZ+2%gGsp0p|+GCLfQbEz-2QYA0d zrR#GglSKmP(x^rPWF1byc+*VzLvEKnwv~;y2lvK?xtMq`i@&3ppwkct*vPN`>Efr) zjV^eDKIav>cfyO4g=-gBXXk>GJNM?#e;Am(eEFn$Q!B|(wF5Ccxh=$v9QL!?xb0hN zb9BEIB*hYt$0T14XYcW2v}%34bK|Q>;kAJTTO&)N>3mq0r0YSS_K#b><>fBw$a(RF zpd@HWp)2OZ6m;F`R1#bb3$jl})%zZmn`Hf)2YJ~ok9Z1xqSwV!5D4T|h;>Dbkij6e zfN(QtLhWj&4z>oVa&Ke;WiYQFxNUG#cyjTibIEWh4pH3{I5ek^%iVNyHL1Jv)rn8Q zh}XDiDL6^-DN+U9RZy_L-<+69{=U_l2@0X1;~nIbQUm&?UAXzF22!x4%Jql!0CP-p z(+5Hp;}dd|)*l)9Y+*=E?(jTAq6}*)H2?%vLBO>cUg}D(lnQ?|2Z6GLtVRAHK*f79 zqhQc+FAHDSu1Rv8-B|X3PME8~G@0oNY2^S6g&$T59C6!FNTmB=fr8OrC*_sZ`4sPn z)IB-RP=vB~s>o2{lssoUw`k!Ock0Z1A%m#2%o`Wg*08L3r)A}=?wIbfx|y3i1ccx4 z2i;_XW@nUzL%Z9jDxH7IQyGshG_Fj5tsU(k(0%X5_D4=rs3D0$K!CrTFmw>c^;F8i zn{I)UXg#azPf25wkAp4WctE2pLkf5G8dDb-taYBfUnx*@!;j2JzNCgfap5r-Z^hq8 z6Di7GKtsp_8ykd&}1SE4cNHW8jSj#yTHfxP0ehJZXlYHmfd*OWx^D9@EAj zH0&^3!giGC3I-RbEeLd~OEQ7YdaJbue*Xlovxq>86fMdK+dpP=&g`Ql^(ozG5whZv zMBWf6qB}j|w>VM{ML(ZJF+;(2Zob?H&!2bb)@_xknH6hWsO5L0U>TM^=Xpf@f!oqp zD^)gPN+RYMZAVs3@F@A!!sZXEaJm*w>Uy!kf=XLbAqc*#9~P6#FqB`NIwLSFaf2oe z!y=;xE$7((+SeUG0ldqbM%_~}U8miYja71H#?r-^i|(O39G3IM_@e{%+E{+L8zj;7 zR1~6|(+5(G;Sii|$vwM@xvK!@1SDeV#!sZ4~TbhCbxz&R11f?}L-SQ44DYxw& z(^}^6+iltOj%l0tdXVu^-l$VNmS3m<1lW6i&p{HEOGEL)sRaiIC^Ze4>)w_1=J$F} z?oo3c&M|ROMEE5mC>I2(b99W!chP5lYeY8bsQoW@)gw3MZBfj;2GRrB zAdf9kZE;}M?<>KNo3J*1Vw1NZU5fg04x>LgsyFg~esW}kE2(^DgujB}FdKn+*wIF{~L{pDBSNH0LE9O=La*JCD z))nzo%*6zxh>Hm}_`@R;$3v20ZY6+D+)5CV7}l4)qBTt;dnWov)T1zRChK7Jk5Ksx0gBzzOmS6<$-K7 z8AH;Z0w+TuHDAlQ0)qXUNoy7qBA2qXm9I+|%^1p5kw~XFSMe97<$HqvJ}{xmqWhke)n&R z$4x`sJ5ZEIy{|kTM=mq6Pv|Ku{!y;xuDuccu6M`f`jl$4b1JG400N#D>}@k1t=aBa zQBTl~UV;F+Dm4vslR?K$$;Jh{FI~HzHD6D+lKerS`{jeYh`0W$boSAb=P_@2Dy#ei zq;OWNG8N+5WQqXOd2>H zH}t>11%oamuEE4n$PCa8CoKZa-xjhVnPZ+F;MH?o6LcEBn(pagpwJcO4V@v4`Nmpn z_@^xFozy&f;py7bmdJ)~EIflB9;+9B^8f$Qe%sLb+31{2s1Or@?ke0}b4a*gM~%Gt zgLCVMF(gJ~@l_5Yzd2A%IF(u?G?c(^g*0&v2H9!a&2J59CpOs)6?Yqdt?VW5xd%%s z;S#T164m)w$NDc(826f5_$?bLOrS*@O&UHqVF{sJ<2NTf>lnc|>y-V&88nqZup9&x zK=8uz?v@naMqjN%chZRYdZEPmpN;A)S?r@YF9M zKQDEE2`zoG#=>5CLZ_nDuAGPMTy8aA&Q%d5reXi+D^O zMFesOE9#Pk#N7xtX#5wUIS(TjSET)&6;ZSLq||i*YGqv{KH{3U%K_j_nX(j&lzWAMV{>QPDJz;_s!cS&aH@ z#gEhMAer~FjlVYS80#bEUGeLv3v?3%x>q1z12^fuL(f$@$9Z~!GWZM&Z=#z19N9uz zvSq(MMT@_92i~nfz;<*xo?;rlj}#uCn!^)@y1#SYDY}NZ7xLDcT4$nbSd^DX z`rSWkkNb7tvq4H;G^I18WujOwYP$qdHU%BG=*lB)d>0wa*`P0@6P*B^U4%kVDk^r_ zzF)`X>ltAUX1D~$)w>}KZACmh17PRlDZne{HY~EX47K zm%#{I*&syRN$pZ)!gQQe!4(ku1Yc`qlcpa&Kgm64-tE@6XZsvygTPC4tZEZz$x_rN zm9`_$OaL7Z{8eQ17LkIjXpd4GUC zf5?4={Ia0`D48mD;rM{*M-5HN;-QJ3*0PH* z#&=bPceyigXax!(NmbgNh4z*umIbm=ZKc4K5bx{VL+gryo!roy>Xb{Eyr&87GYU44 z(VV6(!$MR9UdUf#Ir59Q<}`UZj^e&8;5`To74c+38@sBIlD(0pmY0w3N^J68u&>Q3 zP!`xUY=txu7rI%=npD)<4AjmWThG0fVatbNtT32CS*^yt{H#<(h+;$OE2WW!FC9qA zmOP7=v_&EAAq8baC{t&O&Xy{CiMF8(MC&S}+wd%4TduOj2$x^5=TIFn{LckHDzre# z&PjhZwJIEZP@m#W&$1=2w4(272bSCQys1cXOM>! zuHBWw!0z`x)`Ypm#cl2<1_zMB277nf+z$l--GWhX-nT{4Riv;WWm+8WZFX%>wxQ}2 z(?%fb2_D!sPqC@_Zv8dmxhM!zDDN}BMG9{cI;7m1Tc>2#@?Zz6&YL~|L*XdFM=DS5 zv69-yw^&(FP_YHlnQoG$DK=5DBy(4eMbqp_ULKs*S#{`o?avZcu%f(0>1i+|E=;Ij z>vQKxv&L!oX%cKW?)~5w?~%#LoK(^+$UX6(84fUcL62@^5L!jnw|SRyL&}vai?J^s zmVPgpM}@e#{tDi3U{3cGTxY@tB32mqd2&diHX;chayi^IQj^*5Fj81fET*O*54V<& z^S(+GbMQ3cetc7N?s1$4Dc|N1DWWg7h?Cul@Z`m5`PrX+Ii;@LP2Zv**2s)y+3o*$8!USg zB2!XwVt7c*AcO5aW^_%zX@cuwC`kN@Ol;=V;D#`f&5||mQHyg8>ojP&_^sgdYO)dD zHKF<7UYmQA0#82RLhj;Mo=LW+9{ z?so3-ev>Ssrf+z%HYG9ApyY$hS=t7>>w%0fb)nFI+0l~Y+a+5 zuNW!dcZw67=4rKf%9()guCO*&TzPZuVe06C$j{>gjT0N5921fhWw1XaG2xVH!cC;` zGzFt*itn`W1h)S9jqzhkbeESe_^E)mL0Khb%k#>IXTCIv6nR8bhbQ6hB*cUm?4P-H zZAgFwZWYBs8Fl*c&&+6{qomQHwC{5kW&HNnfRb`MjrJ^)%KHj^b-*`HSX8r=JI!_& z<>A`Bs2!#J@1^&-3Zxkk#e~~9R zh$;;ZNf{Cis}Q~T`hpbet>JQIS$7+>a-%p_46|-AYSt7n7QUW|TGqrjSXi#c)9rUw zuQzuN+-9DApmrTnxIuRuIUioo*WDi}kcyisK0H>KtnBlDO@8MZh#waqk0-qelPY=@ zbNUGFM!3oRZfsstCba!>>yPe_yXHC@ z^93eXap3&TuX)+-qFb>%21gWjL-HNWkB4dpi%jANx8qjI*5FO-oRL-~SMZ?*>jbsC z;jR)#`1jilF$(NBk6Q=xwhlX<-@DpZ?S5da$JlpYSQ$J33Ff1)u!nXj5urWj={w^9%mw4<(*H45mGz z>Z_!N?T-wT%bPg7i~mfQ<%#F;&gS{^!jrCr(`;TADCxVy*E<;>yZhpal78n_$ds-1 zBXZy3oqNQcOC5H!iN^h*nh5Fom{yW~;0!+`}I^S;Lcq@!I{?i@? z#~aq)_P$y7JsON@bLKbU_YeEq1+A!ocUIq>^b_ zjZcY>4^IvcPozz&rG_TSgX6=K+y*Amo7K{kxSKGs!{8uYFElU zqBhREdQ6>XNX8nqop`VRg^!!o?aQ;4yR5fsa^E3eqM{@FxZ%d6p2jUUE-aifFH6p8 zUx@z2L-B@~O9MSc(O*6Z?UxuA6PXgD?GqUm85)-qga2Y9BKxAZV2btet?lj^7Cs<3 zE-pGr3r`?9E;KIs3m0u@Tw*v^IZT_75?(5Cur?_?Iy^KvE|GHANgC~!q_0vS+mE8P zo)rH{s#NLUy#G}ZE=%8OQZMOa!nRf*7g(3uyK3z5Cr+(_r@OS#)5 zCp+1b1F`%7Vfm5=3WVoG*Yea(^w~zOHivHdv0Ute@iV>nC{-3orQ^7r6h8~C>{edw zA`)yOz9d8>p(0*b9w>Z0tP~pSFR}^-@*9|hGy}oS1!`kQQB4%^Ri(AU31wdRGgp)z zsjk`q58Qi;)WY=tju9ixlk+q{Ur!E+j*bmU4tGmPi42Xx-*yU*NeWGjj86`a<$Uml zBvL zYqd+TLKA{vgv77jwCw!|HX%vLVsIseCxv48gQu%opU7A@#XjQCbA=}j2#JmiLnVq* zir&dFR9&%=;<=yz?i63L2FiI!9BU7wJo$WU!Way(hwfXnmGdUaKmk5EtfFGsI)%N6n z2EQ1aeg;kEO&+ilu(Mw%qQk-GU+>U_(ltY2%#tF>agS87j{KYq7X@uB;_-|!nHXgX z*9QN^bqa5v7}+>!ke#uJN{6@e+evjLD}!kdF5j@3c3{?eCFQ7{@$6&hPbR0-tN#!D Coh$_a delta 56164 zcmeFad3aRCw#MB%bVD|XfXqfja6|t2&A2>bdva=YIcupZ&1%)>^e{TD59b)z0ql z+5@fbUDfikPDduXemVK3OS<;ja80)!eO~zds>vT8KPopEJ@D|EcmJ|3IHqO$=VK9l zcGg*b@`0x`e6mi+MIt4+#U)CXX3w2o4312RMD~e9%1gVX7f8O%v-~j>sE>S}!&YER z_=aEuFhRK{VCUsKTb2!?Eg$A$P?;}-4Z%a|+Dhi<&+a;t)~4 zl%njMS&=iTqz!yFxIcIwJ!lHv05%7QqO49`$d4A_i}18lRz4lU!3YL|ZNWXMk;r~v z6LMAI+29f2AS!MJZre8!Ndq^48r5|UCxZvTF96$vUC~#5c242k0y^9VUL7gPom-qc zHxl^}6YLk66)As&k*H_=X;mZIP|sRqCkpE61h6%@o#CkAji4$X?r=vlTk#_FNS4p1 z71guVVZeYiO%sDrDQ(MCR9bteT6TIBYy=JlHSA()QR#hYl980p$S&?UZFctb6nNG0 zBQ{We6AFv3&#+|YRkWgcy9K%MCuhc2z{c<;*+tn?X6KegiV&!Rr$9CQ3$_tf!mGix zpek4bs)0+H1exc2khZG%GhP0Z2U;_AVe+I`2_JR@)Ie8&s%IMd^&>fv^8CE~+>TSH zL?$6ng(o`9Jw3Oeq&O1!4*RQuGxBCn&B>lSmC={T;~s2h8<@}RDt}g9!PJh0(;|8B z8bDrg;gp%VIVJxLcGhW`J?0SWPJ=*oD-(3~EIb2$Saztb<#jjP&x7h_>0#D9kAmWh zW=-!nHE(WYnp&Gz6gls3o4+6!6m6TanNqTT?)==GZ2DN(!SYi;^>G{+){!?|PKw*|OI+(iTQjfi96sT21E@J$ z?eJ2E3xo0X+Gc&(*P1G)uwdGh!ugS$>>ODlatMQ!Y116aR7Hh_vx`fncEo!k9~1eM ze|1nmeEp-@VO)$GKXV$d5*||;OWr|maSnqol)Z{K2YMrCFB)eo@ zac*%DyqqJiU`FoTyprOnxkVV3KtIq%(V3u3_M2G0yocBI~^cK2KWFGreQXs6|K_-61Y(i^Rdvrh}l?_XYO6Xp52B{K@A z7Duvka&n6Z&$EKn_1k8RV-{6xG@Vwry5n#f#G#-juL-F96vuxv#TxiMhc5+QgO+8t zU=Bski$Lv#J*ZWDdxwodbvEgXlcw8R#(?VVl$^PxMI|$fKVy2O_udTKo01uG3+GLr z(RF@_owp0~Y_AeCt?{~8+`oE@Zi7JGCDzTIn_U`d60E0t17}+s7Z&6wtVPb4n_Wbp zi8Mkk8~4b!%?+S>4L+~9Ru<;Y*F^CLY3b|_arYaA+qMrk7OWKxsITW zIHh~G)FKDNqt~_k{Q}$2GmKAedUugsLv80+d#wZ&`IF=85|r-KwyZHOqwbBIYpt~t zUZsnRX6Kc3#XBjbZd_Mln@#R7Q?pBT(2Er16@=a0GtaL0ouF3i7Enf73#z__Bdk?d z!OIl`kYSXSUqC|RN;Y&M-Ph`xH%~D(66uZ&HQpl})?Hxvz3}SzweYHN87N;bD#qY^ zTFdtEbwRmFUa_@g)qJb}DY!2=<;BG_ie^zDYoV<;J-EGLTkr1RnTCgV$tts>9|EeA z?@>sNJ_)Mw^U7_b*~O&=IkWOgUJlyr+qP_r(>jx(Nb5-aUN+cqwymwyVQxWA;nci> z>B+@B;vAd5|GCyfOXhZGTZ=>{xq3cx^26cPo6F9#Jw6}Q98D=@FONhrBW+NS;KuW9 zch4xCJ5~OepI5-{SX|Q7<=?i%=DQ^j`Kf>K^S-^uZ(C|jG_PQ4UQS*lGG$)zltRpS z_Jy{+w;7MBFFrkYb_vdY(?zyF*~i%YesuBluCY6@2c%Nw=KDuvTT6@seeRHv0Z zdJl$gj{Fr`k*yEE)aoUJ+mhhpMr|_|AysZZ82W3;+%RDN)wO2Xa_f%`gRI7F%aUv0 z-OFvylOew}d-iPX6G^$k)}B1=ucI#-$x&BY{GI#*;L{x6`6@eO2Z7Q%Lra6t8aIZ? z#`bb7NUs}||3*Sy`>hE+X?$Soe?7k!8q?IvO(E-B&b?u-e7AfDW082 z;EkjsZ-xAh>m!kazysm;2QPqc57vj*3ZD%xTYLwqSHr;r!M7aW+40*z^vlYdkx+pj zSK2$;7F3{G4K@IC3TGD*og%N@YTf9tHP(UVgUTYL7cyBNFPsH@Djk>;_xIKL{QOt^hT{1)w^1io;$m z|3Js5fU4)i+w5J>I4_~aeC>XI;5BWT@z_1qu0@5#CDS=jT#S~wedjtGn393-8hFh^ z(Y$;*8;Pt8Mm23&wj8OCbP1@rOvc+}98Jd6WV}sA`T4Y>*(sW?x+0OEHd==(nqCx| zt7v-1lG38w;t%e#t?mNlklR45&dnB&q=Jrl1tsZ`G&hS#Yu@{X(UC5Ht;lV*o| zOM;AM1Il*YZ!0}yv+YDvP(zD>s{F?XEl${CQE^G#`UJT;F*mnlZYiD4ojbQ9Au{(( zCx87RTTd~^OilURhpk7<0A-w?EtZwfES^%JQ70pAG6HX>g2pH&*ZV)HSk`66ayrf~ zoX%M|J7-32B=b?LcQn|D{N11i-~KU+ODY|YJe-w|`PrJZMvqIsjNnE>Bbf?@K3Y6C zPaT>*yKstdUvcGF=3(4!!cE?Z7jj>PbF$Nrt!N)0qKJTMH&1$P`e%@lXmR z?~YF}6;0tcgQ_^cw0O>J_VoqZY{bd~H8t(G+YX%uFTIl-dcmL;ZOig@SWh1TDhA{f zvcz)PYT%o}9|5YBAD^>&AAtM8Kk}@#&xfGw*b+UpR@dQuFWBy{ahM$D6+3NjE&}OI zS$P==Z6Vy-X3xTY$$(kx z-Ezj9*^$ZNl4e5ZzZ!`ggW_PY6W9jq2!8!aByu$Pgu@jMOTmMX=NA@CFXZ{fsju5u zbKD!YLzjSxIe)e14?$m+{qapK7bP(?__$@;4*lp)L%71XElvY9?&Nj6*E?3eB4w$zPw#$TO7T9h&A z>%}U0N%{&$P;+y$&*(U%aBAs)b0IG)|3AKw|A!ZHyl8b#>z36wa>q~2Jw1E&yxd4m zVgBsAf?1K|$g=0>6)MoR_}Uu0E+|W3)^OxG-&p&WX6MgF6e-bDJMe&St$Ye7OE&}? z(N=l(%+sY8NyeMxP59>TY=QBv!o2APg>$KCT5-qR`6Y9+>jta$Z&}ulTsh^zV0+L9 z6%M|pTRJM>)1g1S*6G)S(lZVhfpY#_hr_^D@I4$J2NaTl@t?*s2Ir8W0 z+WcqWWr6Z~w%oIz%IPT=eTghhrT&Qsej%X_Oe>t*ktc35F7Un36FxvLQyt10)=18T z*Akrus-X#>I<$#~nu0euOg_xW3zi;mcv<&`w$xz`Q}(sP$}T9)&o0TCkzY7>dUio1 zl6<7GjeG@=o1MMFhk&GSYvhHFkmSFISL<1gt^6~We>EsOJH4{V#ZEC<@G2_PFqVKC z_TBq=;e=iS9t2+qYGOx$n$T_zTY#F_swNiS1P_3J43v51QQv;x$`*D27q(PKSfLBT z1gBw#CxNn8e^47q4~OqlPJ8MpW_VBfO8n^4#AIEO*a$r}v=_PXbEmh(;gz6FK6gfLHf=@jhL;7KA7qi+ zaKXI%NaSMVis3W42al3w7Nn7N0X2eK+S!r&?X7QA!AsuG<)a%ml*0v^6J_L(gKd6O zu*rri4?1Q4)SI?nUAk)Ki+LNqI&N2;;Dz2JgLQodrtI~CpZoOoC#HmLmNyT!E=dnc zvl8Bpppu^%K~+}5FRK%Y^rW>|;2j@NeGm@Im78}rr*t;!>U(}_G!p5JBsJJND(;Vf z8w{5cWM#!;E8zMEy?*E!l%A0ATgG@w5^7SDKN{{hxOyt*nINNY!ha9h@nLB+yiP%7 z--I_as3PaKx{*jPavG=?-UmTxzl7hWo^3fAcv*3;O;FV@;aweM^iQPx><0___x7F+ zD*Gp5sq9dFgDw3ty>UUtfJE$K`2HdPc2EhQ#w494K6WOllf%?YqwbFsGw>{!e53z$);Yg_=LE(Jt!TT@ahJY{LBri__-&@7?y}-5TEGA z37OtGLFKT7_g+xN&m)42;Rz(A!xR3qxQ6Os?uN(x`h=%GaCL&LzHx6xkTD|R-4>LN zNchc&_1(#d28;W}V*}yRgI@hJ{c=(xkw-%L3vd&Ig~Kzwql3zk32$yt#n1bLj8O^y z0Ag@oG*g4D*V2PZBv&I*YboLU?g}zSCzzvBeohW5M<@Jyi2|AAdqEas{5;4QlkkoV zO8I$uP&p>yKb#hc92=IW?1;?G(+wV>Y>5s5Zur(|0FK`@H^o{$Ez^Mzivr&T2 z@jPXDGBwSC)V=t3BF4Sm5;Ib|*!wd;Z$0fWsg356Tzb~OpIbP^?l%LImN`6iZs`$Ag$T&ISzi^Os=#*gb&h%6kr@T8Q@UUoKxbDH;;XQ+l@d^J4 zxOAI$S*Y0A_Kqp)*xt5>OS44IgFBwQI$`Ti1*H=b{$6CV0IiIP`zNrssw%5-W{`18 z*gt+ogUVA9{@g=2L6GkyCt+g3YZjDFO!(suwNuVQrQK`b)Kchh{u^+v^WUVW9+vc) z#lz#N=fcUO!u20?e#K8_$HL{m@8NdRqQTYyaldahN72}2aJ_>qqcZ*Hl%f(_W}hSM z7;rv%(ihH7yXJHOoI(apG$0;Z3pXOzGA`5q#mU1t_0R4Qi43<|qvEj_;EtD@`uiPe zyKb#H8qV5^(BxkTr#`S4CdU1z9ak?{JR$BMb5ycb*(ST1Q-BNLto`9q_rS?+)*N5; zN%fC7+Rk8IlXZZXo(ktI$$VZ9r?JJftYiOxOAq_GPe-doD8y?09fwz+68Fl3(y0mW z;h>VA^@6IYiCFJWk;tH6%YaPo`lY!E@0Fl3H{le^B``4dK@%ID$iI#eWT5>yr@ViOouXioo1QZhL842%0u!Qn!R1SiB( zemgFBps2Th93fRrv0w`0MCgn;34aqZTj|!<(o+d%3iX&GFYfgVs^%nOJ?PGeVDFsF z*tw*Jg{hs@sRI*9*{tf+{iKG3dUXh`gTmC%>eQ9hskf?Ahi4`AimFqU)u|NvXKNWk zYCu@u<<+U()v1F{Naju_)i2arTb=s4I&~}?L73}Z7*w9YW&*FN!pf(_{h#1iM4ITL zchJ%h^m3Gb?)il0ve8KsGo-}?m%N_+bm%#Umu3ARp+`*Yyr0~io)Bb>ah zZW#V5aL0*o4YW#Pg9j<OT74j61h2PL*+2w#BV{yVCGcvjQvvANQK_#+h zkWsBlu}E5uvZs=8o%++^)Jp<6k!e*>d3M782$|}ouh*xij<)Ty7AbrQoTkt2Hjl$q z58eM2uB*+E5tR>E1I!R!1t;S@&f95k6wop(aq9|b30 zlXs=3!VoNA77vYk&jpniVS-bvdKkI<95@Y&$sG}oT?^MWJk)Xtxj5lBVm+xcMu5W% zgJXMSxp}$q*!gf6>by+ zWt{tvscGT}mHrMVN2DX1e|rR_%M<>*Y+H)K;(+VnWMdArOXL0raJD7%{pM4WtC74| zKe*vRud+-(Af;LILuXIq*r7+mZD=8!Rt3kQJ?W`1tST|*vqMd_Sp+p~G#*YHD{F*> zzuIwOUhHFOhNtFZa%~i4;SGqVOv()w^y|%Sp!CXwUvHX?gEqe8z`51Sop3$eP{jv- zFDYEzEvd7nMkTyHeP#?Uuovay*Wz300CWL;sk(A~VZ)Pm@ z=h|N2cO&BdX>fhRJQkUEb5MChBK8I{_QnaBvDU>3R*H5bNZF-{9aERW$q8%r$M=wE z?+Bkn_(zvm>)ETuX>fLoS@_3oaCUyQuY43#-I(wW4l-^^cqav=HzoXE=W(g0ChOoM zPv@Erhlf!1MmQ~uy5Z9Q22O2;p(?fK8FpM@pUhV+c=~}v^NW`(NEf4K?E;U^wEsnEi$Nd&0^9Ui{=pbVak4BKG z3pOBq0w-fLvD}zCoyi&vwhYdUO(fMnylvb_O3Trj@hiA=ID!m4IpVC^b8_Hl3=>$S4R$Nh`oWHdN}|5I?PgNjdy$71CQ*1Aq6NNI)Rd#vIB&URWW z^;J0AHI-?!nBfI`hh+MrNOh-7cxv#^gUbkOW;^#Eg41qH^vaFF$p60zYIh9P2Urhf^kKIGcq@*13+g5xHkro}~eBH@_eyWx5VdnfXs<^Dvh)5UuJ zwq`p@R>yO2@nM zHJp0S!w#OTy}3O2IjeW9_2oFWwg5kmRDx`-7k%TY_rV*Ia}oMwy= z&32f1Ws>7W7dr=Ttj>u3E+?-WEIyAIdzD>B_P}}xoNB-`h=i}ebquyF&h*oaHCtG- zR~TeG#-j`;Bj`UC_ZwVo*GlNqUjLv9jYY_`Vr<=;;JSx>^{$TlpTKp6vzNg53hOGY z?Wyt9GvQRO#(o=+XcC#+qPQQuCK;~SxKangX`Pwn@73uX=y}p2B&s_+lz5K>rB5dO z-;v3%oZT>buWRjLi1{G=T>zIwUU(|=cfs`vIXPb2>#T)w%B;9|N>KF_kD8r~x){kb zaO#`gv#Q`!mmTwQE0a#CZM-1JcqS3MaFq^oy`IU8y+Vp3D5*nk&`v|jD-ANXCH(v0 zH3C}Ai^o2JJ5jlQ@74Ui8o71y)o>?=@+wlC{+#@@8(j(taLXc9M&2gR^65b*mzYTAN8qwr8PEh{tY& z>l$py%8czH#S!YVOuzdYJEKH;j``=n*-cBKVrx+Ke8Tr{tM$jAvGJ zS9d+T)BZb1U4$q?VtuHWHay%}s-w zT+KcH2d@2HwaYDntI>P|uDWHv``z~F7q%C>5RNA~_hkAnk&-VHtnjR+_gLQ}JaNt# z0Vkgb!>@lSoJPq?#e-jjlf^98c%8Kl9?7mW5Kgg)2Q1k3A-LgTEh@LqdfNj1z;XlO z6b_juR^jDv^6Q#a_$ZQbNJ8)PTit7Wz^b99p>XQC-JQ>dI}Xks6*j~5gkz^-?tX$( zym6){z>CmJPp=jmeh1sp+XO%ECxok8cgCb|?IO?@J!S z)R~cRGLALexo~n8mLxe`os+Oa;Hbh6Ra^=wjoLw%ipo^+gD!u2cc!0f%O51b4Q zHd1bI*cyIp&gc<@fIy4?jeSj{7MSGL7L(@t0bjR5_xEoGJ2z!wFp3AG5%|jow z^S~ox=6)<(wYI+$ZU|+0R>S&A-)s$Tuf!L?$xAQ+PYPayv!;`ew%B4h<{fT^<2ZwG zDq9PuzOsepq^H8jR?Hy{9QIHoJO>btIZKX*vxD-kOHYM4E>x8Feg>zuIR@~E=-8n0 zeRi3L?M7`cH4nkb*YGZS5Pc-Mf^-S&11Hb3XQCpPS1ox~Io;)~c<(vg~eCQJok?M_p=($Ahfkr>rKw#>7@TsyTY) z*L&K|jeR^Y7EV2*&gaupVfxxiFzTx=EA)E5`7_pVtSc&+1gBy)NL>q86Tm)(v+g0y zgSRD3rF+N4a6>7_fJVk+yWj?gzoG58-PT8Rpv){dTZ6XyTO7wm^h0{8V;C8SuGY^| zd3YNfMoNCfI%B~uh0}ty$JX<9Bpa1u?}Qr|<|(kWe$I9eyV1R2a0+8I#r1AwQ1w;9 zZ~DBQ13P!q;rdt&p0urj>ly4dnX>5hVv?rgV5g;lZ08lzx-&AA7-$gz++xy^f>v z*mu=Vo5yhrZa17PYNxv8i#9K`uYQ~UZNguUtScJGSVoV*X|C)q+v>fPTyWZ+2Lx5$ zB~s5umW4*ObNi1W>4$^^Aqy(?AGVAQ1q0z^7Ha1}a2Z@8dka@ z#dNq#TOU{aiyX(YTN(GB3`&1Y_+4H}dX46F&8xgwhQ8f`(suFUZ8f(HuBe*Z3s+Fh z&Dvd?+x-XbuGb=w8ECT2^2oIL>vm_bN7!9(Q^;fUq<=%+u#dj2_1D0SA&=lfAp9Aw z7hHqz_aE^$t;2*HUFtb-oyn`Q;|3(M33D|+?$>=Qx#+ikpB{zb{4_PwznYYaai+(! zo`#c0VqP{P$AsHJ?2xxzn24Q4ieFmhW%>`3l4aY3g@2Yj$;Af0qu=PErME^!6;gXCQRJ` zHx905@c#Q&nMveEJs(cm>_Zddu`S9A7xa&$x+Ayovd2HImW}IWa5A5La`P;l7CXj% zBR%>-xS;!G`p1##9V*G&=ffQf$Ki5)dMb?C4A1|u@8A-_!VUbQXOA5^)3P`oD}}Sy z&3j1o3-uS@7mr0g3}=WN;s{dxk)zKg`AWFZ<-LEHs;HOn8+>GK7j7!Cv2eV8Ge6T? zVJc%}zlK0joW*o|Jl5c2)w=M4o~a}fw$hc{9pSVU(NzMNxmB<7dAlk^J>*OHOb1F zr1u5(aQ&)DE~`nt6ehQ%QrDMQ!>UfINv^I*eh?g+lARL8Ln`o+B~O-4hH z*N99h;2-j}TUN$v)|NIuO2zT28 zQi{>x^78M7(=3NK!`OFl#{_!^XZi((V8P(5W537+u&k;qXnZWr&ol$20tXvH#%o$!VM(PZgc0r<-mn^Fnh8p_TAgsNg;PHXW9#DHJEpQN^>u5IY+3twG298Lgcm-4Q*AC* z1vfDKy++nPwhRG}h`boC2W9FSZ+|cDZ-P~q>?;4laa730dps`zD=*|y7r^y}tKLy! z4@nZ9;+paru(iCmEIkU79&8zz>0e1oZLw^4n*I+s?F{yCw;wP1+RcE&oj2T69qg46 zHA)Zh%6yNvgspw`P<1d|_@v!C*OVTLZ67w3hk9kPR9@xHGFuMy&h!V7RAr2!G#)z- zuD988IN^@BfTx+ghq2<~eAnSLn=~XR+vN8oC)(twW-i$?N@{`?)@xp!%qDrVl~$5G zF-&^>O=$<_WesopssmgA`4!4FybCPOUXEV*1GitR>O5v+INb5onwP=JLN?4iSIsFP z?bABxNt(sca6_xhT@R;PaD2{(@4z|lm4JUguNaT8_0ncNzC{u)jnI#V!)4jpzT<>a!?7j50LLps{2sc& zfwl%~i3xDdr3oghYB*~6C!FfTI~eVuZ7oOdu;(;5jfiD3cb*vMVFsgitGsr|I+K zv6k(zTsWQ+N!ea%~DIXG%6zC)|d}`yq@VQVc zXd$GJur73t7(PPJbUfbU57h8EPe1qwRqYZ;?OzD#BTR)Zhm?N>6osyaV$e!RAED%{ zA&vh=NFSk^zYS9UT1fTW%@@OfF!5hd3hN-@yUrDcza23*_VxBNk5Ba) z=Skt8E~7T8zz2|UkIS!(>Ii51F#lthFO>c#kn%ZHhaaJ5hUI!KJBa7d8-9d7^pj=( zfSy^1dK2~TS4g$)m72K=u@^Bb`+4zj7E_ePN2oSAhlC$tJ5q-_`Qc7p8znlz$%Sg| zD2JV#yf#X7jFW@Mhtt{36*$&q2$hUGUa0k$aC~j_%*y^=OS87Wmsl2$wHZa-_pipqBPYjz3wMe1r|a9LIy!L#H`GZES(! zSx)}{9@?O*yOxiiUKf`^<@9665rDs4@aIKRIrFWg<{|uFNJwH_LMo_M@29&(V z?M`qvsEY29)S;5=9bX&S8;5zvn9ccKY8Qp=hn&v;1U<83m}1W`V$Xh*ebJQ_s{NNh zb@Wvy7b>~i@&6l?;ohLU_WBP%wX-KhUktU0uz`GVioGdpfvWH~Cs6C+J%=fv6r*bWUr+O=?m3?#g4CyDtCd)U+VIgYM)Spm$-~eU4~Eujxj{hIv1l0ed0{=x{9e&<5SQ}Y%V;G;V?r*tlq5Au_ z`4{M^o-dt#ZB%_^<3V-j6sIRt{zNCA?Bqh}PX#rs=?-VQ{69e(A!j>g%D=+p3$@}`gR;QQPX1@8amaV0Oi%2%?=-O^$8_^#PLFPcB{ijqY<WT3m0s z0&hFHkTd9boSxjsdrmA=`|msar;`iS-X16a(8-08f9&L+IC*VsOTHflv_GO$8^#D4 z@{#>f(5~4I4DWTI6uN-gyas^!{NJN<`v3ci|7(XeurUloo-oc0Xk1w`0X8Bd*YUMc zqM7_q!F-or8)d0E$Tf9yK}|`C)B7`2J!iQ59!jV{sT2H9(5ra!Oz+D&WvZw|G2&lP z9hrw*`SV@=zoN<01>v0#G9x(ztRtPo5zUkfIgLAL{w&1%7^rV22(-I3DJ zOuBTU!(B$#Dd-;LmD2^QXsAm2PA!Tfs=bi4;ap((uw!4c~ayeYvP(_tNgT@ox?hoIQ( zmf(W*2zsqYFwZPqk0AYC1fNTAhRL`W!ABCTyca>K`B;LhHy{|c0l}GO#RdceHzL?8 z!9p`=BZ8kKxN9SVGE*hNZTBIVbRUAn=Jxv#oV*D^!%YazG2=HONUcDyMS}B;UxDC$ z3GyotEHRZ5%(x#xoBI(gHF@_VXjO?|hXfay)|CjJmSABef@S6z3FbeLvX43Xfs~=% zrTStN_IMD%8xkzHf?W?%WZ8oh3C!*X(YRnUf?k^uTxpi7Nct9ve72b)#$;?k@KLp3 zh4~l(-zLj?2)Nd)0L;LLsbKFzRIt(vdKmfjX0>3IsS?~^Mm++oHn$6IG~QO=CNmx| zsgF{0%T|isV*E!D+%G}?qX^cRN(p8>hM>)32yQodk0EIFID#D#+-X`rj^OFXQ}#8B z9#0wU-DRGUVEz+mbbJDhdravQ2#$CX!J87SHyxftuuFnvPa@c0c1v)16S?LvxpG%oh zr{bKW>NNDsJN4?E!=FQnw)=+DC9lPlH|1A+{Cd4yM~LMAY2Ap|8TPGM-mFfUCtdT- z2Pvts?Tz_0v049JN}8YiucR-pXtF0|dY$IU{}MT8c3vm_SBk3Z$%9Nh&bOCeKcWnm)ny_MrOo z_O{ENJzh##bJet9Cu0Og?Z!dPpLHw^DQiyvMzr0uHtY$Sv`a9*k zXYOZC5VmTXQ=N;@M;b(ZQx|%!L2q)81$ktX?R{c`4UPo!;6!pkb z9nq_I?VW72lc^&eoNSDfsUvI<;paprBeLlGv*Bl~lj*w9&B?}vvQr7G2>2d!_|Y>_ z^-iy#^s=Ie-g@R=_@WTsdk#ORg@x^C6Hb=xWKH0+oNS6|S5>EJuN;@v6mh2N=QGvG znvw45WVuczgY|Nw;owzge z**Q+e`6hg2TW)%;lgVArBdw26$y_3^10sWY26toTjgY0cUL>v4Nj(Yw?f~p)Mqt9{^=X_ zk!zgd%?RWReV}Wd>{eGe3)yu}w#Mllk8Gur-R@*3AiLhl?sR4QBD=xK*6QmFO80|q zbi%uwVt-_}IN9A!HUQbJPIeD6{^<|2L~e7kd!1|$vbs)oA2JPOFqDc+k*EThvi1La zNA`7!_dDTGWR09m-xE+$Z|CvXAHu>9IN5My9HYWd@_Pg$;95GFzDuB_-rVD_E`*;g z$kdrpPyrO$|6!*%8UcTFCHy?%WMklDT79;)$J zh)gfI9|k__WRpnmaLagylTFt6Z$YS_@thN$N;>)Y_IYIdi<}1O)pdPdbb8sO$0Ac8 zdCAG9kUrVz{lm#}khONQmyyZ*Q=tPTll@Clg=n9sLP?+nsEhkDrmJA}%g4?XV`e{hPW z$X=r1EbvDsTR{3bWXFR)IoX+{^^M09z$#?&+q0mRuD;)#-a=%LQ&w^0cVzW7|BE1f zds-i#IaE^ewImc=wE0L@4viYI*@dKU3MGv57$>|4!8cYI>Fi_|!z=PB2z7C?Wu(<%$+|k(C8QPk^y!97R=5<> zXceq_IK9hA&mzt!FdZApu=8@5UNDMGai@4W=}giJOzBP*kX}Jr9|aeU?h0rX=~F>{ zgGRC|p&Oh`bx3vucA&pd@gp;ixt*OwbkCRD32fN%4*KFv?UvE^~xeQ!>HF1>c7z>8My^|0Fv{Jbh2AX zKMKitMmgCU(!W6RozYHqoAM#~ghICJyB!+sWG6b=9q>2m{HOU+9GBvq&}7n@uW?Sc zmh`El^*IUDK<|RGoou|*yBpaQ(keT_$?hSYdi zlHCivuM&LJ0m(K%%bn~rC)1`q{%cxViPBx$dMy8Q) zniFabUqXdyY`UvZYxpxrpP5cCRofY#jeF`;q}7Xd^qT0~3a3M7Kt)htMRsbmp2sg9 za{Xv#Lw#-IL})BDz})3W+cq3YaulTh7qyGo?MF{K?lY>_HQ@{BE9h(J8|Yi;d*}z~ zN9ZT$XJ{|<8x)C|nf0UXyY3>r8+r|T9eM+L6?z$Z1$qhc=()Z|^b9mE#XM9$dSLU% zNbCDf>!Ew04bbi8>-y2O*y$wanMq zvlXMirk!sfZ8G0O+E}!aXybU_tZooJ%6q}=phO$(_}byMqieU;(L?7H9ZMcDv3;W5 zn&|NM5M-dMp=->beWF7qEheeUiY_O*jGPCZ4=sW6Azkrw;g}2QK5-hP`-1KVqoFa- ziO^VR7&IIj0cAjkn6GKMLwl0v<9`=I7ekjomqM38mqP)hn}9LH8b-Ud(Xm;_7 zN<0{92{nWoK~13jpnV{HA@)~jFQns$jvqhk{-YD$JCObW!(C7X>HDF(psSz@p^G5h zU)w>aLuWx{P#!cAIu6Q&640?w96A`%8`EvUHqe1k8gu}pgWId*y>=QubUJ$k+6p}i zJq|qqJq0}tJp*lno`rNkdjZ-B>2Os9&4H#vGoV?}Y)I#+d!YX2^TyGZWfMs1n51J- zKPV2RLmAL$mBdWTVdjLBbw^g?_e=y)g->IwCNK1Sz7NSn7dW}O5^KsqBm$jGmN20@>p`#JOl zq(2Yx7W6js9;5@nKcQElT~OJL{J0ug0bK*>54Gq-u$WQprDLB!pF(d#@6yme=-|tc zUdkT;4TQ9*>s&Po(&6kYWS2takT&^O(ccB_hF*jAK(9k@#v&<^FHFPc(Uzk|z*kXL z&*a8ZZXvjf!EA;Kpls+o`m_YP0J;d$1El3pJxGs2^h6_<_D+HNn|aNn@vle^>kj|@nq2D2ohNIB_=)Mmh1@-r(UV(N) zx{v-0?SNi{*L6~Vk4jfaT^aSKfpVcS(4!virTgI%&>fJjfOkRqyJWihsh1~0`a5Qw zpze_VwwZ2hM?yNeMZjUeaA**uW1!0E;?oO@WJ14UIpqy?c`qQl6}rBRAFH4npjpr< z(0J$ss2|i3>I@wU9Rf9nnnG%zHl2zU#OnyT9@>}DHH9>V9iWD_EL+>EW&dhcU0!~H zbYXb~dYei`8%S3`_d^drTcGQq|LJr+NF5c>G-x_B1Iptsc64hM9aMHZJ=XnTAwrGf zDClUYBP6re=69gC1=4-xQAqcZq-jTy*5Nv7=A?zzU@^&V1SdmNAWcg*=t#xbq`73O z6iijene$!vTcBH^C6JE&N!_Gr)tSejU!i12gfpmID1T7nQ{j^xOm>VvM#Ra#J7I>z zV(4s0%E^jq%Rfi{&K`Qk=aGonzh1O`mmRQr;-;r~PeB`@_0Y}GHP8x3o4{1)BxqG#(PA~J4cV{pfkrbNXOZ8A*~BmXt-jJMqO$Bri)*vbdHvl z>GXRxq|>mn^FbX?W5O)9q?Ry5HMtUWrk;)RX!e!v4khUIe$l90#msT0+ zUIj@{>8qj9SnH-bW^mVN`*!Q#AB4XJq?yQC@E+)H=q~6^v#cxUElsD?lzlt20lF9Z z*W%K768YhvHZX0|PryF{>D=%rqzGOK-48tk=@z6-rUKF~dLN|n(sz1ChP&eB2&J^i zDM(3_j7q0umy%skEe|Q%)vmN^e+>TdI_Al4(ZlM0 z6=99>Ip}t?phxsb^H7gyi?ntK8Z*=j9=!F%R%h%yI@Rlv-l=n^E?v0)e%T}1IxP!@ z<|qWMH?+;Z?UHL!y&mLuMmo;)Iu-?8k{Y7W^_0_Y%{=>z^(kJLPF=c6W1g9V0`HYw zBt1Qe*cZKTW)8pnpe64=l0ro^h4g0gFbX~NhCgQSnbP1~<7T{ZK?yc93JQx6Jr`!R>dquo-z zikOM<=+Km3BIXV-tv*w$0$ma}A9~g~P4-Xmx^?On*4WfENvG3o%uzs^wy`GY-8}Ww zg*Of@9FyY38GR3$%QTBON1K|t>CuDIw3=mtMyI|q=CFtF|1rhu?rb#L+?5`k&MS+} zGk^lKFeBQ{n{U<&E;b)X$Sw7_=wjYHd;T~YTt`(h!mWp%bMCs1BNnKtc-XN=Or1=7v3V&oIxOWI&$RCuEl<<2ME1HT@6MN>ZTe7)6mK#UqHZV5mY$4m zn;G(Gw6*!FXLNsF*lf^?h8yB34Jh;3)CptX?DGIs)LOX}I_81%+BgKz%2^H((9g?XdCtIgX| zxB~^vtN;7*Ri~A_>xBxq5sTh5P0-S26q=%t*?;;!o?gG-qoD%b?;3gDWG86pb#o?q z-apMfa4Fx_Gusl;BV%4F&TH~VMq8OCeWDq>$vUJD%}qkA=6=0k&E>nEo70@y^*+A@h2;JnbhG_Cw*;UpGAU(ws{cqM(`R$^@C6 z>O>p!sr0%s?w08NaNefVy7bEbI@HV1&}a;*FO#7~iU8N~plB_K2KG zP8)JEFTC-MuI;Kc&aQU#?J*CZKobMaMS|&OH5GXmnp6AH;0<`40@1-opFHg4+>{PMi>_Zb`q#D( zx)_OD+N%D?ZH2Y(A6DwQUi@cPN|yR#-@|IV@z(X?0nwK6&XHnbo`S(8lk)STJ>T!G z8H(d#JtBYVqXc}8A7)|y=rC`m+1{V^v%&n{pMif!Me?Sr8kc8vn{;>y~R z?P*~>_}s)l(gwfq^Z;4GhQx^Zd?4}m$5!UDQPEauDXs0w-k$pLVS|Tnd$vY#UsE`U zsMCanpr!8@_I&b_QP(j4i!j2mow`SEFv*c50f!C+P25}EEq+a z>_^)@G2rUf-_t~Q8~H9JM{Yl>*A*AcdFhR(!klnME;Y-jCT+!mw()M4=PcN|@&;D) z#YnUsZZTWcPNjKW@TB>EaI{;;E^Tdn#Y+eNw(*uj&Z0gA0A~1K16X4((~KPwZI|}f z0Vy0pYhG5qa|phXXR3x^-ub5E(CGBE%tLG#yJ_a>^Gbg^rA=tHE{f)Z%soTt&=~W~ zP}G;1RfTL;ErzjKm6|TY$a&NZ8Ai?mQzq^_vtk%KXh)MaoZ%kZ!S?@%uiyHzxaC(B zRC}zQwLWG93O$CPAm@H)G_Q95_7a;7VKzKx{PjGJGb>g16pDt1yXyMSVh5d;WxFf~ zDl|KXGs3#2?FeAE$sfU~gZ=o%5z(%~S4KpStRLQlEf@3uzJ&3dXnlxjIErSIZzn%G|D6Z758XR2#iP4nXgk(qq2P5flRji)=tD&s z*~w?`-TltC?gdmNCt(RWPwg?pX|zp0XZXtn=N?p3(>Ry&=&W-$tb6{TR+QDg&h0DP zEWRMxvQFnnzIpyc8e3qRj0TpNqks`tQcj!ErnS4$&pqp)$7^c5iJZpdjJ|POMKEhn zVNK2kaugL>4{4Zt&LNAwqb+UY4EPBXjAp&;LP6esTH{XhMr}Xt#b(}-gmHb|HSenN zPt7kt+V|*bz`stefB(1bb8fHEime7DGF+- zZttbJm&cB1q)vvrU8Ij$j{*_gjeec^T?PL7{ByAxCi84J(@%^JOZ#hkhGy*Mh4r=O ze%WR$9sP*G%d-aGyE`-X-KJ|WiDC__Zk`!6mUb606YA)Kby?>Q@40$xXtQt+TVhs? zWjPrXWYm{io-=M{w;yz}anWkFX*Z6Jy=5{5Cz-M1sPC^m!A`T;JTAI_+JbazzdEP3 z{`Mb@*7gcp4I}1K^EK6YADKf>!r`wlbCy$g2;uqeBeUit{@;=YCiCQIrt{B=%TA6) zy|f=&*-2Y|%!Fl!-PbFlnb*75L4XQ2zIw5DJeFD5(~c{zvgd8x>*nu? zGDF-^*g`HfT_(`zYBK~#+u-z$9s2C1)t7HvP^0&RDU-r36xch;Z@hic%r8<8eX&O2 zQ!{Eyv_*_6f9q+Un?OWw(#w9=;OTlJP5+PbAFa_n)Oe@RXog7x(uO&`*Uy>Ozspsx zZLQJEHY21k&nZl4@vGmcZL=S06qcHH4@Fx_GoDb@9;>VR{7TIUB|`Z#Aojp~zT zr`mkOd@8+f&|_yRzq5bdq`8U1udC5ZHEk!Ncc96b$i?Eziq#XNm0sFU+#BUNHxAkF zj#GnG@1%HNAkmh=A9T(^zrMM2GI6o#04puJs?kUL%s%%H9a6&J5lMd2pyS)EzIo#n z?_7H+!8Ia>n>Q)S*Sxsqv@Xk{2E~F2Z(KL%^Esb#?Cgp}>u-|7Id_b^zI@-|+%85q z-PtbZ;t@kz9sSe|uhit6>2hxPu6?)BS@9V)wOvS#BFp`6E^1l)z-xVUd*fI!nhxFO zvQBF}bVu7QPjia628lX#KgemMeB{fiT@UGgQF={7&$ygz*WGpB8Fh1yt;u=Kyn8Bf z^)nQ-v@-U8Hh=xm9U?V7`E7vdGd0?(tnomru}_P7>#JrAy`e^4@c z=vY&E50{fS^oYCe`Dg+0k&B8}d}JIs%_!4sWuL~azPjo56mK9B4PquJA8GOVt^e$D z{OTN68~b^A8Cje|%Fo>tx#@@IZ?vx|bhX)j8h+DskoB7_7r*hw>X#nNWPXWs+PpfL z-%;@FXwAm=qx+fioM_Yx%#O~sr;rv=;?p|wd^Ua+?&&Seuqn}a+F$IYU3u_{8=0vk{22Su5 z1J#as^tP8jyX%Ff9LT#6O11S)Hak&p{o|;1*KD}0rY-Z9R&t^(HSXOny5H5m)0MWj za@8`kx&pn63>SsgJj-aJQa72^;_fjIs_d2L|ae87C%k+>f(2jH*3cEq-nzm*L&WyD*|?z zQQ|%@vuX+LAvkTpBa?)9Yr@u+Z0TZCNV{pDQ%X_i*?$*=AL4}2`!jki=)H2$_yzX$1eT>f3jHJCN*ig zT?*~ioL$km+ub}#WTog39_c{z*7dmE#~07ocJpQ20rJfgB?OdRCT(7{b>4e3tash_ z(0k2$Tsh+PaQ1ir5{dlZM%YefCCz!gOvOCRKEQlBkMMq>$v7RZiOD}5u8n!{bd0x^ zjY^@W-Rs9pK74Y!55r4uxWVyg8HFB+nKm%o*zJ|!t8S@^qu_3ue^C?%Ze)6$f!zn1 z>}#T(%u8oP+uC$9Q*VB>ZQ6x-c1E{<9y>Yfu!q^olZW6QCQHv7++&AEX5M_N;Vp*J zXfyLQMQysZcd$9Cl+Jf7wB~Fw_M}^HZTJ5)cja+CX5Zf@PmLHArNxsi1|wVQV{2?* zLyUcA3}!41V<}=#$}X~0wj*tdQkD|Ym@H)-QpuVUsY|oz8z@Xe5s##oVz$~3uhfHGa5GrHqih&J|9|P z7v+OM3;lCkZ>Df7n0W5B`)t(FRXC`ri6^XN(Gq&Y+Y_9I5_v+VesntsPXp<$Cm431 zeqQ=Ss~qKKwAc&QeU^tXcFZm}tv9{Vh?-!ZZ*ve}2sQAA>^Fc3HUV3V7*T!nrjEw} zaD<~7fR%H^v}OA5e0JsHu@a?5nKscB{4ERyN8vPR0Sa%L%EcZEWHygdIG(gwpn=(6 z^t$BTj2F98SxuFD_)x>f>$33t#-_(TN1}#z6x2c}l`T(5l>4E^hMF#voD9n5)7XX38b+RCX}rDujX{s08N&Ul2!AaFE=1d4+Ra5A zX_i1E-&%rH`7{p;Q7M=ehEV}j-zX;RS&0CQC++ct9tkKc?AQjadXisGT zh&JUURcd&-a501(w?Ir-W4nSKDbby<63b1AZV2G>Bs&#&>@xiM)g-QA{|dscl)e~b z1zXs+GC0+ABkd9}a|YsNn|;9CX5reb_;)HZF!vRwwxtAT`Em*c6z-TTG}P)}>-vrE zW>jw}`b?(|OEJL%$ag8|Ia4xTn*aO6&7k{BvA1fI@iM93LjC>yUo1oY=6}gDagO%- zYSw@@CN2$CYa4dYBK-z`MaDa&%$MqKo+H&Zub?v;vRrc0tf1nHkVh)zEk_YRFP9@5 zGmjdsz_3NMLHJ4?@iBu9D-b17-3Q zz4gs%u_r6J9y?VUF!LsAcu!(Aq>;BT`c#4pmbBMbs)LVfq~Vq9hBmx_dPU8M%dThi z8QN&k1_GE*?|sp$FZEjq+y>yxfE)Ltt@r)8Ve!E6K?>SGu=|89!eB1`)>Bvi6?FMlQRG1sW5=2qCL*z*7!!i+SMSmHc*&ML*v7RpDDjYbsNx+n`aM4y=Ka?R z8h{yVLB@gbrkUVo%Xf$#CnrouZ}M~&nrTI#O>sFn%Ca*JJ)7$bJLb z?xArTB%8J;L&b(V@v8Z&_}@w+*$?BWz_U>UFZ!bHhu-969SQf4K4|2Zp%k|PVz3B9 z>M0OsBi~&deedoL9ilbmgE&5_leq~CY?_L@x_@%OvEZW4I^0TXwGp^ow3u<{fn(eK z(aM!KtGLWuqT}Tjc>^au1Ax!n*Q>iV^tl`~M+c}%2~4Oir7^C3xbl17MYi%^tM59R zUenL6R-?>0G<7q^)<}&qXN}x-vG=p^yI3RYN!ggf0YG5l@Md<~@1#eY zC2NaVXR%i8uWp}}HFf`Y2AVfJsWwR+<@+%r29<1KS#ykk5X9{y0d&RThHq7L_GEOp(2MP|B7EHcxAEQjWA)86V3_T;ezYSTDgsLjY? zPy5;5Jr$})$rCt9sptbUmWx96-6wEX16m&F{wmJ_f`iM1jkZ8CWwTG{Rr*g5OdZt{ z45i&cm~&yiYPW(y4KTa^$3scur`yA~2*scT$}2EL)hpI+YOxixRZCE@_{-^0+*Vu> zMUAZAU=^&$p)7f`*HUO+>@AiKbdnqg*K1&~7?6^war3g@f zB76)SfAS{`y@*(&Q5*JzWhadYW50-Rl7F~VStDOb7TRAPlr!+b!|2B9zJYF?N_)4V zuRH*_FN4k_?K<|eR>H+%EDtFk0E3sw^mZFoq~Q_4bx_l~my_!ywp1x;;UcA05#Xi~ z^^U;#`z@t$f&ZL15-$9@qQMy+m3Dg z`Z!J74vn8q;doT3P|>uy zt3{y@+*9W$IA3mnD4&5tqdnv5E@;Yb6!msuM;=iSyeXw}n~9Xmg&RGOf)z01w6LIT zYSx@Kw4v!09VVdE_0jldEi+W}my=*Es+nFN6W}&kD|7 zz4n@XV&ujs1I>86VONA=NSpH~EzosaF$?FZ3{8=F1Y1h<1BZh{3|>{I zK3sfU)0GzTt3LQ-9QKFCMM&+BQ{L_&*DqXF2aKs5L7G_Lz9$)l?2K`&&Rk**I7j%%(GHc!FYWHs*N_Z{I_y!Ky%(H+9+#CZ+(uR?M4F=O#@ZJ97k|bqoNv zY36P$^%Ob|oJO!DTdA!0RJ$>Dj`itYfU7#|E>+(HR`hrHGDGxa*SmKsx6CXN9!2Bw z;i4xf>t8idW&Sm! ze}dK5n|nFeI2C=8b1+-CYzLz^h~5LJJqCEE=0~pXrwSxr3(Xe1;q9A@ zvJT2eqt+cte2^H7rqJZ=q2mTnIGWmayeZZ}WTT=0NAQBq5kuyss>(*Qj!O4IVzsG3 z0+@Z7C-#@y!5vqk`xZP0p2v%I)0f-;D0lK!==pV}YWp#3ZuG@|s3N!9Li5eu79JV4 zE%Yu(VSBOpw2#&@ZUXJx4~imxK<;!$z(ePGxsLQ+&|e4lCDQ|dtLIr3jlq+glP`p3 zG2`<#juGtRDK~CHD3IaZX*c7B6bdzdym@er^Dt|C98JMtKojWp0dzlu%o8#2MBsRh z5Bt94{p)}qV3uGtD_4Gd`XLeYW|9{dCutMI|Fp0#Nk%Vd>)EUzG=p)YzI}i^l$nDBj1^x#+X~VyyS}-GAdA`Cx-$xN)Cl3yKMBNQJ_Xi zqp&eR4#l*iIC=EsPyndpP$;Z)=p=N)iAs-L8W$OK)v=-{vjVM^%nF4SJ;@coD9IIU z6FDEl+Sij=0i2Rqp>T2&oLFMrSGW4QC@k*hvpP=nY zZHy@p0K_eJbJ2rNr@&J&idv-N=^?qK!j4w`wvJlt^}hU9wM=RFPg>AZ4>v+v&gTjMyWl>YaMc(A!V%CoGjX|QX zQO6${hTJM`DhADmjtFDRJD#Z~*xCgqRt8t+i4-jCeQZL`o+dg8Pq`6gA?96PxKlBn z%Fi|1XD?MtrNbuo!8U)wDZ!3zN4Tu3yrTN2kP%(?m9SZFJX}c)*T8`&Jz; zjUEG_o%>c?{VkuH#GJ2srK6*xc9cLyi zudM?-q?rI17?;tS)7W%P%kW!gP-<2(#ln0`m6Sr*{Q`F=PoPkYXTC=^LnPs1Hv|C$|LXRX0{C0lP3(B-n2L#1enZ!IThDWYqThT=9kr{PFWAvb<b3U}b-4h*IRJP|{L$w3 zU*@#y{?WZSpByqjuqh9mCG?q{fnCRd{Sv!-)w{(DO1%I*lZkIE@C{;AO!3*TBQDHU zd&1scM5gH&;RWiBLjD%Nzh%v0_H?P4Rb#huD;-P9l>*bzp%?AuVim<_aLyb%F>dn< zkvTy%GbAVZ4T$iSXvhSUYxW^ylh7I8MgSJM5pXe8!_X+3EqUI5ERYw0C3y|L<;If!Zo}|J566wkHqf+f*+wym)K#_x05brb4=ry| z_=kH99YCzI@|AjJUsA~o>Qkw0CY1GfCn*t zrq`|+A?>PKE3tSKfXycu6a7;5B{)}N)PB6IonKG!9qMX^h>Sy9Ug962sOx3y$&)nt zGG;NahDN!gI_*eY(R0|0w#vEQ(h0}UU*r#f*0QEXQHX2q)f*G5tskTVIHHDKwZ=`S z^p+xiIjO4|M0rfx4S;$840DO?-+fAEA01#F)wlxb#F0H0YpLHgOhG6uz9w0c{}lw- z3TZED+P<>XD7ViqzU);nZ^l(aWsNCrm#S7`lghvOd9{9RcN8`11;36Jy}g2UpalYI zWzp}?I;>nVw-o1Qxwh#^t*)Zm5%eB~{1b3IqdU*_Zx>o=^8sBSp5zUHt)B|0Hg(31 z6DwAV)DcB(;cm45s?9_u*ZQF zGWaf~Qh?i*0l<7a#Ca9HZL}>;0VpPoDSo`lHCu`=-^_pJ;Gxtg9_t3S^i?+RLm;ex zSl%wy`gOv3PN86ZgKqnfTQ=rfa4Z^0vDVDBDLrCU z8eq_gy4-*vrxcE8oC~;k>P4M$@DsFuQs12eGeG4{0BvmhfIOQnJD*8wdzyi zP5jQG{3;zqavs*zU-&^VOMl~4*UC|T&sjj63I{`21!|4$gen^~G5Cwg^-40;9-;TK z+7vxiyVe&q7B?IM*GV(H;eYs^If%nkoRlXyR&|dRKWcSHaOq`b3o5uTwH(f{!wZB} zx!PxBmkkZ0W}U<{Yt8o|87&_N*gVNn>yJ;`*T5+4_djyg`ocRutUKJMozS?z_3b^Prl*EFO#ZPGfFa$dCdIJZTO_Y rv!;CQaWMmKRs72a!6CLrjm!h?XWz03IJL1#rl_#XG^EwGMw$NwAz(u3 diff --git a/test/harness.ts b/test/harness.ts index 6d1c6d36a4..0921b1dcc0 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -5,7 +5,7 @@ import { readFile, readlink, writeFile } from "fs/promises"; import fs, { closeSync, openSync } from "node:fs"; import os from "node:os"; import { dirname, isAbsolute, join } from "path"; -import detect_libc from "detect-libc"; +import detectLibc from "detect-libc"; type Awaitable = T | Promise; @@ -19,7 +19,8 @@ export const isIntelMacOS = isMacOS && process.arch === "x64"; export const isDebug = Bun.version.includes("debug"); export const isCI = process.env.CI !== undefined; export const isBuildKite = process.env.BUILDKITE === "true"; -export const libc_family = detect_libc.familySync(); +export const libcFamily = detectLibc.familySync() as "glibc" | "musl"; +export const isVerbose = process.env.DEBUG === "1"; // Use these to mark a test as flaky or broken. // This will help us keep track of these tests. @@ -1372,7 +1373,7 @@ export function waitForFileToExist(path: string, interval: number) { export function libcPathForDlopen() { switch (process.platform) { case "linux": - switch (libc_family) { + switch (libcFamily) { case "glibc": return "libc.so.6"; case "musl": diff --git a/test/integration/next-pages/test/dev-server-puppeteer.ts b/test/integration/next-pages/test/dev-server-puppeteer.ts index b529afbf68..06bf56b425 100644 --- a/test/integration/next-pages/test/dev-server-puppeteer.ts +++ b/test/integration/next-pages/test/dev-server-puppeteer.ts @@ -1,8 +1,9 @@ import assert from "assert"; import { copyFileSync } from "fs"; import { join } from "path"; -import { ConsoleMessage, Page, launch } from "puppeteer"; - +import type { ConsoleMessage, Page } from "puppeteer"; +import { launch } from "puppeteer"; +import { which } from "bun"; const root = join(import.meta.dir, "../"); copyFileSync(join(root, "src/Counter1.txt"), join(root, "src/Counter.tsx")); @@ -12,28 +13,38 @@ if (process.argv.length > 2) { url = process.argv[2]; } +const browserPath = which("chromium-browser") || which("chromium") || which("chrome") || undefined; +if (!browserPath) { + console.warn("Since a Chromium browser was not found, it will be downloaded by Puppeteer."); +} + const b = await launch({ - // While puppeteer is migrating to their new headless: `true` mode, - // this causes strange issues on macOS in the cloud (AWS and MacStadium). - // - // There is a GitHub issue, but the discussion is unhelpful: - // https://github.com/puppeteer/puppeteer/issues/10153 - // - // Fixes: 'TargetCloseError: Protocol error (Target.setAutoAttach): Target closed' - headless: "shell", + // On macOS, there are issues using the new headless mode. + // "TargetCloseError: Protocol error (Target.setAutoAttach): Target closed" + headless: process.platform === "darwin" ? "shell" : true, + // Inherit the stdout and stderr of the browser process. dumpio: true, + // Prefer to use a pipe to connect to the browser, instead of a WebSocket. pipe: true, + // Disable timeouts. + timeout: 0, + protocolTimeout: 0, + // Specify that chrome should be used, for consistent test results. + // If a browser path is not found, it will be downloaded. + browser: "chrome", + executablePath: browserPath, args: [ - // Fixes: 'dock_plist is not an NSDictionary' + // On Linux, there are issues with the sandbox, so disable it. + // On macOS, this fixes: "dock_plist is not an NSDictionary" "--no-sandbox", - "--single-process", "--disable-setuid-sandbox", + + // On Docker, the default /dev/shm is too small for Chrome, which causes + // crashes when rendering large pages, so disable it. "--disable-dev-shm-usage", - // Fixes: 'Navigating frame was detached' + + // Fixes: "Navigating frame was detached" "--disable-features=site-per-process", - // Uncomment if you want debug logs from Chromium: - // "--enable-logging=stderr", - // "--v=1", ], }); diff --git a/test/js/node/cluster/test-docs-http-server.ts b/test/js/node/cluster/test-docs-http-server.ts index 91547ed7ae..a72498d227 100644 --- a/test/js/node/cluster/test-docs-http-server.ts +++ b/test/js/node/cluster/test-docs-http-server.ts @@ -1,8 +1,14 @@ +import { isBroken, isWindows } from "harness"; import assert from "node:assert"; import cluster from "node:cluster"; import http from "node:http"; import { availableParallelism } from "node:os"; +if (isWindows && isBroken) { + console.log("Skipping on Windows because it does not work when there are more than 1 CPU"); + process.exit(0); +} + const numCPUs = availableParallelism(); let workers = 0; diff --git a/test/package.json b/test/package.json index f643ef682d..5fc461dc32 100644 --- a/test/package.json +++ b/test/package.json @@ -4,7 +4,9 @@ "@types/react": "18.0.28", "@types/react-dom": "18.0.11", "@types/supertest": "2.0.12", - "@types/utf-8-validate": "5.0.0" + "@types/utf-8-validate": "5.0.0", + "@types/ws": "8.5.10", + "@types/puppeteer": "7.0.4" }, "dependencies": { "@azure/service-bus": "7.9.4", @@ -16,7 +18,6 @@ "@remix-run/serve": "2.10.3", "@resvg/resvg-js": "2.4.1", "@swc/core": "1.3.38", - "@types/ws": "8.5.10", "aws-cdk-lib": "2.148.0", "axios": "1.6.8", "body-parser": "1.20.2",