Deflake some build issues

* Disable sccache on Windows
* Add workaround for EBUSY/UNKNOWN spawn errors

.buildkite/ci.yml

@@ -622,8 +622,9 @@ steps:
           - "build\\bun-deps\\*.lib"
         env:
           SCCACHE_DIR: "$$HOME\\.cache\\sccache"
-          ZIG_LOCAL_CACHE_DIR: "$$HOME\\.cache\\zig-cache"
+          CCACHE_DIR: "$$HOME\\.cache\\ccache"
           SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
+          CCACHE_DISABLE: "1"
         command:
           - ".\\scripts\\all-dependencies.ps1"
 
@@ -646,6 +647,11 @@ steps:
           # HACK: See scripts/build-bun-cpp.ps1
           # - "build\\bun-cpp-objects.a"
           - "build\\bun-cpp-objects.a.*"
+        env:
+          SCCACHE_DIR: "$$HOME\\.cache\\sccache"
+          CCACHE_DIR: "$$HOME\\.cache\\ccache"
+          SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
+          CCACHE_DISABLE: "1"
         command:
           - ".\\scripts\\build-bun-cpp.ps1"
 
@@ -665,8 +671,9 @@ steps:
           - "features.json"
         env:
           SCCACHE_DIR: "$$HOME\\.cache\\sccache"
-          ZIG_LOCAL_CACHE_DIR: "$$HOME\\.cache\\zig-cache"
+          CCACHE_DIR: "$$HOME\\.cache\\ccache"
           SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
+          CCACHE_DISABLE: "1"
         command:
           - ".\\scripts\\buildkite-link-bun.ps1"
 
@@ -709,8 +716,9 @@ steps:
           - "build\\bun-deps\\*.lib"
         env:
           SCCACHE_DIR: "$$HOME\\.cache\\sccache"
-          ZIG_LOCAL_CACHE_DIR: "$$HOME\\.cache\\zig-cache"
+          CCACHE_DIR: "$$HOME\\.cache\\ccache"
           SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
+          CCACHE_DISABLE: "1"
           USE_BASELINE_BUILD: "1"
         command:
           - ".\\scripts\\all-dependencies.ps1"
@@ -736,8 +744,9 @@ steps:
           - "build\\bun-cpp-objects.a.*"
         env:
           SCCACHE_DIR: "$$HOME\\.cache\\sccache"
-          ZIG_LOCAL_CACHE_DIR: "$$HOME\\.cache\\zig-cache"
+          CCACHE_DIR: "$$HOME\\.cache\\ccache"
           SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
+          CCACHE_DISABLE: "1"
           USE_BASELINE_BUILD: "1"
         command:
           - ".\\scripts\\build-bun-cpp.ps1"
@@ -758,8 +767,9 @@ steps:
           - "features.json"
         env:
           SCCACHE_DIR: "$$HOME\\.cache\\sccache"
-          ZIG_LOCAL_CACHE_DIR: "$$HOME\\.cache\\zig-cache"
+          CCACHE_DIR: "$$HOME\\.cache\\ccache"
           SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
+          CCACHE_DISABLE: "1"
           USE_BASELINE_BUILD: "1"
         command:
           - ".\\scripts\\buildkite-link-bun.ps1 -Baseline $$True"

.buildkite/scripts/env.sh

@@ -78,6 +78,7 @@ function export_environment() {
   export CCACHE_DIR="$HOME/.cache/ccache/$BUILDKITE_STEP_KEY"
   export SCCACHE_DIR="$HOME/.cache/sccache/$BUILDKITE_STEP_KEY"
   export ZIG_LOCAL_CACHE_DIR="$HOME/.cache/zig-cache/$BUILDKITE_STEP_KEY"
+  export ZIG_GLOBAL_CACHE_DIR="$HOME/.cache/zig-cache/$BUILDKITE_STEP_KEY"
   export BUN_DEPS_CACHE_DIR="$HOME/.cache/bun-deps/$BUILDKITE_STEP_KEY"
   if [ "$(assert_arch)" == "aarch64" ]; then
     export CPU_TARGET="native"
@@ -86,12 +87,6 @@ function export_environment() {
   else
     export CPU_TARGET="haswell"
   fi
-  # LTO is disabled on Windows and macOS.
-  if [[ "$BUILDKITE_STEP_KEY" == *"nolto"* ]] || [[ "$BUILDKITE_STEP_KEY" == *"darwin"* ]] || [[ "$BUILDKITE_STEP_KEY" == *"windows"* ]]; then
-    export USE_LTO="OFF"
-  else
-    export USE_LTO="ON"
-  fi
   if $(buildkite-agent meta-data exists release &> /dev/null); then
     export CMAKE_BUILD_TYPE="$(buildkite-agent meta-data get release)"
   else
@@ -107,11 +102,13 @@ function export_environment() {
   else
     export USE_DEBUG_JSC="OFF"
   fi
-  if [ "$BUILDKITE_CLEAN_CHECKOUT" == "true" ]; then
+  if [ "$BUILDKITE_CLEAN_CHECKOUT" == "true" || "$BUILDKITE_BRANCH" == "main" ]; then
     rm -rf "$CCACHE_DIR"
     rm -rf "$SCCACHE_DIR"
     rm -rf "$ZIG_LOCAL_CACHE_DIR"
+    rm -rf "$ZIG_GLOBAL_CACHE_DIR"
     rm -rf "$BUN_DEPS_CACHE_DIR"
+    export CCACHE_RECACHE="1"
   fi
 }
 

CMakeLists.txt

@@ -487,7 +487,7 @@ if(USE_UNIFIED_SOURCES)
 endif()
 
 # CCache
-find_program(CCACHE_PROGRAM sccache)
+# find_program(CCACHE_PROGRAM sccache)
 find_program(CCACHE_PROGRAM ccache)
 
 if(CCACHE_PROGRAM)

scripts/env.ps1

@@ -35,7 +35,7 @@ $BUN_DEPS_DIR = if ($env:BUN_DEPS_DIR) { $env:BUN_DEPS_DIR } else { Join-Path $B
 $BUN_DEPS_OUT_DIR = if ($env:BUN_DEPS_OUT_DIR) { $env:BUN_DEPS_OUT_DIR } else { Join-Path $BUN_BASE_DIR 'build\bun-deps' }
 
 $CPUS = if ($env:CPUS) { $env:CPUS } else { (Get-CimInstance -Class Win32_Processor).NumberOfCores }
-$Lto = if ($env:USE_LTO) { $env:USE_LTO -eq "1" } else { $True }
+$Lto = if ($env:USE_LTO) { $env:USE_LTO -eq "1" } else { $False }
 $Baseline = if ($env:USE_BASELINE_BUILD) {
   $env:USE_BASELINE_BUILD -eq "1"
 } elseif ($env:BUILDKITE_STEP_KEY -match "baseline") {
@@ -103,7 +103,10 @@ if ($Lto) {
   $CMAKE_FLAGS += "-DUSE_LTO=ON"
 }
 
-if (Get-Command sccache -ErrorAction SilentlyContinue) {
+if (Get-Command ccache -ErrorAction SilentlyContinue) {
+  $CMAKE_FLAGS += "-DCMAKE_C_COMPILER_LAUNCHER=ccache"
+  $CMAKE_FLAGS += "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
+} elseif (Get-Command sccache -ErrorAction SilentlyContinue) {
   # Continue with local compiler if sccache has an error
   $env:SCCACHE_IGNORE_SERVER_IO_ERROR = "1"
 

scripts/runner.node.mjs

@@ -134,36 +134,37 @@ async function printInfo() {
 async function runTests() {
   let execPath;
   if (options["step"]) {
-    execPath = await getExecPathFromBuildKite(options["step"]);
+    downloadLoop: for (let i = 0; i < 10; i++) {
+      execPath = await getExecPathFromBuildKite(options["step"]);
+      for (let j = 0; j < 10; j++) {
+        const { error } = spawnSync(execPath, ["--version"], {
+          encoding: "utf-8",
+          timeout: spawnTimeout,
+          env: {
+            PATH: process.env.PATH,
+            BUN_DEBUG_QUIET_LOGS: 1,
+          },
+        });
+        if (!error) {
+          break;
+        }
+        const { code } = error;
+        if (code === "EBUSY") {
+          console.log("Bun appears to be busy, retrying...");
+          continue;
+        }
+        if (code === "UNKNOWN") {
+          console.log("Bun appears to be corrupted, downloading again...");
+          rmSync(execPath, { force: true });
+          continue downloadLoop;
+        }
+      }
+    }
   } else {
     execPath = getExecPath(options["exec-path"]);
   }
   console.log("Bun:", execPath);
 
-  for (let i = 0; i < 10; i++) {
-    try {
-      const { error } = spawnSync(execPath, ["--version"], {
-        encoding: "utf-8",
-        timeout: spawnTimeout,
-        env: {
-          PATH: process.env.PATH,
-          BUN_DEBUG_QUIET_LOGS: 1,
-        },
-      });
-      if (!error) {
-        break;
-      }
-      throw error;
-    } catch (error) {
-      const { code } = error;
-      if (code === "EBUSY" || code === "UNKNOWN") {
-        console.log(`Bun appears to be busy, retrying... [code: ${code}]`);
-        await new Promise(resolve => setTimeout(resolve, 1000 * (i + 1)));
-        continue;
-      }
-    }
-  }
-
   const revision = getRevision(execPath);
   console.log("Revision:", revision);