diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs
index 296b52dd4e..b4d7e39b49 100755
--- a/.buildkite/ci.mjs
+++ b/.buildkite/ci.mjs
@@ -16,6 +16,7 @@ import {
   getEmoji,
   getEnv,
   getLastSuccessfulBuild,
+  getSecret,
   isBuildkite,
   isBuildManual,
   isFork,
@@ -1203,6 +1204,42 @@ async function main() {
     console.log("Generated options:", options);
   }
 
+  startGroup("Querying GitHub for files...");
+  if (options && isBuildkite) {
+    /** @type {string[]} */
+    let allFiles = [];
+    /** @type {string[]} */
+    let newFiles = [];
+    let prFileCount = 0;
+    try {
+      console.log("on buildkite: collecting new files from PR");
+      const per_page = 50;
+      const { BUILDKITE_PULL_REQUEST } = process.env;
+      for (let i = 1; i <= 10; i++) {
+        const res = await fetch(
+          `https://api.github.com/repos/oven-sh/bun/pulls/${BUILDKITE_PULL_REQUEST}/files?per_page=${per_page}&page=${i}`,
+          { headers: { Authorization: `Bearer ${getSecret("GITHUB_TOKEN")}` } },
+        );
+        const doc = await res.json();
+        console.log(`-> page ${i}, found ${doc.length} items`);
+        if (doc.length === 0) break;
+        for (const { filename, status } of doc) {
+          prFileCount += 1;
+          allFiles.push(filename);
+          if (status !== "added") continue;
+          newFiles.push(filename);
+        }
+        if (doc.length < per_page) break;
+      }
+      console.log(`- PR ${BUILDKITE_PULL_REQUEST}, ${prFileCount} files, ${newFiles.length} new files`);
+    } catch (e) {
+      console.error(e);
+    }
+    if (allFiles.every(filename => filename.startsWith("docs/"))) {
+      return;
+    }
+  }
+
   startGroup("Generating pipeline...");
   const pipeline = await getPipeline(options);
   if (!pipeline) {
diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs
index 972af6791d..abdf911968 100755
--- a/scripts/runner.node.mjs
+++ b/scripts/runner.node.mjs
@@ -185,32 +185,33 @@ if (options["quiet"]) {
   isQuiet = true;
 }
 
+/** @type {string[]} */
+let allFiles = [];
+/** @type {string[]} */
 let newFiles = [];
 let prFileCount = 0;
 if (isBuildkite) {
   try {
     console.log("on buildkite: collecting new files from PR");
     const per_page = 50;
-    for (let i = 1; i <= 5; i++) {
+    const { BUILDKITE_PULL_REQUEST } = process.env;
+    for (let i = 1; i <= 10; i++) {
       const res = await fetch(
-        `https://api.github.com/repos/oven-sh/bun/pulls/${process.env.BUILDKITE_PULL_REQUEST}/files?per_page=${per_page}&page=${i}`,
-        {
-          headers: {
-            Authorization: `Bearer ${getSecret("GITHUB_TOKEN")}`,
-          },
-        },
+        `https://api.github.com/repos/oven-sh/bun/pulls/${BUILDKITE_PULL_REQUEST}/files?per_page=${per_page}&page=${i}`,
+        { headers: { Authorization: `Bearer ${getSecret("GITHUB_TOKEN")}` } },
       );
       const doc = await res.json();
       console.log(`-> page ${i}, found ${doc.length} items`);
       if (doc.length === 0) break;
       for (const { filename, status } of doc) {
         prFileCount += 1;
+        allFiles.push(filename);
         if (status !== "added") continue;
         newFiles.push(filename);
       }
       if (doc.length < per_page) break;
     }
-    console.log(`- PR ${process.env.BUILDKITE_PULL_REQUEST}, ${prFileCount} files, ${newFiles.length} new files`);
+    console.log(`- PR ${BUILDKITE_PULL_REQUEST}, ${prFileCount} files, ${newFiles.length} new files`);
   } catch (e) {
     console.error(e);
   }
@@ -2615,8 +2616,18 @@ export async function main() {
     ]);
   }
 
-  const results = await runTests();
-  const ok = results.every(({ ok }) => ok);
+  let doRunTests = true;
+  if (isCI) {
+    if (allFiles.every(filename => filename.startsWith("docs/"))) {
+      doRunTests = false;
+    }
+  }
+
+  let ok = true;
+  if (doRunTests) {
+    const results = await runTests();
+    ok = results.every(({ ok }) => ok);
+  }
 
   let waitForUser = false;
   while (isCI) {