Add test report (#18772)

scripts/runner.node.mjs

@@ -7,26 +7,35 @@
 // - It cannot use Bun APIs, since it is run using Node.js.
 // - It does not import dependencies, so it's faster to start.
 
+import { spawn, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import {
-  constants as fs,
-  readFileSync,
-  mkdtempSync,
-  existsSync,
-  statSync,
-  mkdirSync,
   accessSync,
   appendFileSync,
+  existsSync,
+  constants as fs,
+  mkdirSync,
+  mkdtempSync,
   readdirSync,
+  readFileSync,
+  statSync,
+  unlink,
+  unlinkSync,
+  writeFileSync,
 } from "node:fs";
-import { spawn, spawnSync } from "node:child_process";
-import { join, basename, dirname, relative, sep } from "node:path";
+import { hostname, userInfo } from "node:os";
+import { basename, dirname, join, relative, sep } from "node:path";
 import { parseArgs } from "node:util";
 import {
+  getBranch,
   getBuildLabel,
   getBuildUrl,
+  getCommit,
   getEnv,
   getFileUrl,
+  getHostname,
   getLoggedInUserCountOrDetails,
+  getSecret,
   getShell,
   getWindowsExitReason,
   isBuildkite,
@@ -41,7 +50,6 @@ import {
   tmpdir,
   unzip,
 } from "./utils.mjs";
-import { userInfo } from "node:os";
 let isQuiet = false;
 const cwd = import.meta.dirname ? dirname(import.meta.dirname) : process.cwd();
 const testsPath = join(cwd, "test");
@@ -116,9 +124,33 @@ const { values: options, positionals: filters } = parseArgs({
       type: "string",
       default: isCI ? "4" : "0", // N retries = N+1 attempts
     },
+    ["junit"]: {
+      type: "boolean",
+      default: isCI, // Always enable JUnit in CI
+    },
+    ["junit-temp-dir"]: {
+      type: "string",
+      default: "junit-reports",
+    },
+    ["junit-upload"]: {
+      type: "boolean",
+      default: isBuildkite,
+    },
   },
 });
 
+const cliOptions = options;
+
+if (cliOptions.junit) {
+  cliOptions["junit-temp-dir"] = join(tmpdir(), cliOptions["junit-temp-dir"]);
+  try {
+    mkdirSync(cliOptions["junit-temp-dir"], { recursive: true });
+  } catch (err) {
+    cliOptions.junit = false;
+    console.error(`Error creating JUnit temp directory: ${err.message}`);
+  }
+}
+
 if (options["quiet"]) {
   isQuiet = true;
 }
@@ -337,6 +369,95 @@ async function runTests() {
     reportOutputToGitHubAction("failing_tests", markdown);
   }
 
+  // Generate and upload JUnit reports if requested
+  if (options["junit"]) {
+    const junitTempDir = options["junit-temp-dir"];
+    mkdirSync(junitTempDir, { recursive: true });
+
+    // Generate JUnit reports for tests that don't use bun test
+    const nonBunTestResults = [...okResults, ...flakyResults, ...failedResults].filter(result => {
+      // Check if this is a test that wasn't run with bun test
+      const isNodeTest =
+        isJavaScript(result.testPath) && !isTestStrict(result.testPath) && !result.testPath.includes("vendor");
+      return isNodeTest;
+    });
+
+    // If we have tests not covered by bun test JUnit reports, generate a report for them
+    if (nonBunTestResults.length > 0) {
+      const nonBunTestJunitPath = join(junitTempDir, "non-bun-test-results.xml");
+      generateJUnitReport(nonBunTestJunitPath, nonBunTestResults);
+      !isQuiet &&
+        console.log(
+          `Generated JUnit report for ${nonBunTestResults.length} non-bun test results at ${nonBunTestJunitPath}`,
+        );
+
+      // Upload this report immediately if we're on BuildKite
+      if (isBuildkite && options["junit-upload"]) {
+        const uploadSuccess = await uploadJUnitToBuildKite(nonBunTestJunitPath);
+        if (uploadSuccess) {
+          // Delete the file after successful upload to prevent redundant uploads
+          try {
+            unlinkSync(nonBunTestJunitPath);
+            !isQuiet && console.log(`Uploaded and deleted non-bun test JUnit report`);
+          } catch (unlinkError) {
+            !isQuiet && console.log(`Uploaded but failed to delete non-bun test JUnit report: ${unlinkError.message}`);
+          }
+        } else {
+          !isQuiet && console.log(`Failed to upload non-bun test JUnit report to BuildKite`);
+        }
+      }
+    }
+
+    // Check for any JUnit reports that may not have been uploaded yet
+    // Since we're deleting files after upload, any remaining files need to be uploaded
+    if (isBuildkite && options["junit-upload"]) {
+      try {
+        // Only process XML files and skip the non-bun test results which we've already uploaded
+        const allJunitFiles = readdirSync(junitTempDir).filter(
+          file => file.endsWith(".xml") && file !== "non-bun-test-results.xml",
+        );
+
+        if (allJunitFiles.length > 0) {
+          !isQuiet && console.log(`Found ${allJunitFiles.length} remaining JUnit reports to upload...`);
+
+          // Process each remaining JUnit file - these are files we haven't processed yet
+          let uploadedCount = 0;
+
+          for (const file of allJunitFiles) {
+            const filePath = join(junitTempDir, file);
+
+            if (existsSync(filePath)) {
+              try {
+                const uploadSuccess = await uploadJUnitToBuildKite(filePath);
+                if (uploadSuccess) {
+                  // Delete the file after successful upload
+                  try {
+                    unlinkSync(filePath);
+                    uploadedCount++;
+                  } catch (unlinkError) {
+                    !isQuiet && console.log(`Uploaded but failed to delete ${file}: ${unlinkError.message}`);
+                  }
+                }
+              } catch (err) {
+                console.error(`Error uploading JUnit file ${file}:`, err);
+              }
+            }
+          }
+
+          if (uploadedCount > 0) {
+            !isQuiet && console.log(`Uploaded and deleted ${uploadedCount} remaining JUnit reports`);
+          } else {
+            !isQuiet && console.log(`No JUnit reports needed to be uploaded`);
+          }
+        } else {
+          !isQuiet && console.log(`No remaining JUnit reports found to upload`);
+        }
+      } catch (err) {
+        console.error(`Error checking for remaining JUnit reports:`, err);
+      }
+    }
+  }
+
   if (!isCI && !isQuiet) {
     console.table({
       "Total Tests": okResults.length + failedResults.length + flakyResults.length,
@@ -653,8 +774,31 @@ async function spawnBunTest(execPath, testPath, options = { cwd }) {
   const absPath = join(options["cwd"], testPath);
   const isReallyTest = isTestStrict(testPath) || absPath.includes("vendor");
   const args = options["args"] ?? [];
+
+  const testArgs = ["test", ...args, `--timeout=${perTestTimeout}`];
+
+  // This will be set if a JUnit file is generated
+  let junitFilePath = null;
+
+  // In CI, we want to use JUnit for all tests
+  // Create a unique filename for each test run using a hash of the test path
+  // This ensures we can run tests in parallel without file conflicts
+  if (cliOptions.junit) {
+    const testHash = createHash("sha1").update(testPath).digest("base64url");
+    const junitTempDir = cliOptions["junit-temp-dir"];
+
+    // Create the JUnit file path
+    junitFilePath = `${junitTempDir}/test-${testHash}.xml`;
+
+    // Add JUnit reporter
+    testArgs.push("--reporter=junit");
+    testArgs.push(`--reporter-outfile=${junitFilePath}`);
+  }
+
+  testArgs.push(absPath);
+
   const { ok, error, stdout } = await spawnBun(execPath, {
-    args: isReallyTest ? ["test", ...args, `--timeout=${perTestTimeout}`, absPath] : [...args, absPath],
+    args: isReallyTest ? testArgs : [...args, absPath],
     cwd: options["cwd"],
     timeout: isReallyTest ? timeout : 30_000,
     env: {
@@ -664,6 +808,25 @@ async function spawnBunTest(execPath, testPath, options = { cwd }) {
     stderr: chunk => pipeTestStdout(process.stderr, chunk),
   });
   const { tests, errors, stdout: stdoutPreview } = parseTestStdout(stdout, testPath);
+
+  // If we generated a JUnit file and we're on BuildKite, upload it immediately
+  if (junitFilePath && isReallyTest && isBuildkite && cliOptions["junit-upload"]) {
+    // Give the file system a moment to finish writing the file
+    if (existsSync(junitFilePath)) {
+      uploadJUnitToBuildKite(junitFilePath)
+        .then(uploadSuccess => {
+          unlink(junitFilePath, () => {
+            if (!uploadSuccess) {
+              console.error(`Failed to upload JUnit report for ${testPath}`);
+            }
+          });
+        })
+        .catch(err => {
+          console.error(`Error uploading JUnit report for ${testPath}:`, err);
+        });
+    }
+  }
+
   return {
     testPath,
     ok,
@@ -1549,6 +1712,260 @@ function onExit(signal) {
   });
 }
 
+let getBuildkiteAnalyticsToken = () => {
+  let token = getSecret("TEST_REPORTING_API", { required: true });
+  getBuildkiteAnalyticsToken = () => token;
+  return token;
+};
+
+/**
+ * Generate a JUnit XML report from test results
+ * @param {string} outfile - The path to write the JUnit XML report to
+ * @param {TestResult[]} results - The test results to include in the report
+ */
+function generateJUnitReport(outfile, results) {
+  !isQuiet && console.log(`Generating JUnit XML report: ${outfile}`);
+
+  // Start the XML document
+  let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
+
+  // Add an overall testsuite container with metadata
+  const totalTests = results.length;
+  const totalFailures = results.filter(r => r.status === "fail").length;
+  const timestamp = new Date().toISOString();
+
+  // Calculate total time
+  const totalTime = results.reduce((sum, result) => {
+    const duration = result.duration || 0;
+    return sum + duration / 1000; // Convert ms to seconds
+  }, 0);
+
+  // Create a unique package name to identify this run
+  const packageName = `bun.internal.${process.env.BUILDKITE_PIPELINE_SLUG || "tests"}`;
+
+  xml += `<testsuites name="${escapeXml(packageName)}" tests="${totalTests}" failures="${totalFailures}" time="${totalTime.toFixed(3)}" timestamp="${timestamp}">\n`;
+
+  // Group results by test file
+  const testSuites = new Map();
+
+  for (const result of results) {
+    const { testPath, ok, status, error, tests, stdoutPreview, stdout, duration = 0 } = result;
+
+    if (!testSuites.has(testPath)) {
+      testSuites.set(testPath, {
+        name: testPath,
+        tests: [],
+        failures: 0,
+        errors: 0,
+        skipped: 0,
+        time: 0,
+        timestamp: timestamp,
+        hostname: getHostname(),
+        stdout: stdout || stdoutPreview || "",
+      });
+    }
+
+    const suite = testSuites.get(testPath);
+
+    // For test suites with granular test information
+    if (tests.length > 0) {
+      for (const test of tests) {
+        const { test: testName, status: testStatus, duration: testDuration = 0, errors: testErrors = [] } = test;
+
+        suite.time += testDuration / 1000; // Convert to seconds
+
+        const testCase = {
+          name: testName,
+          classname: `${packageName}.${testPath.replace(/[\/\\]/g, ".")}`,
+          time: testDuration / 1000, // Convert to seconds
+        };
+
+        if (testStatus === "fail") {
+          suite.failures++;
+
+          // Collect error details
+          let errorMessage = "Test failed";
+          let errorType = "AssertionError";
+          let errorContent = "";
+
+          if (testErrors && testErrors.length > 0) {
+            const primaryError = testErrors[0];
+            errorMessage = primaryError.name || "Test failed";
+            errorType = primaryError.name || "AssertionError";
+            errorContent = primaryError.stack || primaryError.name;
+
+            if (testErrors.length > 1) {
+              errorContent +=
+                "\n\nAdditional errors:\n" +
+                testErrors
+                  .slice(1)
+                  .map(e => e.stack || e.name)
+                  .join("\n");
+            }
+          } else {
+            errorContent = error || "Unknown error";
+          }
+
+          testCase.failure = {
+            message: errorMessage,
+            type: errorType,
+            content: errorContent,
+          };
+        } else if (testStatus === "skip" || testStatus === "todo") {
+          suite.skipped++;
+          testCase.skipped = {
+            message: testStatus === "skip" ? "Test skipped" : "Test marked as todo",
+          };
+        }
+
+        suite.tests.push(testCase);
+      }
+    } else {
+      // For test suites without granular test information (e.g., bun install tests)
+      suite.time += duration / 1000; // Convert to seconds
+
+      const testCase = {
+        name: basename(testPath),
+        classname: `${packageName}.${testPath.replace(/[\/\\]/g, ".")}`,
+        time: duration / 1000, // Convert to seconds
+      };
+
+      if (status === "fail") {
+        suite.failures++;
+        testCase.failure = {
+          message: "Test failed",
+          type: "AssertionError",
+          content: error || "Unknown error",
+        };
+      }
+
+      suite.tests.push(testCase);
+    }
+  }
+
+  // Write each test suite to the XML
+  for (const [name, suite] of testSuites) {
+    xml += `  <testsuite name="${escapeXml(name)}" tests="${suite.tests.length}" failures="${suite.failures}" errors="${suite.errors}" skipped="${suite.skipped}" time="${suite.time.toFixed(3)}" timestamp="${suite.timestamp}" hostname="${escapeXml(suite.hostname)}">\n`;
+
+    // Include system-out if we have stdout
+    if (suite.stdout) {
+      xml += `    <system-out><![CDATA[${suite.stdout}]]></system-out>\n`;
+    }
+
+    // Write each test case
+    for (const test of suite.tests) {
+      xml += `    <testcase name="${escapeXml(test.name)}" classname="${escapeXml(test.classname)}" time="${test.time.toFixed(3)}"`;
+
+      if (test.skipped) {
+        xml += `>\n      <skipped message="${escapeXml(test.skipped.message)}"/>\n    </testcase>\n`;
+      } else if (test.failure) {
+        xml += `>\n`;
+        xml += `      <failure message="${escapeXml(test.failure.message)}" type="${escapeXml(test.failure.type)}"><![CDATA[${test.failure.content}]]></failure>\n`;
+        xml += `    </testcase>\n`;
+      } else {
+        xml += `/>\n`;
+      }
+    }
+
+    xml += `  </testsuite>\n`;
+  }
+
+  xml += `</testsuites>`;
+
+  // Create directory if it doesn't exist
+  const dir = dirname(outfile);
+  mkdirSync(dir, { recursive: true });
+
+  // Write to file
+  writeFileSync(outfile, xml);
+  !isQuiet && console.log(`JUnit XML report written to ${outfile}`);
+}
+
+/**
+ * Upload JUnit XML report to BuildKite Test Analytics
+ * @param {string} junitFile - Path to the JUnit XML file to upload
+ * @returns {Promise<boolean>} - Whether the upload was successful
+ */
+async function uploadJUnitToBuildKite(junitFile) {
+  const fileName = basename(junitFile);
+  !isQuiet && console.log(`Uploading JUnit file "${fileName}" to BuildKite Test Analytics...`);
+
+  // Get BuildKite environment variables for run_env fields
+  const buildId = getEnv("BUILDKITE_BUILD_ID", false);
+  const buildUrl = getEnv("BUILDKITE_BUILD_URL", false);
+  const branch = getBranch();
+  const commit = getCommit();
+  const buildNumber = getEnv("BUILDKITE_BUILD_NUMBER", false);
+  const jobId = getEnv("BUILDKITE_JOB_ID", false);
+  const message = getEnv("BUILDKITE_MESSAGE", false);
+
+  try {
+    // Add a unique test suite identifier to help with correlation in BuildKite
+    const testId = fileName.replace(/\.xml$/, "");
+
+    // Use fetch and FormData instead of curl
+    const formData = new FormData();
+
+    // Add the JUnit file data
+    formData.append("data", new Blob([readFileSync(junitFile)]), fileName);
+    formData.append("format", "junit");
+    formData.append("run_env[CI]", "buildkite");
+
+    // Add additional fields
+    if (buildId) formData.append("run_env[key]", buildId);
+    if (buildUrl) formData.append("run_env[url]", buildUrl);
+    if (branch) formData.append("run_env[branch]", branch);
+    if (commit) formData.append("run_env[commit_sha]", commit);
+    if (buildNumber) formData.append("run_env[number]", buildNumber);
+    if (jobId) formData.append("run_env[job_id]", jobId);
+    if (message) formData.append("run_env[message]", message);
+
+    // Add custom tags
+    formData.append("tags[runtime]", "bun");
+    formData.append("tags[suite]", testId);
+
+    // Add additional context information specific to this run
+    formData.append("run_env[source]", "junit-import");
+    formData.append("run_env[collector]", "bun-runner");
+
+    const url = "https://analytics-api.buildkite.com/v1/uploads";
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Authorization": `Token token="${getBuildkiteAnalyticsToken()}"`,
+      },
+      body: formData,
+    });
+
+    if (response.ok) {
+      !isQuiet && console.log(`JUnit file "${fileName}" successfully uploaded to BuildKite Test Analytics`);
+      return true;
+    } else {
+      const errorText = await response.text();
+      console.error(`Failed to upload JUnit file "${fileName}": HTTP ${response.status}`, errorText);
+      return false;
+    }
+  } catch (error) {
+    console.error(`Error uploading JUnit file "${fileName}":`, error);
+    return false;
+  }
+}
+
+/**
+ * Escape XML special characters
+ * @param {string} str - String to escape
+ * @returns {string} - Escaped string
+ */
+function escapeXml(str) {
+  if (typeof str !== "string") return "";
+  return str
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
 export async function main() {
   for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
     process.on(signal, () => onExit(signal));