diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs
index ceb2d5ca5f..24bfd90332 100755
--- a/scripts/runner.node.mjs
+++ b/scripts/runner.node.mjs
@@ -7,26 +7,35 @@
// - It cannot use Bun APIs, since it is run using Node.js.
// - It does not import dependencies, so it's faster to start.
+import { spawn, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
import {
- constants as fs,
- readFileSync,
- mkdtempSync,
- existsSync,
- statSync,
- mkdirSync,
accessSync,
appendFileSync,
+ existsSync,
+ constants as fs,
+ mkdirSync,
+ mkdtempSync,
readdirSync,
+ readFileSync,
+ statSync,
+ unlink,
+ unlinkSync,
+ writeFileSync,
} from "node:fs";
-import { spawn, spawnSync } from "node:child_process";
-import { join, basename, dirname, relative, sep } from "node:path";
+import { hostname, userInfo } from "node:os";
+import { basename, dirname, join, relative, sep } from "node:path";
import { parseArgs } from "node:util";
import {
+ getBranch,
getBuildLabel,
getBuildUrl,
+ getCommit,
getEnv,
getFileUrl,
+ getHostname,
getLoggedInUserCountOrDetails,
+ getSecret,
getShell,
getWindowsExitReason,
isBuildkite,
@@ -41,7 +50,6 @@ import {
tmpdir,
unzip,
} from "./utils.mjs";
-import { userInfo } from "node:os";
let isQuiet = false;
const cwd = import.meta.dirname ? dirname(import.meta.dirname) : process.cwd();
const testsPath = join(cwd, "test");
@@ -116,9 +124,33 @@ const { values: options, positionals: filters } = parseArgs({
type: "string",
default: isCI ? "4" : "0", // N retries = N+1 attempts
},
+ ["junit"]: {
+ type: "boolean",
+ default: isCI, // Always enable JUnit in CI
+ },
+ ["junit-temp-dir"]: {
+ type: "string",
+ default: "junit-reports",
+ },
+ ["junit-upload"]: {
+ type: "boolean",
+ default: isBuildkite,
+ },
},
});
+const cliOptions = options;
+
+if (cliOptions.junit) {
+ cliOptions["junit-temp-dir"] = join(tmpdir(), cliOptions["junit-temp-dir"]);
+ try {
+ mkdirSync(cliOptions["junit-temp-dir"], { recursive: true });
+ } catch (err) {
+ cliOptions.junit = false;
+ console.error(`Error creating JUnit temp directory: ${err.message}`);
+ }
+}
+
if (options["quiet"]) {
isQuiet = true;
}
@@ -337,6 +369,95 @@ async function runTests() {
reportOutputToGitHubAction("failing_tests", markdown);
}
+ // Generate and upload JUnit reports if requested
+ if (options["junit"]) {
+ const junitTempDir = options["junit-temp-dir"];
+ mkdirSync(junitTempDir, { recursive: true });
+
+ // Generate JUnit reports for tests that don't use bun test
+ const nonBunTestResults = [...okResults, ...flakyResults, ...failedResults].filter(result => {
+ // Check if this is a test that wasn't run with bun test
+ const isNodeTest =
+ isJavaScript(result.testPath) && !isTestStrict(result.testPath) && !result.testPath.includes("vendor");
+ return isNodeTest;
+ });
+
+ // If we have tests not covered by bun test JUnit reports, generate a report for them
+ if (nonBunTestResults.length > 0) {
+ const nonBunTestJunitPath = join(junitTempDir, "non-bun-test-results.xml");
+ generateJUnitReport(nonBunTestJunitPath, nonBunTestResults);
+ !isQuiet &&
+ console.log(
+ `Generated JUnit report for ${nonBunTestResults.length} non-bun test results at ${nonBunTestJunitPath}`,
+ );
+
+ // Upload this report immediately if we're on BuildKite
+ if (isBuildkite && options["junit-upload"]) {
+ const uploadSuccess = await uploadJUnitToBuildKite(nonBunTestJunitPath);
+ if (uploadSuccess) {
+ // Delete the file after successful upload to prevent redundant uploads
+ try {
+ unlinkSync(nonBunTestJunitPath);
+ !isQuiet && console.log(`Uploaded and deleted non-bun test JUnit report`);
+ } catch (unlinkError) {
+ !isQuiet && console.log(`Uploaded but failed to delete non-bun test JUnit report: ${unlinkError.message}`);
+ }
+ } else {
+ !isQuiet && console.log(`Failed to upload non-bun test JUnit report to BuildKite`);
+ }
+ }
+ }
+
+ // Check for any JUnit reports that may not have been uploaded yet
+ // Since we're deleting files after upload, any remaining files need to be uploaded
+ if (isBuildkite && options["junit-upload"]) {
+ try {
+ // Only process XML files and skip the non-bun test results which we've already uploaded
+ const allJunitFiles = readdirSync(junitTempDir).filter(
+ file => file.endsWith(".xml") && file !== "non-bun-test-results.xml",
+ );
+
+ if (allJunitFiles.length > 0) {
+ !isQuiet && console.log(`Found ${allJunitFiles.length} remaining JUnit reports to upload...`);
+
+ // Process each remaining JUnit file - these are files we haven't processed yet
+ let uploadedCount = 0;
+
+ for (const file of allJunitFiles) {
+ const filePath = join(junitTempDir, file);
+
+ if (existsSync(filePath)) {
+ try {
+ const uploadSuccess = await uploadJUnitToBuildKite(filePath);
+ if (uploadSuccess) {
+ // Delete the file after successful upload
+ try {
+ unlinkSync(filePath);
+ uploadedCount++;
+ } catch (unlinkError) {
+ !isQuiet && console.log(`Uploaded but failed to delete ${file}: ${unlinkError.message}`);
+ }
+ }
+ } catch (err) {
+ console.error(`Error uploading JUnit file ${file}:`, err);
+ }
+ }
+ }
+
+ if (uploadedCount > 0) {
+ !isQuiet && console.log(`Uploaded and deleted ${uploadedCount} remaining JUnit reports`);
+ } else {
+ !isQuiet && console.log(`No JUnit reports needed to be uploaded`);
+ }
+ } else {
+ !isQuiet && console.log(`No remaining JUnit reports found to upload`);
+ }
+ } catch (err) {
+ console.error(`Error checking for remaining JUnit reports:`, err);
+ }
+ }
+ }
+
if (!isCI && !isQuiet) {
console.table({
"Total Tests": okResults.length + failedResults.length + flakyResults.length,
@@ -653,8 +774,31 @@ async function spawnBunTest(execPath, testPath, options = { cwd }) {
const absPath = join(options["cwd"], testPath);
const isReallyTest = isTestStrict(testPath) || absPath.includes("vendor");
const args = options["args"] ?? [];
+
+ const testArgs = ["test", ...args, `--timeout=${perTestTimeout}`];
+
+ // This will be set if a JUnit file is generated
+ let junitFilePath = null;
+
+ // In CI, we want to use JUnit for all tests
+ // Create a unique filename for each test run using a hash of the test path
+ // This ensures we can run tests in parallel without file conflicts
+ if (cliOptions.junit) {
+ const testHash = createHash("sha1").update(testPath).digest("base64url");
+ const junitTempDir = cliOptions["junit-temp-dir"];
+
+ // Create the JUnit file path
+ junitFilePath = `${junitTempDir}/test-${testHash}.xml`;
+
+ // Add JUnit reporter
+ testArgs.push("--reporter=junit");
+ testArgs.push(`--reporter-outfile=${junitFilePath}`);
+ }
+
+ testArgs.push(absPath);
+
const { ok, error, stdout } = await spawnBun(execPath, {
- args: isReallyTest ? ["test", ...args, `--timeout=${perTestTimeout}`, absPath] : [...args, absPath],
+ args: isReallyTest ? testArgs : [...args, absPath],
cwd: options["cwd"],
timeout: isReallyTest ? timeout : 30_000,
env: {
@@ -664,6 +808,25 @@ async function spawnBunTest(execPath, testPath, options = { cwd }) {
stderr: chunk => pipeTestStdout(process.stderr, chunk),
});
const { tests, errors, stdout: stdoutPreview } = parseTestStdout(stdout, testPath);
+
+ // If we generated a JUnit file and we're on BuildKite, upload it immediately
+ if (junitFilePath && isReallyTest && isBuildkite && cliOptions["junit-upload"]) {
+ // Give the file system a moment to finish writing the file
+ if (existsSync(junitFilePath)) {
+ uploadJUnitToBuildKite(junitFilePath)
+ .then(uploadSuccess => {
+ unlink(junitFilePath, () => {
+ if (!uploadSuccess) {
+ console.error(`Failed to upload JUnit report for ${testPath}`);
+ }
+ });
+ })
+ .catch(err => {
+ console.error(`Error uploading JUnit report for ${testPath}:`, err);
+ });
+ }
+ }
+
return {
testPath,
ok,
@@ -1549,6 +1712,260 @@ function onExit(signal) {
});
}
+let getBuildkiteAnalyticsToken = () => {
+ let token = getSecret("TEST_REPORTING_API", { required: true });
+ getBuildkiteAnalyticsToken = () => token;
+ return token;
+};
+
+/**
+ * Generate a JUnit XML report from test results
+ * @param {string} outfile - The path to write the JUnit XML report to
+ * @param {TestResult[]} results - The test results to include in the report
+ */
+function generateJUnitReport(outfile, results) {
+ !isQuiet && console.log(`Generating JUnit XML report: ${outfile}`);
+
+ // Start the XML document
+ let xml = '\n';
+
+ // Add an overall testsuite container with metadata
+ const totalTests = results.length;
+ const totalFailures = results.filter(r => r.status === "fail").length;
+ const timestamp = new Date().toISOString();
+
+ // Calculate total time
+ const totalTime = results.reduce((sum, result) => {
+ const duration = result.duration || 0;
+ return sum + duration / 1000; // Convert ms to seconds
+ }, 0);
+
+ // Create a unique package name to identify this run
+ const packageName = `bun.internal.${process.env.BUILDKITE_PIPELINE_SLUG || "tests"}`;
+
+ xml += `\n`;
+
+ // Group results by test file
+ const testSuites = new Map();
+
+ for (const result of results) {
+ const { testPath, ok, status, error, tests, stdoutPreview, stdout, duration = 0 } = result;
+
+ if (!testSuites.has(testPath)) {
+ testSuites.set(testPath, {
+ name: testPath,
+ tests: [],
+ failures: 0,
+ errors: 0,
+ skipped: 0,
+ time: 0,
+ timestamp: timestamp,
+ hostname: getHostname(),
+ stdout: stdout || stdoutPreview || "",
+ });
+ }
+
+ const suite = testSuites.get(testPath);
+
+ // For test suites with granular test information
+ if (tests.length > 0) {
+ for (const test of tests) {
+ const { test: testName, status: testStatus, duration: testDuration = 0, errors: testErrors = [] } = test;
+
+ suite.time += testDuration / 1000; // Convert to seconds
+
+ const testCase = {
+ name: testName,
+ classname: `${packageName}.${testPath.replace(/[\/\\]/g, ".")}`,
+ time: testDuration / 1000, // Convert to seconds
+ };
+
+ if (testStatus === "fail") {
+ suite.failures++;
+
+ // Collect error details
+ let errorMessage = "Test failed";
+ let errorType = "AssertionError";
+ let errorContent = "";
+
+ if (testErrors && testErrors.length > 0) {
+ const primaryError = testErrors[0];
+ errorMessage = primaryError.name || "Test failed";
+ errorType = primaryError.name || "AssertionError";
+ errorContent = primaryError.stack || primaryError.name;
+
+ if (testErrors.length > 1) {
+ errorContent +=
+ "\n\nAdditional errors:\n" +
+ testErrors
+ .slice(1)
+ .map(e => e.stack || e.name)
+ .join("\n");
+ }
+ } else {
+ errorContent = error || "Unknown error";
+ }
+
+ testCase.failure = {
+ message: errorMessage,
+ type: errorType,
+ content: errorContent,
+ };
+ } else if (testStatus === "skip" || testStatus === "todo") {
+ suite.skipped++;
+ testCase.skipped = {
+ message: testStatus === "skip" ? "Test skipped" : "Test marked as todo",
+ };
+ }
+
+ suite.tests.push(testCase);
+ }
+ } else {
+ // For test suites without granular test information (e.g., bun install tests)
+ suite.time += duration / 1000; // Convert to seconds
+
+ const testCase = {
+ name: basename(testPath),
+ classname: `${packageName}.${testPath.replace(/[\/\\]/g, ".")}`,
+ time: duration / 1000, // Convert to seconds
+ };
+
+ if (status === "fail") {
+ suite.failures++;
+ testCase.failure = {
+ message: "Test failed",
+ type: "AssertionError",
+ content: error || "Unknown error",
+ };
+ }
+
+ suite.tests.push(testCase);
+ }
+ }
+
+ // Write each test suite to the XML
+ for (const [name, suite] of testSuites) {
+ xml += ` \n`;
+
+ // Include system-out if we have stdout
+ if (suite.stdout) {
+ xml += ` \n`;
+ }
+
+ // Write each test case
+ for (const test of suite.tests) {
+ xml += ` \n \n \n`;
+ } else if (test.failure) {
+ xml += `>\n`;
+ xml += ` \n`;
+ xml += ` \n`;
+ } else {
+ xml += `/>\n`;
+ }
+ }
+
+ xml += ` \n`;
+ }
+
+ xml += ``;
+
+ // Create directory if it doesn't exist
+ const dir = dirname(outfile);
+ mkdirSync(dir, { recursive: true });
+
+ // Write to file
+ writeFileSync(outfile, xml);
+ !isQuiet && console.log(`JUnit XML report written to ${outfile}`);
+}
+
+/**
+ * Upload JUnit XML report to BuildKite Test Analytics
+ * @param {string} junitFile - Path to the JUnit XML file to upload
+ * @returns {Promise} - Whether the upload was successful
+ */
+async function uploadJUnitToBuildKite(junitFile) {
+ const fileName = basename(junitFile);
+ !isQuiet && console.log(`Uploading JUnit file "${fileName}" to BuildKite Test Analytics...`);
+
+ // Get BuildKite environment variables for run_env fields
+ const buildId = getEnv("BUILDKITE_BUILD_ID", false);
+ const buildUrl = getEnv("BUILDKITE_BUILD_URL", false);
+ const branch = getBranch();
+ const commit = getCommit();
+ const buildNumber = getEnv("BUILDKITE_BUILD_NUMBER", false);
+ const jobId = getEnv("BUILDKITE_JOB_ID", false);
+ const message = getEnv("BUILDKITE_MESSAGE", false);
+
+ try {
+ // Add a unique test suite identifier to help with correlation in BuildKite
+ const testId = fileName.replace(/\.xml$/, "");
+
+ // Use fetch and FormData instead of curl
+ const formData = new FormData();
+
+ // Add the JUnit file data
+ formData.append("data", new Blob([readFileSync(junitFile)]), fileName);
+ formData.append("format", "junit");
+ formData.append("run_env[CI]", "buildkite");
+
+ // Add additional fields
+ if (buildId) formData.append("run_env[key]", buildId);
+ if (buildUrl) formData.append("run_env[url]", buildUrl);
+ if (branch) formData.append("run_env[branch]", branch);
+ if (commit) formData.append("run_env[commit_sha]", commit);
+ if (buildNumber) formData.append("run_env[number]", buildNumber);
+ if (jobId) formData.append("run_env[job_id]", jobId);
+ if (message) formData.append("run_env[message]", message);
+
+ // Add custom tags
+ formData.append("tags[runtime]", "bun");
+ formData.append("tags[suite]", testId);
+
+ // Add additional context information specific to this run
+ formData.append("run_env[source]", "junit-import");
+ formData.append("run_env[collector]", "bun-runner");
+
+ const url = "https://analytics-api.buildkite.com/v1/uploads";
+ const response = await fetch(url, {
+ method: "POST",
+ headers: {
+ "Authorization": `Token token="${getBuildkiteAnalyticsToken()}"`,
+ },
+ body: formData,
+ });
+
+ if (response.ok) {
+ !isQuiet && console.log(`JUnit file "${fileName}" successfully uploaded to BuildKite Test Analytics`);
+ return true;
+ } else {
+ const errorText = await response.text();
+ console.error(`Failed to upload JUnit file "${fileName}": HTTP ${response.status}`, errorText);
+ return false;
+ }
+ } catch (error) {
+ console.error(`Error uploading JUnit file "${fileName}":`, error);
+ return false;
+ }
+}
+
+/**
+ * Escape XML special characters
+ * @param {string} str - String to escape
+ * @returns {string} - Escaped string
+ */
+function escapeXml(str) {
+ if (typeof str !== "string") return "";
+ return str
+ .replace(/&/g, "&")
+ .replace(//g, ">")
+ .replace(/"/g, """)
+ .replace(/'/g, "'");
+}
+
export async function main() {
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
process.on(signal, () => onExit(signal));