diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs index ceb2d5ca5f..24bfd90332 100755 --- a/scripts/runner.node.mjs +++ b/scripts/runner.node.mjs @@ -7,26 +7,35 @@ // - It cannot use Bun APIs, since it is run using Node.js. // - It does not import dependencies, so it's faster to start. +import { spawn, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { - constants as fs, - readFileSync, - mkdtempSync, - existsSync, - statSync, - mkdirSync, accessSync, appendFileSync, + existsSync, + constants as fs, + mkdirSync, + mkdtempSync, readdirSync, + readFileSync, + statSync, + unlink, + unlinkSync, + writeFileSync, } from "node:fs"; -import { spawn, spawnSync } from "node:child_process"; -import { join, basename, dirname, relative, sep } from "node:path"; +import { hostname, userInfo } from "node:os"; +import { basename, dirname, join, relative, sep } from "node:path"; import { parseArgs } from "node:util"; import { + getBranch, getBuildLabel, getBuildUrl, + getCommit, getEnv, getFileUrl, + getHostname, getLoggedInUserCountOrDetails, + getSecret, getShell, getWindowsExitReason, isBuildkite, @@ -41,7 +50,6 @@ import { tmpdir, unzip, } from "./utils.mjs"; -import { userInfo } from "node:os"; let isQuiet = false; const cwd = import.meta.dirname ? dirname(import.meta.dirname) : process.cwd(); const testsPath = join(cwd, "test"); @@ -116,9 +124,33 @@ const { values: options, positionals: filters } = parseArgs({ type: "string", default: isCI ? "4" : "0", // N retries = N+1 attempts }, + ["junit"]: { + type: "boolean", + default: isCI, // Always enable JUnit in CI + }, + ["junit-temp-dir"]: { + type: "string", + default: "junit-reports", + }, + ["junit-upload"]: { + type: "boolean", + default: isBuildkite, + }, }, }); +const cliOptions = options; + +if (cliOptions.junit) { + cliOptions["junit-temp-dir"] = join(tmpdir(), cliOptions["junit-temp-dir"]); + try { + mkdirSync(cliOptions["junit-temp-dir"], { recursive: true }); + } catch (err) { + cliOptions.junit = false; + console.error(`Error creating JUnit temp directory: ${err.message}`); + } +} + if (options["quiet"]) { isQuiet = true; } @@ -337,6 +369,95 @@ async function runTests() { reportOutputToGitHubAction("failing_tests", markdown); } + // Generate and upload JUnit reports if requested + if (options["junit"]) { + const junitTempDir = options["junit-temp-dir"]; + mkdirSync(junitTempDir, { recursive: true }); + + // Generate JUnit reports for tests that don't use bun test + const nonBunTestResults = [...okResults, ...flakyResults, ...failedResults].filter(result => { + // Check if this is a test that wasn't run with bun test + const isNodeTest = + isJavaScript(result.testPath) && !isTestStrict(result.testPath) && !result.testPath.includes("vendor"); + return isNodeTest; + }); + + // If we have tests not covered by bun test JUnit reports, generate a report for them + if (nonBunTestResults.length > 0) { + const nonBunTestJunitPath = join(junitTempDir, "non-bun-test-results.xml"); + generateJUnitReport(nonBunTestJunitPath, nonBunTestResults); + !isQuiet && + console.log( + `Generated JUnit report for ${nonBunTestResults.length} non-bun test results at ${nonBunTestJunitPath}`, + ); + + // Upload this report immediately if we're on BuildKite + if (isBuildkite && options["junit-upload"]) { + const uploadSuccess = await uploadJUnitToBuildKite(nonBunTestJunitPath); + if (uploadSuccess) { + // Delete the file after successful upload to prevent redundant uploads + try { + unlinkSync(nonBunTestJunitPath); + !isQuiet && console.log(`Uploaded and deleted non-bun test JUnit report`); + } catch (unlinkError) { + !isQuiet && console.log(`Uploaded but failed to delete non-bun test JUnit report: ${unlinkError.message}`); + } + } else { + !isQuiet && console.log(`Failed to upload non-bun test JUnit report to BuildKite`); + } + } + } + + // Check for any JUnit reports that may not have been uploaded yet + // Since we're deleting files after upload, any remaining files need to be uploaded + if (isBuildkite && options["junit-upload"]) { + try { + // Only process XML files and skip the non-bun test results which we've already uploaded + const allJunitFiles = readdirSync(junitTempDir).filter( + file => file.endsWith(".xml") && file !== "non-bun-test-results.xml", + ); + + if (allJunitFiles.length > 0) { + !isQuiet && console.log(`Found ${allJunitFiles.length} remaining JUnit reports to upload...`); + + // Process each remaining JUnit file - these are files we haven't processed yet + let uploadedCount = 0; + + for (const file of allJunitFiles) { + const filePath = join(junitTempDir, file); + + if (existsSync(filePath)) { + try { + const uploadSuccess = await uploadJUnitToBuildKite(filePath); + if (uploadSuccess) { + // Delete the file after successful upload + try { + unlinkSync(filePath); + uploadedCount++; + } catch (unlinkError) { + !isQuiet && console.log(`Uploaded but failed to delete ${file}: ${unlinkError.message}`); + } + } + } catch (err) { + console.error(`Error uploading JUnit file ${file}:`, err); + } + } + } + + if (uploadedCount > 0) { + !isQuiet && console.log(`Uploaded and deleted ${uploadedCount} remaining JUnit reports`); + } else { + !isQuiet && console.log(`No JUnit reports needed to be uploaded`); + } + } else { + !isQuiet && console.log(`No remaining JUnit reports found to upload`); + } + } catch (err) { + console.error(`Error checking for remaining JUnit reports:`, err); + } + } + } + if (!isCI && !isQuiet) { console.table({ "Total Tests": okResults.length + failedResults.length + flakyResults.length, @@ -653,8 +774,31 @@ async function spawnBunTest(execPath, testPath, options = { cwd }) { const absPath = join(options["cwd"], testPath); const isReallyTest = isTestStrict(testPath) || absPath.includes("vendor"); const args = options["args"] ?? []; + + const testArgs = ["test", ...args, `--timeout=${perTestTimeout}`]; + + // This will be set if a JUnit file is generated + let junitFilePath = null; + + // In CI, we want to use JUnit for all tests + // Create a unique filename for each test run using a hash of the test path + // This ensures we can run tests in parallel without file conflicts + if (cliOptions.junit) { + const testHash = createHash("sha1").update(testPath).digest("base64url"); + const junitTempDir = cliOptions["junit-temp-dir"]; + + // Create the JUnit file path + junitFilePath = `${junitTempDir}/test-${testHash}.xml`; + + // Add JUnit reporter + testArgs.push("--reporter=junit"); + testArgs.push(`--reporter-outfile=${junitFilePath}`); + } + + testArgs.push(absPath); + const { ok, error, stdout } = await spawnBun(execPath, { - args: isReallyTest ? ["test", ...args, `--timeout=${perTestTimeout}`, absPath] : [...args, absPath], + args: isReallyTest ? testArgs : [...args, absPath], cwd: options["cwd"], timeout: isReallyTest ? timeout : 30_000, env: { @@ -664,6 +808,25 @@ async function spawnBunTest(execPath, testPath, options = { cwd }) { stderr: chunk => pipeTestStdout(process.stderr, chunk), }); const { tests, errors, stdout: stdoutPreview } = parseTestStdout(stdout, testPath); + + // If we generated a JUnit file and we're on BuildKite, upload it immediately + if (junitFilePath && isReallyTest && isBuildkite && cliOptions["junit-upload"]) { + // Give the file system a moment to finish writing the file + if (existsSync(junitFilePath)) { + uploadJUnitToBuildKite(junitFilePath) + .then(uploadSuccess => { + unlink(junitFilePath, () => { + if (!uploadSuccess) { + console.error(`Failed to upload JUnit report for ${testPath}`); + } + }); + }) + .catch(err => { + console.error(`Error uploading JUnit report for ${testPath}:`, err); + }); + } + } + return { testPath, ok, @@ -1549,6 +1712,260 @@ function onExit(signal) { }); } +let getBuildkiteAnalyticsToken = () => { + let token = getSecret("TEST_REPORTING_API", { required: true }); + getBuildkiteAnalyticsToken = () => token; + return token; +}; + +/** + * Generate a JUnit XML report from test results + * @param {string} outfile - The path to write the JUnit XML report to + * @param {TestResult[]} results - The test results to include in the report + */ +function generateJUnitReport(outfile, results) { + !isQuiet && console.log(`Generating JUnit XML report: ${outfile}`); + + // Start the XML document + let xml = '\n'; + + // Add an overall testsuite container with metadata + const totalTests = results.length; + const totalFailures = results.filter(r => r.status === "fail").length; + const timestamp = new Date().toISOString(); + + // Calculate total time + const totalTime = results.reduce((sum, result) => { + const duration = result.duration || 0; + return sum + duration / 1000; // Convert ms to seconds + }, 0); + + // Create a unique package name to identify this run + const packageName = `bun.internal.${process.env.BUILDKITE_PIPELINE_SLUG || "tests"}`; + + xml += `\n`; + + // Group results by test file + const testSuites = new Map(); + + for (const result of results) { + const { testPath, ok, status, error, tests, stdoutPreview, stdout, duration = 0 } = result; + + if (!testSuites.has(testPath)) { + testSuites.set(testPath, { + name: testPath, + tests: [], + failures: 0, + errors: 0, + skipped: 0, + time: 0, + timestamp: timestamp, + hostname: getHostname(), + stdout: stdout || stdoutPreview || "", + }); + } + + const suite = testSuites.get(testPath); + + // For test suites with granular test information + if (tests.length > 0) { + for (const test of tests) { + const { test: testName, status: testStatus, duration: testDuration = 0, errors: testErrors = [] } = test; + + suite.time += testDuration / 1000; // Convert to seconds + + const testCase = { + name: testName, + classname: `${packageName}.${testPath.replace(/[\/\\]/g, ".")}`, + time: testDuration / 1000, // Convert to seconds + }; + + if (testStatus === "fail") { + suite.failures++; + + // Collect error details + let errorMessage = "Test failed"; + let errorType = "AssertionError"; + let errorContent = ""; + + if (testErrors && testErrors.length > 0) { + const primaryError = testErrors[0]; + errorMessage = primaryError.name || "Test failed"; + errorType = primaryError.name || "AssertionError"; + errorContent = primaryError.stack || primaryError.name; + + if (testErrors.length > 1) { + errorContent += + "\n\nAdditional errors:\n" + + testErrors + .slice(1) + .map(e => e.stack || e.name) + .join("\n"); + } + } else { + errorContent = error || "Unknown error"; + } + + testCase.failure = { + message: errorMessage, + type: errorType, + content: errorContent, + }; + } else if (testStatus === "skip" || testStatus === "todo") { + suite.skipped++; + testCase.skipped = { + message: testStatus === "skip" ? "Test skipped" : "Test marked as todo", + }; + } + + suite.tests.push(testCase); + } + } else { + // For test suites without granular test information (e.g., bun install tests) + suite.time += duration / 1000; // Convert to seconds + + const testCase = { + name: basename(testPath), + classname: `${packageName}.${testPath.replace(/[\/\\]/g, ".")}`, + time: duration / 1000, // Convert to seconds + }; + + if (status === "fail") { + suite.failures++; + testCase.failure = { + message: "Test failed", + type: "AssertionError", + content: error || "Unknown error", + }; + } + + suite.tests.push(testCase); + } + } + + // Write each test suite to the XML + for (const [name, suite] of testSuites) { + xml += ` \n`; + + // Include system-out if we have stdout + if (suite.stdout) { + xml += ` \n`; + } + + // Write each test case + for (const test of suite.tests) { + xml += ` \n \n \n`; + } else if (test.failure) { + xml += `>\n`; + xml += ` \n`; + xml += ` \n`; + } else { + xml += `/>\n`; + } + } + + xml += ` \n`; + } + + xml += ``; + + // Create directory if it doesn't exist + const dir = dirname(outfile); + mkdirSync(dir, { recursive: true }); + + // Write to file + writeFileSync(outfile, xml); + !isQuiet && console.log(`JUnit XML report written to ${outfile}`); +} + +/** + * Upload JUnit XML report to BuildKite Test Analytics + * @param {string} junitFile - Path to the JUnit XML file to upload + * @returns {Promise} - Whether the upload was successful + */ +async function uploadJUnitToBuildKite(junitFile) { + const fileName = basename(junitFile); + !isQuiet && console.log(`Uploading JUnit file "${fileName}" to BuildKite Test Analytics...`); + + // Get BuildKite environment variables for run_env fields + const buildId = getEnv("BUILDKITE_BUILD_ID", false); + const buildUrl = getEnv("BUILDKITE_BUILD_URL", false); + const branch = getBranch(); + const commit = getCommit(); + const buildNumber = getEnv("BUILDKITE_BUILD_NUMBER", false); + const jobId = getEnv("BUILDKITE_JOB_ID", false); + const message = getEnv("BUILDKITE_MESSAGE", false); + + try { + // Add a unique test suite identifier to help with correlation in BuildKite + const testId = fileName.replace(/\.xml$/, ""); + + // Use fetch and FormData instead of curl + const formData = new FormData(); + + // Add the JUnit file data + formData.append("data", new Blob([readFileSync(junitFile)]), fileName); + formData.append("format", "junit"); + formData.append("run_env[CI]", "buildkite"); + + // Add additional fields + if (buildId) formData.append("run_env[key]", buildId); + if (buildUrl) formData.append("run_env[url]", buildUrl); + if (branch) formData.append("run_env[branch]", branch); + if (commit) formData.append("run_env[commit_sha]", commit); + if (buildNumber) formData.append("run_env[number]", buildNumber); + if (jobId) formData.append("run_env[job_id]", jobId); + if (message) formData.append("run_env[message]", message); + + // Add custom tags + formData.append("tags[runtime]", "bun"); + formData.append("tags[suite]", testId); + + // Add additional context information specific to this run + formData.append("run_env[source]", "junit-import"); + formData.append("run_env[collector]", "bun-runner"); + + const url = "https://analytics-api.buildkite.com/v1/uploads"; + const response = await fetch(url, { + method: "POST", + headers: { + "Authorization": `Token token="${getBuildkiteAnalyticsToken()}"`, + }, + body: formData, + }); + + if (response.ok) { + !isQuiet && console.log(`JUnit file "${fileName}" successfully uploaded to BuildKite Test Analytics`); + return true; + } else { + const errorText = await response.text(); + console.error(`Failed to upload JUnit file "${fileName}": HTTP ${response.status}`, errorText); + return false; + } + } catch (error) { + console.error(`Error uploading JUnit file "${fileName}":`, error); + return false; + } +} + +/** + * Escape XML special characters + * @param {string} str - String to escape + * @returns {string} - Escaped string + */ +function escapeXml(str) { + if (typeof str !== "string") return ""; + return str + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + export async function main() { for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) { process.on(signal, () => onExit(signal));