From 7e9fa4ab08d9250c72795565c173cf7349de4018 Mon Sep 17 00:00:00 2001 From: robobun Date: Fri, 16 Jan 2026 15:37:31 -0800 Subject: [PATCH] feat(scripts): enhance buildkite-failures.ts to fetch and save full logs (#26177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Fetches complete logs from BuildKite's public API (no token required) - Saves logs to `/tmp/bun-build-{number}-{platform}-{step}.log` - Shows log file path in output for each failed job - Displays brief error summary (unique errors, max 5) - Adds help text with usage examples (`--help`) - Groups failures by type (build/test/other) - Shows annotation counts with link to view full annotations - Documents usage in CLAUDE.md ## Test plan - [x] Tested with build #35051 (9 failed jobs) - [x] Verified logs saved to `/tmp/bun-build-35051-*.log` - [x] Verified error extraction and deduplication works - [x] Verified `--help` flag shows usage 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Bot Co-authored-by: Claude Opus 4.5 --- CLAUDE.md | 21 ++ scripts/buildkite-failures.ts | 395 +++++++++++++++++++++++++++++++--- 2 files changed, 385 insertions(+), 31 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0c9f70c345..7fc53decb4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -211,3 +211,24 @@ Built-in JavaScript modules use special syntax and are organized as: 12. **Branch names must start with `claude/`** - This is a requirement for the CI to work. **ONLY** push up changes after running `bun bd test ` and ensuring your tests pass. + +## Debugging CI Failures + +Use `scripts/buildkite-failures.ts` to fetch and analyze CI build failures: + +```bash +# View failures for current branch +bun run scripts/buildkite-failures.ts + +# View failures for a specific build number +bun run scripts/buildkite-failures.ts 35051 + +# View failures for a GitHub PR +bun run scripts/buildkite-failures.ts #26173 +bun run scripts/buildkite-failures.ts https://github.com/oven-sh/bun/pull/26173 + +# Wait for build to complete (polls every 10s until pass/fail) +bun run scripts/buildkite-failures.ts --wait +``` + +The script fetches logs from BuildKite's public API and saves complete logs to `/tmp/bun-build-{number}-{platform}-{step}.log`. It displays a summary of errors and the file path for each failed job. Use `--wait` to poll continuously until the build completes or fails. diff --git a/scripts/buildkite-failures.ts b/scripts/buildkite-failures.ts index fa506f83f4..4bc0bb5eca 100755 --- a/scripts/buildkite-failures.ts +++ b/scripts/buildkite-failures.ts @@ -49,9 +49,42 @@ const colors = { // Parse command line arguments const args = process.argv.slice(2); + +// Show help +if (args.includes("--help") || args.includes("-h")) { + console.log(`Usage: bun run scripts/buildkite-failures.ts [options] [build-id|branch|pr-url|buildkite-url] + +Shows detailed error information from BuildKite build failures. +Full logs are saved to /tmp/bun-build-{number}-{platform}-{step}.log + +Arguments: + build-id BuildKite build number (e.g., 35051) + branch Git branch name (e.g., main, claude/fix-bug) + pr-url GitHub PR URL (e.g., https://github.com/oven-sh/bun/pull/26173) + buildkite-url BuildKite build URL + #number GitHub PR number (e.g., #26173) + (none) Uses current git branch + +Options: + --flaky, -f Include flaky test annotations + --warnings, -w Include warning annotations + --wait Poll continuously until build completes or fails + --help, -h Show this help message + +Examples: + bun run scripts/buildkite-failures.ts # Current branch + bun run scripts/buildkite-failures.ts main # Main branch + bun run scripts/buildkite-failures.ts 35051 # Build #35051 + bun run scripts/buildkite-failures.ts #26173 # PR #26173 + bun run scripts/buildkite-failures.ts --wait # Wait for current branch build to complete +`); + process.exit(0); +} + const showWarnings = args.includes("--warnings") || args.includes("-w"); const showFlaky = args.includes("--flaky") || args.includes("-f"); -const inputArg = args.find(arg => !arg.startsWith("-")); +const waitMode = args.includes("--wait"); +const inputArg = args.find(arg => !arg.startsWith("-") && !arg.startsWith("--")); // Determine what type of input we have let buildNumber = null; @@ -114,38 +147,133 @@ if (!buildNumber) { buildNumber = match[1]; } -// Fetch build JSON -const buildResponse = await fetch(`https://buildkite.com/bun/bun/builds/${buildNumber}.json`); -const build = await buildResponse.json(); +// Helper to format time ago +function formatTimeAgo(dateStr: string | null): string { + if (!dateStr) return "not started"; + const date = new Date(dateStr); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffSecs = Math.floor(diffMs / 1000); + const diffMins = Math.floor(diffSecs / 60); + const diffHours = Math.floor(diffMins / 60); + const diffDays = Math.floor(diffHours / 24); -// Calculate time ago -const buildTime = new Date(build.started_at); -const now = new Date(); -const diffMs = now.getTime() - buildTime.getTime(); -const diffSecs = Math.floor(diffMs / 1000); -const diffMins = Math.floor(diffSecs / 60); -const diffHours = Math.floor(diffMins / 60); -const diffDays = Math.floor(diffHours / 24); - -let timeAgo; -if (diffDays > 0) { - timeAgo = `${diffDays} day${diffDays !== 1 ? "s" : ""} ago`; -} else if (diffHours > 0) { - timeAgo = `${diffHours} hour${diffHours !== 1 ? "s" : ""} ago`; -} else if (diffMins > 0) { - timeAgo = `${diffMins} minute${diffMins !== 1 ? "s" : ""} ago`; -} else { - timeAgo = `${diffSecs} second${diffSecs !== 1 ? "s" : ""} ago`; + if (diffDays > 0) return `${diffDays} day${diffDays !== 1 ? "s" : ""} ago`; + if (diffHours > 0) return `${diffHours} hour${diffHours !== 1 ? "s" : ""} ago`; + if (diffMins > 0) return `${diffMins} minute${diffMins !== 1 ? "s" : ""} ago`; + return `${diffSecs} second${diffSecs !== 1 ? "s" : ""} ago`; } +// Helper to clear line for updates +const clearLine = isTTY ? "\x1b[2K\r" : ""; + +// Poll for build status +let build: any; +let pollCount = 0; +const pollInterval = 10000; // 10 seconds + +while (true) { + // Fetch build JSON + const buildResponse = await fetch(`https://buildkite.com/bun/bun/builds/${buildNumber}.json`); + build = await buildResponse.json(); + + // Check for failed jobs first (even if build is still running) + const failedJobsEarly = + build.jobs?.filter( + (job: any) => job.exit_status && job.exit_status > 0 && !job.soft_failed && job.type === "script", + ) || []; + + // In wait mode with failures, stop polling and show failures + if (waitMode && failedJobsEarly.length > 0) { + if (pollCount > 0) { + process.stdout.write(clearLine); + } + break; + } + + // Calculate time ago (use created_at as fallback for scheduled/pending builds) + const timeAgo = formatTimeAgo(build.started_at || build.created_at); + + // Check if build passed + if (build.state === "passed") { + if (pollCount > 0) { + process.stdout.write(clearLine); + } + console.log(`${timeAgo} - build #${buildNumber} https://buildkite.com/bun/bun/builds/${buildNumber}\n`); + console.log(`${colors.green}✅ Passed!${colors.reset}`); + process.exit(0); + } + + // Check if build was canceled + if (build.state === "canceled" || build.state === "canceling") { + if (pollCount > 0) { + process.stdout.write(clearLine); + } + console.log(`${timeAgo} - build #${buildNumber} https://buildkite.com/bun/bun/builds/${buildNumber}\n`); + console.log(`${colors.dim}🚫 Build was canceled${colors.reset}`); + process.exit(0); + } + + // Check if build is pending/running/scheduled + if (build.state === "scheduled" || build.state === "running" || build.state === "creating") { + const runningJobs = build.jobs?.filter((job: any) => job.state === "running") || []; + const pendingJobs = build.jobs?.filter((job: any) => job.state === "scheduled" || job.state === "waiting") || []; + const passedJobs = build.jobs?.filter((job: any) => job.state === "passed") || []; + const totalJobs = build.jobs?.filter((job: any) => job.type === "script")?.length || 0; + + if (waitMode) { + // In wait mode, show a single updating line + let statusMsg = ""; + if (build.state === "scheduled" || build.state === "creating") { + statusMsg = `⏳ Waiting... (scheduled ${formatTimeAgo(build.created_at)})`; + } else { + statusMsg = `🔄 Running... ${passedJobs.length}/${totalJobs} passed, ${runningJobs.length} running`; + } + process.stdout.write(`${clearLine}${colors.dim}${statusMsg}${colors.reset}`); + pollCount++; + await Bun.sleep(pollInterval); + continue; + } else { + // Not in wait mode, show full status and exit + console.log(`${timeAgo} - build #${buildNumber} https://buildkite.com/bun/bun/builds/${buildNumber}\n`); + + if (build.state === "scheduled" || build.state === "creating") { + console.log(`${colors.dim}⏳ Build is scheduled/pending${colors.reset}`); + if (build.created_at) { + console.log(`${colors.dim} Created: ${formatTimeAgo(build.created_at)}${colors.reset}`); + } + } else { + console.log(`${colors.dim}🔄 Build is running${colors.reset}`); + if (build.started_at) { + console.log(`${colors.dim} Started: ${formatTimeAgo(build.started_at)}${colors.reset}`); + } + console.log( + `${colors.dim} Progress: ${passedJobs.length}/${totalJobs} jobs passed, ${runningJobs.length} running, ${pendingJobs.length} pending${colors.reset}`, + ); + + if (runningJobs.length > 0) { + console.log(`\n${colors.dim}Running jobs:${colors.reset}`); + for (const job of runningJobs.slice(0, 5)) { + const name = job.name || job.label || "Unknown"; + console.log(` ${colors.dim}• ${name}${colors.reset}`); + } + if (runningJobs.length > 5) { + console.log(` ${colors.dim}... and ${runningJobs.length - 5} more${colors.reset}`); + } + } + } + process.exit(0); + } + } + + // Build is in a terminal state (failed, etc.) - break out of loop + break; +} + +// Print header for failed build +const timeAgo = formatTimeAgo(build.started_at || build.created_at); console.log(`${timeAgo} - build #${buildNumber} https://buildkite.com/bun/bun/builds/${buildNumber}\n`); -// Check if build passed -if (build.state === "passed") { - console.log(`${colors.green}✅ Passed!${colors.reset}`); - process.exit(0); -} - // Get failed jobs const failedJobs = build.jobs?.filter(job => job.exit_status && job.exit_status > 0 && !job.soft_failed && job.type === "script") || []; @@ -734,7 +862,212 @@ if (registerRequestIndex !== -1) { console.log(` https://buildkite.com/bun/bun/builds/${buildNumber}#annotations`); } } else { - console.log(`\n${colors.red}${colors.bold}${failedJobs.length} job failures${colors.reset}\n`); - console.log("View detailed results at:"); - console.log(` https://buildkite.com/bun/bun/builds/${buildNumber}#annotations`); + // No annotations found - show detailed job failure information + if (failedJobs.length > 0) { + console.log(`\n${colors.red}${colors.bold}${failedJobs.length} job failures${colors.reset}\n`); + + // Show annotation counts if available + const annotationCounts = build.annotation_counts_by_style; + if (annotationCounts) { + const errors = annotationCounts.error || 0; + const warnings = annotationCounts.warning || 0; + if (errors > 0 || warnings > 0) { + const parts = []; + if (errors > 0) parts.push(`${errors} error${errors !== 1 ? "s" : ""}`); + if (warnings > 0) parts.push(`${warnings} warning${warnings !== 1 ? "s" : ""}`); + console.log( + `${colors.dim}Annotations: ${parts.join(", ")} - view at https://buildkite.com/bun/bun/builds/${buildNumber}#annotations${colors.reset}\n`, + ); + } + } + + // Group jobs by type + const buildJobs = failedJobs.filter(job => (job.name || job.label || "").includes("build-")); + const testJobs = failedJobs.filter(job => (job.name || job.label || "").includes("test")); + const otherJobs = failedJobs.filter( + job => !(job.name || job.label || "").includes("build-") && !(job.name || job.label || "").includes("test"), + ); + + // Display build failures + if (buildJobs.length > 0) { + console.log( + `${colors.bgRed}${colors.white}${colors.bold} Build Failures (${buildJobs.length}) ${colors.reset}\n`, + ); + for (const job of buildJobs) { + const name = (job.name || job.label || "Unknown").replace(/^:([^:]+):/, (_, emoji) => { + const platform = emoji.toLowerCase(); + return platformMap[platform] || `:${emoji}:`; + }); + const duration = + job.started_at && job.finished_at + ? `${((new Date(job.finished_at).getTime() - new Date(job.started_at).getTime()) / 1000).toFixed(0)}s` + : "N/A"; + console.log(` ${colors.red}✗${colors.reset} ${name}`); + console.log(` ${colors.dim}Duration: ${duration} | Exit: ${job.exit_status}${colors.reset}`); + console.log(` ${colors.dim}https://buildkite.com${job.path}${colors.reset}`); + console.log(); + } + } + + // Display test failures + if (testJobs.length > 0) { + console.log(`${colors.bgBlue}${colors.white}${colors.bold} Test Failures (${testJobs.length}) ${colors.reset}\n`); + for (const job of testJobs) { + const name = (job.name || job.label || "Unknown").replace(/^:([^:]+):/, (_, emoji) => { + const platform = emoji.toLowerCase(); + return platformMap[platform] || `:${emoji}:`; + }); + const duration = + job.started_at && job.finished_at + ? `${((new Date(job.finished_at).getTime() - new Date(job.started_at).getTime()) / 1000).toFixed(0)}s` + : "N/A"; + console.log(` ${colors.red}✗${colors.reset} ${name}`); + console.log(` ${colors.dim}Duration: ${duration} | Exit: ${job.exit_status}${colors.reset}`); + console.log(` ${colors.dim}https://buildkite.com${job.path}${colors.reset}`); + console.log(); + } + } + + // Display other failures + if (otherJobs.length > 0) { + console.log( + `${colors.bgBlue}${colors.white}${colors.bold} Other Failures (${otherJobs.length}) ${colors.reset}\n`, + ); + for (const job of otherJobs) { + const name = (job.name || job.label || "Unknown").replace(/^:([^:]+):/, (_, emoji) => { + const platform = emoji.toLowerCase(); + return platformMap[platform] || `:${emoji}:`; + }); + const duration = + job.started_at && job.finished_at + ? `${((new Date(job.finished_at).getTime() - new Date(job.started_at).getTime()) / 1000).toFixed(0)}s` + : "N/A"; + console.log(` ${colors.red}✗${colors.reset} ${name}`); + console.log(` ${colors.dim}Duration: ${duration} | Exit: ${job.exit_status}${colors.reset}`); + console.log(` ${colors.dim}https://buildkite.com${job.path}${colors.reset}`); + console.log(); + } + } + + // Fetch and display logs for all failed jobs + // Use the public BuildKite log endpoint + console.log(`${colors.dim}Fetching logs for ${failedJobs.length} failed jobs...${colors.reset}\n`); + + for (const job of failedJobs) { + const name = (job.name || job.label || "Unknown").replace(/^:([^:]+):/, (_, emoji) => { + const platform = emoji.toLowerCase(); + return platformMap[platform] || `:${emoji}:`; + }); + + // Create a sanitized filename from the job name + // e.g., ":darwin: aarch64 - build-cpp" -> "darwin-aarch64-build-cpp" + const sanitizedName = (job.name || job.label || "unknown") + .replace(/^:([^:]+):\s*/, "$1-") // :darwin: -> darwin- + .replace(/\s+-\s+/g, "-") // " - " -> "-" + .replace(/[^a-zA-Z0-9-]/g, "-") // Replace other chars with - + .replace(/-+/g, "-") // Collapse multiple - + .replace(/^-|-$/g, "") // Remove leading/trailing - + .toLowerCase(); + + const logFilePath = `/tmp/bun-build-${buildNumber}-${sanitizedName}.log`; + + try { + const logResponse = await fetch( + `https://buildkite.com/organizations/bun/pipelines/bun/builds/${buildNumber}/jobs/${job.id}/log`, + ); + + if (logResponse.ok) { + const logData = await logResponse.json(); + let output = logData.output || ""; + + // Convert HTML to readable text (without ANSI codes for file output) + const plainOutput = output + // Remove timestamp tags + .replace(/]*>[^<]*<\/time>/g, "") + // Remove all span tags + .replace(/]*>([^<]*)<\/span>/g, "$1") + // Remove remaining HTML tags + .replace(/<[^>]+>/g, "") + // Decode HTML entities + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(///g, "/") + .replace(/ /g, " "); + + // Write the full log to a file + await Bun.write(logFilePath, plainOutput); + + // Extract unique error messages for display + const lines = plainOutput.split("\n"); + const uniqueErrors = new Set(); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Look for actual error messages + const isError = + (line.includes("error:") && !line.includes('error: script "') && !line.includes("error: exit")) || + line.includes("fatal error:") || + line.includes("panic:") || + line.includes("undefined reference"); + + if (isError) { + // Extract just the error message part (remove path prefixes and timestamps) + const errorMsg = line + .replace(/^.*?\d{4}-\d{2}-\d{2}T[\d:.]+Z/, "") // Remove timestamps + .replace(/^.*?\/[^\s]*:\d+:\d+:\s*/, "") // Remove file paths + .trim(); + + if (errorMsg && !uniqueErrors.has(errorMsg)) { + uniqueErrors.add(errorMsg); + } + } + } + + // Display job info with log file path + console.log(`${colors.bgBlue}${colors.white}${colors.bold} ${name} ${colors.reset}`); + console.log(` ${colors.dim}Log: ${logFilePath}${colors.reset}`); + + if (uniqueErrors.size > 0) { + console.log(` ${colors.red}Errors (${uniqueErrors.size}):${colors.reset}`); + let count = 0; + for (const err of uniqueErrors) { + if (count >= 5) { + console.log(` ${colors.dim}... and ${uniqueErrors.size - 5} more${colors.reset}`); + break; + } + console.log(` ${colors.red}•${colors.reset} ${err.slice(0, 120)}${err.length > 120 ? "..." : ""}`); + count++; + } + } else { + // Show last few lines as a preview + const lastLines = lines.slice(-5).filter(l => l.trim()); + if (lastLines.length > 0) { + console.log(` ${colors.dim}Last output:${colors.reset}`); + for (const line of lastLines) { + console.log(` ${colors.dim}${line.slice(0, 100)}${line.length > 100 ? "..." : ""}${colors.reset}`); + } + } + } + + if (logData.truncated) { + console.log(` ${colors.dim}(Log was truncated by BuildKite)${colors.reset}`); + } + } else { + console.log(`${colors.bgBlue}${colors.white}${colors.bold} ${name} ${colors.reset}`); + console.log(` ${colors.dim}Failed to fetch log: ${logResponse.status}${colors.reset}`); + } + } catch (e) { + console.log(`${colors.bgBlue}${colors.white}${colors.bold} ${name} ${colors.reset}`); + console.log(` ${colors.dim}Error fetching log: ${e.message}${colors.reset}`); + } + console.log(); + } + } else { + console.log("View detailed results at:"); + console.log(` https://buildkite.com/bun/bun/builds/${buildNumber}#annotations`); + } }