From c6076f2e4ea948e3c89150e8c206cdbbeccb80aa Mon Sep 17 00:00:00 2001 From: Ashcon Partovi Date: Wed, 2 Apr 2025 12:51:04 -0700 Subject: [PATCH] Fix updating dependency not cloning and re-building (#18708) --- cmake/Globals.cmake | 9 +++- scripts/build.mjs | 58 +++++++++++++++++--- scripts/runner.node.mjs | 47 ++-------------- scripts/utils.mjs | 117 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 179 insertions(+), 52 deletions(-) diff --git a/cmake/Globals.cmake b/cmake/Globals.cmake index 5d71f72642..e9a74a1380 100644 --- a/cmake/Globals.cmake +++ b/cmake/Globals.cmake @@ -633,7 +633,7 @@ function(register_repository) set(GIT_PATH ${VENDOR_PATH}/${GIT_NAME}) endif() - set(GIT_EFFECTIVE_OUTPUTS) + set(GIT_EFFECTIVE_OUTPUTS ${GIT_PATH}/.ref) foreach(output ${GIT_OUTPUTS}) list(APPEND GIT_EFFECTIVE_OUTPUTS ${GIT_PATH}/${output}) endforeach() @@ -751,11 +751,17 @@ function(register_cmake_command) list(APPEND MAKE_EFFECTIVE_ARGS --fresh) endif() + set(MAKE_SOURCES) + if(TARGET clone-${MAKE_TARGET}) + list(APPEND MAKE_SOURCES ${MAKE_CWD}/.ref) + endif() + register_command( COMMENT "Configuring ${MAKE_TARGET}" TARGET configure-${MAKE_TARGET} COMMAND ${CMAKE_COMMAND} ${MAKE_EFFECTIVE_ARGS} CWD ${MAKE_CWD} + SOURCES ${MAKE_SOURCES} OUTPUTS ${MAKE_BUILD_PATH}/CMakeCache.txt ) @@ -807,6 +813,7 @@ function(register_cmake_command) TARGETS configure-${MAKE_TARGET} COMMAND ${CMAKE_COMMAND} ${MAKE_BUILD_ARGS} CWD ${MAKE_CWD} + SOURCES ${MAKE_SOURCES} ARTIFACTS ${MAKE_ARTIFACTS} ) diff --git a/scripts/build.mjs b/scripts/build.mjs index 6fc38545a6..38da226cec 100755 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -2,8 +2,15 @@ import { spawn as nodeSpawn } from "node:child_process"; import { existsSync, readFileSync, mkdirSync, cpSync, chmodSync } from "node:fs"; -import { basename, join, resolve } from "node:path"; -import { isCI, printEnvironment, startGroup } from "./utils.mjs"; +import { basename, join, relative, resolve } from "node:path"; +import { + formatAnnotationToHtml, + isCI, + parseAnnotations, + printEnvironment, + reportAnnotationToBuildKite, + startGroup, +} from "./utils.mjs"; // https://cmake.org/cmake/help/latest/manual/cmake.1.html#generate-a-project-buildsystem const generateFlags = [ @@ -222,16 +229,24 @@ async function spawn(command, args, options, label) { timestamp = Date.now(); }); + let stdoutBuffer = ""; + let done; if (pipe) { const stdout = new Promise(resolve => { subprocess.stdout.on("end", resolve); - subprocess.stdout.on("data", data => process.stdout.write(data)); + subprocess.stdout.on("data", data => { + stdoutBuffer += data.toString(); + process.stdout.write(data); + }); }); const stderr = new Promise(resolve => { subprocess.stderr.on("end", resolve); - subprocess.stderr.on("data", data => process.stderr.write(data)); + subprocess.stderr.on("data", data => { + stdoutBuffer += data.toString(); + process.stderr.write(data); + }); }); done = Promise.all([stdout, stderr]); @@ -252,9 +267,38 @@ async function spawn(command, args, options, label) { return; } - if (error) { - console.error(error); - } else if (signalCode) { + let annotated; + try { + const { annotations } = parseAnnotations(stdoutBuffer); + for (const annotation of annotations) { + const content = formatAnnotationToHtml(annotation); + reportAnnotationToBuildKite({ + priority: 10, + label: annotation.title || annotation.filename, + content, + }); + annotated = true; + } + } catch (error) { + console.error(`Failed to parse annotations:`, error); + } + + if (!annotated) { + const content = formatAnnotationToHtml({ + filename: relative(process.cwd(), import.meta.filename), + title: "build failed", + content: stdoutBuffer, + source: "build", + level: "error", + }); + reportAnnotationToBuildKite({ + priority: 10, + label: "build failed", + content, + }); + } + + if (signalCode) { console.error(`Command killed: ${signalCode}`); } else { console.error(`Command exited: code ${exitCode}`); diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs index afad1c2268..ceb2d5ca5f 100755 --- a/scripts/runner.node.mjs +++ b/scripts/runner.node.mjs @@ -36,6 +36,7 @@ import { isWindows, isX64, printEnvironment, + reportAnnotationToBuildKite, startGroup, tmpdir, unzip, @@ -1333,7 +1334,9 @@ function formatTestToMarkdown(result, concise) { if (error) { markdown += ` - ${error}`; } - markdown += ` on ${platform}`; + if (platform) { + markdown += ` on ${platform}`; + } if (concise) { markdown += "\n"; @@ -1396,48 +1399,6 @@ function listArtifactsFromBuildKite(glob, step) { return []; } -/** - * @typedef {object} BuildkiteAnnotation - * @property {string} [context] - * @property {string} label - * @property {string} content - * @property {"error" | "warning" | "info"} [style] - * @property {number} [priority] - * @property {number} [attempt] - */ - -/** - * @param {BuildkiteAnnotation} annotation - */ -function reportAnnotationToBuildKite({ context, label, content, style = "error", priority = 3, attempt = 0 }) { - const { error, status, signal, stderr } = spawnSync( - "buildkite-agent", - ["annotate", "--append", "--style", `${style}`, "--context", `${context || label}`, "--priority", `${priority}`], - { - input: content, - stdio: ["pipe", "ignore", "pipe"], - encoding: "utf-8", - timeout: spawnTimeout, - cwd, - }, - ); - if (status === 0) { - return; - } - if (attempt > 0) { - const cause = error ?? signal ?? `code ${status}`; - throw new Error(`Failed to create annotation: ${label}`, { cause }); - } - const buildLabel = getTestLabel(); - const buildUrl = getBuildUrl(); - const platform = buildUrl ? `${buildLabel}` : buildLabel; - let errorMessage = `
${label} - annotation error on ${platform}`; - if (stderr) { - errorMessage += `\n\n\`\`\`terminal\n${escapeCodeBlock(stderr)}\n\`\`\`\n\n
\n\n`; - } - reportAnnotationToBuildKite({ label: `${label}-error`, content: errorMessage, attempt: attempt + 1 }); -} - /** * @param {string} name * @param {string} value diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 18aa8c51c8..a4ab086229 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -2366,6 +2366,81 @@ export function parseAnnotation(options, context) { }; } +/** + * @typedef {Object} AnnotationFormatOptions + * @property {boolean} [concise] + * @property {boolean} [buildkite] + */ + +/** + * @param {Annotation} annotation + * @param {AnnotationFormatOptions} [options] + * @returns {string} + */ +export function formatAnnotationToHtml(annotation, options = {}) { + const { title, content, source, level, filename, line } = annotation; + const { concise, buildkite = isBuildkite } = options; + + let html; + if (concise) { + html = "
  • "; + } else { + html = "
    "; + } + + if (filename) { + const filePath = filename.replace(/\\/g, "/"); + const fileUrl = getFileUrl(filePath, line); + if (fileUrl) { + html += `${filePath}`; + } else { + html += `${filePath}`; + } + html += " - "; + } + + if (title) { + html += title; + } else if (source) { + if (level) { + html += `${source} ${level}`; + } else { + html += source; + } + } else if (level) { + html += level; + } else { + html += "unknown error"; + } + + const buildLabel = getBuildLabel(); + if (buildLabel) { + html += " on "; + const buildUrl = getBuildUrl(); + if (buildUrl) { + html += `${buildLabel}`; + } else { + html += buildLabel; + } + } + + if (concise) { + html += "
    • \n"; + } else { + html += "\n\n"; + if (buildkite) { + const preview = escapeCodeBlock(content); + html += `\`\`\`terminal\n${preview}\n\`\`\`\n`; + } else { + const preview = escapeHtml(stripAnsi(content)); + html += `
    ${preview}
    \n`; + } + html += "\n\n\n\n"; + } + + return html; +} + /** * @typedef {Object} AnnotationResult * @property {Annotation[]} annotations @@ -2399,7 +2474,7 @@ export function parseAnnotations(content, options = {}) { let length = 0; let match; - while (i + length <= originalLines.length && length < maxLength) { + while (i + length < originalLines.length && length < maxLength) { const originalLine = originalLines[i + length++]; const line = stripAnsi(originalLine).trim(); const patternMatch = pattern.exec(line); @@ -2548,6 +2623,46 @@ export function parseAnnotations(content, options = {}) { }; } +/** + * @typedef {object} BuildkiteAnnotation + * @property {string} [context] + * @property {string} label + * @property {string} content + * @property {"error" | "warning" | "info"} [style] + * @property {number} [priority] + * @property {number} [attempt] + */ + +/** + * @param {BuildkiteAnnotation} annotation + */ +export function reportAnnotationToBuildKite({ context, label, content, style = "error", priority = 3, attempt = 0 }) { + const { error, status, signal, stderr } = nodeSpawnSync( + "buildkite-agent", + ["annotate", "--append", "--style", `${style}`, "--context", `${context || label}`, "--priority", `${priority}`], + { + input: content, + stdio: ["pipe", "ignore", "pipe"], + encoding: "utf-8", + timeout: 5_000, + }, + ); + if (status === 0) { + return; + } + if (attempt > 0) { + const cause = error ?? signal ?? `code ${status}`; + throw new Error(`Failed to create annotation: ${label}`, { cause }); + } + const errorContent = formatAnnotationToHtml({ + title: "annotation error", + content: stderr || "", + source: "buildkite", + level: "error", + }); + reportAnnotationToBuildKite({ label: `${label}-error`, content: errorContent, attempt: attempt + 1 }); +} + /** * @param {object} obj * @param {number} indent