diff --git a/src/cli.zig b/src/cli.zig index 85c5199069..b73cfec894 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -362,6 +362,7 @@ pub const Command = struct { file_reporter: ?TestCommand.FileReporter = null, reporter_outfile: ?[]const u8 = null, + agent: bool = false, }; pub const Debugger = union(enum) { diff --git a/src/cli/Arguments.zig b/src/cli/Arguments.zig index 03ec522e31..b40d3bb1b7 100644 --- a/src/cli/Arguments.zig +++ b/src/cli/Arguments.zig @@ -192,6 +192,7 @@ pub const test_only_params = [_]ParamType{ clap.parseParam("-t, --test-name-pattern Run only tests with a name that matches the given regex.") catch unreachable, clap.parseParam("--reporter Specify the test reporter. Currently --reporter=junit is the only supported format.") catch unreachable, clap.parseParam("--reporter-outfile The output file used for the format from --reporter.") catch unreachable, + clap.parseParam("--agent Use agent reporter (only prints errors and summary).") catch unreachable, }; pub const test_params = test_only_params ++ runtime_params_ ++ transpiler_params_ ++ base_params_; @@ -481,6 +482,7 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C ctx.test_options.update_snapshots = args.flag("--update-snapshots"); ctx.test_options.run_todo = args.flag("--todo"); ctx.test_options.only = args.flag("--only"); + ctx.test_options.agent = args.flag("--agent"); } ctx.args.absolute_working_dir = cwd; @@ -674,7 +676,8 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C Output.errGeneric("Invalid value for --console-depth: \"{s}\". Must be a positive integer\n", .{depth_str}); Global.exit(1); }; - ctx.runtime_options.console_depth = depth; + // Treat depth=0 as maxInt(u16) for infinite depth + ctx.runtime_options.console_depth = if (depth == 0) std.math.maxInt(u16) else depth; } if (args.option("--dns-result-order")) |order| { diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig index 26a22eacd2..98bc765c29 100644 --- a/src/cli/test_command.zig +++ b/src/cli/test_command.zig @@ -824,15 +824,18 @@ pub const CommandLineReporter = struct { } pub fn handleTestPass(cb: *TestRunner.Callback, id: Test.ID, file: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { - const writer = Output.errorWriterBuffered(); - defer Output.flush(); - var this: *CommandLineReporter = @fieldParentPtr("callback", cb); - writeTestStatusLine(.pass, &writer); + // In agent mode, don't print pass status + if (!this.jest.test_options.agent) { + const writer = Output.errorWriterBuffered(); + defer Output.flush(); - const line_number = this.jest.tests.items(.line_number)[id]; - printTestLine(.pass, label, elapsed_ns, parent, expectations, false, writer, file, this.file_reporter, line_number); + writeTestStatusLine(.pass, &writer); + + const line_number = this.jest.tests.items(.line_number)[id]; + printTestLine(.pass, label, elapsed_ns, parent, expectations, false, writer, file, this.file_reporter, line_number); + } this.jest.tests.items(.status)[id] = TestRunner.Test.Status.pass; this.summary().pass += 1; @@ -840,8 +843,6 @@ pub const CommandLineReporter = struct { } pub fn handleTestFail(cb: *TestRunner.Callback, id: Test.ID, file: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { - var writer_ = Output.errorWriterBuffered(); - defer Output.flush(); var this: *CommandLineReporter = @fieldParentPtr("callback", cb); // when the tests fail, we want to repeat the failures at the end @@ -853,13 +854,23 @@ pub const CommandLineReporter = struct { const line_number = this.jest.tests.items(.line_number)[id]; printTestLine(.fail, label, elapsed_ns, parent, expectations, false, writer, file, this.file_reporter, line_number); + // In agent mode, immediately print failures + if (this.jest.test_options.agent) { + var writer_ = Output.errorWriterBuffered(); + defer Output.flush(); + writer_.writeAll(this.failures_to_repeat_buf.items[initial_length..]) catch {}; + } else { + // In normal mode, output to stderr at the end + var writer_ = Output.errorWriterBuffered(); + defer Output.flush(); + writer_.writeAll(this.failures_to_repeat_buf.items[initial_length..]) catch {}; + } + // We must always reset the colors because (skip) will have set them to if (Output.enable_ansi_colors_stderr) { writer.writeAll(Output.prettyFmt("", true)) catch {}; } - writer_.writeAll(this.failures_to_repeat_buf.items[initial_length..]) catch {}; - // this.updateDots(); this.summary().fail += 1; this.summary().expectations += expectations; @@ -876,7 +887,8 @@ pub const CommandLineReporter = struct { var this: *CommandLineReporter = @fieldParentPtr("callback", cb); // If you do it.only, don't report the skipped tests because its pretty noisy - if (jest.Jest.runner != null and !jest.Jest.runner.?.only) { + // In agent mode, don't print skipped tests + if (jest.Jest.runner != null and !jest.Jest.runner.?.only and !this.jest.test_options.agent) { var writer_ = Output.errorWriterBuffered(); defer Output.flush(); // when the tests skip, we want to repeat the failures at the end @@ -921,21 +933,24 @@ pub const CommandLineReporter = struct { } pub fn handleTestTodo(cb: *TestRunner.Callback, id: Test.ID, file: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void { - var writer_ = Output.errorWriterBuffered(); - var this: *CommandLineReporter = @fieldParentPtr("callback", cb); - // when the tests skip, we want to repeat the failures at the end - // so that you can see them better when there are lots of tests that ran - const initial_length = this.todos_to_repeat_buf.items.len; - var writer = this.todos_to_repeat_buf.writer(bun.default_allocator); + // In agent mode, don't print todo status + if (!this.jest.test_options.agent) { + var writer_ = Output.errorWriterBuffered(); - writeTestStatusLine(.todo, &writer); - const line_number = this.jest.tests.items(.line_number)[id]; - printTestLine(.todo, label, elapsed_ns, parent, expectations, true, writer, file, this.file_reporter, line_number); + // when the tests skip, we want to repeat the failures at the end + // so that you can see them better when there are lots of tests that ran + const initial_length = this.todos_to_repeat_buf.items.len; + var writer = this.todos_to_repeat_buf.writer(bun.default_allocator); - writer_.writeAll(this.todos_to_repeat_buf.items[initial_length..]) catch {}; - Output.flush(); + writeTestStatusLine(.todo, &writer); + const line_number = this.jest.tests.items(.line_number)[id]; + printTestLine(.todo, label, elapsed_ns, parent, expectations, true, writer, file, this.file_reporter, line_number); + + writer_.writeAll(this.todos_to_repeat_buf.items[initial_length..]) catch {}; + Output.flush(); + } // this.updateDots(); this.summary().todo += 1; @@ -1307,6 +1322,12 @@ pub const TestCommand = struct { pub fn exec(ctx: Command.Context) !void { Output.is_github_action = Output.isGithubAction(); + // Disable ANSI colors in agent mode + if (ctx.test_options.agent) { + Output.enable_ansi_colors_stderr = false; + Output.enable_ansi_colors_stdout = false; + } + // print the version so you know its doing stuff if it takes a sec Output.prettyln("bun test v" ++ Global.package_json_version_with_sha ++ "", .{}); Output.flush(); @@ -1530,7 +1551,8 @@ pub const TestCommand = struct { const write_snapshots_success = try jest.Jest.runner.?.snapshots.writeInlineSnapshots(); try jest.Jest.runner.?.snapshots.writeSnapshotFile(); var coverage_options = ctx.test_options.coverage; - if (reporter.summary().pass > 20) { + // In agent mode, don't print repeat buffers since errors are printed immediately + if (reporter.summary().pass > 20 and !ctx.test_options.agent) { if (reporter.summary().skip > 0) { Output.prettyError("\n{d} tests skipped:\n", .{reporter.summary().skip}); Output.flush(); @@ -1737,7 +1759,11 @@ pub const TestCommand = struct { } const summary = reporter.summary(); - if (failed_to_find_any_tests or summary.didLabelFilterOutAllTests() or summary.fail > 0 or (coverage_options.enabled and coverage_options.fractions.failing and coverage_options.fail_on_low_coverage) or !write_snapshots_success) { + // In agent mode, exit with code 1 when no tests are run + const no_tests_run = (summary.pass + summary.fail + summary.skip + summary.todo) == 0; + const should_exit_with_error = failed_to_find_any_tests or summary.didLabelFilterOutAllTests() or summary.fail > 0 or (coverage_options.enabled and coverage_options.fractions.failing and coverage_options.fail_on_low_coverage) or !write_snapshots_success or (ctx.test_options.agent and no_tests_run); + + if (should_exit_with_error) { Global.exit(1); } else if (reporter.jest.unhandled_errors_between_tests > 0) { Global.exit(reporter.jest.unhandled_errors_between_tests); diff --git a/test/cli/test/agent.test.ts b/test/cli/test/agent.test.ts new file mode 100644 index 0000000000..861cdebe8a --- /dev/null +++ b/test/cli/test/agent.test.ts @@ -0,0 +1,338 @@ +import { test, expect } from "bun:test"; +import { bunExe, tempDirWithFiles } from "harness"; +import { join } from "path"; + +test("--agent flag: only prints errors and summary", async () => { + const dir = tempDirWithFiles("agent-test-1", { + "pass.test.js": ` + import { test, expect } from "bun:test"; + + test("passing test", () => { + expect(1 + 1).toBe(2); + }); + `, + "fail.test.js": ` + import { test, expect } from "bun:test"; + + test("failing test", () => { + expect(1 + 1).toBe(3); + }); + `, + "skip.test.js": ` + import { test, expect } from "bun:test"; + + test.skip("skipped test", () => { + expect(1 + 1).toBe(2); + }); + `, + "todo.test.js": ` + import { test, expect } from "bun:test"; + + test.todo("todo test", () => { + expect(1 + 1).toBe(2); + }); + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", "--agent"], + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + // Should exit with code 1 because tests failed + expect(exitCode).toBe(1); + + // Should not contain ANSI color codes + expect(stderr).not.toContain("\u001b["); + expect(stdout).not.toContain("\u001b["); + + // Should contain failure output + expect(stderr).toContain("failing test"); + expect(stderr).toContain("Expected:"); + expect(stderr).toContain("Received:"); + + // Should NOT contain pass/skip/todo individual test output + expect(stderr).not.toContain("passing test"); + expect(stderr).not.toContain("skipped test"); + expect(stderr).not.toContain("todo test"); + + // Should contain summary with counts + expect(stderr).toContain("1 pass"); + expect(stderr).toContain("1 skip"); + expect(stderr).toContain("1 todo"); + expect(stderr).toContain("1 fail"); + + // Should contain total test count + expect(stderr).toContain("Ran 4 test"); +}); + +test("--agent flag: exits with code 1 when no tests are run", async () => { + const dir = tempDirWithFiles("agent-test-2", { + "not-a-test.js": `console.log("not a test");`, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", "--agent"], + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + // Should exit with code 1 when no tests are found + expect(exitCode).toBe(1); + + // Should not contain ANSI color codes + expect(stderr).not.toContain("\u001b["); + expect(stdout).not.toContain("\u001b["); +}); + +test("--agent flag: with only passing tests", async () => { + const dir = tempDirWithFiles("agent-test-3", { + "pass1.test.js": ` + import { test, expect } from "bun:test"; + + test("passing test 1", () => { + expect(1 + 1).toBe(2); + }); + `, + "pass2.test.js": ` + import { test, expect } from "bun:test"; + + test("passing test 2", () => { + expect(2 + 2).toBe(4); + }); + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", "--agent"], + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + // Should exit with code 0 when all tests pass + expect(exitCode).toBe(0); + + // Should not contain ANSI color codes + expect(stderr).not.toContain("\u001b["); + expect(stdout).not.toContain("\u001b["); + + // Should NOT contain individual test pass output + expect(stderr).not.toContain("passing test 1"); + expect(stderr).not.toContain("passing test 2"); + + // Should contain summary with counts + expect(stderr).toContain("2 pass"); + expect(stderr).toContain("0 fail"); + + // Should contain total test count + expect(stderr).toContain("Ran 2 test"); +}); + +test("--agent flag: with test filters", async () => { + const dir = tempDirWithFiles("agent-test-4", { + "test1.test.js": ` + import { test, expect } from "bun:test"; + + test("matching test", () => { + expect(1 + 1).toBe(2); + }); + + test("other test", () => { + expect(2 + 2).toBe(4); + }); + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", "--agent", "-t", "matching"], + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + // Should exit with code 0 when filtered tests pass + expect(exitCode).toBe(0); + + // Should not contain ANSI color codes + expect(stderr).not.toContain("\u001b["); + expect(stdout).not.toContain("\u001b["); + + // Should contain summary with counts (only 1 test should run) + expect(stderr).toContain("1 pass"); + expect(stderr).toContain("0 fail"); + + // Should contain total test count + expect(stderr).toContain("Ran 1 test"); +}); + +test("--agent flag: with many failures (tests immediate output)", async () => { + const dir = tempDirWithFiles("agent-test-5", { + "fail1.test.js": ` + import { test, expect } from "bun:test"; + + test("fail 1", () => { + expect(1).toBe(2); + }); + `, + "fail2.test.js": ` + import { test, expect } from "bun:test"; + + test("fail 2", () => { + expect(2).toBe(3); + }); + `, + "fail3.test.js": ` + import { test, expect } from "bun:test"; + + test("fail 3", () => { + expect(3).toBe(4); + }); + `, + // Add many passing tests to trigger the repeat buffer logic + ...Array.from({ length: 25 }, (_, i) => ({ + [`pass${i}.test.js`]: ` + import { test, expect } from "bun:test"; + + test("pass ${i}", () => { + expect(${i}).toBe(${i}); + }); + `, + })).reduce((acc, obj) => ({ ...acc, ...obj }), {}), + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", "--agent"], + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + // Should exit with code 1 because tests failed + expect(exitCode).toBe(1); + + // Should not contain ANSI color codes + expect(stderr).not.toContain("\u001b["); + expect(stdout).not.toContain("\u001b["); + + // Should contain failure output (printed immediately) + expect(stderr).toContain("fail 1"); + expect(stderr).toContain("fail 2"); + expect(stderr).toContain("fail 3"); + + // Should NOT contain repeat buffer headers (since agent mode disables them) + expect(stderr).not.toContain("tests failed:"); + + // Should contain summary with counts + expect(stderr).toContain("25 pass"); + expect(stderr).toContain("3 fail"); + + // Should contain total test count + expect(stderr).toContain("Ran 28 test"); +}); + +test("normal mode vs agent mode comparison", async () => { + const dir = tempDirWithFiles("agent-test-6", { + "test.test.js": ` + import { test, expect } from "bun:test"; + + test("passing test", () => { + expect(1 + 1).toBe(2); + }); + + test("failing test", () => { + expect(1 + 1).toBe(3); + }); + + test.skip("skipped test", () => { + expect(1 + 1).toBe(2); + }); + `, + }); + + // Run in normal mode + await using normalProc = Bun.spawn({ + cmd: [bunExe(), "test"], + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [normalStdout, normalStderr, normalExitCode] = await Promise.all([ + new Response(normalProc.stdout).text(), + new Response(normalProc.stderr).text(), + normalProc.exited, + ]); + + // Run in agent mode + await using agentProc = Bun.spawn({ + cmd: [bunExe(), "test", "--agent"], + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [agentStdout, agentStderr, agentExitCode] = await Promise.all([ + new Response(agentProc.stdout).text(), + new Response(agentProc.stderr).text(), + agentProc.exited, + ]); + + // Both should exit with the same code + expect(normalExitCode).toBe(agentExitCode); + expect(normalExitCode).toBe(1); // Because tests failed + + // Agent mode should not contain ANSI color codes (even if normal mode might not have them in CI) + expect(agentStderr).not.toContain("\u001b["); + + // Normal mode should show individual test results, agent mode should not + expect(normalStderr).toContain("(pass) passing test"); + expect(normalStderr).toContain("(skip) skipped test"); + expect(agentStderr).not.toContain("(pass) passing test"); + expect(agentStderr).not.toContain("(skip) skipped test"); + + // Both should contain failure output + expect(normalStderr).toContain("failing test"); + expect(agentStderr).toContain("failing test"); + + // Both should contain summary counts + expect(normalStderr).toContain("1 pass"); + expect(normalStderr).toContain("1 fail"); + expect(normalStderr).toContain("1 skip"); + expect(agentStderr).toContain("1 pass"); + expect(agentStderr).toContain("1 fail"); + expect(agentStderr).toContain("1 skip"); +}); \ No newline at end of file