mirrors/bun.sh
1
0
Fork 0
mirror of https://github.com/oven-sh/bun synced 2026-02-14 12:51:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(test): add --retry flag and emit separate testcase entries for retries in JUnit XML (#26866)

Browse Source 
#### (Copies commits from #26447)

## Summary
- Add a global `--retry <N>` flag to `bun test` that sets a default
retry count for all tests (overridable by per-test `{ retry: N }`). Also
configurable via `[test] retry = N` in bunfig.toml.
- When a test passes after one or more retries, the JUnit XML reporter
emits a separate `<testcase>` entry for each failed attempt (with
`<failure>`), followed by the final passing `<testcase>`. This gives
flaky test detection tools per-attempt timing and result data using
standard JUnit XML that all CI systems can parse.

## Test plan
- `bun bd test test/js/junit-reporter/junit.test.js` — verifies separate
`<testcase>` entries appear in JUnit XML for tests that pass after retry
- `bun bd test test/cli/test/retry-flag.test.ts` — verifies the
`--retry` CLI flag applies a default retry count to all tests

## Changelog
<!-- CHANGELOG:START -->
- Added `--retry <N>` flag to `bun test` to set a default retry count
for all tests
- Added `[test] retry` option to bunfig.toml
- JUnit XML reporter now emits separate `<testcase>` entries for each
retry attempt, providing CI visibility into flaky tests
<!-- CHANGELOG:END -->

---------

Co-authored-by: Chris Lloyd <chrislloyd@anthropic.com>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Alistair Smith
2026-02-10 10:58:21 -08:00
committed by GitHub
parent 746771d495
commit 099b5e430c
14 changed files with 516 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
completions/bun-cli.json
View File

@@ -20,12 +20,7 @@
"completionType": "javascript_files"
}
],
"examples": [
"bun run ./index.js",
"bun run ./index.tsx",
"bun run dev",
"bun run lint"
],
"examples": ["bun run ./index.js", "bun run ./index.tsx", "bun run dev", "bun run lint"],
"usage": "Usage: bun run [flags] <file or script>",
"documentationUrl": "https://bun.com/docs/cli/run",
"dynamicCompletions": {
@@ -63,6 +58,14 @@
"required": false,
"multiple": false
},
{
"name": "retry",
"description": "Default retry count for all tests, overridden by per-test { retry: N }",
"hasValue": true,
"valueType": "val",
"required": false,
"multiple": false
},
{
"name": "only",
"description": "Only run tests that are marked with \"test.only()\"",
@@ -152,11 +155,7 @@
"completionType": "test_files"
}
],
"examples": [
"bun test",
"bun test foo bar",
"bun test --test-name-pattern baz"
],
"examples": ["bun test", "bun test foo bar", "bun test --test-name-pattern baz"],
"usage": "Usage: bun test [flags] [<patterns>]",
"documentationUrl": "https://bun.com/docs/cli/test",
"dynamicCompletions": {
@@ -199,9 +198,7 @@
"examples": [],
"usage": "Usage: bunx [flags] <package><@version> [flags and arguments for the package]",
"dynamicCompletions": {},
"aliases": [
"bunx"
]
"aliases": ["bunx"]
},
"repl": {
"name": "repl",
@@ -232,10 +229,7 @@
"completionType": "script"
}
],
"examples": [
"bun exec \"echo hi\"",
"bun exec \"echo \\\"hey friends\\\"!\""
],
"examples": ["bun exec \"echo hi\"", "bun exec \"echo \\\"hey friends\\\"!\""],
"usage": "Usage: bun exec <script>",
"dynamicCompletions": {}
},
@@ -545,14 +539,9 @@
"type": "string"
}
],
"examples": [
"bun install",
"bun install --production"
],
"examples": ["bun install", "bun install --production"],
"usage": "Usage: bun install [flags] <name>@<version>",
"aliases": [
"i"
],
"aliases": ["i"],
"documentationUrl": "https://bun.com/docs/cli/install.",
"dynamicCompletions": {}
},
@@ -864,9 +853,7 @@
"bun add --peer esbuild"
],
"usage": "Usage: bun add [flags] <package><@version>",
"aliases": [
"a"
],
"aliases": ["a"],
"documentationUrl": "https://bun.com/docs/cli/add.",
"dynamicCompletions": {
"packages": true
@@ -1126,13 +1113,9 @@
"completionType": "installed_package"
}
],
"examples": [
"bun remove ts-node"
],
"examples": ["bun remove ts-node"],
"usage": "Usage: bun remove [flags] [<packages>]",
"aliases": [
"rm"
],
"aliases": ["rm"],
"documentationUrl": "https://bun.com/docs/cli/remove.",
"dynamicCompletions": {
"packages": true
@@ -1422,12 +1405,7 @@
"type": "string"
}
],
"examples": [
"bun update",
"bun update --latest",
"bun update -i",
"bun update zod jquery@3"
],
"examples": ["bun update", "bun update --latest", "bun update -i", "bun update zod jquery@3"],
"usage": "Usage: bun update [flags] <name>@<version>",
"documentationUrl": "https://bun.com/docs/cli/update.",
"dynamicCompletions": {}
@@ -1452,10 +1430,7 @@
"type": "string"
}
],
"examples": [
"bun audit",
"bun audit --json"
],
"examples": ["bun audit", "bun audit --json"],
"usage": "Usage: bun audit [flags]",
"documentationUrl": "https://bun.com/docs/install/audit.",
"dynamicCompletions": {}
@@ -1997,10 +1972,7 @@
"completionType": "package"
}
],
"examples": [
"bun link",
"bun link <package>"
],
"examples": ["bun link", "bun link <package>"],
"usage": "Usage: bun link [flags] [<packages>]",
"documentationUrl": "https://bun.com/docs/cli/link.",
"dynamicCompletions": {}
@@ -2252,9 +2224,7 @@
"type": "string"
}
],
"examples": [
"bun unlink"
],
"examples": ["bun unlink"],
"usage": "Usage: bun unlink [flags]",
"documentationUrl": "https://bun.com/docs/cli/unlink.",
"dynamicCompletions": {}
@@ -3248,11 +3218,7 @@
"completionType": "package"
}
],
"examples": [
"bun info react",
"bun info react@18.0.0",
"bun info react version --json"
],
"examples": ["bun info react", "bun info react@18.0.0", "bun info react version --json"],
"usage": "Usage: bun info [flags] <package>[@<version>]",
"documentationUrl": "https://bun.com/docs/cli/info.",
"dynamicCompletions": {}
@@ -3603,12 +3569,7 @@
"type": "string"
}
],
"examples": [
"bun init",
"bun init --yes",
"bun init --react",
"bun init --react=tailwind my-app"
],
"examples": ["bun init", "bun init --yes", "bun init --react", "bun init --react=tailwind my-app"],
"usage": "Usage: bun init [flags] [<folder>]",
"dynamicCompletions": {}
},
@@ -3621,9 +3582,7 @@
"usage": "Usage:",
"documentationUrl": "https://bun.com/docs/cli/bun-create",
"dynamicCompletions": {},
"aliases": [
"c"
]
"aliases": ["c"]
},
"upgrade": {
"name": "upgrade",
@@ -3637,10 +3596,7 @@
"type": "string"
}
],
"examples": [
"bun upgrade",
"bun upgrade --stable"
],
"examples": ["bun upgrade", "bun upgrade --stable"],
"usage": "Usage: bun upgrade [flags]",
"documentationUrl": "https://bun.com/docs/installation#upgrading",
"dynamicCompletions": {}
@@ -3743,9 +3699,7 @@
"description": "Configure auto-install behavior. One of \"auto\" (default, auto-installs when no node_modules), \"fallback\" (missing packages only), \"force\" (always).",
"hasValue": true,
"valueType": "val",
"choices": [
"auto"
],
"choices": ["auto"],
"required": false,
"multiple": false
},
@@ -3827,12 +3781,7 @@
"description": "Set the default order of DNS lookup results. Valid orders: verbatim (default), ipv4first, ipv6first",
"hasValue": true,
"valueType": "val",
"choices": [
"verbatim",
"(default)",
"ipv4first",
"ipv6first"
],
"choices": ["verbatim", "(default)", "ipv4first", "ipv6first"],
"required": false,
"multiple": false
},
@@ -3898,9 +3847,7 @@
"description": "One of \"strict\", \"throw\", \"warn\", \"none\", or \"warn-with-error-code\"",
"hasValue": true,
"valueType": "val",
"choices": [
"strict"
],
"choices": ["strict"],
"required": false,
"multiple": false
},
@@ -4023,4 +3970,4 @@
"files": "bun getcompletes j"
}
}
}
}

1
completions/bun.zsh
View File

@@ -665,6 +665,7 @@ _bun_test_completion() {
'--timeout[Set the per-test timeout in milliseconds, default is 5000.]:timeout' \
'--update-snapshots[Update snapshot files]' \
'--rerun-each[Re-run each test file <NUMBER> times, helps catch certain bugs]:rerun' \
'--retry[Default retry count for all tests]:retry' \
'--todo[Include tests that are marked with "test.todo()"]' \
'--coverage[Generate a coverage profile]' \
'--bail[Exit the test suite after <NUMBER> failures. If you do not specify a number, it defaults to 1.]:bail' \

11
docs/runtime/bunfig.mdx
View File

@@ -298,6 +298,17 @@ This is useful for catching flaky tests or non-deterministic behavior. Each test
The `--rerun-each` CLI flag will override this setting when specified.
### `test.retry`
Default retry count for all tests. Failed tests will be retried up to this many times. Per-test `{ retry: N }` overrides this value. Default `0` (no retries).
```toml title="bunfig.toml" icon="settings"
[test]
retry = 3
```
The `--retry` CLI flag will override this setting when specified.
### `test.concurrentTestGlob`
Specify a glob pattern to automatically run matching test files with concurrent test execution enabled. Test files matching this pattern will behave as if the `--concurrent` flag was passed, running all tests within those files concurrently.

5
docs/snippets/cli/test.mdx
View File

@@ -14,6 +14,11 @@ bun test <patterns>
Re-run each test file <code>NUMBER</code> times, helps catch certain bugs
</ParamField>
<ParamField path="--retry" type="number">
Default retry count for all tests. Failed tests will be retried up to <code>NUMBER</code> times. Overridden by
per-test <code>{`{ retry: N }`}</code>
</ParamField>
<ParamField path="--concurrent" type="boolean">
Treat all tests as <code>test.concurrent()</code> tests
</ParamField>

11
docs/test/configuration.mdx
View File

@@ -222,6 +222,17 @@ randomize = true
seed = 2444615283
```
#### retry
Default retry count for all tests. Failed tests will be retried up to this many times. Per-test `{ retry: N }` overrides this value. Default `0` (no retries).
```toml title="bunfig.toml" icon="settings"
[test]
retry = 3
```
The `--retry` CLI flag will override this setting when specified.
#### rerunEach
Re-run each test file multiple times to identify flaky tests:

29
docs/test/index.mdx
View File

@@ -201,6 +201,35 @@ test.failing.each([1, 2, 3])("chained qualifiers %d", input => {
});
```
## Retry failed tests
Use the `--retry` flag to automatically retry failed tests up to a given number of times. If a test fails and then passes on a subsequent attempt, it is reported as passing.
```sh terminal icon="terminal"
bun test --retry 3
```
Per-test `{ retry: N }` overrides the global `--retry` value:
```ts
// Uses the global --retry value
test("uses global retry", () => {
/* ... */
});
// Overrides --retry with its own value
test("custom retry", { retry: 1 }, () => {
/* ... */
});
```
You can also set this in `bunfig.toml`:
```toml title="bunfig.toml" icon="settings"
[test]
retry = 3
```
## Rerun tests
Use the `--rerun-each` flag to run each test multiple times. This is useful for detecting flaky or non-deterministic test failures.

27
src/bun.js/test/Execution.zig
View File

@@ -78,6 +78,8 @@ pub const ExecutionSequence = struct {
test_entry: ?*ExecutionEntry,
remaining_repeat_count: u32,
remaining_retry_count: u32,
flaky_attempt_count: usize = 0,
flaky_attempts_buf: [MAX_FLAKY_ATTEMPTS]FlakyAttempt = std.mem.zeroes([MAX_FLAKY_ATTEMPTS]FlakyAttempt),
result: Result = .pending,
executing: bool = false,
started_at: bun.timespec = .epoch,
@@ -106,6 +108,17 @@ pub const ExecutionSequence = struct {
};
}
pub const MAX_FLAKY_ATTEMPTS: usize = 16;
pub const FlakyAttempt = struct {
result: Result,
elapsed_ns: u64,
};
pub fn flakyAttempts(this: *const ExecutionSequence) []const FlakyAttempt {
return this.flaky_attempts_buf[0..this.flaky_attempt_count];
}
fn entryMode(this: ExecutionSequence) bun_test.ScopeMode {
if (this.test_entry) |entry| return entry.base.mode;
return .normal;
@@ -480,6 +493,14 @@ fn advanceSequence(this: *Execution, sequence: *ExecutionSequence, group: *Concu
// Handle retry logic: if test failed and we have retries remaining, retry it
if (test_failed and sequence.remaining_retry_count > 0) {
if (sequence.flaky_attempt_count < ExecutionSequence.MAX_FLAKY_ATTEMPTS) {
const elapsed_ns = if (sequence.started_at.eql(&.epoch)) 0 else sequence.started_at.sinceNow(.force_real_time);
sequence.flaky_attempts_buf[sequence.flaky_attempt_count] = .{
.result = sequence.result,
.elapsed_ns = elapsed_ns,
};
sequence.flaky_attempt_count += 1;
}
sequence.remaining_retry_count -= 1;
this.resetSequence(sequence);
return;
@@ -604,13 +625,17 @@ pub fn resetSequence(this: *Execution, sequence: *ExecutionSequence) void {
}
}
// Preserve the current remaining_repeat_count and remaining_retry_count
// Preserve retry/repeat counts and flaky attempt history across reset
const saved_flaky_attempt_count = sequence.flaky_attempt_count;
const saved_flaky_attempts_buf = sequence.flaky_attempts_buf;
sequence.* = .init(.{
.first_entry = sequence.first_entry,
.test_entry = sequence.test_entry,
.retry_count = sequence.remaining_retry_count,
.repeat_count = sequence.remaining_repeat_count,
});
sequence.flaky_attempt_count = saved_flaky_attempt_count;
sequence.flaky_attempts_buf = saved_flaky_attempts_buf;
_ = this;
}

15
src/bun.js/test/ScopeFunctions.zig
View File

@@ -249,7 +249,7 @@ fn enqueueDescribeOrTestCallback(this: *ScopeFunctions, bunTest: *bun_test.BunTe
_ = try bunTest.collection.active_scope.appendTest(bunTest.gpa, description, if (matches_filter) callback else null, .{
.has_done_parameter = has_done_parameter,
.timeout = options.timeout,
.retry_count = options.retry,
.retry_count = options.retry orelse 0,
.repeat_count = options.repeats,
}, base, .collection);
},
@@ -295,7 +295,7 @@ const ParseArgumentsResult = struct {
};
const ParseArgumentsOptions = struct {
timeout: u32 = 0,
retry: u32 = 0,
retry: ?u32 = null,
repeats: u32 = 0,
};
pub const CallbackMode = enum { require, allow };
@@ -391,7 +391,7 @@ pub fn parseArguments(globalThis: *jsc.JSGlobalObject, callframe: *jsc.CallFrame
if (!repeats.isNumber()) {
return globalThis.throwPretty("{f}() expects repeats to be a number", .{signature});
}
if (result.options.retry != 0) {
if (result.options.retry != null and result.options.retry.? != 0) {
return globalThis.throwPretty("{f}(): Cannot set both retry and repeats", .{signature});
}
result.options.repeats = std.math.lossyCast(u32, repeats.asNumber());
@@ -404,6 +404,15 @@ pub fn parseArguments(globalThis: *jsc.JSGlobalObject, callframe: *jsc.CallFrame
result.description = if (description.isUndefinedOrNull()) null else try getDescription(gpa, globalThis, description, signature);
if (result.options.retry == null) {
if (bun.jsc.Jest.Jest.runner) |runner| {
result.options.retry = runner.test_options.retry;
}
}
if ((result.options.retry orelse 0) != 0 and result.options.repeats != 0) {
return globalThis.throwPretty("{f}(): Cannot set both retry and repeats", .{signature});
}
const default_timeout_ms: ?u32 = if (bun.jsc.Jest.Jest.runner) |runner| if (runner.default_timeout_ms != 0) runner.default_timeout_ms else null else null;
const override_timeout_ms: ?u32 = if (bun.jsc.Jest.Jest.runner) |runner| if (runner.default_timeout_override != std.math.maxInt(u32)) runner.default_timeout_override else null else null;
const timeout_option_ms: ?u32 = if (timeout_option) |timeout| std.math.lossyCast(u32, timeout) else null;

13
src/bunfig.zig
View File

@@ -399,9 +399,22 @@ pub const Bunfig = struct {
if (test_.get("rerunEach")) |expr| {
try this.expect(expr, .e_number);
if (this.ctx.test_options.retry != 0) {
try this.addError(expr.loc, "\"rerunEach\" cannot be used with \"retry\"");
return;
}
this.ctx.test_options.repeat_count = expr.data.e_number.toU32();
}
if (test_.get("retry")) |expr| {
try this.expect(expr, .e_number);
if (this.ctx.test_options.repeat_count != 0) {
try this.addError(expr.loc, "\"retry\" cannot be used with \"rerunEach\"");
return;
}
this.ctx.test_options.retry = expr.data.e_number.toU32();
}
if (test_.get("concurrentTestGlob")) |expr| {
switch (expr.data) {
.e_string => |str| {

1
src/cli.zig
View File

@@ -341,6 +341,7 @@ pub const Command = struct {
default_timeout_ms: u32 = 5 * std.time.ms_per_s,
update_snapshots: bool = false,
repeat_count: u32 = 0,
retry: u32 = 0,
run_todo: bool = false,
only: bool = false,
pass_with_no_tests: bool = false,

13
src/cli/Arguments.zig
View File

@@ -220,6 +220,7 @@ pub const test_only_params = [_]ParamType{
clap.parseParam("--timeout <NUMBER> Set the per-test timeout in milliseconds, default is 5000.") catch unreachable,
clap.parseParam("-u, --update-snapshots Update snapshot files") catch unreachable,
clap.parseParam("--rerun-each <NUMBER> Re-run each test file <NUMBER> times, helps catch certain bugs") catch unreachable,
clap.parseParam("--retry <NUMBER> Default retry count for all tests, overridden by per-test { retry: N }") catch unreachable,
clap.parseParam("--todo Include tests that are marked with \"test.todo()\"") catch unreachable,
clap.parseParam("--only Run only tests that are marked with \"test.only()\" or \"describe.only()\"") catch unreachable,
clap.parseParam("--pass-with-no-tests Exit with code 0 when no tests are found") catch unreachable,
@@ -567,6 +568,18 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C
};
}
}
if (args.option("--retry")) |retry_count| {
if (retry_count.len > 0) {
ctx.test_options.retry = std.fmt.parseInt(u32, retry_count, 10) catch |e| {
Output.prettyErrorln("<r><red>error<r>: --retry expects a number: {s}", .{@errorName(e)});
Global.exit(1);
};
}
}
if (ctx.test_options.retry != 0 and ctx.test_options.repeat_count != 0) {
Output.prettyErrorln("<r><red>error<r>: --retry cannot be used with --rerun-each", .{});
Global.exit(1);
}
if (args.option("--test-name-pattern")) |namePattern| {
ctx.test_options.test_filter_pattern = namePattern;
const regex = RegularExpression.init(bun.String.fromBytes(namePattern), RegularExpression.Flags.none) catch {

3
src/cli/test_command.zig
View File

@@ -854,6 +854,9 @@ pub const CommandLineReporter = struct {
}
}
for (sequence.flakyAttempts()) |attempt| {
bun.handleOom(junit.writeTestCase(attempt.result, filename, display_label, concatenated_describe_scopes.items, 0, attempt.elapsed_ns, line_number));
}
bun.handleOom(junit.writeTestCase(status, filename, display_label, concatenated_describe_scopes.items, assertions, elapsed_ns, line_number));
}
}

315
test/cli/test/retry-flag.test.ts Normal file
View File

@@ -0,0 +1,315 @@
import { expect, test } from "bun:test";
import { bunEnv, bunExe, tempDir } from "harness";
test("--retry retries failed tests", async () => {
using dir = tempDir("retry-flag", {
"flaky.test.ts": `
import { test, expect } from "bun:test";
let count = 0;
test("flaky test", () => {
count++;
if (count < 3) throw new Error("fail attempt " + count);
expect(true).toBe(true);
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "--retry", "3", "flaky.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toContain("flaky test");
expect(stderr).toContain("attempt 3");
expect(exitCode).toBe(0);
});
test("per-test { retry } overrides --retry", async () => {
using dir = tempDir("retry-override", {
"override.test.ts": `
import { test, expect } from "bun:test";
let countA = 0;
let countB = 0;
// Per-test retry=1 overrides --retry 5. Fails twice, so retry=1 not enough.
test("limited retry", { retry: 1 }, () => {
countA++;
if (countA < 3) throw new Error("fail attempt " + countA);
});
// Uses global --retry 5 default, fails once then passes.
test("default retry", () => {
countB++;
if (countB < 2) throw new Error("fail attempt " + countB);
expect(true).toBe(true);
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "--retry", "5", "override.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toContain("limited retry");
expect(stderr).toContain("default retry");
expect(exitCode).not.toBe(0);
});
test("--retry works with describe blocks and beforeEach/afterEach hooks", async () => {
using dir = tempDir("retry-hooks", {
"hooks.test.ts": `
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
let hookLog: string[] = [];
let attempt = 0;
describe("suite with hooks", () => {
beforeEach(() => {
hookLog.push("before");
});
afterEach(() => {
hookLog.push("after");
});
test("flaky with hooks", () => {
attempt++;
hookLog.push("test:" + attempt);
if (attempt < 3) throw new Error("fail attempt " + attempt);
// On passing attempt, verify hooks ran each time
expect(hookLog).toEqual([
"before", "test:1", "after",
"before", "test:2", "after",
"before", "test:3",
]);
});
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "--retry", "3", "hooks.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toContain("flaky with hooks");
expect(exitCode).toBe(0);
});
test("--retry with multiple tests where only some need retries", async () => {
using dir = tempDir("retry-multi", {
"multi.test.ts": `
import { test, expect } from "bun:test";
let flakyCount = 0;
test("always passes", () => {
expect(1 + 1).toBe(2);
});
test("flaky test", () => {
flakyCount++;
if (flakyCount < 2) throw new Error("fail");
expect(true).toBe(true);
});
test("always fails", () => {
throw new Error("permanent failure");
});
test("another passing test", () => {
expect("hello").toBe("hello");
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "--retry", "3", "multi.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
// "always fails" should still fail after all retries
expect(exitCode).not.toBe(0);
// Both passing tests and the flaky test should show as passing
expect(stderr).toContain("always passes");
expect(stderr).toContain("flaky test");
expect(stderr).toContain("always fails");
expect(stderr).toContain("another passing test");
});
test("--retry with async tests", async () => {
using dir = tempDir("retry-async", {
"async.test.ts": `
import { test, expect } from "bun:test";
let count = 0;
test("async flaky test", async () => {
count++;
await Bun.sleep(1);
if (count < 3) throw new Error("async fail attempt " + count);
expect(true).toBe(true);
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "--retry", "3", "async.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toContain("async flaky test");
expect(stderr).toContain("attempt 3");
expect(exitCode).toBe(0);
});
test("--retry with nested describe blocks", async () => {
using dir = tempDir("retry-nested", {
"nested.test.ts": `
import { test, expect, describe, beforeEach } from "bun:test";
let outerSetup = 0;
let innerSetup = 0;
let attempt = 0;
describe("outer", () => {
beforeEach(() => { outerSetup++; });
describe("inner", () => {
beforeEach(() => { innerSetup++; });
test("deeply nested flaky", () => {
attempt++;
if (attempt < 2) throw new Error("fail");
// Both outer and inner beforeEach should have run for each attempt
expect(outerSetup).toBe(2);
expect(innerSetup).toBe(2);
});
});
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "--retry", "3", "nested.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toContain("deeply nested flaky");
expect(exitCode).toBe(0);
});
test("--retry past MAX_FLAKY_ATTEMPTS (16) still retries correctly", async () => {
using dir = tempDir("retry-max", {
"max.test.ts": `
import { test, expect } from "bun:test";
let count = 0;
// Fails 19 times, passes on attempt 20 -- well past the 16-entry buffer
test("fails many times", { retry: 20 }, () => {
count++;
if (count < 20) throw new Error("fail attempt " + count);
expect(true).toBe(true);
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "max.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toContain("fails many times");
expect(exitCode).toBe(0);
});
test("--retry with test.skip and test.todo does not retry them", async () => {
using dir = tempDir("retry-skip-todo", {
"skip-todo.test.ts": `
import { test, expect } from "bun:test";
test.skip("skipped test", () => {
throw new Error("should not run");
});
test.todo("todo test");
let count = 0;
test("normal flaky", () => {
count++;
if (count < 2) throw new Error("fail");
expect(true).toBe(true);
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "--retry", "3", "skip-todo.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toContain("normal flaky");
expect(stderr).toContain("skipped test");
expect(stderr).toContain("todo test");
expect(exitCode).toBe(0);
});
test("bunfig.toml retry works equivalently", async () => {
using dir = tempDir("retry-bunfig", {
"bunfig.toml": `
[test]
retry = 3
`,
"flaky.test.ts": `
import { test, expect } from "bun:test";
let count = 0;
test("flaky via bunfig", () => {
count++;
if (count < 3) throw new Error("fail attempt " + count);
expect(true).toBe(true);
});
`,
});
await using proc = Bun.spawn({
cmd: [bunExe(), "test", "flaky.test.ts"],
env: bunEnv,
cwd: String(dir),
stderr: "pipe",
});
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(stderr).toContain("flaky via bunfig");
expect(stderr).toContain("attempt 3");
expect(exitCode).toBe(0);
});

47
test/js/junit-reporter/junit.test.js
View File

@@ -314,6 +314,53 @@ describe("junit reporter", () => {
expect(xmlContent1).toContain("line=");
expect(xmlContent2).toContain("line=");
});
it("should emit separate testcase entries for each retry attempt", async () => {
const tmpDir = tempDirWithFiles("junit-retry", {
"package.json": "{}",
"flaky.test.js": `
import { test, expect } from "bun:test";
let attempt = 0;
test("flaky test", { retry: 3 }, () => {
attempt++;
if (attempt < 3) {
throw new Error("flaky failure attempt " + attempt);
}
expect(true).toBe(true);
});
test("stable test", () => {
expect(1 + 1).toBe(2);
});
`,
});
const junitPath = `${tmpDir}/junit.xml`;
await using proc = spawn([bunExe(), "test", "--reporter=junit", "--reporter-outfile", junitPath], {
cwd: tmpDir,
env: { ...bunEnv, BUN_DEBUG_QUIET_LOGS: "1" },
stdout: "pipe",
stderr: "pipe",
});
await proc.exited;
const xmlContent = await file(junitPath).text();
// Each retry attempt should be a separate testcase with the same name
const flakyTestCases = [...xmlContent.matchAll(/<testcase[^>]*name="flaky test"[^>]*>/g)];
expect(flakyTestCases).toHaveLength(3);
// The first two should have <failure> elements, the last should be self-closing
const flakyEntries = [...xmlContent.matchAll(/<testcase[^>]*name="flaky test"[^/]*(?:\/>|>[\s\S]*?<\/testcase>)/g)];
expect(flakyEntries).toHaveLength(3);
expect(flakyEntries[0][0]).toContain("<failure");
expect(flakyEntries[1][0]).toContain("<failure");
expect(flakyEntries[2][0]).not.toContain("<failure");
expect(xmlContent).toContain('name="stable test"');
expect(proc.exitCode).toBe(0);
});
});
function filterJunitXmlOutput(xmlContent) {