diff --git a/docs/runtime/bunfig.md b/docs/runtime/bunfig.md index 532908f9bc..9ec995161d 100644 --- a/docs/runtime/bunfig.md +++ b/docs/runtime/bunfig.md @@ -249,6 +249,46 @@ This is useful for: The `--concurrent` CLI flag will override this setting when specified. +### `test.randomize` + +Run tests in random order. Default `false`. + +```toml +[test] +randomize = true +``` + +This helps catch bugs related to test interdependencies by running tests in a different order each time. When combined with `seed`, the random order becomes reproducible. + +The `--randomize` CLI flag will override this setting when specified. + +### `test.seed` + +Set the random seed for test randomization. This option requires `randomize` to be `true`. + +```toml +[test] +randomize = true +seed = 2444615283 +``` + +Using a seed makes the randomized test order reproducible across runs, which is useful for debugging flaky tests. When you encounter a test failure with randomization enabled, you can use the same seed to reproduce the exact test order. + +The `--seed` CLI flag will override this setting when specified. + +### `test.rerunEach` + +Re-run each test file a specified number of times. Default `0` (run once). + +```toml +[test] +rerunEach = 3 +``` + +This is useful for catching flaky tests or non-deterministic behavior. Each test file will be executed the specified number of times. + +The `--rerun-each` CLI flag will override this setting when specified. + ## Package manager Package management is a complex issue; to support a range of use cases, the behavior of `bun install` can be configured under the `[install]` section. diff --git a/src/bunfig.zig b/src/bunfig.zig index a43f6e62fa..2041454161 100644 --- a/src/bunfig.zig +++ b/src/bunfig.zig @@ -329,6 +329,33 @@ pub const Bunfig = struct { this.ctx.test_options.coverage.skip_test_files = expr.data.e_boolean.value; } + var randomize_from_config: ?bool = null; + + if (test_.get("randomize")) |expr| { + try this.expect(expr, .e_boolean); + randomize_from_config = expr.data.e_boolean.value; + this.ctx.test_options.randomize = expr.data.e_boolean.value; + } + + if (test_.get("seed")) |expr| { + try this.expect(expr, .e_number); + const seed_value = expr.data.e_number.toU32(); + + // Validate that randomize is true when seed is specified + // Either randomize must be set to true in this config, or already enabled + const has_randomize_true = (randomize_from_config orelse this.ctx.test_options.randomize); + if (!has_randomize_true) { + try this.addError(expr.loc, "\"seed\" can only be used when \"randomize\" is true"); + } + + this.ctx.test_options.seed = seed_value; + } + + if (test_.get("rerunEach")) |expr| { + try this.expect(expr, .e_number); + this.ctx.test_options.repeat_count = expr.data.e_number.toU32(); + } + if (test_.get("concurrentTestGlob")) |expr| { switch (expr.data) { .e_string => |str| { diff --git a/test/cli/__snapshots__/bunfig-test-options.test.ts.snap b/test/cli/__snapshots__/bunfig-test-options.test.ts.snap new file mode 100644 index 0000000000..9ec9a35afa --- /dev/null +++ b/test/cli/__snapshots__/bunfig-test-options.test.ts.snap @@ -0,0 +1,11 @@ +// Bun Snapshot v1, https://bun.sh/docs/test/snapshots + +exports[`bunfig.toml test options randomize with seed produces consistent order 1`] = ` +[ + "echo", + "alpha", + "bravo", + "charlie", + "delta", +] +`; diff --git a/test/cli/bunfig-test-options.test.ts b/test/cli/bunfig-test-options.test.ts new file mode 100644 index 0000000000..e21c23848b --- /dev/null +++ b/test/cli/bunfig-test-options.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, tempDirWithFiles } from "harness"; + +describe("bunfig.toml test options", () => { + test("randomize with seed produces consistent order", async () => { + const dir = tempDirWithFiles("bunfig-test-randomize-seed", { + "test.test.ts": ` + import { test, expect } from "bun:test"; + test("alpha", () => { + console.log("RUNNING: alpha"); + expect(1).toBe(1); + }); + test("bravo", () => { + console.log("RUNNING: bravo"); + expect(2).toBe(2); + }); + test("charlie", () => { + console.log("RUNNING: charlie"); + expect(3).toBe(3); + }); + test("delta", () => { + console.log("RUNNING: delta"); + expect(4).toBe(4); + }); + test("echo", () => { + console.log("RUNNING: echo"); + expect(5).toBe(5); + }); + `, + "bunfig.toml": `[test]\nrandomize = true\nseed = 2444615283`, + }); + + // Run twice to verify same order + const outputs: string[] = []; + for (let i = 0; i < 2; i++) { + await using proc = Bun.spawn({ + cmd: [bunExe(), "test"], + env: bunEnv, + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + expect(exitCode).toBe(0); + outputs.push(stdout + stderr); + } + + // Extract the order tests ran in + const extractOrder = (output: string) => { + const matches = output.matchAll(/RUNNING: (\w+)/g); + return Array.from(matches, m => m[1]); + }; + + const order1 = extractOrder(outputs[0]); + const order2 = extractOrder(outputs[1]); + + // Should have all 5 tests + expect(order1.length).toBe(5); + expect(order2.length).toBe(5); + + // Order should be identical across runs + expect(order1).toEqual(order2); + + // Order should NOT be alphabetical (tests randomization is working) + const alphabetical = ["alpha", "bravo", "charlie", "delta", "echo"]; + expect(order1).not.toEqual(alphabetical); + + // Snapshot the actual order for regression testing + expect(order1).toMatchSnapshot(); + }); + + test("seed without randomize errors", async () => { + const dir = tempDirWithFiles("bunfig-test-seed-no-randomize", { + "test.test.ts": ` + import { test, expect } from "bun:test"; + test("test 1", () => expect(1).toBe(1)); + `, + "bunfig.toml": `[test]\nseed = 2444615283`, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test"], + env: bunEnv, + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + expect(exitCode).toBe(1); + const output = stdout + stderr; + expect(output).toContain("seed"); + expect(output).toContain("randomize"); + }); + + test("seed with randomize=false errors", async () => { + const dir = tempDirWithFiles("bunfig-test-seed-randomize-false", { + "test.test.ts": ` + import { test, expect } from "bun:test"; + test("test 1", () => expect(1).toBe(1)); + `, + "bunfig.toml": `[test]\nrandomize = false\nseed = 2444615283`, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test"], + env: bunEnv, + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + expect(exitCode).toBe(1); + const output = stdout + stderr; + expect(output).toContain("seed"); + expect(output).toContain("randomize"); + }); + + test("rerunEach option works", async () => { + const dir = tempDirWithFiles("bunfig-test-rerun-each", { + "test.test.ts": ` + import { test, expect } from "bun:test"; + let counter = 0; + test("test 1", () => { + counter++; + expect(counter).toBeGreaterThan(0); + }); + `, + "bunfig.toml": `[test]\nrerunEach = 3`, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test"], + env: bunEnv, + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + expect(exitCode).toBe(0); + const output = stdout + stderr; + // With rerunEach = 3, the test file should run 3 times + // So we should see "3 pass" (1 test * 3 runs) + expect(output).toContain("3 pass"); + }); + + test("all test options together", async () => { + const dir = tempDirWithFiles("bunfig-test-all-options", { + "test.test.ts": ` + import { test, expect } from "bun:test"; + test("test 1", () => expect(1).toBe(1)); + test("test 2", () => expect(2).toBe(2)); + `, + "bunfig.toml": `[test]\nrandomize = true\nseed = 12345\nrerunEach = 2`, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "test"], + env: bunEnv, + cwd: dir, + stderr: "pipe", + stdout: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + + expect(exitCode).toBe(0); + const output = stdout + stderr; + // 2 tests * 2 reruns = 4 total test runs + expect(output).toContain("4 pass"); + }); +});