From c4519c75521220ade3d125ac328f1fb9dbdfe9b2 Mon Sep 17 00:00:00 2001 From: pfg Date: Thu, 25 Sep 2025 23:47:46 -0700 Subject: [PATCH] Add --randomize --seed flag (#22987) Outputs the seed when randomizing. Adds --seed flag to reproduce a random order. Seeds might not produce the same order across operating systems / bun versions. Fixes #11847 --------- Co-authored-by: Claude Bot Co-authored-by: Claude Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- docs/cli/test.md | 30 +++ src/bun.js/test/Order.zig | 8 +- src/bun.js/test/bun_test.zig | 7 +- src/bun.js/test/jest.zig | 2 +- src/cli.zig | 1 + src/cli/Arguments.zig | 9 + src/cli/test_command.zig | 17 +- test/cli/test/test-randomize.fixture.ts | 3 + test/cli/test/test-randomize.test.ts | 323 ++++++------------------ 9 files changed, 149 insertions(+), 251 deletions(-) create mode 100644 test/cli/test/test-randomize.fixture.ts diff --git a/docs/cli/test.md b/docs/cli/test.md index bd636bcc39..e3729b8b62 100644 --- a/docs/cli/test.md +++ b/docs/cli/test.md @@ -117,6 +117,36 @@ Use the `--rerun-each` flag to run each test multiple times. This is useful for $ bun test --rerun-each 100 ``` +## Randomize test execution order + +Use the `--randomize` flag to run tests in a random order. This helps detect tests that depend on shared state or execution order. + +```sh +$ bun test --randomize +``` + +When using `--randomize`, the seed used for randomization will be displayed in the test summary: + +```sh +$ bun test --randomize +# ... test output ... + --seed=12345 + 2 pass + 8 fail +Ran 10 tests across 2 files. [50.00ms] +``` + +### Reproducible random order with `--seed` + +Use the `--seed` flag to specify a seed for the randomization. This allows you to reproduce the same test order when debugging order-dependent failures. + +```sh +# Reproduce a previous randomized run +$ bun test --seed 123456 +``` + +The `--seed` flag implies `--randomize`, so you don't need to specify both. Using the same seed value will always produce the same test execution order, making it easier to debug intermittent failures caused by test interdependencies. + ## Bail out with `--bail` Use the `--bail` flag to abort the test run early after a pre-determined number of test failures. By default Bun will run all tests and report all failures, but sometimes in CI environments it's preferable to terminate earlier to reduce CPU usage. diff --git a/src/bun.js/test/Order.zig b/src/bun.js/test/Order.zig index 3adae8f64b..9f220fafc8 100644 --- a/src/bun.js/test/Order.zig +++ b/src/bun.js/test/Order.zig @@ -37,8 +37,8 @@ pub const AllOrderResult = struct { } }; pub const Config = struct { - always_use_hooks: bool = false, - randomize: bool = false, + always_use_hooks: bool, + randomize: ?std.Random, }; pub fn generateAllOrder(this: *Order, entries: []const *ExecutionEntry, _: Config) bun.JSError!AllOrderResult { const start = this.groups.items.len; @@ -63,9 +63,7 @@ pub fn generateOrderDescribe(this: *Order, current: *DescribeScope, cfg: Config) const beforeall_order: AllOrderResult = if (use_hooks) try generateAllOrder(this, current.beforeAll.items, cfg) else .empty; // shuffle entries if randomize flag is set - if (cfg.randomize) { - var prng = std.Random.DefaultPrng.init(bun.fastRandom()); - const random = prng.random(); + if (cfg.randomize) |random| { random.shuffle(TestScheduleEntry, current.entries.items); } diff --git a/src/bun.js/test/bun_test.zig b/src/bun.js/test/bun_test.zig index 1150c8c916..f25c65fb06 100644 --- a/src/bun.js/test/bun_test.zig +++ b/src/bun.js/test/bun_test.zig @@ -514,8 +514,11 @@ pub const BunTest = struct { defer order.deinit(); const has_filter = if (this.reporter) |reporter| if (reporter.jest.filter_regex) |_| true else false else false; - const should_randomize = if (this.reporter) |reporter| reporter.jest.randomize else false; - const cfg: Order.Config = .{ .always_use_hooks = this.collection.root_scope.base.only == .no and !has_filter, .randomize = should_randomize }; + const should_randomize: ?std.Random = if (this.reporter) |reporter| reporter.jest.randomize else null; + const cfg: Order.Config = .{ + .always_use_hooks = this.collection.root_scope.base.only == .no and !has_filter, + .randomize = should_randomize, + }; const beforeall_order: Order.AllOrderResult = if (cfg.always_use_hooks or this.collection.root_scope.base.has_callback) try order.generateAllOrder(this.buntest.hook_scope.beforeAll.items, cfg) else .empty; try order.generateOrderDescribe(this.collection.root_scope, cfg); beforeall_order.setFailureSkipTo(&order); diff --git a/src/bun.js/test/jest.zig b/src/bun.js/test/jest.zig index 013bbcb5f9..8ec2dbdfd4 100644 --- a/src/bun.js/test/jest.zig +++ b/src/bun.js/test/jest.zig @@ -55,7 +55,7 @@ pub const TestRunner = struct { only: bool = false, run_todo: bool = false, concurrent: bool = false, - randomize: bool = false, + randomize: ?std.Random = null, concurrent_test_glob: ?[]const []const u8 = null, last_file: u64 = 0, bail: u32 = 0, diff --git a/src/cli.zig b/src/cli.zig index f192591414..4461e8f92c 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -340,6 +340,7 @@ pub const Command = struct { only: bool = false, concurrent: bool = false, randomize: bool = false, + seed: ?u32 = null, concurrent_test_glob: ?[]const []const u8 = null, bail: u32 = 0, coverage: TestCommand.CodeCoverageOptions = .{}, diff --git a/src/cli/Arguments.zig b/src/cli/Arguments.zig index 1828060979..324c8165e2 100644 --- a/src/cli/Arguments.zig +++ b/src/cli/Arguments.zig @@ -198,6 +198,7 @@ pub const test_only_params = [_]ParamType{ clap.parseParam("--todo Include tests that are marked with \"test.todo()\"") catch unreachable, clap.parseParam("--concurrent Treat all tests as `test.concurrent()` tests") catch unreachable, clap.parseParam("--randomize Run tests in random order") catch unreachable, + clap.parseParam("--seed Set the random seed for test randomization") catch unreachable, clap.parseParam("--coverage Generate a coverage profile") catch unreachable, clap.parseParam("--coverage-reporter ... Report coverage in 'text' and/or 'lcov'. Defaults to 'text'.") catch unreachable, clap.parseParam("--coverage-dir Directory for coverage files. Defaults to 'coverage'.") catch unreachable, @@ -497,6 +498,14 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C ctx.test_options.run_todo = args.flag("--todo"); ctx.test_options.concurrent = args.flag("--concurrent"); ctx.test_options.randomize = args.flag("--randomize"); + + if (args.option("--seed")) |seed_str| { + ctx.test_options.randomize = true; + ctx.test_options.seed = std.fmt.parseInt(u32, seed_str, 10) catch { + Output.prettyErrorln("error: Invalid seed value: {s}", .{seed_str}); + std.process.exit(1); + }; + } } ctx.args.absolute_working_dir = cwd; diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig index 924c415445..e155123ab8 100644 --- a/src/cli/test_command.zig +++ b/src/cli/test_command.zig @@ -1280,6 +1280,11 @@ pub const TestCommand = struct { bun.jsc.initialize(false); HTTPThread.init(&.{}); + const enable_random = ctx.test_options.randomize; + const seed: u32 = if (enable_random) ctx.test_options.seed orelse @truncate(bun.fastRandom()) else 0; // seed is limited to u32 so storing it in js doesn't lose precision + var random_instance: ?std.Random.DefaultPrng = if (enable_random) std.Random.DefaultPrng.init(seed) else null; + const random = if (random_instance) |*instance| instance.random() else null; + var snapshot_file_buf = std.ArrayList(u8).init(ctx.allocator); var snapshot_values = Snapshots.ValuesHashMap.init(ctx.allocator); var snapshot_counts = bun.StringHashMap(usize).init(ctx.allocator); @@ -1301,7 +1306,7 @@ pub const TestCommand = struct { .allocator = ctx.allocator, .default_timeout_ms = ctx.test_options.default_timeout_ms, .concurrent = ctx.test_options.concurrent, - .randomize = ctx.test_options.randomize, + .randomize = random, .concurrent_test_glob = ctx.test_options.concurrent_test_glob, .run_todo = ctx.test_options.run_todo, .only = ctx.test_options.only, @@ -1475,6 +1480,11 @@ pub const TestCommand = struct { const search_count = scanner.search_count; if (test_files.len > 0) { + // Randomize the order of test files if --randomize flag is set + if (random) |rand| { + rand.shuffle(PathString, test_files); + } + vm.hot_reload = ctx.debug.hot_reload; switch (vm.hot_reload) { @@ -1607,6 +1617,11 @@ pub const TestCommand = struct { const did_label_filter_out_all_tests = summary.didLabelFilterOutAllTests() and reporter.jest.unhandled_errors_between_tests == 0; if (!did_label_filter_out_all_tests) { + // Display the random seed if tests were randomized + if (random != null) { + Output.prettyError(" --seed={d}\n", .{seed}); + } + if (summary.pass > 0) { Output.prettyError("", .{}); } diff --git a/test/cli/test/test-randomize.fixture.ts b/test/cli/test/test-randomize.fixture.ts new file mode 100644 index 0000000000..a7d9f18359 --- /dev/null +++ b/test/cli/test/test-randomize.fixture.ts @@ -0,0 +1,3 @@ +test.each(Array.from({ length: 100 }, (_, i) => i + 1))("many %d", item => { + console.log(item); +}); diff --git a/test/cli/test/test-randomize.test.ts b/test/cli/test/test-randomize.test.ts index b53dfc2804..d5754ba0d7 100644 --- a/test/cli/test/test-randomize.test.ts +++ b/test/cli/test/test-randomize.test.ts @@ -1,256 +1,95 @@ import { expect, test } from "bun:test"; -import { bunEnv, bunExe, tempDir } from "harness"; -import { join } from "path"; +import { bunEnv, bunExe, tempDirWithFiles } from "harness"; -test("--randomize flag randomizes test execution order", async () => { - // Create a test file with multiple tests that output their names - using dir = tempDir("test-randomize", {}); - const testFile = join(String(dir), "order.test.js"); +// test: +// --randomize randomizes +// output produces a seed which produces the same result +// --seed produces the same result twice - await Bun.write( - testFile, - ` - import { test } from "bun:test"; - - test("test-01", () => { - console.log("test-01"); - }); - - test("test-02", () => { - console.log("test-02"); - }); - - test("test-03", () => { - console.log("test-03"); - }); - - test("test-04", () => { - console.log("test-04"); - }); - - test("test-05", () => { - console.log("test-05"); - }); - - test("test-06", () => { - console.log("test-06"); - }); - - test("test-07", () => { - console.log("test-07"); - }); - - test("test-08", () => { - console.log("test-08"); - }); - `, - ); - - // Run without --randomize to get the default order - await using defaultProc = Bun.spawn({ - cmd: [bunExe(), "test", testFile], +const unsortedOrder = Array.from({ length: 100 }, (_, i) => i + 1); +async function runFixture(flags: string[]): Promise<{ order: number[]; seed: number | null }> { + const proc = await Bun.spawn([bunExe(), "test", ...flags], { env: bunEnv, - stdout: "pipe", - stderr: "pipe", - cwd: String(dir), + stdio: ["pipe", "pipe", "pipe"], }); + const exitCode = await proc.exited; + const stdout = await proc.stdout.text(); + const stderr = await proc.stderr.text(); + expect(exitCode).toBe(0); + const stdoutOrder = stdout + .split("\n") + .map(l => l.trim()) + .filter(l => l && !isNaN(+l)) + .map(l => +l); + const seed = stderr.includes("--seed") ? +(stderr.match(/--seed=(-?\d+)/)?.[1] + "") : null; + return { order: stdoutOrder, seed: seed }; +} - const [defaultOut, defaultErr, defaultExit] = await Promise.all([ - defaultProc.stdout.text(), - defaultProc.stderr.text(), - defaultProc.exited, +const sortNumbers = (a: number, b: number) => a - b; +test("--randomize and --seed work", async () => { + const fixture = import.meta.dir + "/test-randomize.fixture.ts"; + + // with --randomize + const { order: randomizedOrder, seed: randomizedSeed } = await runFixture([fixture, "--randomize"]); + expect(randomizedSeed).toBeFinite(); + expect(randomizedOrder.toSorted(sortNumbers)).toEqual(unsortedOrder); + expect(randomizedOrder).not.toEqual(unsortedOrder); + + // different randomized run is different + const { order: differentRandomizedOrder, seed: differentRandomizedSeed } = await runFixture([fixture, "--randomize"]); + expect(differentRandomizedOrder.toSorted(sortNumbers)).toEqual(unsortedOrder); + expect(differentRandomizedOrder).not.toEqual(unsortedOrder); + expect(differentRandomizedOrder).not.toEqual(randomizedOrder); + expect(differentRandomizedSeed).not.toEqual(randomizedSeed); + + // with same seed as first run + const { order: seededOrder, seed: seededSeed } = await runFixture([fixture, "--seed", "" + randomizedSeed]); + expect(seededOrder).toEqual(randomizedOrder); + expect(seededSeed).toEqual(randomizedSeed); + + // with both randomize and seed parameter + const { order: randomizedAndSeededOrder, seed: randomizedAndSeededSeed } = await runFixture([ + fixture, + "--randomize", + "--seed", + "" + randomizedSeed, ]); + expect(randomizedAndSeededOrder).toEqual(randomizedOrder); + expect(randomizedAndSeededSeed).toEqual(randomizedSeed); - expect(defaultExit).toBe(0); + // without seed + const { order: unseededOrder, seed: unseededSeed } = await runFixture([fixture]); + expect(unseededOrder).toEqual(unsortedOrder); + expect(unseededSeed).toBeNull(); +}); - // Extract test execution order from output - const defaultTests = defaultOut.match(/test-\d+/g) || []; - expect(defaultTests.length).toBe(8); - - // Run multiple times WITH --randomize to find a different order - let foundDifferentOrder = false; - const maxAttempts = 20; // Increase attempts since randomization might occasionally match - - for (let i = 0; i < maxAttempts; i++) { - await using randomProc = Bun.spawn({ - cmd: [bunExe(), "test", testFile, "--randomize"], - env: bunEnv, - stdout: "pipe", - stderr: "pipe", - cwd: String(dir), - }); - - const [randomOut, randomErr, randomExit] = await Promise.all([ - randomProc.stdout.text(), - randomProc.stderr.text(), - randomProc.exited, - ]); - - expect(randomExit).toBe(0); - - const randomTests = randomOut.match(/test-\d+/g) || []; - expect(randomTests.length).toBe(8); - - // Check if all tests ran (just different order) - const sortedRandom = [...randomTests].sort(); - const sortedDefault = [...defaultTests].sort(); - expect(sortedRandom).toEqual(sortedDefault); - - // Check if order is different - const orderIsDifferent = randomTests.some((test, index) => test !== defaultTests[index]); - if (orderIsDifferent) { - foundDifferentOrder = true; - break; - } - } - - // With 8 tests and 20 attempts, the probability of not finding a different order - // by pure chance is (1/8!)^20 which is astronomically small - expect(foundDifferentOrder).toBe(true); -}, 30000); // 30 second timeout for this test - -test("--randomize flag works with describe blocks", async () => { - using dir = tempDir("test-randomize-describe", {}); - const testFile = join(String(dir), "describe.test.js"); - - await Bun.write( - testFile, - ` - import { test, describe } from "bun:test"; - - describe("Suite-A", () => { - test("A1", () => { - console.log("A1"); - }); - - test("A2", () => { - console.log("A2"); - }); - - test("A3", () => { - console.log("A3"); - }); - }); - - describe("Suite-B", () => { - test("B1", () => { - console.log("B1"); - }); - - test("B2", () => { - console.log("B2"); - }); - }); - - describe("Suite-C", () => { - test("C1", () => { - console.log("C1"); - }); - - test("C2", () => { - console.log("C2"); - }); - }); - `, +test("randomizes order of files", async () => { + const dir = tempDirWithFiles( + "randomize-order-of-files", + Object.fromEntries( + Array.from({ length: 20 }, (_, i) => [ + `test${i + 1}.test.ts`, + `test("test ${i + 1}", () => { console.log(${i + 1}); });`, + ]), + ), ); - // Run without --randomize - await using defaultProc = Bun.spawn({ - cmd: [bunExe(), "test", testFile], - env: bunEnv, - stdout: "pipe", - stderr: "pipe", - cwd: String(dir), - }); + const { order: unrandomizedOrder, seed: unrandomizedSeed } = await runFixture([dir]); + const { order: anotherUnrandomizedOrder, seed: anotherUnrandomizedSeed } = await runFixture([dir]); + expect(unrandomizedSeed).toBeNull(); + expect(anotherUnrandomizedSeed).toBeNull(); + expect(anotherUnrandomizedOrder).toEqual(unrandomizedOrder); - const [defaultOut, defaultErr, defaultExit] = await Promise.all([ - defaultProc.stdout.text(), - defaultProc.stderr.text(), - defaultProc.exited, - ]); + const { order: randomizedOrder, seed: randomizedSeed } = await runFixture([dir, "--randomize"]); + expect(randomizedSeed).toBeFinite(); + expect(unrandomizedOrder).not.toEqual(randomizedOrder); - expect(defaultExit).toBe(0); + const { order: anotherRandomizedOrder, seed: anotherRandomizedSeed } = await runFixture([dir, "--randomize"]); + expect(anotherRandomizedOrder).not.toEqual(randomizedOrder); + expect(anotherRandomizedSeed).not.toEqual(randomizedSeed); - const defaultTests = defaultOut.match(/[ABC]\d/g) || []; - expect(defaultTests.length).toBe(7); - - // Run with --randomize multiple times - let foundDifferentOrder = false; - - for (let i = 0; i < 20; i++) { - await using randomProc = Bun.spawn({ - cmd: [bunExe(), "test", testFile, "--randomize"], - env: bunEnv, - stdout: "pipe", - stderr: "pipe", - cwd: String(dir), - }); - - const [randomOut, randomErr, randomExit] = await Promise.all([ - randomProc.stdout.text(), - randomProc.stderr.text(), - randomProc.exited, - ]); - - expect(randomExit).toBe(0); - - const randomTests = randomOut.match(/[ABC]\d/g) || []; - expect(randomTests.length).toBe(7); - - // Verify all tests ran - expect([...randomTests].sort()).toEqual([...defaultTests].sort()); - - // Check if order is different - const orderIsDifferent = randomTests.some((test, index) => test !== defaultTests[index]); - if (orderIsDifferent) { - foundDifferentOrder = true; - break; - } - } - - expect(foundDifferentOrder).toBe(true); -}, 30000); - -test("without --randomize flag tests run in consistent order", async () => { - using dir = tempDir("test-consistent", {}); - const testFile = join(String(dir), "consistent.test.js"); - - await Bun.write( - testFile, - ` - import { test } from "bun:test"; - - test("test-1", () => { console.log("1"); }); - test("test-2", () => { console.log("2"); }); - test("test-3", () => { console.log("3"); }); - test("test-4", () => { console.log("4"); }); - test("test-5", () => { console.log("5"); }); - `, - ); - - const runs = []; - - // Run 5 times without --randomize - for (let i = 0; i < 5; i++) { - await using proc = Bun.spawn({ - cmd: [bunExe(), "test", testFile], - env: bunEnv, - stdout: "pipe", - stderr: "pipe", - cwd: String(dir), - }); - - const [out, err, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - - expect(exitCode).toBe(0); - - const order = out.match(/\d/g) || []; - runs.push(order.join("")); - } - - // All runs should have the same order - const firstRun = runs[0]; - for (const run of runs) { - expect(run).toBe(firstRun); - } -}, 20000); + // test with --seed + const { order: seededOrder, seed: seededSeed } = await runFixture([dir, "--seed", "" + randomizedSeed]); + expect(seededOrder).toEqual(randomizedOrder); + expect(seededSeed).toEqual(randomizedSeed); +});