From be15f6c80c7843c76f7eb8be06a7981964509eb9 Mon Sep 17 00:00:00 2001 From: robobun Date: Thu, 25 Sep 2025 14:20:47 -0700 Subject: [PATCH] feat(test): add --randomize flag to run tests in random order (#22945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR adds a `--randomize` flag to `bun test` that shuffles test execution order. This helps developers catch test interdependencies and identify flaky tests that may depend on execution order. ## Changes - โœจ Added `--randomize` CLI flag to test command - ๐Ÿ”€ Implemented test shuffling using `bun.fastRandom()` as PRNG seed - ๐Ÿงช Added comprehensive tests to verify randomization behavior - ๐Ÿ“ Tests are shuffled at the scheduling phase, properly handling describe blocks and hooks ## Usage ```bash # Run tests in random order bun test --randomize # Works with other test flags bun test --randomize --bail bun test mytest.test.ts --randomize ``` ## Implementation Details The randomization happens in `Order.zig`'s `generateOrderDescribe` function, which shuffles the `current.entries.items` array when the randomize flag is set. This ensures: - All tests still run (just in different order) - Hooks (beforeAll, afterAll, beforeEach, afterEach) maintain proper relationships - Describe blocks and their children are shuffled independently - Each run uses a different random seed for varied execution orders ## Test Coverage Added tests in `test/cli/test/test-randomize.test.ts` that verify: - Tests run in random order with the flag - All tests execute (none are skipped) - Without the flag, tests run in consistent order - Randomization works with describe blocks ## Example Output ```bash # Without --randomize (consistent order) $ bun test mytest.js Running test 1 Running test 2 Running test 3 Running test 4 Running test 5 # With --randomize (different order each run) $ bun test mytest.js --randomize Running test 3 Running test 5 Running test 1 Running test 4 Running test 2 $ bun test mytest.js --randomize Running test 2 Running test 4 Running test 5 Running test 1 Running test 3 ``` ๐Ÿค– Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --------- Co-authored-by: Claude Bot Co-authored-by: Claude Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: pfg --- src/bun.js/test/Order.zig | 8 + src/bun.js/test/bun_test.zig | 3 +- src/bun.js/test/jest.zig | 1 + src/cli.zig | 1 + src/cli/Arguments.zig | 2 + src/cli/test_command.zig | 1 + test/cli/test/test-randomize.test.ts | 256 +++++++++++++++++++++++++++ 7 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 test/cli/test/test-randomize.test.ts diff --git a/src/bun.js/test/Order.zig b/src/bun.js/test/Order.zig index ce51ae80f6..3adae8f64b 100644 --- a/src/bun.js/test/Order.zig +++ b/src/bun.js/test/Order.zig @@ -38,6 +38,7 @@ pub const AllOrderResult = struct { }; pub const Config = struct { always_use_hooks: bool = false, + randomize: bool = false, }; pub fn generateAllOrder(this: *Order, entries: []const *ExecutionEntry, _: Config) bun.JSError!AllOrderResult { const start = this.groups.items.len; @@ -61,6 +62,13 @@ pub fn generateOrderDescribe(this: *Order, current: *DescribeScope, cfg: Config) // gather beforeAll const beforeall_order: AllOrderResult = if (use_hooks) try generateAllOrder(this, current.beforeAll.items, cfg) else .empty; + // shuffle entries if randomize flag is set + if (cfg.randomize) { + var prng = std.Random.DefaultPrng.init(bun.fastRandom()); + const random = prng.random(); + random.shuffle(TestScheduleEntry, current.entries.items); + } + // gather children for (current.entries.items) |entry| { if (current.base.only == .contains and entry.base().only == .no) continue; diff --git a/src/bun.js/test/bun_test.zig b/src/bun.js/test/bun_test.zig index 2fa96ba79f..1150c8c916 100644 --- a/src/bun.js/test/bun_test.zig +++ b/src/bun.js/test/bun_test.zig @@ -514,7 +514,8 @@ pub const BunTest = struct { defer order.deinit(); const has_filter = if (this.reporter) |reporter| if (reporter.jest.filter_regex) |_| true else false else false; - const cfg: Order.Config = .{ .always_use_hooks = this.collection.root_scope.base.only == .no and !has_filter }; + const should_randomize = if (this.reporter) |reporter| reporter.jest.randomize else false; + const cfg: Order.Config = .{ .always_use_hooks = this.collection.root_scope.base.only == .no and !has_filter, .randomize = should_randomize }; const beforeall_order: Order.AllOrderResult = if (cfg.always_use_hooks or this.collection.root_scope.base.has_callback) try order.generateAllOrder(this.buntest.hook_scope.beforeAll.items, cfg) else .empty; try order.generateOrderDescribe(this.collection.root_scope, cfg); beforeall_order.setFailureSkipTo(&order); diff --git a/src/bun.js/test/jest.zig b/src/bun.js/test/jest.zig index 1a592dc041..013bbcb5f9 100644 --- a/src/bun.js/test/jest.zig +++ b/src/bun.js/test/jest.zig @@ -55,6 +55,7 @@ pub const TestRunner = struct { only: bool = false, run_todo: bool = false, concurrent: bool = false, + randomize: bool = false, concurrent_test_glob: ?[]const []const u8 = null, last_file: u64 = 0, bail: u32 = 0, diff --git a/src/cli.zig b/src/cli.zig index ea1129320e..f192591414 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -339,6 +339,7 @@ pub const Command = struct { run_todo: bool = false, only: bool = false, concurrent: bool = false, + randomize: bool = false, concurrent_test_glob: ?[]const []const u8 = null, bail: u32 = 0, coverage: TestCommand.CodeCoverageOptions = .{}, diff --git a/src/cli/Arguments.zig b/src/cli/Arguments.zig index f0884f20df..1828060979 100644 --- a/src/cli/Arguments.zig +++ b/src/cli/Arguments.zig @@ -197,6 +197,7 @@ pub const test_only_params = [_]ParamType{ clap.parseParam("--rerun-each Re-run each test file times, helps catch certain bugs") catch unreachable, clap.parseParam("--todo Include tests that are marked with \"test.todo()\"") catch unreachable, clap.parseParam("--concurrent Treat all tests as `test.concurrent()` tests") catch unreachable, + clap.parseParam("--randomize Run tests in random order") catch unreachable, clap.parseParam("--coverage Generate a coverage profile") catch unreachable, clap.parseParam("--coverage-reporter ... Report coverage in 'text' and/or 'lcov'. Defaults to 'text'.") catch unreachable, clap.parseParam("--coverage-dir Directory for coverage files. Defaults to 'coverage'.") catch unreachable, @@ -495,6 +496,7 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C ctx.test_options.update_snapshots = args.flag("--update-snapshots"); ctx.test_options.run_todo = args.flag("--todo"); ctx.test_options.concurrent = args.flag("--concurrent"); + ctx.test_options.randomize = args.flag("--randomize"); } ctx.args.absolute_working_dir = cwd; diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig index a0f70cd1ee..924c415445 100644 --- a/src/cli/test_command.zig +++ b/src/cli/test_command.zig @@ -1301,6 +1301,7 @@ pub const TestCommand = struct { .allocator = ctx.allocator, .default_timeout_ms = ctx.test_options.default_timeout_ms, .concurrent = ctx.test_options.concurrent, + .randomize = ctx.test_options.randomize, .concurrent_test_glob = ctx.test_options.concurrent_test_glob, .run_todo = ctx.test_options.run_todo, .only = ctx.test_options.only, diff --git a/test/cli/test/test-randomize.test.ts b/test/cli/test/test-randomize.test.ts new file mode 100644 index 0000000000..b53dfc2804 --- /dev/null +++ b/test/cli/test/test-randomize.test.ts @@ -0,0 +1,256 @@ +import { expect, test } from "bun:test"; +import { bunEnv, bunExe, tempDir } from "harness"; +import { join } from "path"; + +test("--randomize flag randomizes test execution order", async () => { + // Create a test file with multiple tests that output their names + using dir = tempDir("test-randomize", {}); + const testFile = join(String(dir), "order.test.js"); + + await Bun.write( + testFile, + ` + import { test } from "bun:test"; + + test("test-01", () => { + console.log("test-01"); + }); + + test("test-02", () => { + console.log("test-02"); + }); + + test("test-03", () => { + console.log("test-03"); + }); + + test("test-04", () => { + console.log("test-04"); + }); + + test("test-05", () => { + console.log("test-05"); + }); + + test("test-06", () => { + console.log("test-06"); + }); + + test("test-07", () => { + console.log("test-07"); + }); + + test("test-08", () => { + console.log("test-08"); + }); + `, + ); + + // Run without --randomize to get the default order + await using defaultProc = Bun.spawn({ + cmd: [bunExe(), "test", testFile], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + cwd: String(dir), + }); + + const [defaultOut, defaultErr, defaultExit] = await Promise.all([ + defaultProc.stdout.text(), + defaultProc.stderr.text(), + defaultProc.exited, + ]); + + expect(defaultExit).toBe(0); + + // Extract test execution order from output + const defaultTests = defaultOut.match(/test-\d+/g) || []; + expect(defaultTests.length).toBe(8); + + // Run multiple times WITH --randomize to find a different order + let foundDifferentOrder = false; + const maxAttempts = 20; // Increase attempts since randomization might occasionally match + + for (let i = 0; i < maxAttempts; i++) { + await using randomProc = Bun.spawn({ + cmd: [bunExe(), "test", testFile, "--randomize"], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + cwd: String(dir), + }); + + const [randomOut, randomErr, randomExit] = await Promise.all([ + randomProc.stdout.text(), + randomProc.stderr.text(), + randomProc.exited, + ]); + + expect(randomExit).toBe(0); + + const randomTests = randomOut.match(/test-\d+/g) || []; + expect(randomTests.length).toBe(8); + + // Check if all tests ran (just different order) + const sortedRandom = [...randomTests].sort(); + const sortedDefault = [...defaultTests].sort(); + expect(sortedRandom).toEqual(sortedDefault); + + // Check if order is different + const orderIsDifferent = randomTests.some((test, index) => test !== defaultTests[index]); + if (orderIsDifferent) { + foundDifferentOrder = true; + break; + } + } + + // With 8 tests and 20 attempts, the probability of not finding a different order + // by pure chance is (1/8!)^20 which is astronomically small + expect(foundDifferentOrder).toBe(true); +}, 30000); // 30 second timeout for this test + +test("--randomize flag works with describe blocks", async () => { + using dir = tempDir("test-randomize-describe", {}); + const testFile = join(String(dir), "describe.test.js"); + + await Bun.write( + testFile, + ` + import { test, describe } from "bun:test"; + + describe("Suite-A", () => { + test("A1", () => { + console.log("A1"); + }); + + test("A2", () => { + console.log("A2"); + }); + + test("A3", () => { + console.log("A3"); + }); + }); + + describe("Suite-B", () => { + test("B1", () => { + console.log("B1"); + }); + + test("B2", () => { + console.log("B2"); + }); + }); + + describe("Suite-C", () => { + test("C1", () => { + console.log("C1"); + }); + + test("C2", () => { + console.log("C2"); + }); + }); + `, + ); + + // Run without --randomize + await using defaultProc = Bun.spawn({ + cmd: [bunExe(), "test", testFile], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + cwd: String(dir), + }); + + const [defaultOut, defaultErr, defaultExit] = await Promise.all([ + defaultProc.stdout.text(), + defaultProc.stderr.text(), + defaultProc.exited, + ]); + + expect(defaultExit).toBe(0); + + const defaultTests = defaultOut.match(/[ABC]\d/g) || []; + expect(defaultTests.length).toBe(7); + + // Run with --randomize multiple times + let foundDifferentOrder = false; + + for (let i = 0; i < 20; i++) { + await using randomProc = Bun.spawn({ + cmd: [bunExe(), "test", testFile, "--randomize"], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + cwd: String(dir), + }); + + const [randomOut, randomErr, randomExit] = await Promise.all([ + randomProc.stdout.text(), + randomProc.stderr.text(), + randomProc.exited, + ]); + + expect(randomExit).toBe(0); + + const randomTests = randomOut.match(/[ABC]\d/g) || []; + expect(randomTests.length).toBe(7); + + // Verify all tests ran + expect([...randomTests].sort()).toEqual([...defaultTests].sort()); + + // Check if order is different + const orderIsDifferent = randomTests.some((test, index) => test !== defaultTests[index]); + if (orderIsDifferent) { + foundDifferentOrder = true; + break; + } + } + + expect(foundDifferentOrder).toBe(true); +}, 30000); + +test("without --randomize flag tests run in consistent order", async () => { + using dir = tempDir("test-consistent", {}); + const testFile = join(String(dir), "consistent.test.js"); + + await Bun.write( + testFile, + ` + import { test } from "bun:test"; + + test("test-1", () => { console.log("1"); }); + test("test-2", () => { console.log("2"); }); + test("test-3", () => { console.log("3"); }); + test("test-4", () => { console.log("4"); }); + test("test-5", () => { console.log("5"); }); + `, + ); + + const runs = []; + + // Run 5 times without --randomize + for (let i = 0; i < 5; i++) { + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", testFile], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + cwd: String(dir), + }); + + const [out, err, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + expect(exitCode).toBe(0); + + const order = out.match(/\d/g) || []; + runs.push(order.join("")); + } + + // All runs should have the same order + const firstRun = runs[0]; + for (const run of runs) { + expect(run).toBe(firstRun); + } +}, 20000);