feat(test): add --randomize flag to run tests in random order (#22945)

## Summary

This PR adds a `--randomize` flag to `bun test` that shuffles test
execution order. This helps developers catch test interdependencies and
identify flaky tests that may depend on execution order.

## Changes

- ✨ Added `--randomize` CLI flag to test command
- 🔀 Implemented test shuffling using `bun.fastRandom()` as PRNG seed
- 🧪 Added comprehensive tests to verify randomization behavior
- 📝 Tests are shuffled at the scheduling phase, properly handling
describe blocks and hooks

## Usage

```bash
# Run tests in random order
bun test --randomize

# Works with other test flags
bun test --randomize --bail
bun test mytest.test.ts --randomize
```

## Implementation Details

The randomization happens in `Order.zig`'s `generateOrderDescribe`
function, which shuffles the `current.entries.items` array when the
randomize flag is set. This ensures:

- All tests still run (just in different order)
- Hooks (beforeAll, afterAll, beforeEach, afterEach) maintain proper
relationships
- Describe blocks and their children are shuffled independently
- Each run uses a different random seed for varied execution orders

## Test Coverage

Added tests in `test/cli/test/test-randomize.test.ts` that verify:
- Tests run in random order with the flag
- All tests execute (none are skipped)
- Without the flag, tests run in consistent order
- Randomization works with describe blocks

## Example Output

```bash
# Without --randomize (consistent order)
$ bun test mytest.js
Running test 1
Running test 2
Running test 3
Running test 4
Running test 5

# With --randomize (different order each run)
$ bun test mytest.js --randomize
Running test 3
Running test 5
Running test 1
Running test 4
Running test 2

$ bun test mytest.js --randomize
Running test 2
Running test 4
Running test 5
Running test 1
Running test 3
```

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

---------

Co-authored-by: Claude Bot <claude-bot@bun.sh>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: pfg <pfg@pfg.pw>

src/bun.js/test/Order.zig

@@ -38,6 +38,7 @@ pub const AllOrderResult = struct {
 };
 pub const Config = struct {
     always_use_hooks: bool = false,
+    randomize: bool = false,
 };
 pub fn generateAllOrder(this: *Order, entries: []const *ExecutionEntry, _: Config) bun.JSError!AllOrderResult {
     const start = this.groups.items.len;
@@ -61,6 +62,13 @@ pub fn generateOrderDescribe(this: *Order, current: *DescribeScope, cfg: Config)
     // gather beforeAll
     const beforeall_order: AllOrderResult = if (use_hooks) try generateAllOrder(this, current.beforeAll.items, cfg) else .empty;
 
+    // shuffle entries if randomize flag is set
+    if (cfg.randomize) {
+        var prng = std.Random.DefaultPrng.init(bun.fastRandom());
+        const random = prng.random();
+        random.shuffle(TestScheduleEntry, current.entries.items);
+    }
+
     // gather children
     for (current.entries.items) |entry| {
         if (current.base.only == .contains and entry.base().only == .no) continue;

src/bun.js/test/bun_test.zig

@@ -514,7 +514,8 @@ pub const BunTest = struct {
                 defer order.deinit();
 
                 const has_filter = if (this.reporter) |reporter| if (reporter.jest.filter_regex) |_| true else false else false;
-                const cfg: Order.Config = .{ .always_use_hooks = this.collection.root_scope.base.only == .no and !has_filter };
+                const should_randomize = if (this.reporter) |reporter| reporter.jest.randomize else false;
+                const cfg: Order.Config = .{ .always_use_hooks = this.collection.root_scope.base.only == .no and !has_filter, .randomize = should_randomize };
                 const beforeall_order: Order.AllOrderResult = if (cfg.always_use_hooks or this.collection.root_scope.base.has_callback) try order.generateAllOrder(this.buntest.hook_scope.beforeAll.items, cfg) else .empty;
                 try order.generateOrderDescribe(this.collection.root_scope, cfg);
                 beforeall_order.setFailureSkipTo(&order);

src/bun.js/test/jest.zig

@@ -55,6 +55,7 @@ pub const TestRunner = struct {
     only: bool = false,
     run_todo: bool = false,
     concurrent: bool = false,
+    randomize: bool = false,
     concurrent_test_glob: ?[]const []const u8 = null,
     last_file: u64 = 0,
     bail: u32 = 0,

src/cli.zig

@@ -339,6 +339,7 @@ pub const Command = struct {
         run_todo: bool = false,
         only: bool = false,
         concurrent: bool = false,
+        randomize: bool = false,
         concurrent_test_glob: ?[]const []const u8 = null,
         bail: u32 = 0,
         coverage: TestCommand.CodeCoverageOptions = .{},

src/cli/Arguments.zig

@@ -197,6 +197,7 @@ pub const test_only_params = [_]ParamType{
     clap.parseParam("--rerun-each <NUMBER>            Re-run each test file <NUMBER> times, helps catch certain bugs") catch unreachable,
     clap.parseParam("--todo                           Include tests that are marked with \"test.todo()\"") catch unreachable,
     clap.parseParam("--concurrent                     Treat all tests as `test.concurrent()` tests") catch unreachable,
+    clap.parseParam("--randomize                      Run tests in random order") catch unreachable,
     clap.parseParam("--coverage                       Generate a coverage profile") catch unreachable,
     clap.parseParam("--coverage-reporter <STR>...     Report coverage in 'text' and/or 'lcov'. Defaults to 'text'.") catch unreachable,
     clap.parseParam("--coverage-dir <STR>             Directory for coverage files. Defaults to 'coverage'.") catch unreachable,
@@ -495,6 +496,7 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C
         ctx.test_options.update_snapshots = args.flag("--update-snapshots");
         ctx.test_options.run_todo = args.flag("--todo");
         ctx.test_options.concurrent = args.flag("--concurrent");
+        ctx.test_options.randomize = args.flag("--randomize");
     }
 
     ctx.args.absolute_working_dir = cwd;

src/cli/test_command.zig

@@ -1301,6 +1301,7 @@ pub const TestCommand = struct {
                 .allocator = ctx.allocator,
                 .default_timeout_ms = ctx.test_options.default_timeout_ms,
                 .concurrent = ctx.test_options.concurrent,
+                .randomize = ctx.test_options.randomize,
                 .concurrent_test_glob = ctx.test_options.concurrent_test_glob,
                 .run_todo = ctx.test_options.run_todo,
                 .only = ctx.test_options.only,

test/cli/test/test-randomize.test.ts

@@ -0,0 +1,256 @@
+import { expect, test } from "bun:test";
+import { bunEnv, bunExe, tempDir } from "harness";
+import { join } from "path";
+
+test("--randomize flag randomizes test execution order", async () => {
+  // Create a test file with multiple tests that output their names
+  using dir = tempDir("test-randomize", {});
+  const testFile = join(String(dir), "order.test.js");
+
+  await Bun.write(
+    testFile,
+    `
+    import { test } from "bun:test";
+
+    test("test-01", () => {
+      console.log("test-01");
+    });
+
+    test("test-02", () => {
+      console.log("test-02");
+    });
+
+    test("test-03", () => {
+      console.log("test-03");
+    });
+
+    test("test-04", () => {
+      console.log("test-04");
+    });
+
+    test("test-05", () => {
+      console.log("test-05");
+    });
+
+    test("test-06", () => {
+      console.log("test-06");
+    });
+
+    test("test-07", () => {
+      console.log("test-07");
+    });
+
+    test("test-08", () => {
+      console.log("test-08");
+    });
+  `,
+  );
+
+  // Run without --randomize to get the default order
+  await using defaultProc = Bun.spawn({
+    cmd: [bunExe(), "test", testFile],
+    env: bunEnv,
+    stdout: "pipe",
+    stderr: "pipe",
+    cwd: String(dir),
+  });
+
+  const [defaultOut, defaultErr, defaultExit] = await Promise.all([
+    defaultProc.stdout.text(),
+    defaultProc.stderr.text(),
+    defaultProc.exited,
+  ]);
+
+  expect(defaultExit).toBe(0);
+
+  // Extract test execution order from output
+  const defaultTests = defaultOut.match(/test-\d+/g) || [];
+  expect(defaultTests.length).toBe(8);
+
+  // Run multiple times WITH --randomize to find a different order
+  let foundDifferentOrder = false;
+  const maxAttempts = 20; // Increase attempts since randomization might occasionally match
+
+  for (let i = 0; i < maxAttempts; i++) {
+    await using randomProc = Bun.spawn({
+      cmd: [bunExe(), "test", testFile, "--randomize"],
+      env: bunEnv,
+      stdout: "pipe",
+      stderr: "pipe",
+      cwd: String(dir),
+    });
+
+    const [randomOut, randomErr, randomExit] = await Promise.all([
+      randomProc.stdout.text(),
+      randomProc.stderr.text(),
+      randomProc.exited,
+    ]);
+
+    expect(randomExit).toBe(0);
+
+    const randomTests = randomOut.match(/test-\d+/g) || [];
+    expect(randomTests.length).toBe(8);
+
+    // Check if all tests ran (just different order)
+    const sortedRandom = [...randomTests].sort();
+    const sortedDefault = [...defaultTests].sort();
+    expect(sortedRandom).toEqual(sortedDefault);
+
+    // Check if order is different
+    const orderIsDifferent = randomTests.some((test, index) => test !== defaultTests[index]);
+    if (orderIsDifferent) {
+      foundDifferentOrder = true;
+      break;
+    }
+  }
+
+  // With 8 tests and 20 attempts, the probability of not finding a different order
+  // by pure chance is (1/8!)^20 which is astronomically small
+  expect(foundDifferentOrder).toBe(true);
+}, 30000); // 30 second timeout for this test
+
+test("--randomize flag works with describe blocks", async () => {
+  using dir = tempDir("test-randomize-describe", {});
+  const testFile = join(String(dir), "describe.test.js");
+
+  await Bun.write(
+    testFile,
+    `
+    import { test, describe } from "bun:test";
+
+    describe("Suite-A", () => {
+      test("A1", () => {
+        console.log("A1");
+      });
+
+      test("A2", () => {
+        console.log("A2");
+      });
+
+      test("A3", () => {
+        console.log("A3");
+      });
+    });
+
+    describe("Suite-B", () => {
+      test("B1", () => {
+        console.log("B1");
+      });
+
+      test("B2", () => {
+        console.log("B2");
+      });
+    });
+
+    describe("Suite-C", () => {
+      test("C1", () => {
+        console.log("C1");
+      });
+
+      test("C2", () => {
+        console.log("C2");
+      });
+    });
+  `,
+  );
+
+  // Run without --randomize
+  await using defaultProc = Bun.spawn({
+    cmd: [bunExe(), "test", testFile],
+    env: bunEnv,
+    stdout: "pipe",
+    stderr: "pipe",
+    cwd: String(dir),
+  });
+
+  const [defaultOut, defaultErr, defaultExit] = await Promise.all([
+    defaultProc.stdout.text(),
+    defaultProc.stderr.text(),
+    defaultProc.exited,
+  ]);
+
+  expect(defaultExit).toBe(0);
+
+  const defaultTests = defaultOut.match(/[ABC]\d/g) || [];
+  expect(defaultTests.length).toBe(7);
+
+  // Run with --randomize multiple times
+  let foundDifferentOrder = false;
+
+  for (let i = 0; i < 20; i++) {
+    await using randomProc = Bun.spawn({
+      cmd: [bunExe(), "test", testFile, "--randomize"],
+      env: bunEnv,
+      stdout: "pipe",
+      stderr: "pipe",
+      cwd: String(dir),
+    });
+
+    const [randomOut, randomErr, randomExit] = await Promise.all([
+      randomProc.stdout.text(),
+      randomProc.stderr.text(),
+      randomProc.exited,
+    ]);
+
+    expect(randomExit).toBe(0);
+
+    const randomTests = randomOut.match(/[ABC]\d/g) || [];
+    expect(randomTests.length).toBe(7);
+
+    // Verify all tests ran
+    expect([...randomTests].sort()).toEqual([...defaultTests].sort());
+
+    // Check if order is different
+    const orderIsDifferent = randomTests.some((test, index) => test !== defaultTests[index]);
+    if (orderIsDifferent) {
+      foundDifferentOrder = true;
+      break;
+    }
+  }
+
+  expect(foundDifferentOrder).toBe(true);
+}, 30000);
+
+test("without --randomize flag tests run in consistent order", async () => {
+  using dir = tempDir("test-consistent", {});
+  const testFile = join(String(dir), "consistent.test.js");
+
+  await Bun.write(
+    testFile,
+    `
+    import { test } from "bun:test";
+
+    test("test-1", () => { console.log("1"); });
+    test("test-2", () => { console.log("2"); });
+    test("test-3", () => { console.log("3"); });
+    test("test-4", () => { console.log("4"); });
+    test("test-5", () => { console.log("5"); });
+  `,
+  );
+
+  const runs = [];
+
+  // Run 5 times without --randomize
+  for (let i = 0; i < 5; i++) {
+    await using proc = Bun.spawn({
+      cmd: [bunExe(), "test", testFile],
+      env: bunEnv,
+      stdout: "pipe",
+      stderr: "pipe",
+      cwd: String(dir),
+    });
+
+    const [out, err, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+
+    expect(exitCode).toBe(0);
+
+    const order = out.match(/\d/g) || [];
+    runs.push(order.join(""));
+  }
+
+  // All runs should have the same order
+  const firstRun = runs[0];
+  for (const run of runs) {
+    expect(run).toBe(firstRun);
+  }
+}, 20000);