mirrors/bun.sh
1
0
Fork 0
mirror of https://github.com/oven-sh/bun synced 2026-02-02 15:08:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add --randomize --seed flag (#22987)

Browse Source 
Outputs the seed when randomizing. Adds --seed flag to reproduce a
random order. Seeds might not produce the same order across operating
systems / bun versions.

Fixes #11847

---------

Co-authored-by: Claude Bot <claude-bot@bun.sh>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
pfg
2025-09-25 23:47:46 -07:00
committed by GitHub
parent 656747bcf1
commit c4519c7552
9 changed files with 149 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
docs/cli/test.md
View File

@@ -117,6 +117,36 @@ Use the `--rerun-each` flag to run each test multiple times. This is useful for
$ bun test --rerun-each 100
```
## Randomize test execution order
Use the `--randomize` flag to run tests in a random order. This helps detect tests that depend on shared state or execution order.
```sh
$ bun test --randomize
```
When using `--randomize`, the seed used for randomization will be displayed in the test summary:
```sh
$ bun test --randomize
# ... test output ...
--seed=12345
2 pass
8 fail
Ran 10 tests across 2 files. [50.00ms]
```
### Reproducible random order with `--seed`
Use the `--seed` flag to specify a seed for the randomization. This allows you to reproduce the same test order when debugging order-dependent failures.
```sh
# Reproduce a previous randomized run
$ bun test --seed 123456
```
The `--seed` flag implies `--randomize`, so you don't need to specify both. Using the same seed value will always produce the same test execution order, making it easier to debug intermittent failures caused by test interdependencies.
## Bail out with `--bail`
Use the `--bail` flag to abort the test run early after a pre-determined number of test failures. By default Bun will run all tests and report all failures, but sometimes in CI environments it's preferable to terminate earlier to reduce CPU usage.

8
src/bun.js/test/Order.zig
View File

@@ -37,8 +37,8 @@ pub const AllOrderResult = struct {
}
};
pub const Config = struct {
always_use_hooks: bool = false,
randomize: bool = false,
always_use_hooks: bool,
randomize: ?std.Random,
};
pub fn generateAllOrder(this: *Order, entries: []const *ExecutionEntry, _: Config) bun.JSError!AllOrderResult {
const start = this.groups.items.len;
@@ -63,9 +63,7 @@ pub fn generateOrderDescribe(this: *Order, current: *DescribeScope, cfg: Config)
const beforeall_order: AllOrderResult = if (use_hooks) try generateAllOrder(this, current.beforeAll.items, cfg) else .empty;
// shuffle entries if randomize flag is set
if (cfg.randomize) {
var prng = std.Random.DefaultPrng.init(bun.fastRandom());
const random = prng.random();
if (cfg.randomize) |random| {
random.shuffle(TestScheduleEntry, current.entries.items);
}

7
src/bun.js/test/bun_test.zig
View File

@@ -514,8 +514,11 @@ pub const BunTest = struct {
defer order.deinit();
const has_filter = if (this.reporter) |reporter| if (reporter.jest.filter_regex) |_| true else false else false;
const should_randomize = if (this.reporter) |reporter| reporter.jest.randomize else false;
const cfg: Order.Config = .{ .always_use_hooks = this.collection.root_scope.base.only == .no and !has_filter, .randomize = should_randomize };
const should_randomize: ?std.Random = if (this.reporter) |reporter| reporter.jest.randomize else null;
const cfg: Order.Config = .{
.always_use_hooks = this.collection.root_scope.base.only == .no and !has_filter,
.randomize = should_randomize,
};
const beforeall_order: Order.AllOrderResult = if (cfg.always_use_hooks or this.collection.root_scope.base.has_callback) try order.generateAllOrder(this.buntest.hook_scope.beforeAll.items, cfg) else .empty;
try order.generateOrderDescribe(this.collection.root_scope, cfg);
beforeall_order.setFailureSkipTo(&order);

2
src/bun.js/test/jest.zig
View File

@@ -55,7 +55,7 @@ pub const TestRunner = struct {
only: bool = false,
run_todo: bool = false,
concurrent: bool = false,
randomize: bool = false,
randomize: ?std.Random = null,
concurrent_test_glob: ?[]const []const u8 = null,
last_file: u64 = 0,
bail: u32 = 0,

1
src/cli.zig
View File

@@ -340,6 +340,7 @@ pub const Command = struct {
only: bool = false,
concurrent: bool = false,
randomize: bool = false,
seed: ?u32 = null,
concurrent_test_glob: ?[]const []const u8 = null,
bail: u32 = 0,
coverage: TestCommand.CodeCoverageOptions = .{},

9
src/cli/Arguments.zig
View File

@@ -198,6 +198,7 @@ pub const test_only_params = [_]ParamType{
clap.parseParam("--todo Include tests that are marked with \"test.todo()\"") catch unreachable,
clap.parseParam("--concurrent Treat all tests as `test.concurrent()` tests") catch unreachable,
clap.parseParam("--randomize Run tests in random order") catch unreachable,
clap.parseParam("--seed <INT> Set the random seed for test randomization") catch unreachable,
clap.parseParam("--coverage Generate a coverage profile") catch unreachable,
clap.parseParam("--coverage-reporter <STR>... Report coverage in 'text' and/or 'lcov'. Defaults to 'text'.") catch unreachable,
clap.parseParam("--coverage-dir <STR> Directory for coverage files. Defaults to 'coverage'.") catch unreachable,
@@ -497,6 +498,14 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C
ctx.test_options.run_todo = args.flag("--todo");
ctx.test_options.concurrent = args.flag("--concurrent");
ctx.test_options.randomize = args.flag("--randomize");
if (args.option("--seed")) |seed_str| {
ctx.test_options.randomize = true;
ctx.test_options.seed = std.fmt.parseInt(u32, seed_str, 10) catch {
Output.prettyErrorln("<red>error<r>: Invalid seed value: {s}", .{seed_str});
std.process.exit(1);
};
}
}
ctx.args.absolute_working_dir = cwd;

17
src/cli/test_command.zig
View File

@@ -1280,6 +1280,11 @@ pub const TestCommand = struct {
bun.jsc.initialize(false);
HTTPThread.init(&.{});
const enable_random = ctx.test_options.randomize;
const seed: u32 = if (enable_random) ctx.test_options.seed orelse @truncate(bun.fastRandom()) else 0; // seed is limited to u32 so storing it in js doesn't lose precision
var random_instance: ?std.Random.DefaultPrng = if (enable_random) std.Random.DefaultPrng.init(seed) else null;
const random = if (random_instance) |*instance| instance.random() else null;
var snapshot_file_buf = std.ArrayList(u8).init(ctx.allocator);
var snapshot_values = Snapshots.ValuesHashMap.init(ctx.allocator);
var snapshot_counts = bun.StringHashMap(usize).init(ctx.allocator);
@@ -1301,7 +1306,7 @@ pub const TestCommand = struct {
.allocator = ctx.allocator,
.default_timeout_ms = ctx.test_options.default_timeout_ms,
.concurrent = ctx.test_options.concurrent,
.randomize = ctx.test_options.randomize,
.randomize = random,
.concurrent_test_glob = ctx.test_options.concurrent_test_glob,
.run_todo = ctx.test_options.run_todo,
.only = ctx.test_options.only,
@@ -1475,6 +1480,11 @@ pub const TestCommand = struct {
const search_count = scanner.search_count;
if (test_files.len > 0) {
// Randomize the order of test files if --randomize flag is set
if (random) |rand| {
rand.shuffle(PathString, test_files);
}
vm.hot_reload = ctx.debug.hot_reload;
switch (vm.hot_reload) {
@@ -1607,6 +1617,11 @@ pub const TestCommand = struct {
const did_label_filter_out_all_tests = summary.didLabelFilterOutAllTests() and reporter.jest.unhandled_errors_between_tests == 0;
if (!did_label_filter_out_all_tests) {
// Display the random seed if tests were randomized
if (random != null) {
Output.prettyError(" <r>--seed={d}<r>\n", .{seed});
}
if (summary.pass > 0) {
Output.prettyError("<r><green>", .{});
}

3
test/cli/test/test-randomize.fixture.ts Normal file
View File

@@ -0,0 +1,3 @@
test.each(Array.from({ length: 100 }, (_, i) => i + 1))("many %d", item => {
console.log(item);
});

323
test/cli/test/test-randomize.test.ts
View File

@@ -1,256 +1,95 @@
import { expect, test } from "bun:test";
import { bunEnv, bunExe, tempDir } from "harness";
import { join } from "path";
import { bunEnv, bunExe, tempDirWithFiles } from "harness";
test("--randomize flag randomizes test execution order", async () => {
// Create a test file with multiple tests that output their names
using dir = tempDir("test-randomize", {});
const testFile = join(String(dir), "order.test.js");
// test:
// --randomize randomizes
// output produces a seed which produces the same result
// --seed produces the same result twice
await Bun.write(
testFile,
`
import { test } from "bun:test";
test("test-01", () => {
console.log("test-01");
});
test("test-02", () => {
console.log("test-02");
});
test("test-03", () => {
console.log("test-03");
});
test("test-04", () => {
console.log("test-04");
});
test("test-05", () => {
console.log("test-05");
});
test("test-06", () => {
console.log("test-06");
});
test("test-07", () => {
console.log("test-07");
});
test("test-08", () => {
console.log("test-08");
});
`,
);
// Run without --randomize to get the default order
await using defaultProc = Bun.spawn({
cmd: [bunExe(), "test", testFile],
const unsortedOrder = Array.from({ length: 100 }, (_, i) => i + 1);
async function runFixture(flags: string[]): Promise<{ order: number[]; seed: number | null }> {
const proc = await Bun.spawn([bunExe(), "test", ...flags], {
env: bunEnv,
stdout: "pipe",
stderr: "pipe",
cwd: String(dir),
stdio: ["pipe", "pipe", "pipe"],
});
const exitCode = await proc.exited;
const stdout = await proc.stdout.text();
const stderr = await proc.stderr.text();
expect(exitCode).toBe(0);
const stdoutOrder = stdout
.split("\n")
.map(l => l.trim())
.filter(l => l && !isNaN(+l))
.map(l => +l);
const seed = stderr.includes("--seed") ? +(stderr.match(/--seed=(-?\d+)/)?.[1] + "") : null;
return { order: stdoutOrder, seed: seed };
}
const [defaultOut, defaultErr, defaultExit] = await Promise.all([
defaultProc.stdout.text(),
defaultProc.stderr.text(),
defaultProc.exited,
const sortNumbers = (a: number, b: number) => a - b;
test("--randomize and --seed work", async () => {
const fixture = import.meta.dir + "/test-randomize.fixture.ts";
// with --randomize
const { order: randomizedOrder, seed: randomizedSeed } = await runFixture([fixture, "--randomize"]);
expect(randomizedSeed).toBeFinite();
expect(randomizedOrder.toSorted(sortNumbers)).toEqual(unsortedOrder);
expect(randomizedOrder).not.toEqual(unsortedOrder);
// different randomized run is different
const { order: differentRandomizedOrder, seed: differentRandomizedSeed } = await runFixture([fixture, "--randomize"]);
expect(differentRandomizedOrder.toSorted(sortNumbers)).toEqual(unsortedOrder);
expect(differentRandomizedOrder).not.toEqual(unsortedOrder);
expect(differentRandomizedOrder).not.toEqual(randomizedOrder);
expect(differentRandomizedSeed).not.toEqual(randomizedSeed);
// with same seed as first run
const { order: seededOrder, seed: seededSeed } = await runFixture([fixture, "--seed", "" + randomizedSeed]);
expect(seededOrder).toEqual(randomizedOrder);
expect(seededSeed).toEqual(randomizedSeed);
// with both randomize and seed parameter
const { order: randomizedAndSeededOrder, seed: randomizedAndSeededSeed } = await runFixture([
fixture,
"--randomize",
"--seed",
"" + randomizedSeed,
]);
expect(randomizedAndSeededOrder).toEqual(randomizedOrder);
expect(randomizedAndSeededSeed).toEqual(randomizedSeed);
expect(defaultExit).toBe(0);
// without seed
const { order: unseededOrder, seed: unseededSeed } = await runFixture([fixture]);
expect(unseededOrder).toEqual(unsortedOrder);
expect(unseededSeed).toBeNull();
});
// Extract test execution order from output
const defaultTests = defaultOut.match(/test-\d+/g) || [];
expect(defaultTests.length).toBe(8);
// Run multiple times WITH --randomize to find a different order
let foundDifferentOrder = false;
const maxAttempts = 20; // Increase attempts since randomization might occasionally match
for (let i = 0; i < maxAttempts; i++) {
await using randomProc = Bun.spawn({
cmd: [bunExe(), "test", testFile, "--randomize"],
env: bunEnv,
stdout: "pipe",
stderr: "pipe",
cwd: String(dir),
});
const [randomOut, randomErr, randomExit] = await Promise.all([
randomProc.stdout.text(),
randomProc.stderr.text(),
randomProc.exited,
]);
expect(randomExit).toBe(0);
const randomTests = randomOut.match(/test-\d+/g) || [];
expect(randomTests.length).toBe(8);
// Check if all tests ran (just different order)
const sortedRandom = [...randomTests].sort();
const sortedDefault = [...defaultTests].sort();
expect(sortedRandom).toEqual(sortedDefault);
// Check if order is different
const orderIsDifferent = randomTests.some((test, index) => test !== defaultTests[index]);
if (orderIsDifferent) {
foundDifferentOrder = true;
break;
}
}
// With 8 tests and 20 attempts, the probability of not finding a different order
// by pure chance is (1/8!)^20 which is astronomically small
expect(foundDifferentOrder).toBe(true);
}, 30000); // 30 second timeout for this test
test("--randomize flag works with describe blocks", async () => {
using dir = tempDir("test-randomize-describe", {});
const testFile = join(String(dir), "describe.test.js");
await Bun.write(
testFile,
`
import { test, describe } from "bun:test";
describe("Suite-A", () => {
test("A1", () => {
console.log("A1");
});
test("A2", () => {
console.log("A2");
});
test("A3", () => {
console.log("A3");
});
});
describe("Suite-B", () => {
test("B1", () => {
console.log("B1");
});
test("B2", () => {
console.log("B2");
});
});
describe("Suite-C", () => {
test("C1", () => {
console.log("C1");
});
test("C2", () => {
console.log("C2");
});
});
`,
test("randomizes order of files", async () => {
const dir = tempDirWithFiles(
"randomize-order-of-files",
Object.fromEntries(
Array.from({ length: 20 }, (_, i) => [
`test${i + 1}.test.ts`,
`test("test ${i + 1}", () => { console.log(${i + 1}); });`,
]),
),
);
// Run without --randomize
await using defaultProc = Bun.spawn({
cmd: [bunExe(), "test", testFile],
env: bunEnv,
stdout: "pipe",
stderr: "pipe",
cwd: String(dir),
});
const { order: unrandomizedOrder, seed: unrandomizedSeed } = await runFixture([dir]);
const { order: anotherUnrandomizedOrder, seed: anotherUnrandomizedSeed } = await runFixture([dir]);
expect(unrandomizedSeed).toBeNull();
expect(anotherUnrandomizedSeed).toBeNull();
expect(anotherUnrandomizedOrder).toEqual(unrandomizedOrder);
const [defaultOut, defaultErr, defaultExit] = await Promise.all([
defaultProc.stdout.text(),
defaultProc.stderr.text(),
defaultProc.exited,
]);
const { order: randomizedOrder, seed: randomizedSeed } = await runFixture([dir, "--randomize"]);
expect(randomizedSeed).toBeFinite();
expect(unrandomizedOrder).not.toEqual(randomizedOrder);
expect(defaultExit).toBe(0);
const { order: anotherRandomizedOrder, seed: anotherRandomizedSeed } = await runFixture([dir, "--randomize"]);
expect(anotherRandomizedOrder).not.toEqual(randomizedOrder);
expect(anotherRandomizedSeed).not.toEqual(randomizedSeed);
const defaultTests = defaultOut.match(/[ABC]\d/g) || [];
expect(defaultTests.length).toBe(7);
// Run with --randomize multiple times
let foundDifferentOrder = false;
for (let i = 0; i < 20; i++) {
await using randomProc = Bun.spawn({
cmd: [bunExe(), "test", testFile, "--randomize"],
env: bunEnv,
stdout: "pipe",
stderr: "pipe",
cwd: String(dir),
});
const [randomOut, randomErr, randomExit] = await Promise.all([
randomProc.stdout.text(),
randomProc.stderr.text(),
randomProc.exited,
]);
expect(randomExit).toBe(0);
const randomTests = randomOut.match(/[ABC]\d/g) || [];
expect(randomTests.length).toBe(7);
// Verify all tests ran
expect([...randomTests].sort()).toEqual([...defaultTests].sort());
// Check if order is different
const orderIsDifferent = randomTests.some((test, index) => test !== defaultTests[index]);
if (orderIsDifferent) {
foundDifferentOrder = true;
break;
}
}
expect(foundDifferentOrder).toBe(true);
}, 30000);
test("without --randomize flag tests run in consistent order", async () => {
using dir = tempDir("test-consistent", {});
const testFile = join(String(dir), "consistent.test.js");
await Bun.write(
testFile,
`
import { test } from "bun:test";
test("test-1", () => { console.log("1"); });
test("test-2", () => { console.log("2"); });
test("test-3", () => { console.log("3"); });
test("test-4", () => { console.log("4"); });
test("test-5", () => { console.log("5"); });
`,
);
const runs = [];
// Run 5 times without --randomize
for (let i = 0; i < 5; i++) {
await using proc = Bun.spawn({
cmd: [bunExe(), "test", testFile],
env: bunEnv,
stdout: "pipe",
stderr: "pipe",
cwd: String(dir),
});
const [out, err, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
expect(exitCode).toBe(0);
const order = out.match(/\d/g) || [];
runs.push(order.join(""));
}
// All runs should have the same order
const firstRun = runs[0];
for (const run of runs) {
expect(run).toBe(firstRun);
}
}, 20000);
// test with --seed
const { order: seededOrder, seed: seededSeed } = await runFixture([dir, "--seed", "" + randomizedSeed]);
expect(seededOrder).toEqual(randomizedOrder);
expect(seededSeed).toEqual(randomizedSeed);
});