From 250d30eb7dc52ce8f302e3a903e1a41853d6714d Mon Sep 17 00:00:00 2001 From: pfg Date: Fri, 26 Sep 2025 16:39:08 -0700 Subject: [PATCH] Concurrent limit `--max-concurrency`, defaults to 20 (#22944) ### What does this PR do? Adds a max-concurrency flag to limit the amount of concurrent tests that run at once. Defaults to 20. Jest and Vitest both default to 5. ### How did you verify your code works? Tests --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Jarred Sumner --- docs/cli/test.md | 79 ++++++++++++++++++++++ src/bun.js/test/Execution.zig | 36 ++++++++-- src/bun.js/test/bun_test.zig | 4 +- src/bun.js/test/jest.zig | 1 + src/cli.zig | 1 + src/cli/Arguments.zig | 10 +++ src/cli/test_command.zig | 1 + test/js/bun/test/concurrent-max.fixture.ts | 49 ++++++++++++++ test/js/bun/test/concurrent.test.ts | 67 ++++++++++++++++++ test/js/sql/sql-mysql.test.ts | 3 + 10 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 test/js/bun/test/concurrent-max.fixture.ts diff --git a/docs/cli/test.md b/docs/cli/test.md index e3729b8b62..627f48978b 100644 --- a/docs/cli/test.md +++ b/docs/cli/test.md @@ -109,6 +109,85 @@ Use the `--timeout` flag to specify a _per-test_ timeout in milliseconds. If a t $ bun test --timeout 20 ``` +## Concurrent test execution + +By default, Bun runs all tests sequentially within each test file. You can enable concurrent execution to run async tests in parallel, significantly speeding up test suites with independent tests. + +### `--concurrent` flag + +Use the `--concurrent` flag to run all tests concurrently within their respective files: + +```sh +$ bun test --concurrent +``` + +When this flag is enabled, all tests will run in parallel unless explicitly marked with `test.serial`. + +### `--max-concurrency` flag + +Control the maximum number of tests running simultaneously with the `--max-concurrency` flag: + +```sh +# Limit to 4 concurrent tests +$ bun test --concurrent --max-concurrency 4 + +# Default: 20 +$ bun test --concurrent +``` + +This helps prevent resource exhaustion when running many concurrent tests. The default value is 20. + +### `test.concurrent` + +Mark individual tests to run concurrently, even when the `--concurrent` flag is not used: + +```ts +import { test, expect } from "bun:test"; + +// These tests run in parallel with each other +test.concurrent("concurrent test 1", async () => { + await fetch("/api/endpoint1"); + expect(true).toBe(true); +}); + +test.concurrent("concurrent test 2", async () => { + await fetch("/api/endpoint2"); + expect(true).toBe(true); +}); + +// This test runs sequentially +test("sequential test", () => { + expect(1 + 1).toBe(2); +}); +``` + +### `test.serial` + +Force tests to run sequentially, even when the `--concurrent` flag is enabled: + +```ts +import { test, expect } from "bun:test"; + +let sharedState = 0; + +// These tests must run in order +test.serial("first serial test", () => { + sharedState = 1; + expect(sharedState).toBe(1); +}); + +test.serial("second serial test", () => { + // Depends on the previous test + expect(sharedState).toBe(1); + sharedState = 2; +}); + +// This test can run concurrently if --concurrent is enabled +test("independent test", () => { + expect(true).toBe(true); +}); +``` + ## Rerun tests Use the `--rerun-each` flag to run each test multiple times. This is useful for detecting flaky or non-deterministic test failures. diff --git a/src/bun.js/test/Execution.zig b/src/bun.js/test/Execution.zig index c57ae6414c..2c02034bd6 100644 --- a/src/bun.js/test/Execution.zig +++ b/src/bun.js/test/Execution.zig @@ -44,6 +44,8 @@ group_index: usize, pub const ConcurrentGroup = struct { sequence_start: usize, sequence_end: usize, + /// Index of the next sequence that has not been started yet + next_sequence_index: usize, executing: bool, remaining_incomplete_entries: usize, /// used by beforeAll to skip directly to afterAll if it fails @@ -56,6 +58,7 @@ pub const ConcurrentGroup = struct { .executing = false, .remaining_incomplete_entries = sequence_end - sequence_start, .failure_skip_to = next_index, + .next_sequence_index = 0, }; } pub fn tryExtend(this: *ConcurrentGroup, next_sequence_start: usize, next_sequence_end: usize) bool { @@ -243,11 +246,24 @@ pub fn step(buntest_strong: bun_test.BunTestPtr, globalThis: *jsc.JSGlobalObject bun.assert(sequence.active_index < sequence.entries(this).len); this.advanceSequence(sequence, group); - const sequence_result = try stepSequence(buntest_strong, globalThis, sequence, group, sequence_index, &now); + const sequence_result = try stepSequence(buntest_strong, globalThis, group, sequence_index, &now); switch (sequence_result) { .done => {}, .execute => |exec| return .{ .waiting = .{ .timeout = exec.timeout } }, } + // this sequence is complete; execute the next sequence + while (group.next_sequence_index < group.sequences(this).len) : (group.next_sequence_index += 1) { + const target_sequence = &group.sequences(this)[group.next_sequence_index]; + if (target_sequence.executing) continue; + const sequence_status = try stepSequence(buntest_strong, globalThis, group, group.next_sequence_index, &now); + switch (sequence_status) { + .done => continue, + .execute => |exec| { + return .{ .waiting = .{ .timeout = exec.timeout } }; + }, + } + } + // all sequences have started if (group.remaining_incomplete_entries == 0) { return try stepGroup(buntest_strong, globalThis, &now); } @@ -299,14 +315,21 @@ fn stepGroupOne(buntest_strong: bun_test.BunTestPtr, globalThis: *jsc.JSGlobalOb const buntest = buntest_strong.get(); const this = &buntest.execution; var final_status: AdvanceStatus = .done; - for (group.sequences(this), 0..) |*sequence, sequence_index| { - const sequence_status = try stepSequence(buntest_strong, globalThis, sequence, group, sequence_index, now); + const concurrent_limit = if (buntest.reporter) |reporter| reporter.jest.max_concurrency else blk: { + bun.assert(false); // probably can't get here because reporter is only set null when the file is exited + break :blk 20; + }; + var active_count: usize = 0; + for (0..group.sequences(this).len) |sequence_index| { + const sequence_status = try stepSequence(buntest_strong, globalThis, group, sequence_index, now); switch (sequence_status) { .done => {}, .execute => |exec| { const prev_timeout: bun.timespec = if (final_status == .execute) final_status.execute.timeout else .epoch; const this_timeout = exec.timeout; final_status = .{ .execute = .{ .timeout = prev_timeout.minIgnoreEpoch(this_timeout) } }; + active_count += 1; + if (concurrent_limit != 0 and active_count >= concurrent_limit) break; }, } } @@ -320,18 +343,19 @@ const AdvanceSequenceStatus = union(enum) { timeout: bun.timespec = .epoch, }, }; -fn stepSequence(buntest_strong: bun_test.BunTestPtr, globalThis: *jsc.JSGlobalObject, sequence: *ExecutionSequence, group: *ConcurrentGroup, sequence_index: usize, now: *bun.timespec) !AdvanceSequenceStatus { +fn stepSequence(buntest_strong: bun_test.BunTestPtr, globalThis: *jsc.JSGlobalObject, group: *ConcurrentGroup, sequence_index: usize, now: *bun.timespec) !AdvanceSequenceStatus { while (true) { - return try stepSequenceOne(buntest_strong, globalThis, sequence, group, sequence_index, now) orelse continue; + return try stepSequenceOne(buntest_strong, globalThis, group, sequence_index, now) orelse continue; } } /// returns null if the while loop should continue -fn stepSequenceOne(buntest_strong: bun_test.BunTestPtr, globalThis: *jsc.JSGlobalObject, sequence: *ExecutionSequence, group: *ConcurrentGroup, sequence_index: usize, now: *bun.timespec) !?AdvanceSequenceStatus { +fn stepSequenceOne(buntest_strong: bun_test.BunTestPtr, globalThis: *jsc.JSGlobalObject, group: *ConcurrentGroup, sequence_index: usize, now: *bun.timespec) !?AdvanceSequenceStatus { groupLog.begin(@src()); defer groupLog.end(); const buntest = buntest_strong.get(); const this = &buntest.execution; + const sequence = &group.sequences(this)[sequence_index]; if (sequence.executing) { const active_entry = sequence.activeEntry(this) orelse { bun.debugAssert(false); // sequence is executing with no active entry diff --git a/src/bun.js/test/bun_test.zig b/src/bun.js/test/bun_test.zig index f25c65fb06..d4a93b45c1 100644 --- a/src/bun.js/test/bun_test.zig +++ b/src/bun.js/test/bun_test.zig @@ -152,12 +152,12 @@ pub const BunTest = struct { arena_allocator: std.heap.ArenaAllocator, arena: std.mem.Allocator, file_id: jsc.Jest.TestRunner.File.ID, - /// null if the runner has moved on to the next file + /// null if the runner has moved on to the next file but a strong reference to BunTest is stll keeping it alive reporter: ?*test_command.CommandLineReporter, timer: bun.api.Timer.EventLoopTimer = .{ .next = .epoch, .tag = .BunTest }, result_queue: ResultQueue, /// Whether tests in this file should default to concurrent execution - default_concurrent: bool = false, + default_concurrent: bool, phase: enum { collection, diff --git a/src/bun.js/test/jest.zig b/src/bun.js/test/jest.zig index 8ec2dbdfd4..521c3d064d 100644 --- a/src/bun.js/test/jest.zig +++ b/src/bun.js/test/jest.zig @@ -59,6 +59,7 @@ pub const TestRunner = struct { concurrent_test_glob: ?[]const []const u8 = null, last_file: u64 = 0, bail: u32 = 0, + max_concurrency: u32, allocator: std.mem.Allocator, diff --git a/src/cli.zig b/src/cli.zig index e5fbda4598..0fada12099 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -348,6 +348,7 @@ pub const Command = struct { coverage: TestCommand.CodeCoverageOptions = .{}, test_filter_pattern: ?[]const u8 = null, test_filter_regex: ?*RegularExpression = null, + max_concurrency: u32 = 20, file_reporter: ?TestCommand.FileReporter = null, reporter_outfile: ?[]const u8 = null, diff --git a/src/cli/Arguments.zig b/src/cli/Arguments.zig index 324c8165e2..0a3ac55964 100644 --- a/src/cli/Arguments.zig +++ b/src/cli/Arguments.zig @@ -206,6 +206,7 @@ pub const test_only_params = [_]ParamType{ clap.parseParam("-t, --test-name-pattern Run only tests with a name that matches the given regex.") catch unreachable, clap.parseParam("--reporter Test output reporter format. Available: 'junit' (requires --reporter-outfile). Default: console output.") catch unreachable, clap.parseParam("--reporter-outfile Output file path for the reporter format (required with --reporter).") catch unreachable, + clap.parseParam("--max-concurrency Maximum number of concurrent tests to execute at once. Default is 20.") catch unreachable, }; pub const test_params = test_only_params ++ runtime_params_ ++ transpiler_params_ ++ base_params_; @@ -417,6 +418,15 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C } } + if (args.option("--max-concurrency")) |max_concurrency| { + if (max_concurrency.len > 0) { + ctx.test_options.max_concurrency = std.fmt.parseInt(u32, max_concurrency, 10) catch { + Output.prettyErrorln("error: Invalid max-concurrency: \"{s}\"", .{max_concurrency}); + Global.exit(1); + }; + } + } + if (!ctx.test_options.coverage.enabled) { ctx.test_options.coverage.enabled = args.flag("--coverage"); } diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig index e155123ab8..318cb518c7 100644 --- a/src/cli/test_command.zig +++ b/src/cli/test_command.zig @@ -1311,6 +1311,7 @@ pub const TestCommand = struct { .run_todo = ctx.test_options.run_todo, .only = ctx.test_options.only, .bail = ctx.test_options.bail, + .max_concurrency = ctx.test_options.max_concurrency, .filter_regex = ctx.test_options.test_filter_regex, .snapshots = Snapshots{ .allocator = ctx.allocator, diff --git a/test/js/bun/test/concurrent-max.fixture.ts b/test/js/bun/test/concurrent-max.fixture.ts new file mode 100644 index 0000000000..1838e9cdd7 --- /dev/null +++ b/test/js/bun/test/concurrent-max.fixture.ts @@ -0,0 +1,49 @@ +import { test, expect, afterAll, beforeEach, afterEach } from "bun:test"; + +// Track concurrent executions +let currentlyExecuting = 0; +const executionLog: number[] = []; + +beforeEach(() => { + currentlyExecuting++; + executionLog.push(currentlyExecuting); +}); +afterEach(() => currentlyExecuting--); + +function queue(fn: () => void) { + resolveQueue.push(fn); + if (!timeout) { + const set = () => + setTimeout(() => { + const cb = resolveQueue.shift(); + if (!cb) { + timeout = false; + return; + } + cb(); + set(); + }, 0); + set(); + timeout = true; + } else { + timeout = true; + } +} + +const resolveQueue: (() => void)[] = []; +let timeout: boolean = false; + +test.concurrent.each(Array.from({ length: 100 }, (_, i) => i + 1))(`concurrent test %d`, (i, done) => { + console.log(`start test ${i}`); + // Small delay to ensure tests overlap + queue(() => { + console.log(`end test ${i}`); + done(); + }); +}); + +// afterAll to report the max concurrency observed +afterAll(() => { + // Log execution pattern + console.log("Execution pattern: " + JSON.stringify(executionLog)); +}); diff --git a/test/js/bun/test/concurrent.test.ts b/test/js/bun/test/concurrent.test.ts index 05d03eb532..df1fe530c2 100644 --- a/test/js/bun/test/concurrent.test.ts +++ b/test/js/bun/test/concurrent.test.ts @@ -57,3 +57,70 @@ test("concurrent order", async () => { } `); }); + +test("max-concurrency limits concurrent tests", async () => { + // Test with max-concurrency=3 + const result = await Bun.spawn({ + cmd: [bunExe(), "test", "--max-concurrency", "3", import.meta.dir + "/concurrent-max.fixture.ts"], + stdout: "pipe", + stderr: "pipe", + env: bunEnv, + }); + const exitCode = await result.exited; + const stdout = await result.stdout.text(); + + expect(exitCode).toBe(0); + + // Extract max concurrent value from output + const maxMatch = stdout.match(/Execution pattern: ([^\n]+)/); + expect(maxMatch).toBeTruthy(); + const executionPattern = JSON.parse(maxMatch![1]); + + // Should be 1,2,3,3,3,3,3,... + const expected = Array.from({ length: 100 }, (_, i) => Math.min(i + 1, 3)); + expect(executionPattern).toEqual(expected); +}); + +test("max-concurrency default is 20", async () => { + const result = await Bun.spawn({ + cmd: [bunExe(), "test", import.meta.dir + "/concurrent-max.fixture.ts"], + stdout: "pipe", + stderr: "pipe", + env: bunEnv, + }); + const exitCode = await result.exited; + const stdout = await result.stdout.text(); + + expect(exitCode).toBe(0); + + // Extract max concurrent value from output + const maxMatch = stdout.match(/Execution pattern: ([^\n]+)/); + expect(maxMatch).toBeTruthy(); + const executionPattern = JSON.parse(maxMatch![1]); + + // Should be 1,2,3,...,18,19,20,20,20,20,20,20,... + const expected = Array.from({ length: 100 }, (_, i) => Math.min(i + 1, 20)); + expect(executionPattern).toEqual(expected); +}); + +test("zero removes max-concurrency", async () => { + const result = await Bun.spawn({ + cmd: [bunExe(), "test", "--max-concurrency", "0", import.meta.dir + "/concurrent-max.fixture.ts"], + stdout: "pipe", + stderr: "pipe", + env: bunEnv, + }); + const exitCode = await result.exited; + const stdout = await result.stdout.text(); + + expect(exitCode).toBe(0); + + // Extract max concurrent value from output + const maxMatch = stdout.match(/Execution pattern: ([^\n]+)/); + expect(maxMatch).toBeTruthy(); + const executionPattern = JSON.parse(maxMatch![1]); + + // Should be 1,2,3,...,18,19,20,20,20,20,20,20,... + const expected = Array.from({ length: 100 }, (_, i) => i + 1); + expect(executionPattern).toEqual(expected); +}); diff --git a/test/js/sql/sql-mysql.test.ts b/test/js/sql/sql-mysql.test.ts index e3a6e6a4e4..ca60698759 100644 --- a/test/js/sql/sql-mysql.test.ts +++ b/test/js/sql/sql-mysql.test.ts @@ -287,11 +287,13 @@ if (isDockerEnabled()) { }); test("Create table", async () => { + await using sql = new SQL({ ...getOptions(), max: 1 }); await sql`create table test_my_table(id int)`; await sql`drop table test_my_table`; }); test("Drop table", async () => { + await using sql = new SQL({ ...getOptions(), max: 1 }); await sql`create table drop_table_test(id int)`; await sql`drop table drop_table_test`; // Verify that table is dropped @@ -498,6 +500,7 @@ if (isDockerEnabled()) { }); test("Prepared transaction", async () => { + await using sql = new SQL({ ...getOptions(), max: 1 }); await sql`create table test_prepared_transaction (a int)`; try {