From 4830e2d8174fa36997c0bbb439058cfb69cb6b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?TATSUNO=20=E2=80=9CTaz=E2=80=9D=20Yasuhiro?= Date: Sat, 22 Jun 2024 18:03:19 +0900 Subject: [PATCH] Implement initial LCOV reporter (no function names support) (#11883) Co-authored-by: Jarred Sumner Co-authored-by: dave caruso --- docs/test/coverage.md | 18 + src/bunfig.zig | 28 ++ src/cli.zig | 20 + src/cli/test_command.zig | 279 ++++++++++---- src/sourcemap/CodeCoverage.zig | 345 +++++++++++------- .../test/__snapshots__/coverage.test.ts.snap | 27 ++ test/cli/test/coverage.test.ts | 36 ++ 7 files changed, 554 insertions(+), 199 deletions(-) create mode 100644 test/cli/test/__snapshots__/coverage.test.ts.snap diff --git a/docs/test/coverage.md b/docs/test/coverage.md index 2d7d92b5f9..bd24f391e6 100644 --- a/docs/test/coverage.md +++ b/docs/test/coverage.md @@ -63,3 +63,21 @@ Internally, Bun transpiles all files by default, so Bun automatically generates [test] coverageIgnoreSourcemaps = true # default false ``` + +### Coverage reporters + +By default, coverage reports will be printed to the console. + +You can specify the reporters and the directory where the reports will be saved. +This is needed, especially when you integrate coverages with tools like CodeCov, CodeClimate, Coveralls and so on. + +```toml +coverageReporters = ["console", "lcov"] # default ["console"] +coverageDir = "path/to/somewhere" # default "coverage" +``` + +| Reporter | Description | +|-----------|-------------| +| `console` | Prints a text summary of the coverage to the console. | +| `lcov` | Save coverage in [lcov](https://github.com/linux-test-project/lcov) format. | + diff --git a/src/bunfig.zig b/src/bunfig.zig index 6b4167fcfb..66b12e1d6b 100644 --- a/src/bunfig.zig +++ b/src/bunfig.zig @@ -21,6 +21,7 @@ const Npm = @import("./install/npm.zig"); const PackageManager = @import("./install/install.zig").PackageManager; const PackageJSON = @import("./resolver/package_json.zig").PackageJSON; const resolver = @import("./resolver/resolver.zig"); +const TestCommand = @import("./cli/test_command.zig").TestCommand; pub const MacroImportReplacementMap = bun.StringArrayHashMap(string); pub const MacroMap = bun.StringArrayHashMapUnmanaged(MacroImportReplacementMap); pub const BundlePackageOverride = bun.StringArrayHashMapUnmanaged(options.BundleOverride); @@ -55,6 +56,11 @@ pub const Bunfig = struct { return error.@"Invalid Bunfig"; } + fn addErrorFormat(this: *Parser, loc: logger.Loc, allocator: std.mem.Allocator, comptime text: string, args: anytype) !void { + this.log.addErrorFmt(this.source, loc, allocator, text, args) catch unreachable; + return error.@"Invalid Bunfig"; + } + fn parseRegistryURLString(this: *Parser, str: *js_ast.E.String) !Api.NpmRegistry { const url = URL.parse(str.data); var registry = std.mem.zeroes(Api.NpmRegistry); @@ -252,6 +258,28 @@ pub const Bunfig = struct { this.ctx.test_options.coverage.enabled = expr.data.e_boolean.value; } + if (test_.get("coverageReporters")) |expr| { + this.ctx.test_options.coverage.reporters = .{ .console = false, .lcov = false }; + try this.expect(expr, .e_array); + const items = expr.data.e_array.items.slice(); + for (items) |item| { + try this.expectString(item); + const item_str = item.asString(bun.default_allocator) orelse ""; + if (bun.strings.eqlComptime(item_str, "console")) { + this.ctx.test_options.coverage.reporters.console = true; + } else if (bun.strings.eqlComptime(item_str, "lcov")) { + this.ctx.test_options.coverage.reporters.lcov = true; + } else { + try this.addErrorFormat(item.loc, allocator, "Invalid coverage reporter \"{s}\"", .{item_str}); + } + } + } + + if (test_.get("coverageDir")) |expr| { + try this.expectString(expr); + this.ctx.test_options.coverage.reports_directory = try expr.data.e_string.string(allocator); + } + if (test_.get("coverageThreshold")) |expr| outer: { if (expr.data == .e_number) { this.ctx.test_options.coverage.fractions.functions = expr.data.e_number.value; diff --git a/src/cli.zig b/src/cli.zig index 94b1089422..09a4b617b4 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -249,6 +249,8 @@ pub const Arguments = struct { clap.parseParam("--only Only run tests that are marked with \"test.only()\"") catch unreachable, clap.parseParam("--todo Include tests that are marked with \"test.todo()\"") catch unreachable, clap.parseParam("--coverage Generate a coverage profile") catch unreachable, + clap.parseParam("--coverage-reporter ... Report coverage in 'console' and/or 'lcov'. Defaults to 'console'.") catch unreachable, + clap.parseParam("--coverage-dir Directory for coverage files. Defaults to 'coverage'.") catch unreachable, clap.parseParam("--bail ? Exit the test suite after failures. If you do not specify a number, it defaults to 1.") catch unreachable, clap.parseParam("-t, --test-name-pattern Run only tests with a name that matches the given regex.") catch unreachable, }; @@ -441,6 +443,24 @@ pub const Arguments = struct { ctx.test_options.coverage.enabled = args.flag("--coverage"); } + if (args.options("--coverage-reporter").len > 0) { + ctx.test_options.coverage.reporters = .{ .console = false, .lcov = false }; + for (args.options("--coverage-reporter")) |reporter| { + if (bun.strings.eqlComptime(reporter, "console")) { + ctx.test_options.coverage.reporters.console = true; + } else if (bun.strings.eqlComptime(reporter, "lcov")) { + ctx.test_options.coverage.reporters.lcov = true; + } else { + Output.prettyErrorln("error: --coverage-reporter received invalid reporter: \"{s}\"", .{reporter}); + Global.exit(1); + } + } + } + + if (args.option("--coverage-dir")) |dir| { + ctx.test_options.coverage.reports_directory = dir; + } + if (args.option("--bail")) |bail| { if (bail.len > 0) { ctx.test_options.bail = std.fmt.parseInt(u32, bail, 10) catch |e| { diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig index e02df273f0..89fdccda7c 100644 --- a/src/cli/test_command.zig +++ b/src/cli/test_command.zig @@ -42,7 +42,7 @@ const jest = JSC.Jest; const TestRunner = JSC.Jest.TestRunner; const Snapshots = JSC.Snapshot.Snapshots; const Test = TestRunner.Test; - +const CodeCoverageReport = bun.sourcemap.CodeCoverageReport; const uws = bun.uws; fn fmtStatusTextLine(comptime status: @Type(.EnumLiteral), comptime emoji_or_color: bool) []const u8 { @@ -271,23 +271,17 @@ pub const CommandLineReporter = struct { Output.printStartEnd(bun.start_time, std.time.nanoTimestamp()); } - pub fn printCodeCoverage(this: *CommandLineReporter, vm: *JSC.VirtualMachine, opts: *TestCommand.CodeCoverageOptions, comptime enable_ansi_colors: bool) !void { - const trace = bun.tracy.traceNamed(@src(), "TestCommand.printCodeCoverage"); - defer trace.end(); + pub fn generateCodeCoverage(this: *CommandLineReporter, vm: *JSC.VirtualMachine, opts: *TestCommand.CodeCoverageOptions, comptime reporters: TestCommand.Reporters, comptime enable_ansi_colors: bool) !void { + if (comptime !reporters.console and !reporters.lcov) { + return; + } - _ = this; var map = bun.sourcemap.ByteRangeMapping.map orelse return; var iter = map.valueIterator(); - var max_filepath_length: usize = "All files".len; - const relative_dir = vm.bundler.fs.top_level_dir; - var byte_ranges = try std.ArrayList(bun.sourcemap.ByteRangeMapping).initCapacity(bun.default_allocator, map.count()); while (iter.next()) |entry| { - const value: bun.sourcemap.ByteRangeMapping = entry.*; - const utf8 = value.source_url.slice(); - byte_ranges.appendAssumeCapacity(value); - max_filepath_length = @max(bun.path.relative(relative_dir, utf8).len, max_filepath_length); + byte_ranges.appendAssumeCapacity(entry.*); } if (byte_ranges.items.len == 0) { @@ -296,25 +290,65 @@ pub const CommandLineReporter = struct { std.sort.pdq(bun.sourcemap.ByteRangeMapping, byte_ranges.items, void{}, bun.sourcemap.ByteRangeMapping.isLessThan); - iter = map.valueIterator(); - var writer = Output.errorWriter(); + try this.printCodeCoverage(vm, opts, byte_ranges.items, reporters, enable_ansi_colors); + } + + pub fn printCodeCoverage(this: *CommandLineReporter, vm: *JSC.VirtualMachine, opts: *TestCommand.CodeCoverageOptions, byte_ranges: []bun.sourcemap.ByteRangeMapping, comptime reporters: TestCommand.Reporters, comptime enable_ansi_colors: bool) !void { + _ = this; // autofix + const trace = bun.tracy.traceNamed(@src(), comptime brk: { + if (reporters.console and reporters.lcov) { + break :brk "TestCommand.printCodeCoverageLCovAndConsole"; + } + + if (reporters.console) { + break :brk "TestCommand.printCodeCoverageConsole"; + } + + if (reporters.lcov) { + break :brk "TestCommand.printCodeCoverageLCov"; + } + + @compileError("No reporters enabled"); + }); + defer trace.end(); + + if (comptime !reporters.console and !reporters.lcov) { + @compileError("No reporters enabled"); + } + + const relative_dir = vm.bundler.fs.top_level_dir; + + // --- Console --- + const max_filepath_length: usize = if (reporters.console) brk: { + var len = "All files".len; + for (byte_ranges) |*entry| { + const utf8 = entry.source_url.slice(); + len = @max(bun.path.relative(relative_dir, utf8).len, len); + } + + break :brk len; + } else 0; + + var console = Output.errorWriter(); const base_fraction = opts.fractions; var failing = false; - writer.writeAll(Output.prettyFmt("", enable_ansi_colors)) catch return; - writer.writeByteNTimes('-', max_filepath_length + 2) catch return; - writer.writeAll(Output.prettyFmt("|---------|---------|-------------------\n", enable_ansi_colors)) catch return; - writer.writeAll("File") catch return; - writer.writeByteNTimes(' ', max_filepath_length - "File".len + 1) catch return; - // writer.writeAll(Output.prettyFmt(" | % Funcs | % Blocks | % Lines | Uncovered Line #s\n", enable_ansi_colors)) catch return; - writer.writeAll(Output.prettyFmt(" | % Funcs | % Lines | Uncovered Line #s\n", enable_ansi_colors)) catch return; - writer.writeAll(Output.prettyFmt("", enable_ansi_colors)) catch return; - writer.writeByteNTimes('-', max_filepath_length + 2) catch return; - writer.writeAll(Output.prettyFmt("|---------|---------|-------------------\n", enable_ansi_colors)) catch return; + if (comptime reporters.console) { + console.writeAll(Output.prettyFmt("", enable_ansi_colors)) catch return; + console.writeByteNTimes('-', max_filepath_length + 2) catch return; + console.writeAll(Output.prettyFmt("|---------|---------|-------------------\n", enable_ansi_colors)) catch return; + console.writeAll("File") catch return; + console.writeByteNTimes(' ', max_filepath_length - "File".len + 1) catch return; + // writer.writeAll(Output.prettyFmt(" | % Funcs | % Blocks | % Lines | Uncovered Line #s\n", enable_ansi_colors)) catch return; + console.writeAll(Output.prettyFmt(" | % Funcs | % Lines | Uncovered Line #s\n", enable_ansi_colors)) catch return; + console.writeAll(Output.prettyFmt("", enable_ansi_colors)) catch return; + console.writeByteNTimes('-', max_filepath_length + 2) catch return; + console.writeAll(Output.prettyFmt("|---------|---------|-------------------\n", enable_ansi_colors)) catch return; + } - var coverage_buffer = bun.MutableString.initEmpty(bun.default_allocator); - var coverage_buffer_buffer = coverage_buffer.bufferedWriter(); - var coverage_writer = coverage_buffer_buffer.writer(); + var console_buffer = bun.MutableString.initEmpty(bun.default_allocator); + var console_buffer_buffer = console_buffer.bufferedWriter(); + var console_writer = console_buffer_buffer.writer(); var avg = bun.sourcemap.CoverageFraction{ .functions = 0.0, @@ -322,50 +356,149 @@ pub const CommandLineReporter = struct { .stmts = 0.0, }; var avg_count: f64 = 0; + // --- Console --- - for (byte_ranges.items) |*entry| { - var report = bun.sourcemap.CodeCoverageReport.generate(vm.global, bun.default_allocator, entry, opts.ignore_sourcemap) orelse continue; - defer report.deinit(bun.default_allocator); - var fraction = base_fraction; - report.writeFormat(max_filepath_length, &fraction, relative_dir, coverage_writer, enable_ansi_colors) catch continue; - avg.functions += fraction.functions; - avg.lines += fraction.lines; - avg.stmts += fraction.stmts; - avg_count += 1.0; - if (fraction.failing) { - failing = true; - } + // --- LCOV --- + var lcov_name_buf: bun.PathBuffer = undefined; + const lcov_file, const lcov_name, const lcov_buffered_writer, const lcov_writer = brk: { + if (comptime !reporters.lcov) break :brk .{ {}, {}, {}, {} }; - coverage_writer.writeAll("\n") catch continue; - } - - { - avg.functions /= avg_count; - avg.lines /= avg_count; - avg.stmts /= avg_count; - - try bun.sourcemap.CodeCoverageReport.writeFormatWithValues( - "All files", - max_filepath_length, - avg, - base_fraction, - failing, - writer, - false, - enable_ansi_colors, + // Ensure the directory exists + var fs = bun.JSC.Node.NodeFS{}; + _ = fs.mkdirRecursive( + .{ + .path = bun.JSC.Node.PathLike{ + .encoded_slice = JSC.ZigString.Slice.fromUTF8NeverFree(opts.reports_directory), + }, + .always_return_none = true, + }, + .sync, ); - try writer.writeAll(Output.prettyFmt(" |\n", enable_ansi_colors)); + // Write the lcov.info file to a temporary file we atomically rename to the final name after it succeeds + var base64_bytes: [8]u8 = undefined; + var shortname_buf: [512]u8 = undefined; + bun.rand(&base64_bytes); + const tmpname = std.fmt.bufPrintZ(&shortname_buf, ".lcov.info.{s}.tmp", .{bun.fmt.fmtSliceHexLower(&base64_bytes)}) catch unreachable; + const path = bun.path.joinAbsStringBufZ(relative_dir, &lcov_name_buf, &.{ opts.reports_directory, tmpname }, .auto); + const file = bun.sys.File.openat( + std.fs.cwd(), + path, + bun.O.CREAT | bun.O.WRONLY | bun.O.TRUNC | bun.O.CLOEXEC, + 0o644, + ); + + switch (file) { + .err => |err| { + Output.err(.lcovCoverageError, "Failed to create lcov file", .{}); + Output.printError("\n{s}", .{err}); + Global.exit(1); + }, + .result => |f| { + const buffered = buffered_writer: { + const writer = f.writer(); + // Heap-allocate the buffered writer because we want a stable memory address + 64 KB is kind of a lot. + const ptr = try bun.default_allocator.create(std.io.BufferedWriter(64 * 1024, bun.sys.File.Writer)); + ptr.* = .{ + .end = 0, + .unbuffered_writer = writer, + }; + break :buffered_writer ptr; + }; + + break :brk .{ + f, + path, + buffered, + buffered.writer(), + }; + }, + } + }; + errdefer { + if (comptime reporters.lcov) { + lcov_file.close(); + _ = bun.sys.unlink( + lcov_name, + ); + } + } + // --- LCOV --- + + for (byte_ranges) |*entry| { + var report = CodeCoverageReport.generate(vm.global, bun.default_allocator, entry, opts.ignore_sourcemap) orelse continue; + defer report.deinit(bun.default_allocator); + + if (comptime reporters.console) { + var fraction = base_fraction; + CodeCoverageReport.Console.writeFormat(&report, max_filepath_length, &fraction, relative_dir, console_writer, enable_ansi_colors) catch continue; + avg.functions += fraction.functions; + avg.lines += fraction.lines; + avg.stmts += fraction.stmts; + avg_count += 1.0; + if (fraction.failing) { + failing = true; + } + + console_writer.writeAll("\n") catch continue; + } + + if (comptime reporters.lcov) { + CodeCoverageReport.Lcov.writeFormat( + &report, + relative_dir, + lcov_writer, + ) catch continue; + } } - coverage_buffer_buffer.flush() catch return; - try writer.writeAll(coverage_buffer.list.items); - try writer.writeAll(Output.prettyFmt("", enable_ansi_colors)); - writer.writeByteNTimes('-', max_filepath_length + 2) catch return; - writer.writeAll(Output.prettyFmt("|---------|---------|-------------------\n", enable_ansi_colors)) catch return; + if (comptime reporters.console) { + { + avg.functions /= avg_count; + avg.lines /= avg_count; + avg.stmts /= avg_count; - opts.fractions.failing = failing; - Output.flush(); + try CodeCoverageReport.Console.writeFormatWithValues( + "All files", + max_filepath_length, + avg, + base_fraction, + failing, + console, + false, + enable_ansi_colors, + ); + + try console.writeAll(Output.prettyFmt(" |\n", enable_ansi_colors)); + } + + console_buffer_buffer.flush() catch return; + try console.writeAll(console_buffer.list.items); + try console.writeAll(Output.prettyFmt("", enable_ansi_colors)); + console.writeByteNTimes('-', max_filepath_length + 2) catch return; + console.writeAll(Output.prettyFmt("|---------|---------|-------------------\n", enable_ansi_colors)) catch return; + + opts.fractions.failing = failing; + Output.flush(); + } + + if (comptime reporters.lcov) { + try lcov_buffered_writer.flush(); + lcov_file.close(); + bun.C.moveFileZ( + bun.toFD(std.fs.cwd()), + lcov_name, + bun.toFD(std.fs.cwd()), + bun.path.joinAbsStringZ( + relative_dir, + &.{ opts.reports_directory, "lcov.info" }, + .auto, + ), + ) catch |err| { + Output.err(err, "Failed to save lcov.info file", .{}); + Global.exit(1); + }; + } } }; @@ -572,11 +705,21 @@ pub const TestCommand = struct { pub const name = "test"; pub const CodeCoverageOptions = struct { skip_test_files: bool = !Environment.allow_assert, + reporters: Reporters = .{ .console = true, .lcov = false }, + reports_directory: string = "coverage", fractions: bun.sourcemap.CoverageFraction = .{}, ignore_sourcemap: bool = false, enabled: bool = false, fail_on_low_coverage: bool = false, }; + pub const Reporter = enum { + console, + lcov, + }; + const Reporters = struct { + console: bool, + lcov: bool, + }; pub fn exec(ctx: Command.Context) !void { if (comptime is_bindgen) unreachable; @@ -859,7 +1002,13 @@ pub const TestCommand = struct { if (coverage.enabled) { switch (Output.enable_ansi_colors_stderr) { - inline else => |colors| reporter.printCodeCoverage(vm, &coverage, colors) catch {}, + inline else => |colors| switch (coverage.reporters.console) { + inline else => |console| switch (coverage.reporters.lcov) { + inline else => |lcov| { + try reporter.generateCodeCoverage(vm, &coverage, .{ .console = console, .lcov = lcov }, colors); + }, + }, + }, } } diff --git a/src/sourcemap/CodeCoverage.zig b/src/sourcemap/CodeCoverage.zig index 78197c16b1..be2deaafed 100644 --- a/src/sourcemap/CodeCoverage.zig +++ b/src/sourcemap/CodeCoverage.zig @@ -3,6 +3,7 @@ const std = @import("std"); const LineOffsetTable = bun.sourcemap.LineOffsetTable; const SourceMap = bun.sourcemap; const Bitset = bun.bit_set.DynamicBitSetUnmanaged; +const LinesHits = @import("../baby_list.zig").BabyList(u32); const Output = bun.Output; const prettyFmt = Output.prettyFmt; @@ -24,17 +25,13 @@ pub const CodeCoverageReport = struct { source_url: bun.JSC.ZigString.Slice, executable_lines: Bitset, lines_which_have_executed: Bitset, + line_hits: LinesHits = .{}, functions: std.ArrayListUnmanaged(Block), functions_which_have_executed: Bitset, stmts_which_have_executed: Bitset, stmts: std.ArrayListUnmanaged(Block), total_lines: u32 = 0, - pub const Block = struct { - start_line: u32 = 0, - end_line: u32 = 0, - }; - pub fn linesCoverageFraction(this: *const CodeCoverageReport) f64 { var intersected = this.executable_lines.clone(bun.default_allocator) catch bun.outOfMemory(); defer intersected.deinit(bun.default_allocator); @@ -68,156 +65,213 @@ pub const CodeCoverageReport = struct { return (@as(f64, @floatFromInt(this.functions_which_have_executed.count())) / total_count); } - pub fn writeFormatWithValues( - filename: []const u8, - max_filename_length: usize, - vals: CoverageFraction, - failing: CoverageFraction, - failed: bool, - writer: anytype, - indent_name: bool, - comptime enable_colors: bool, - ) !void { - if (comptime enable_colors) { - if (failed) { - try writer.writeAll(comptime prettyFmt("", true)); - } else { - try writer.writeAll(comptime prettyFmt("", true)); + pub const Console = struct { + pub fn writeFormatWithValues( + filename: []const u8, + max_filename_length: usize, + vals: CoverageFraction, + failing: CoverageFraction, + failed: bool, + writer: anytype, + indent_name: bool, + comptime enable_colors: bool, + ) !void { + if (comptime enable_colors) { + if (failed) { + try writer.writeAll(comptime prettyFmt("", true)); + } else { + try writer.writeAll(comptime prettyFmt("", true)); + } } - } - if (indent_name) { - try writer.writeAll(" "); - } - - try writer.writeAll(filename); - try writer.writeByteNTimes(' ', (max_filename_length - filename.len + @as(usize, @intFromBool(!indent_name)))); - try writer.writeAll(comptime prettyFmt(" | ", enable_colors)); - - if (comptime enable_colors) { - if (vals.functions < failing.functions) { - try writer.writeAll(comptime prettyFmt("", true)); - } else { - try writer.writeAll(comptime prettyFmt("", true)); + if (indent_name) { + try writer.writeAll(" "); } - } - try writer.print("{d: >7.2}", .{vals.functions * 100.0}); - // try writer.writeAll(comptime prettyFmt(" | ", enable_colors)); - // if (comptime enable_colors) { - // // if (vals.stmts < failing.stmts) { - // try writer.writeAll(comptime prettyFmt("", true)); - // // } else { - // // try writer.writeAll(comptime prettyFmt("", true)); - // // } - // } - // try writer.print("{d: >8.2}", .{vals.stmts * 100.0}); - try writer.writeAll(comptime prettyFmt(" | ", enable_colors)); + try writer.writeAll(filename); + try writer.writeByteNTimes(' ', (max_filename_length - filename.len + @as(usize, @intFromBool(!indent_name)))); + try writer.writeAll(comptime prettyFmt(" | ", enable_colors)); - if (comptime enable_colors) { - if (vals.lines < failing.lines) { - try writer.writeAll(comptime prettyFmt("", true)); - } else { - try writer.writeAll(comptime prettyFmt("", true)); + if (comptime enable_colors) { + if (vals.functions < failing.functions) { + try writer.writeAll(comptime prettyFmt("", true)); + } else { + try writer.writeAll(comptime prettyFmt("", true)); + } } + + try writer.print("{d: >7.2}", .{vals.functions * 100.0}); + // try writer.writeAll(comptime prettyFmt(" | ", enable_colors)); + // if (comptime enable_colors) { + // // if (vals.stmts < failing.stmts) { + // try writer.writeAll(comptime prettyFmt("", true)); + // // } else { + // // try writer.writeAll(comptime prettyFmt("", true)); + // // } + // } + // try writer.print("{d: >8.2}", .{vals.stmts * 100.0}); + try writer.writeAll(comptime prettyFmt(" | ", enable_colors)); + + if (comptime enable_colors) { + if (vals.lines < failing.lines) { + try writer.writeAll(comptime prettyFmt("", true)); + } else { + try writer.writeAll(comptime prettyFmt("", true)); + } + } + + try writer.print("{d: >7.2}", .{vals.lines * 100.0}); } - try writer.print("{d: >7.2}", .{vals.lines * 100.0}); - } + pub fn writeFormat( + report: *const CodeCoverageReport, + max_filename_length: usize, + fraction: *CoverageFraction, + base_path: []const u8, + writer: anytype, + comptime enable_colors: bool, + ) !void { + const failing = fraction.*; + const fns = report.functionCoverageFraction(); + const lines = report.linesCoverageFraction(); + const stmts = report.stmtsCoverageFraction(); + fraction.functions = fns; + fraction.lines = lines; + fraction.stmts = stmts; - pub fn writeFormat( - report: *const CodeCoverageReport, - max_filename_length: usize, - fraction: *CoverageFraction, - base_path: []const u8, - writer: anytype, - comptime enable_colors: bool, - ) !void { - const failing = fraction.*; - const fns = report.functionCoverageFraction(); - const lines = report.linesCoverageFraction(); - const stmts = report.stmtsCoverageFraction(); - fraction.functions = fns; - fraction.lines = lines; - fraction.stmts = stmts; + const failed = fns < failing.functions or lines < failing.lines; // or stmts < failing.stmts; + fraction.failing = failed; - const failed = fns < failing.functions or lines < failing.lines; // or stmts < failing.stmts; - fraction.failing = failed; + var filename = report.source_url.slice(); + if (base_path.len > 0) { + filename = bun.path.relative(base_path, filename); + } - var filename = report.source_url.slice(); - if (base_path.len > 0) { - filename = bun.path.relative(base_path, filename); - } + try writeFormatWithValues( + filename, + max_filename_length, + fraction.*, + failing, + failed, + writer, + true, + enable_colors, + ); - try writeFormatWithValues( - filename, - max_filename_length, - fraction.*, - failing, - failed, - writer, - true, - enable_colors, - ); + try writer.writeAll(comptime prettyFmt(" | ", enable_colors)); - try writer.writeAll(comptime prettyFmt(" | ", enable_colors)); + var executable_lines_that_havent_been_executed = report.lines_which_have_executed.clone(bun.default_allocator) catch bun.outOfMemory(); + defer executable_lines_that_havent_been_executed.deinit(bun.default_allocator); + executable_lines_that_havent_been_executed.toggleAll(); - var executable_lines_that_havent_been_executed = report.lines_which_have_executed.clone(bun.default_allocator) catch bun.outOfMemory(); - defer executable_lines_that_havent_been_executed.deinit(bun.default_allocator); - executable_lines_that_havent_been_executed.toggleAll(); + // This sets statements in executed scopes + executable_lines_that_havent_been_executed.setIntersection(report.executable_lines); - // This sets statements in executed scopes - executable_lines_that_havent_been_executed.setIntersection(report.executable_lines); + var iter = executable_lines_that_havent_been_executed.iterator(.{}); + var start_of_line_range: usize = 0; + var prev_line: usize = 0; + var is_first = true; - var iter = executable_lines_that_havent_been_executed.iterator(.{}); - var start_of_line_range: usize = 0; - var prev_line: usize = 0; - var is_first = true; + while (iter.next()) |next_line| { + if (next_line == (prev_line + 1)) { + prev_line = next_line; + continue; + } else if (is_first and start_of_line_range == 0 and prev_line == 0) { + start_of_line_range = next_line; + prev_line = next_line; + continue; + } + + if (is_first) { + is_first = false; + } else { + try writer.print(comptime prettyFmt(",", enable_colors), .{}); + } + + if (start_of_line_range == prev_line) { + try writer.print(comptime prettyFmt("{d}", enable_colors), .{start_of_line_range + 1}); + } else { + try writer.print(comptime prettyFmt("{d}-{d}", enable_colors), .{ start_of_line_range + 1, prev_line + 1 }); + } - while (iter.next()) |next_line| { - if (next_line == (prev_line + 1)) { prev_line = next_line; - continue; - } else if (is_first and start_of_line_range == 0 and prev_line == 0) { start_of_line_range = next_line; - prev_line = next_line; - continue; } - if (is_first) { - is_first = false; - } else { - try writer.print(comptime prettyFmt(",", enable_colors), .{}); - } + if (prev_line != start_of_line_range) { + if (is_first) { + is_first = false; + } else { + try writer.print(comptime prettyFmt(",", enable_colors), .{}); + } - if (start_of_line_range == prev_line) { - try writer.print(comptime prettyFmt("{d}", enable_colors), .{start_of_line_range + 1}); - } else { - try writer.print(comptime prettyFmt("{d}-{d}", enable_colors), .{ start_of_line_range + 1, prev_line + 1 }); - } - - prev_line = next_line; - start_of_line_range = next_line; - } - - if (prev_line != start_of_line_range) { - if (is_first) { - is_first = false; - } else { - try writer.print(comptime prettyFmt(",", enable_colors), .{}); - } - - if (start_of_line_range == prev_line) { - try writer.print(comptime prettyFmt("{d}", enable_colors), .{start_of_line_range + 1}); - } else { - try writer.print(comptime prettyFmt("{d}-{d}", enable_colors), .{ start_of_line_range + 1, prev_line + 1 }); + if (start_of_line_range == prev_line) { + try writer.print(comptime prettyFmt("{d}", enable_colors), .{start_of_line_range + 1}); + } else { + try writer.print(comptime prettyFmt("{d}-{d}", enable_colors), .{ start_of_line_range + 1, prev_line + 1 }); + } } } - } + }; + + pub const Lcov = struct { + pub fn writeFormat( + report: *const CodeCoverageReport, + base_path: []const u8, + writer: anytype, + ) !void { + var filename = report.source_url.slice(); + if (base_path.len > 0) { + filename = bun.path.relative(base_path, filename); + } + + // TN: test name + // Empty value appears fine. For example, `TN:`. + try writer.writeAll("TN:\n"); + + // SF: Source File path + // For example, `SF:path/to/source.ts` + try writer.print("SF:{s}\n", .{filename}); + + // ** Per-function coverage not supported yet, since JSC does not support function names yet. ** + // FN: line number,function name + + // FNF: functions found + try writer.print("FNF:{d}\n", .{report.functions.items.len}); + + // FNH: functions hit + try writer.print("FNH:{d}\n", .{report.functions_which_have_executed.count()}); + + var executable_lines_that_have_been_executed = report.lines_which_have_executed.clone(bun.default_allocator) catch bun.outOfMemory(); + defer executable_lines_that_have_been_executed.deinit(bun.default_allocator); + // This sets statements in executed scopes + executable_lines_that_have_been_executed.setIntersection(report.executable_lines); + var iter = executable_lines_that_have_been_executed.iterator(.{}); + + // ** Branch coverage not supported yet, since JSC does not support those yet. ** // + // BRDA: line, block, (expressions,count)+ + // BRF: branches found + // BRH: branches hit + const line_hits = report.line_hits.slice(); + while (iter.next()) |line| { + // DA: line number, hit count + try writer.print("DA:{d},{d}\n", .{ line + 1, line_hits[line] }); + } + + // LF: lines found + try writer.print("LF:{d}\n", .{report.total_lines}); + + // LH: lines hit + try writer.print("LH:{d}\n", .{executable_lines_that_have_been_executed.count()}); + + try writer.writeAll("end_of_record\n"); + } + }; pub fn deinit(this: *CodeCoverageReport, allocator: std.mem.Allocator) void { this.executable_lines.deinit(allocator); this.lines_which_have_executed.deinit(allocator); + this.line_hits.deinitWithAllocator(allocator); this.functions.deinit(allocator); this.stmts.deinit(allocator); this.functions_which_have_executed.deinit(allocator); @@ -260,7 +314,7 @@ pub const CodeCoverageReport = struct { return; } - this.result.* = this.byte_range_mapping.generateCodeCoverageReportFromBlocks( + this.result.* = this.byte_range_mapping.generateReportFromBlocks( this.allocator, this.byte_range_mapping.source_url, blocks, @@ -357,7 +411,7 @@ pub const ByteRangeMapping = struct { return entry; } - pub fn generateCodeCoverageReportFromBlocks( + pub fn generateReportFromBlocks( this: *ByteRangeMapping, allocator: std.mem.Allocator, source_url: bun.JSC.ZigString.Slice, @@ -370,8 +424,9 @@ pub const ByteRangeMapping = struct { var executable_lines: Bitset = Bitset{}; var lines_which_have_executed: Bitset = Bitset{}; const parsed_mappings_ = bun.JSC.VirtualMachine.get().source_mappings.get(source_url.slice()); + var line_hits = LinesHits{}; - var functions = std.ArrayListUnmanaged(CodeCoverageReport.Block){}; + var functions = std.ArrayListUnmanaged(Block){}; try functions.ensureTotalCapacityPrecise(allocator, function_blocks.len); errdefer functions.deinit(allocator); var functions_which_have_executed: Bitset = try Bitset.initEmpty(allocator, function_blocks.len); @@ -379,7 +434,7 @@ pub const ByteRangeMapping = struct { var stmts_which_have_executed: Bitset = try Bitset.initEmpty(allocator, blocks.len); errdefer stmts_which_have_executed.deinit(allocator); - var stmts = std.ArrayListUnmanaged(CodeCoverageReport.Block){}; + var stmts = std.ArrayListUnmanaged(Block){}; try stmts.ensureTotalCapacityPrecise(allocator, function_blocks.len); errdefer stmts.deinit(allocator); @@ -391,6 +446,13 @@ pub const ByteRangeMapping = struct { line_count = @truncate(line_starts.len); executable_lines = try Bitset.initEmpty(allocator, line_count); lines_which_have_executed = try Bitset.initEmpty(allocator, line_count); + line_hits = try LinesHits.initCapacity(allocator, line_count); + line_hits.len = line_count; + const line_hits_slice = line_hits.slice(); + @memset(line_hits_slice, 0); + + errdefer line_hits.deinitWithAllocator(allocator); + for (blocks, 0..) |block, i| { if (block.endOffset < 0 or block.startOffset < 0) continue; // does not map to anything @@ -415,6 +477,7 @@ pub const ByteRangeMapping = struct { executable_lines.set(line); if (has_executed) { lines_which_have_executed.set(line); + line_hits_slice[line] += 1; } } @@ -455,6 +518,7 @@ pub const ByteRangeMapping = struct { // functions that have executed have non-executable lines in them and thats fine. if (!did_fn_execute) { const end = @min(max_line, line_count); + @memset(line_hits_slice[min_line..end], 0); for (min_line..end) |line| { executable_lines.set(line); lines_which_have_executed.unset(line); @@ -473,6 +537,11 @@ pub const ByteRangeMapping = struct { line_count = @as(u32, @truncate(parsed_mapping.input_line_count)) + 1; executable_lines = try Bitset.initEmpty(allocator, line_count); lines_which_have_executed = try Bitset.initEmpty(allocator, line_count); + line_hits = try LinesHits.initCapacity(allocator, line_count); + line_hits.len = line_count; + const line_hits_slice = line_hits.slice(); + @memset(line_hits_slice, 0); + errdefer line_hits.deinitWithAllocator(allocator); for (blocks, 0..) |block, i| { if (block.endOffset < 0 or block.startOffset < 0) continue; // does not map to anything @@ -499,6 +568,7 @@ pub const ByteRangeMapping = struct { executable_lines.set(line); if (has_executed) { lines_which_have_executed.set(line); + line_hits_slice[line] += 1; } min_line = @min(min_line, line); @@ -557,6 +627,7 @@ pub const ByteRangeMapping = struct { for (min_line..end) |line| { executable_lines.set(line); lines_which_have_executed.unset(line); + line_hits_slice[line] = 0; } } @@ -571,11 +642,12 @@ pub const ByteRangeMapping = struct { unreachable; } - return CodeCoverageReport{ + return .{ .source_url = source_url, .functions = functions, .executable_lines = executable_lines, .lines_which_have_executed = lines_which_have_executed, + .line_hits = line_hits, .total_lines = line_count, .stmts = stmts, .functions_which_have_executed = functions_which_have_executed, @@ -600,7 +672,7 @@ pub const ByteRangeMapping = struct { } var url_slice = source_url.toUTF8(bun.default_allocator); defer url_slice.deinit(); - var report = this.generateCodeCoverageReportFromBlocks(bun.default_allocator, url_slice, blocks, function_blocks, ignore_sourcemap) catch { + var report = this.generateReportFromBlocks(bun.default_allocator, url_slice, blocks, function_blocks, ignore_sourcemap) catch { globalThis.throwOutOfMemory(); return .zero; }; @@ -613,7 +685,7 @@ pub const ByteRangeMapping = struct { var buffered_writer = mutable_str.bufferedWriter(); var writer = buffered_writer.writer(); - report.writeFormat(source_url.utf8ByteLength(), &coverage_fraction, "", &writer, false) catch { + CodeCoverageReport.Console.writeFormat(&report, source_url.utf8ByteLength(), &coverage_fraction, "", &writer, false) catch { globalThis.throwOutOfMemory(); return .zero; }; @@ -655,3 +727,8 @@ pub const CoverageFraction = struct { failing: bool = false, }; + +pub const Block = struct { + start_line: u32 = 0, + end_line: u32 = 0, +}; diff --git a/test/cli/test/__snapshots__/coverage.test.ts.snap b/test/cli/test/__snapshots__/coverage.test.ts.snap new file mode 100644 index 0000000000..df0e950452 --- /dev/null +++ b/test/cli/test/__snapshots__/coverage.test.ts.snap @@ -0,0 +1,27 @@ +// Bun Snapshot v1, https://goo.gl/fbAQLP + +exports[`lcov coverage reporter 1`] = ` +"TN: +SF:demo1.ts +FNF:1 +FNH:0 +DA:2,19 +DA:3,16 +DA:4,1 +LF:5 +LH:3 +end_of_record +TN: +SF:demo2.ts +FNF:2 +FNH:1 +DA:2,26 +DA:4,10 +DA:6,10 +DA:11,1 +DA:14,9 +LF:15 +LH:5 +end_of_record +" +`; diff --git a/test/cli/test/coverage.test.ts b/test/cli/test/coverage.test.ts index af72217d02..d5bb7bdb7a 100644 --- a/test/cli/test/coverage.test.ts +++ b/test/cli/test/coverage.test.ts @@ -1,6 +1,7 @@ import { test, expect } from "bun:test"; import { tempDirWithFiles, bunExe, bunEnv } from "harness"; import path from "path"; +import { readFileSync } from "node:fs"; test("coverage crash", () => { const dir = tempDirWithFiles("cov", { @@ -18,3 +19,38 @@ test("coverage crash", () => { expect(result.exitCode).toBe(0); expect(result.signalCode).toBeUndefined(); }); + +test("lcov coverage reporter", () => { + const dir = tempDirWithFiles("cov", { + "demo2.ts": ` +import { Y } from "./demo1"; + +export function covered() { + // this function IS covered + return Y; +} + +export function uncovered() { + // this function is not covered + return 42; +} + +covered(); +`, + "demo1.ts": ` +export class Y { +#hello; +}; + `, + }); + const result = Bun.spawnSync([bunExe(), "test", "--coverage", "--coverage-reporter", "lcov", "./demo2.ts"], { + cwd: dir, + env: { + ...bunEnv, + }, + stdio: ["inherit", "inherit", "inherit"], + }); + expect(result.exitCode).toBe(0); + expect(result.signalCode).toBeUndefined(); + expect(readFileSync(path.join(dir, "coverage", "lcov.info"), "utf-8")).toMatchSnapshot(); +});