mirror of
https://github.com/oven-sh/bun
synced 2026-02-10 02:48:50 +00:00
ff01dfa03d
We don't want to add an extra pass over the input to convert it to UTF16. And JS engines storing strings as UTF-16 is more expensive anyway, so the runtime win here probably isn't as big (though open to evidence to the contrary!)
471 lines
19 KiB
Zig
471 lines
19 KiB
Zig
const std = @import("std");
|
|
const logger = @import("logger.zig");
|
|
const js_lexer = @import("js_lexer.zig");
|
|
const importRecord = @import("import_record.zig");
|
|
const js_ast = @import("js_ast.zig");
|
|
const options = @import("options.zig");
|
|
const alloc = @import("alloc.zig");
|
|
|
|
const fs = @import("fs.zig");
|
|
usingnamespace @import("global.zig");
|
|
usingnamespace @import("ast/base.zig");
|
|
usingnamespace js_ast.G;
|
|
|
|
const expect = std.testing.expect;
|
|
const ImportKind = importRecord.ImportKind;
|
|
const BindingNodeIndex = js_ast.BindingNodeIndex;
|
|
|
|
const StmtNodeIndex = js_ast.StmtNodeIndex;
|
|
const ExprNodeIndex = js_ast.ExprNodeIndex;
|
|
const ExprNodeList = js_ast.ExprNodeList;
|
|
const StmtNodeList = js_ast.StmtNodeList;
|
|
const BindingNodeList = js_ast.BindingNodeList;
|
|
const assert = std.debug.assert;
|
|
|
|
const LocRef = js_ast.LocRef;
|
|
const S = js_ast.S;
|
|
const B = js_ast.B;
|
|
const G = js_ast.G;
|
|
const T = js_lexer.T;
|
|
const E = js_ast.E;
|
|
const Stmt = js_ast.Stmt;
|
|
const Expr = js_ast.Expr;
|
|
const Binding = js_ast.Binding;
|
|
const Symbol = js_ast.Symbol;
|
|
const Level = js_ast.Op.Level;
|
|
const Op = js_ast.Op;
|
|
const Scope = js_ast.Scope;
|
|
const locModuleScope = logger.Loc.Empty;
|
|
const Lexer = js_lexer.Lexer;
|
|
|
|
fn JSONLikeParser(opts: js_lexer.JSONOptions) type {
|
|
return struct {
|
|
lexer: Lexer,
|
|
source: *const logger.Source,
|
|
log: *logger.Log,
|
|
allocator: *std.mem.Allocator,
|
|
|
|
pub fn init(allocator: *std.mem.Allocator, source: *const logger.Source, log: *logger.Log) !Parser {
|
|
if (opts.allow_comments) {
|
|
return Parser{
|
|
.lexer = try Lexer.initTSConfig(log, source, allocator),
|
|
.allocator = allocator,
|
|
.log = log,
|
|
.source = source,
|
|
};
|
|
} else {
|
|
return Parser{
|
|
.lexer = try Lexer.initJSON(log, source, allocator),
|
|
.allocator = allocator,
|
|
.log = log,
|
|
.source = source,
|
|
};
|
|
}
|
|
}
|
|
|
|
const Parser = @This();
|
|
|
|
pub fn e(p: *Parser, t: anytype, loc: logger.Loc) Expr {
|
|
const Type = @TypeOf(t);
|
|
if (@typeInfo(Type) == .Pointer) {
|
|
return Expr.init(std.meta.Child(Type), t, loc);
|
|
} else {
|
|
return Expr.alloc(p.allocator, Type, t, loc);
|
|
}
|
|
}
|
|
pub fn parseExpr(p: *Parser, comptime maybe_auto_quote: bool) anyerror!Expr {
|
|
const loc = p.lexer.loc();
|
|
|
|
switch (p.lexer.token) {
|
|
.t_false => {
|
|
try p.lexer.next();
|
|
return p.e(E.Boolean{
|
|
.value = false,
|
|
}, loc);
|
|
},
|
|
.t_true => {
|
|
try p.lexer.next();
|
|
return p.e(E.Boolean{
|
|
.value = true,
|
|
}, loc);
|
|
},
|
|
.t_null => {
|
|
try p.lexer.next();
|
|
return p.e(E.Null{}, loc);
|
|
},
|
|
.t_string_literal => {
|
|
var str: E.String = p.lexer.toEString();
|
|
|
|
try p.lexer.next();
|
|
return p.e(str, loc);
|
|
},
|
|
.t_numeric_literal => {
|
|
const value = p.lexer.number;
|
|
try p.lexer.next();
|
|
return p.e(E.Number{ .value = value }, loc);
|
|
},
|
|
.t_minus => {
|
|
try p.lexer.next();
|
|
const value = p.lexer.number;
|
|
try p.lexer.expect(.t_numeric_literal);
|
|
return p.e(E.Number{ .value = -value }, loc);
|
|
},
|
|
.t_open_bracket => {
|
|
try p.lexer.next();
|
|
var is_single_line = !p.lexer.has_newline_before;
|
|
var exprs = std.ArrayList(Expr).init(p.allocator);
|
|
|
|
while (p.lexer.token != .t_close_bracket) {
|
|
if (exprs.items.len > 0) {
|
|
if (p.lexer.has_newline_before) {
|
|
is_single_line = false;
|
|
}
|
|
|
|
if (!try p.parseMaybeTrailingComma(.t_close_bracket)) {
|
|
break;
|
|
}
|
|
|
|
if (p.lexer.has_newline_before) {
|
|
is_single_line = false;
|
|
}
|
|
}
|
|
|
|
exprs.append(try p.parseExpr(false)) catch unreachable;
|
|
}
|
|
|
|
if (p.lexer.has_newline_before) {
|
|
is_single_line = false;
|
|
}
|
|
try p.lexer.expect(.t_close_bracket);
|
|
return p.e(E.Array{ .items = exprs.toOwnedSlice() }, loc);
|
|
},
|
|
.t_open_brace => {
|
|
try p.lexer.next();
|
|
var is_single_line = !p.lexer.has_newline_before;
|
|
var properties = std.ArrayList(G.Property).init(p.allocator);
|
|
var duplicates = std.AutoHashMap(u64, void).init(p.allocator);
|
|
defer duplicates.deinit();
|
|
|
|
while (p.lexer.token != .t_close_brace) {
|
|
if (properties.items.len > 0) {
|
|
if (p.lexer.has_newline_before) {
|
|
is_single_line = false;
|
|
}
|
|
if (!try p.parseMaybeTrailingComma(.t_close_brace)) {
|
|
break;
|
|
}
|
|
if (p.lexer.has_newline_before) {
|
|
is_single_line = false;
|
|
}
|
|
}
|
|
|
|
var str = p.lexer.toEString();
|
|
const hash_key = str.hash();
|
|
const duplicate_get_or_put = duplicates.getOrPut(hash_key) catch unreachable;
|
|
duplicate_get_or_put.key_ptr.* = hash_key;
|
|
|
|
var key_range = p.lexer.range();
|
|
// Warn about duplicate keys
|
|
if (duplicate_get_or_put.found_existing) {
|
|
p.log.addRangeWarningFmt(p.source, key_range, p.allocator, "Duplicate key \"{s}\" in object literal", .{p.lexer.string_literal_slice}) catch unreachable;
|
|
}
|
|
|
|
var key = p.e(str, key_range.loc);
|
|
try p.lexer.expect(.t_string_literal);
|
|
|
|
try p.lexer.expect(.t_colon);
|
|
var value = try p.parseExpr(false);
|
|
properties.append(G.Property{ .key = key, .value = value }) catch unreachable;
|
|
}
|
|
|
|
if (p.lexer.has_newline_before) {
|
|
is_single_line = false;
|
|
}
|
|
try p.lexer.expect(.t_close_brace);
|
|
return p.e(E.Object{
|
|
.properties = properties.toOwnedSlice(),
|
|
.is_single_line = is_single_line,
|
|
}, loc);
|
|
},
|
|
else => {
|
|
if (comptime maybe_auto_quote) {
|
|
p.lexer = try Lexer.initJSON(p.log, p.source, p.allocator);
|
|
try p.lexer.parseStringLiteral(0);
|
|
return p.parseExpr(false);
|
|
}
|
|
|
|
try p.lexer.unexpected();
|
|
if (comptime isDebug) {
|
|
@breakpoint();
|
|
}
|
|
return error.ParserError;
|
|
},
|
|
}
|
|
}
|
|
|
|
pub fn parseMaybeTrailingComma(p: *Parser, closer: T) !bool {
|
|
const comma_range = p.lexer.range();
|
|
try p.lexer.expect(.t_comma);
|
|
|
|
if (p.lexer.token == closer) {
|
|
if (!opts.allow_trailing_commas) {
|
|
p.log.addRangeError(p.source, comma_range, "JSON does not support trailing commas") catch unreachable;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
};
|
|
}
|
|
|
|
const JSONParser = JSONLikeParser(js_lexer.JSONOptions{});
|
|
const DotEnvJSONParser = JSONLikeParser(js_lexer.JSONOptions{
|
|
.starts_with_string = true,
|
|
.allow_trailing_commas = true,
|
|
});
|
|
const TSConfigParser = JSONLikeParser(js_lexer.JSONOptions{ .allow_comments = true, .allow_trailing_commas = true });
|
|
var empty_string = E.String{ .utf8 = "" };
|
|
var empty_object = E.Object{};
|
|
var empty_array = E.Array{ .items = &[_]ExprNodeIndex{} };
|
|
var empty_string_data = Expr.Data{ .e_string = &empty_string };
|
|
var empty_object_data = Expr.Data{ .e_object = &empty_object };
|
|
var empty_array_data = Expr.Data{ .e_array = &empty_array };
|
|
|
|
pub fn ParseJSON(source: *const logger.Source, log: *logger.Log, allocator: *std.mem.Allocator) !Expr {
|
|
var parser = try JSONParser.init(allocator, source, log);
|
|
switch (source.contents.len) {
|
|
// This is to be consisntent with how disabled JS files are handled
|
|
0 => {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_object_data };
|
|
},
|
|
// This is a fast pass I guess
|
|
2 => {
|
|
if (strings.eqlComptime(source.contents[0..1], "\"\"") or strings.eqlComptime(source.contents[0..1], "''")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_string_data };
|
|
} else if (strings.eqlComptime(source.contents[0..1], "{}")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_object_data };
|
|
} else if (strings.eqlComptime(source.contents[0..1], "[]")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_array_data };
|
|
}
|
|
},
|
|
else => {},
|
|
}
|
|
|
|
return try parser.parseExpr(false);
|
|
}
|
|
|
|
pub const JSONParseResult = struct {
|
|
expr: Expr,
|
|
tag: Tag,
|
|
|
|
pub const Tag = enum {
|
|
expr,
|
|
ascii,
|
|
empty,
|
|
};
|
|
};
|
|
|
|
pub fn ParseJSONForBundling(source: *const logger.Source, log: *logger.Log, allocator: *std.mem.Allocator) !JSONParseResult {
|
|
var parser = try JSONParser.init(allocator, source, log);
|
|
switch (source.contents.len) {
|
|
// This is to be consisntent with how disabled JS files are handled
|
|
0 => {
|
|
return JSONParseResult{ .expr = Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_object_data }, .tag = .empty };
|
|
},
|
|
// This is a fast pass I guess
|
|
2 => {
|
|
if (strings.eqlComptime(source.contents[0..1], "\"\"") or strings.eqlComptime(source.contents[0..1], "''")) {
|
|
return JSONParseResult{ .expr = Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_string_data }, .tag = .expr };
|
|
} else if (strings.eqlComptime(source.contents[0..1], "{}")) {
|
|
return JSONParseResult{ .expr = Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_object_data }, .tag = .expr };
|
|
} else if (strings.eqlComptime(source.contents[0..1], "[]")) {
|
|
return JSONParseResult{ .expr = Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_array_data }, .tag = .expr };
|
|
}
|
|
},
|
|
else => {},
|
|
}
|
|
|
|
const result = try parser.parseExpr(false);
|
|
return JSONParseResult{
|
|
.tag = if (parser.lexer.is_ascii_only) JSONParseResult.Tag.ascii else JSONParseResult.Tag.expr,
|
|
.expr = result,
|
|
};
|
|
}
|
|
|
|
// threadlocal var env_json_auto_quote_buffer: MutableString = undefined;
|
|
// threadlocal var env_json_auto_quote_buffer_loaded: bool = false;
|
|
pub fn ParseEnvJSON(source: *const logger.Source, log: *logger.Log, allocator: *std.mem.Allocator) !Expr {
|
|
var parser = try DotEnvJSONParser.init(allocator, source, log);
|
|
switch (source.contents.len) {
|
|
// This is to be consisntent with how disabled JS files are handled
|
|
0 => {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_object_data };
|
|
},
|
|
// This is a fast pass I guess
|
|
2 => {
|
|
if (strings.eqlComptime(source.contents[0..1], "\"\"") or strings.eqlComptime(source.contents[0..1], "''")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_string_data };
|
|
} else if (strings.eqlComptime(source.contents[0..1], "{}")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_object_data };
|
|
} else if (strings.eqlComptime(source.contents[0..1], "[]")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_array_data };
|
|
}
|
|
},
|
|
else => {},
|
|
}
|
|
|
|
switch (source.contents[0]) {
|
|
'{', '[', '0'...'9', '"', '\'' => {
|
|
return try parser.parseExpr(false);
|
|
},
|
|
else => {
|
|
switch (parser.lexer.token) {
|
|
.t_true => {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = .{ .e_boolean = E.Boolean{ .value = true } } };
|
|
},
|
|
.t_false => {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = .{ .e_boolean = E.Boolean{ .value = false } } };
|
|
},
|
|
.t_null => {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = .{ .e_null = E.Null{} } };
|
|
},
|
|
.t_identifier => {
|
|
if (strings.eqlComptime(parser.lexer.identifier, "undefined")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = .{ .e_undefined = E.Undefined{} } };
|
|
}
|
|
|
|
return try parser.parseExpr(true);
|
|
},
|
|
else => {
|
|
return try parser.parseExpr(true);
|
|
},
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
pub fn ParseTSConfig(source: *const logger.Source, log: *logger.Log, allocator: *std.mem.Allocator) !Expr {
|
|
switch (source.contents.len) {
|
|
// This is to be consisntent with how disabled JS files are handled
|
|
0 => {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_object_data };
|
|
},
|
|
// This is a fast pass I guess
|
|
2 => {
|
|
if (strings.eqlComptime(source.contents[0..1], "\"\"") or strings.eqlComptime(source.contents[0..1], "''")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_string_data };
|
|
} else if (strings.eqlComptime(source.contents[0..1], "{}")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_object_data };
|
|
} else if (strings.eqlComptime(source.contents[0..1], "[]")) {
|
|
return Expr{ .loc = logger.Loc{ .start = 0 }, .data = empty_array_data };
|
|
}
|
|
},
|
|
else => {},
|
|
}
|
|
|
|
var parser = try TSConfigParser.init(allocator, source, log);
|
|
|
|
return parser.parseExpr(false);
|
|
}
|
|
|
|
const duplicateKeyJson = "{ \"name\": \"valid\", \"name\": \"invalid\" }";
|
|
|
|
const js_printer = @import("js_printer.zig");
|
|
const renamer = @import("renamer.zig");
|
|
const SymbolList = [][]Symbol;
|
|
|
|
fn expectPrintedJSON(_contents: string, expected: string) !void {
|
|
var contents = alloc.dynamic.alloc(u8, _contents.len + 1) catch unreachable;
|
|
std.mem.copy(u8, contents, _contents);
|
|
contents[contents.len - 1] = ';';
|
|
var log = logger.Log.init(alloc.dynamic);
|
|
defer log.msgs.deinit();
|
|
|
|
var source = logger.Source.initPathString(
|
|
"source.json",
|
|
contents,
|
|
);
|
|
const expr = try ParseJSON(&source, &log, alloc.dynamic);
|
|
var stmt = Stmt.alloc(alloc.dynamic, S.SExpr{ .value = expr }, logger.Loc{ .start = 0 });
|
|
|
|
var part = js_ast.Part{
|
|
.stmts = &([_]Stmt{stmt}),
|
|
};
|
|
const tree = js_ast.Ast.initTest(&([_]js_ast.Part{part}));
|
|
var symbols: SymbolList = &([_][]Symbol{tree.symbols});
|
|
var symbol_map = js_ast.Symbol.Map.initList(symbols);
|
|
if (log.msgs.items.len > 0) {
|
|
Global.panic("--FAIL--\nExpr {s}\nLog: {s}\n--FAIL--", .{ expr, log.msgs.items[0].data.text });
|
|
}
|
|
var linker = @import("linker.zig").Linker{};
|
|
|
|
const result = js_printer.printAst(alloc.dynamic, tree, symbol_map, &source, true, js_printer.Options{ .to_module_ref = Ref{ .inner_index = 0 } }, &linker) catch unreachable;
|
|
|
|
var js = result.js;
|
|
|
|
if (js.len > 1) {
|
|
while (js[js.len - 1] == '\n') {
|
|
js = js[0 .. js.len - 1];
|
|
}
|
|
|
|
if (js[js.len - 1] == ';') {
|
|
js = js[0 .. js.len - 1];
|
|
}
|
|
}
|
|
|
|
std.testing.expectEqualStrings(expected, js);
|
|
}
|
|
|
|
test "ParseJSON" {
|
|
try alloc.setup(default_allocator);
|
|
try expectPrintedJSON("true", "true");
|
|
try expectPrintedJSON("false", "false");
|
|
try expectPrintedJSON("1", "1");
|
|
try expectPrintedJSON("10", "10");
|
|
try expectPrintedJSON("100", "100");
|
|
try expectPrintedJSON("100.1", "100.1");
|
|
try expectPrintedJSON("19.1", "19.1");
|
|
try expectPrintedJSON("19.12", "19.12");
|
|
try expectPrintedJSON("3.4159820837456", "3.4159820837456");
|
|
try expectPrintedJSON("-10000.25", "-10000.25");
|
|
try expectPrintedJSON("\"hi\"", "\"hi\"");
|
|
try expectPrintedJSON("{\"hi\": 1, \"hey\": \"200\", \"boom\": {\"yo\": true}}", "({\"hi\": 1, \"hey\": \"200\", \"boom\": {\"yo\": true}})");
|
|
try expectPrintedJSON("{\"hi\": \"hey\"}", "({hi: \"hey\"})");
|
|
try expectPrintedJSON("{\"hi\": [\"hey\", \"yo\"]}", "({hi:[\"hey\",\"yo\"]})");
|
|
// TODO: emoji?
|
|
}
|
|
|
|
test "ParseJSON DuplicateKey warning" {
|
|
alloc.setup(std.heap.page_allocator) catch unreachable;
|
|
var log = logger.Log.init(alloc.dynamic);
|
|
|
|
var source = logger.Source.initPathString(
|
|
"package.json",
|
|
duplicateKeyJson,
|
|
);
|
|
const expr = try ParseJSON(&source, &log, alloc.dynamic);
|
|
|
|
const tag = @as(Expr.Tag, expr.data);
|
|
expect(tag == .e_object);
|
|
const object = expr.data.e_object;
|
|
std.testing.expectEqual(@as(usize, 2), object.properties.len);
|
|
const name1 = object.properties[0];
|
|
expect(name1.key != null);
|
|
expect(name1.value != null);
|
|
expect(Expr.Tag.e_string == @as(Expr.Tag, name1.value.?.data));
|
|
expect(Expr.Tag.e_string == @as(Expr.Tag, name1.key.?.data));
|
|
expect(strings.eqlUtf16("name", name1.key.?.data.e_string.value));
|
|
expect(strings.eqlUtf16("valid", name1.value.?.data.e_string.value));
|
|
|
|
const name2 = object.properties[1];
|
|
expect(name2.key != null);
|
|
expect(name2.value != null);
|
|
expect(Expr.Tag.e_string == @as(Expr.Tag, name2.value.?.data));
|
|
expect(Expr.Tag.e_string == @as(Expr.Tag, name2.key.?.data));
|
|
expect(strings.eqlUtf16("name", name2.key.?.data.e_string.value));
|
|
std.testing.expectEqualStrings("invalid", try name2.value.?.data.e_string.string(alloc.dynamic));
|
|
|
|
std.testing.expectEqual(@as(usize, 1), log.msgs.items.len);
|
|
}
|