Files
bun.sh/src/defines.zig
Jarred Sumner 6362414d65 Bun gets a new bundler (#2312)
* alright now just gotta try running it

* fix a gajillion compiler errors

* even more code

* okay i fixed more errors

* wip

* Update launch.json

* Update string_builder.zig

* `fast_debug_build_mode` makes debug build 2x faster

* Update bundle_v2.zig

* more code!

* It bundles!

* Rename `Bun.Transpiler` to `Bun.Bundler`

* `import()` expressions almost work

* wip attempt to get import() expr to work

* Bundle namespace imports

* Attempt to fix the issue with import() unsuccessfully

* consider current working directory when resolving relative paths (#2313)

* consider current working directory when resolving relative paths

fixes #2298

* comment test

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

* support `expect().toThrow(/pattern/)` (#2314)

- fix time-zone-dependent test failure

* fix missing `Blob` error messages on Linux (#2315)

* fix & clean up tests (#2318)

- skip flaky tests when running as `root`
- use `expect().toThrow()`
- clean up temporary files after tests

* feat(tty): add some `tty.WriteStream` methods to `process.{stdout, stderr}` (#2320)

* feat(stdio): add some `tty.WriteStream` methods

* chore(builtins): add process builtin gen'd code

* Fix docker install command

* `bun test` on macOS in GitHub Actions (#2322)

* Fixes #2323

* throw invalid parameter errors in `crypto.scryptSync` (#2331)

* throw invalid parameter errors

* remove comptime, add empty buffer function

* remove error_name comptime

* Add reference documentation for bun:test (#2327)

* Reorganize tests (#2332)

* Fix html-rewriter.test.js

* fix the wrong thing being incremented in hmr example (#2334)

* Add more test harness

* Improve Benchmarking page, small fixes (#2339)

* Improve benchmarking page

* WIP

* Add typescript instructions to hot

* Document preload in Plugins. Fix loader in plugin types.

* Fix typo

* Fix links

* run prettier

* Document openInEditor

* improve `Buffer` compatibility with Node.js (#2341)

* improve `Buffer` compatibility with Node.js

* use `memmove()`
allow `encoding` to be `undefined`

* run `bun test` after macOS builds (#2343)

* "binary" is an alias of "latin1"

Fixes https://github.com/oven-sh/bun/issues/2110

* More spec compliant `Blob.prototype.type` (#2340)

* Make `Blob.prototype. type` more spec compliant

* Add a few more checks for isNumber()

* Fix `make headers`

* Safer JSValue.isString()

* More tests for blob.slice

* Make `Blob.prototype.type` more spec compliant

* Add isASCII check

* Fix types

* Fix failing type test

* Update blob.zig

* Update blob.zig

* Fix .eql check on empty values

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

* Fix bug in test runner

* Support `import()` expressions

* Implement `require()`

* clean up bit_set.zig slightly

* Move some things around

* misc cleanup

* Cleanup some things

* Fix a lot of stuff

* Fix `module.exports.fn = fn;` in ESM entry point

* Fix crash due when printing file

* Fix issue with class names

* Fix issue with `export default identifier`

* Update js_parser.zig

* optimization: inline single-property object acceses and arrays

* Fix undefined memory in renamed symbols list

* Handle call target

* wip

* Inline it

* Fix undefined memory issue when reclaiming blocks in ast

* Halt linking on any parse errors

* alias

* Rename `enable_bundling` to `enable_legacy_bundling`

* Workaround anonymous struct literal zig bug

* Use slower approach (without bitset) because it doesn't break after 8 symbols

* Fix incorrectly-renaming statically defined symbols

* Handle more edgecases in our bit_set fork

* Reduce number of allocations for `define`

* Do not rename unbound symbols

* Clean up dot defines a little more

* Make the generated names prettier

* Workaround runtime symbol missing issue

* Fail the build on errors

* Support export * from

* Support `--outfile`

* partially fix renaming

* fanicer symbol renaming impl

* misc, extremely revertible cleanup

* Fix up some bugs with symbol renaming

* formatting

* Update launch.json

* Parse `__PURE__` comments

* clean up simd code for pure comments

* changes to merge

* workaround runtime issue

* Fix issue with `export * as` not propagating correctly

* Make all top-level declarations `var` when bundling

* Fix missing prefix

* Fix assigning to stack copy

* Fix missing runtime symbol

* Fix bug with namespace exports

* Dramatically reduce allocations

* Update launch.json

* Add missing flags

* Update js_parser.zig

* small cleanup

* Make the export name better

* Fix unnecessary `var foo = foo`

* Implement CommonJS -> ESM conversion

* Implement module redirects

* Port esbuild bundler tests for new bundler (#2380)

* started porting esbuild tests

* clean up test names and api before moving on

* port tests using a program i wrote

* replace todo generated comment

* fix generated tests not including some files

* work on tests

* [github web editor] add define, external, inject, minifySyntax, minifyWhitespace options.

* get most of the todo comments out of the way, but expectBundled does not handle most of the cases

* continue working on esbuild tests

* use test.skip for unsupported tests

* Fixups for test runner

* Hoist imports & exports

* Fix test

* Hoist classes

* bundler test refining, 51/835

* Fix runtime require

* bundler test refining, 81/835

* bundler test refining, 93/835

* Make the test work in any timezone

* feat(expect): update toBeInstanceOf (#2396)

* feat: update instanceof binding

* fix: according to PR comments

* Rename `expectObjectTypeCount` to `expectMaxObjectTypeCount`

* Fix socket tests with connection errors (#2403)

* release pending activity with connection error handler

* unref poll_ref

* remove trailing comma

* Organize Dockerfiles for official status

* Remove test Dockerfile

* Remove old Docker workflow

* Feat(test): add toMatch (#2404)

* Fix various fetch/response/request tests (#2416)

* fix most fetch tests, skip a few

* fastGet, toValueGC, and invalid init

* bigint unreachable, range error, log process as process

* remove extra fetch_headers

* remove js_type parameter, check isObject()

* throw invalid mime type error, use enum literal

* switch back to promise rejection

* RangeError pascal case

* Fix several bugs (#2418)

* utf16 codepoint with replacement character

* Fix test failure with `TextEncoder("ascii')`

* Add missing type

* Fix Response.prototype.bodyUsed and Request.prototype.bodyUsed

* Fix bug with scrypt error not clearing

* Update server.zig

* oopsie

* 💅

* docs: Use correct url in the 'Issues' link in README header (#2420)

* Fix crash when rendering error page and the server or network is slow

* [fetch] Make the default body value `null` when unspecified

This is better aligned with the fetch spec

* Make node-net tests less flaky

* [node:net] Fix issue with `listen` callback firing before it's listening

* Always clear timers in node test harness

* Fix out of bounds access

Repro'd in Buffer tests

* Update UWS

cc @cirospaciari

* Make this test more thorough

* Hanging abort test

* 0 length body is a null stream

* Several bug fixes (#2427)

* Fix test

* Fix segfault when unexpected type is passed in `expect().toThrow`

* Fix issues with request constructor

* Don't bother cloning headers when its empty

* woops

* more tests

* fix incorrect test

* Make the fetch error messages better

* Update response.zig

* Fix test that failed on macOS

* Fix test

* Remove extra hash table lookups

* Support running dummy registry directly

cc @alexlamsl

* Update test

* Update test

* fixup

* Workaround crash in test runner

* Fixup test

* Fixup test

* Update os.test.js

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

* Remove usages of port numbers in tests

* Set -O2 and -fno-rtti

* Remove -g

* Prevent undefined memory access

* [bun test] Implement `--rerun-each` flag to run each test N times

* Reduce number of module scopes created

* add some extra abort checks into streams (#2430)

* add some checks to avoid UAF

* avoid multiple calls to finalize if endFromJS is called more than once

* fix no-op comment

* mark as requested_end on abort

* remove requested_end from abort

* remove unnecessary check (#2432)

* Fix bug with scoped aliased dependencies in bun install on macOS

* remove `addLog`, remove `--prominent-compile-errors`

* Finish the upgrade

* Optional chaining flag

* Implement same_target_becomes_destructuring optimization

* bundler test refining, 109/835

* Reset bindings

* Support multiple entry points

* Implement `--entry-names` flag

* Use a tempdir with a better name

* prettier

* Log file name

* Update js_parser.zig

* Mark all bun builtins as external

* Make resolve errors actually errors

* Update bundler_default.test.ts

* Fix `await import(foo)`

* WIP react server components

* Do more stuff at runtime

* ✂️

* Support automatic JSX imports

* Use a module cache for now

* Update tsconfig.base.json

* Fix ThisOutsideFunctionNotRenamed

* woopsie

* moar cpu

* clamp it

* fixup

* Add a bunch of assertions

* Bun uses automatic runtime by default

* Parse Import Attributes

* Add a note about Valgrind

* Update developing.md

* Fix up code splitting for React Server Components

* Implement client component manifest

* Fix crash with --react-server-components and no client components

* Backport 4d31e3c917

* Update launch.json

* Fix for latest zig

* Workaround bug with ?[]const string

Occasionally saw alignment errors in this code

Workaround https://github.com/ziglang/zig/issues/15085

related: https://github.com/ziglang/zig/pull/15089

* switch to regular slice

* Avoid initializing named_imports and named_exports as undefined

* Reduce usages of `undefined`

* Add more assertions

* --watch wip

* Update javascript.zig

* Possibly fix the race condition

* Faster `do`

* bump allocator

* Reduce the size of `Symbol` slightly

* Alphabetically sort runtime import symbols, for determinism

* Prepare for code splitting

* handle overlapping stdout

* pure

* clean up some things

* Fix bug with `$$typeof`

* Address CommonJS -> ESM hoisting bug

* Support `"use server"` in manifest

* Implement `"use server"`

* Fix importing bun builtins when bundling

* Make `commonjs_to_esm` a feature flag, fix some splitting bugs

* ✂️

* fixme remove this

* Fix crash in longestCommonPath

* Chunking! Just need to do import paths now.

* Import paths work...now trying to figure out how to make runtime symbols work

* add workaround

* Replace `bun bun` with `bun build`

* Fix crash with dual package hazard

* Fix many CommonJS <> ESM interop bugs

* Support package.json `"sideEffects"`

also skip loading unnecessary package.json data in `bun run`

* add a not good --watch implementation

* bundler test refining, 140/831

* remove accidentally committed file

* do not return status code 1 on successful bundles

* bundler test refining, 159/830

* pass exit code to exitOrWatch

* clean up help menu

-remove two spaces to line up bun build
-moved all <r> tags to the end of the text they are colorizing
-moved other colors to the start of the text they colorize
-removed unneeded <r> tags, keeping only one at the start of the block

* importstar is fully ported

* wip

* you can run code in this branch now

* Disable this transform

* organize and document bundler tests

* Fix double import

* Fix sloppy mode function declarations

* Disable our CommonJS transform for now

* add `assertNotPresent` to make splitting cases easier

* Bump!

* Update bun.d.ts

* use import.meta.require in runtime code

* Disable this again

* Fix dirname

* Fix ESM -> CJS wrapper

* 💅

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>
Co-authored-by: Alex Lam S.L <alexlamsl@gmail.com>
Co-authored-by: Derrick Farris <mr.dcfarris@gmail.com>
Co-authored-by: Ashcon Partovi <ashcon@partovi.net>
Co-authored-by: Dylan Conway <35280289+dylan-conway@users.noreply.github.com>
Co-authored-by: pfg <pfg@pfg.pw>
Co-authored-by: Colin McDonnell <colinmcd94@gmail.com>
Co-authored-by: dave caruso <me@paperdave.net>
Co-authored-by: zhiyuan <32867472+zhiyuang@users.noreply.github.com>
Co-authored-by: Dylan Conway <dylan.conway567@gmail.com>
Co-authored-by: Kamil Ogórek <kamil.ogorek@gmail.com>
Co-authored-by: Ciro Spaciari <ciro.spaciari@gmail.com>
2023-04-07 20:08:01 -07:00

330 lines
13 KiB
Zig

const std = @import("std");
const js_ast = bun.JSAst;
const logger = @import("bun").logger;
const js_lexer = bun.js_lexer;
const json_parser = bun.JSON;
const fs = @import("fs.zig");
const bun = @import("bun");
const string = bun.string;
const Output = bun.Output;
const Global = bun.Global;
const Environment = bun.Environment;
const strings = bun.strings;
const MutableString = bun.MutableString;
const stringZ = bun.stringZ;
const default_allocator = bun.default_allocator;
const C = bun.C;
const Ref = @import("ast/base.zig").Ref;
const GlobalDefinesKey = @import("./defines-table.zig").GlobalDefinesKey;
const table = @import("./defines-table.zig");
const Globals = struct {
pub const Undefined = js_ast.E.Undefined{};
pub const UndefinedPtr = &Globals.Undefined;
pub const NaN = js_ast.E.Number{ .value = std.math.nan(f64) };
pub const NanPtr = &Globals.NaN;
pub const Infinity = js_ast.E.Number{ .value = std.math.inf(f64) };
pub const InfinityPtr = &Globals.Infinity;
pub const UndefinedData = js_ast.Expr.Data{ .e_undefined = Globals.UndefinedPtr };
pub const NaNData = js_ast.Expr.Data{ .e_number = Globals.NanPtr };
pub const InfinityData = js_ast.Expr.Data{ .e_number = Globals.InfinityPtr };
};
const defines_path = fs.Path.initWithNamespace("defines.json", "internal");
pub const RawDefines = bun.StringArrayHashMap(string);
pub const UserDefines = bun.StringHashMap(DefineData);
pub const UserDefinesArray = bun.StringArrayHashMap(DefineData);
pub const DefineData = struct {
value: js_ast.Expr.Data,
valueless: bool = false,
original_name: ?string = null,
// True if accessing this value is known to not have any side effects. For
// example, a bare reference to "Object.create" can be removed because it
// does not have any observable side effects.
can_be_removed_if_unused: bool = false,
// True if a call to this value is known to not have any side effects. For
// example, a bare call to "Object()" can be removed because it does not
// have any observable side effects.
call_can_be_unwrapped_if_unused: bool = false,
pub fn isUndefined(self: *const DefineData) bool {
return self.valueless;
}
pub fn merge(a: DefineData, b: DefineData) DefineData {
return DefineData{
.value = b.value,
.can_be_removed_if_unused = a.can_be_removed_if_unused,
.call_can_be_unwrapped_if_unused = a.call_can_be_unwrapped_if_unused,
};
}
pub fn from_mergable_input(defines: RawDefines, user_defines: *UserDefines, log: *logger.Log, allocator: std.mem.Allocator) !void {
try user_defines.ensureUnusedCapacity(@truncate(u32, defines.count()));
var iter = defines.iterator();
while (iter.next()) |entry| {
var splitter = std.mem.split(u8, entry.key_ptr.*, ".");
while (splitter.next()) |part| {
if (!js_lexer.isIdentifier(part)) {
if (strings.eql(part, entry.key_ptr)) {
try log.addErrorFmt(null, logger.Loc{}, allocator, "define key \"{s}\" must be a valid identifier", .{entry.key_ptr.*});
} else {
try log.addErrorFmt(null, logger.Loc{}, allocator, "define key \"{s}\" contains invalid identifier \"{s}\"", .{ part, entry.value_ptr.* });
}
break;
}
}
if (js_lexer.isIdentifier(entry.value_ptr.*) and !js_lexer.Keywords.has(entry.value_ptr.*)) {
// Special-case undefined. it's not an identifier here
// https://github.com/evanw/esbuild/issues/1407
if (strings.eqlComptime(entry.value_ptr.*, "undefined")) {
user_defines.putAssumeCapacity(
entry.key_ptr.*,
DefineData{
.value = js_ast.Expr.Data{ .e_undefined = js_ast.E.Undefined{} },
.original_name = entry.value_ptr.*,
.can_be_removed_if_unused = true,
},
);
} else {
const ident = js_ast.E.Identifier{ .ref = Ref.None, .can_be_removed_if_unused = true };
user_defines.putAssumeCapacity(
entry.key_ptr.*,
DefineData{
.value = js_ast.Expr.Data{ .e_identifier = ident },
.original_name = entry.value_ptr.*,
.can_be_removed_if_unused = true,
},
);
}
// user_defines.putAssumeCapacity(
// entry.key_ptr,
// DefineData{ .value = js_ast.Expr.Data{.e_identifier = } },
// );
continue;
}
var _log = log;
var source = logger.Source{
.contents = entry.value_ptr.*,
.path = defines_path,
.key_path = fs.Path.initWithNamespace("defines", "internal"),
};
var expr = try json_parser.ParseEnvJSON(&source, _log, allocator);
var data: js_ast.Expr.Data = undefined;
switch (expr.data) {
.e_missing => {
data = .{ .e_missing = js_ast.E.Missing{} };
},
// We must copy so we don't recycle
.e_string => {
data = .{ .e_string = try allocator.create(js_ast.E.String) };
data.e_string.* = try expr.data.e_string.clone(allocator);
},
.e_null, .e_boolean, .e_number => {
data = expr.data;
},
// We must copy so we don't recycle
.e_object => |obj| {
expr.data.e_object = try allocator.create(js_ast.E.Object);
expr.data.e_object.* = obj.*;
data = expr.data;
},
// We must copy so we don't recycle
.e_array => |obj| {
expr.data.e_array = try allocator.create(js_ast.E.Array);
expr.data.e_array.* = obj.*;
data = expr.data;
},
else => {
continue;
},
}
user_defines.putAssumeCapacity(entry.key_ptr.*, DefineData{
.value = data,
});
}
}
pub fn from_input(defines: RawDefines, log: *logger.Log, allocator: std.mem.Allocator) !UserDefines {
var user_defines = UserDefines.init(allocator);
try from_mergable_input(defines, &user_defines, log, allocator);
return user_defines;
}
};
fn arePartsEqual(a: []const string, b: []const string) bool {
if (a.len != b.len) {
return false;
}
var i: usize = 0;
while (i < a.len) : (i += 1) {
if (!strings.eql(a[i], b[i])) {
return false;
}
}
return true;
}
pub const IdentifierDefine = DefineData;
pub const DotDefine = struct {
parts: []const string,
data: DefineData,
};
// var nan_val = try allocator.create(js_ast.E.Number);
const nan_val = js_ast.E.Number{ .value = std.math.nan_f64 };
const inf_val = js_ast.E.Number{ .value = std.math.inf_f64 };
pub const Define = struct {
identifiers: bun.StringHashMap(IdentifierDefine),
dots: bun.StringHashMap([]DotDefine),
allocator: std.mem.Allocator,
pub fn forIdentifier(this: *const Define, name: []const u8) ?IdentifierDefine {
if (this.identifiers.get(name)) |data| {
return data;
}
return table.pure_global_identifier_map.get(name);
}
pub fn insertFromIterator(define: *Define, allocator: std.mem.Allocator, comptime Iterator: type, iter: Iterator) !void {
outer: while (iter.next()) |user_define| {
const user_define_key = user_define.key_ptr.*;
// If it has a dot, then it's a DotDefine.
// e.g. process.env.NODE_ENV
if (strings.lastIndexOfChar(user_define_key, '.')) |last_dot| {
const tail = user_define_key[last_dot + 1 .. user_define_key.len];
const remainder = user_define_key[0..last_dot];
const count = std.mem.count(u8, remainder, ".") + 1;
var parts = try allocator.alloc(string, count + 1);
var splitter = std.mem.split(u8, remainder, ".");
var i: usize = 0;
while (splitter.next()) |split| : (i += 1) {
parts[i] = split;
}
parts[i] = tail;
var initial_values: []DotDefine = &([_]DotDefine{});
// "NODE_ENV"
var gpe_entry = try define.dots.getOrPut(tail);
if (gpe_entry.found_existing) {
for (gpe_entry.value_ptr.*) |*part| {
// ["process", "env"] === ["process", "env"] (if that actually worked)
if (arePartsEqual(part.parts, parts)) {
part.data = part.data.merge(user_define.value_ptr.*);
continue :outer;
}
}
initial_values = gpe_entry.value_ptr.*;
}
var list = try std.ArrayList(DotDefine).initCapacity(allocator, initial_values.len + 1);
if (initial_values.len > 0) {
list.appendSliceAssumeCapacity(initial_values);
}
list.appendAssumeCapacity(DotDefine{
.data = user_define.value_ptr.*,
// TODO: do we need to allocate this?
.parts = parts,
});
gpe_entry.value_ptr.* = try list.toOwnedSlice();
} else {
// e.g. IS_BROWSER
try define.identifiers.put(user_define_key, user_define.value_ptr.*);
}
}
}
pub fn init(allocator: std.mem.Allocator, _user_defines: ?UserDefines, string_defines: ?UserDefinesArray) !*@This() {
var define = try allocator.create(Define);
define.allocator = allocator;
define.identifiers = bun.StringHashMap(IdentifierDefine).init(allocator);
define.dots = bun.StringHashMap([]DotDefine).init(allocator);
try define.dots.ensureTotalCapacity(124);
const value_define = DefineData{
.value = .{ .e_undefined = .{} },
.valueless = true,
.can_be_removed_if_unused = true,
};
// Step 1. Load the globals into the hash tables
for (GlobalDefinesKey) |global| {
const key = global[global.len - 1];
var gpe = try define.dots.getOrPut(key);
if (gpe.found_existing) {
var list = try std.ArrayList(DotDefine).initCapacity(allocator, gpe.value_ptr.*.len + 1);
list.appendSliceAssumeCapacity(gpe.value_ptr.*);
list.appendAssumeCapacity(DotDefine{
.parts = global[0..global.len],
.data = value_define,
});
gpe.value_ptr.* = try list.toOwnedSlice();
} else {
var list = try std.ArrayList(DotDefine).initCapacity(allocator, 1);
list.appendAssumeCapacity(DotDefine{
.parts = global[0..global.len],
.data = value_define,
});
gpe.value_ptr.* = try list.toOwnedSlice();
}
}
// Step 3. Load user data into hash tables
// At this stage, user data has already been validated.
if (_user_defines) |user_defines| {
var iter = user_defines.iterator();
try define.insertFromIterator(allocator, @TypeOf(&iter), &iter);
}
// Step 4. Load environment data into hash tables.
// These are only strings. We do not parse them as JSON.
if (string_defines) |string_defines_| {
var iter = string_defines_.iterator();
try define.insertFromIterator(allocator, @TypeOf(&iter), &iter);
}
return define;
}
};
const expect = std.testing.expect;
test "UserDefines" {
js_ast.Stmt.Data.Store.create(default_allocator);
js_ast.Expr.Data.Store.create(default_allocator);
var orig = RawDefines.init(default_allocator);
try orig.put("process.env.NODE_ENV", "\"development\"");
try orig.put("globalThis", "window");
var log = logger.Log.init(default_allocator);
var data = try DefineData.from_input(orig, &log, default_allocator);
try expect(data.contains("process.env.NODE_ENV"));
try expect(data.contains("globalThis"));
const globalThis = data.get("globalThis");
const val = data.get("process.env.NODE_ENV");
try expect(val != null);
try expect(val.?.value.e_string.eql([]const u8, "development"));
try std.testing.expectEqualStrings(globalThis.?.original_name.?, "window");
}