From 32ddf343eeff744471aab373b48c6a9236381503 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Wed, 13 Nov 2024 18:19:12 -0800 Subject: [PATCH] bake: csr, streaming ssr, serve integration, safer jsvalue functions, &more (#14900) Co-authored-by: paperdave Co-authored-by: Jarred Sumner --- cmake/CompilerFlags.cmake | 2 +- cmake/targets/BuildBun.cmake | 1 + docs/dev/bundev.md | 11 - docs/dev/cra.md | 31 - docs/dev/css.md | 77 - docs/dev/discord.md | 26 - docs/rfcs/README.md | 4 - src/ArenaAllocator.zig | 248 --- src/bake/BakeGlobalObject.cpp | 104 +- src/bake/BakeGlobalObject.h | 13 - src/bake/BakeProduction.cpp | 24 +- src/bake/BakeSourceProvider.cpp | 27 +- src/bake/BakeSourceProvider.h | 5 - src/bake/DevServer.zig | 1378 +++++++++-------- src/bake/FrameworkRouter.zig | 1188 +++++++++++++- src/bake/bake.d.ts | 487 ++++-- src/bake/bake.private.d.ts | 86 +- src/bake/bake.zig | 672 +++++--- src/bake/bun-framework-react/client.tsx | 177 +++ src/bake/bun-framework-react/index.ts | 40 + src/bake/bun-framework-react/server.tsx | 117 ++ src/bake/bun-framework-react/ssr.tsx | 366 +++++ src/bake/bun-framework-rsc/client.tsx | 41 - src/bake/bun-framework-rsc/index.ts | 27 - src/bake/bun-framework-rsc/server.tsx | 127 -- src/bake/bun-framework-rsc/ssr.tsx | 16 - src/bake/hmr-module.ts | 41 +- src/bake/hmr-runtime-server.ts | 61 +- src/bake/incremental_visualizer.html | 60 +- src/bake/package.json | 3 +- src/bake/production.zig | 473 ++++-- src/bake/tsconfig.json | 7 +- src/bun.js/ConsoleObject.zig | 18 +- src/bun.js/RuntimeTranspilerCache.zig | 3 +- src/bun.js/api/BunObject.zig | 22 +- src/bun.js/api/JSBundler.zig | 57 +- src/bun.js/api/JSTranspiler.zig | 28 +- src/bun.js/api/filesystem_router.classes.ts | 19 + src/bun.js/api/filesystem_router.zig | 5 +- src/bun.js/api/server.zig | 317 +++- src/bun.js/bindings/JS2Native.h | 3 - src/bun.js/bindings/ObjectBindings.cpp | 4 +- src/bun.js/bindings/bindings.cpp | 24 +- src/bun.js/bindings/bindings.zig | 588 +++++-- .../bindings/generated_classes_list.zig | 1 + src/bun.js/bindings/headers.h | 2 +- src/bun.js/bindings/sizes.zig | 80 +- src/bun.js/event_loop.zig | 60 +- src/bun.js/javascript.zig | 5 +- src/bun.js/javascript_core_c_api.zig | 2 +- src/bun.js/node/node_fs.zig | 16 +- src/bun.js/node/node_fs_stat_watcher.zig | 4 +- src/bun.js/node/node_zlib_binding.zig | 2 +- src/bun.js/node/types.zig | 2 +- src/bun.js/node/util/parse_args.zig | 11 +- src/bun.js/node/util/validators.zig | 6 +- src/bun.js/test/pretty_format.zig | 6 +- src/bun.zig | 68 +- src/bundler/bundle_v2.zig | 27 +- src/bundler/entry_points.zig | 27 +- src/cli.zig | 35 +- src/codegen/bake-codegen.ts | 30 +- src/codegen/bundle-modules.ts | 2 +- src/codegen/generate-classes.ts | 15 +- src/codegen/generate-js2native.ts | 41 +- src/codegen/replacements.ts | 20 +- src/crash_handler.zig | 39 + src/css/values/color_js.zig | 2 +- src/defines.zig | 7 + src/deps/uws.zig | 2 +- src/feature_flags.zig | 3 + src/gen_classes_lib.zig | 52 + src/ini.zig | 64 +- src/js/builtins.d.ts | 3 + src/js/builtins/Bake.ts | 102 +- src/js/builtins/ReadableStreamInternals.ts | 5 +- src/js/internal-for-testing.ts | 8 + src/js_parser.zig | 25 + src/js_printer.zig | 4 +- src/jsc.zig | 2 + src/napi/napi.zig | 10 +- src/resolver/resolver.zig | 16 +- src/string.zig | 16 + src/string_immutable.zig | 21 + test/bake/framework-router.test.ts | 63 + test/bundler/expectBundled.ts | 1 + test/js/bun/http/bun-server.test.ts | 2 +- test/js/node/fs/fs.test.ts | 2 +- test/js/node/path/parse-format.test.js | 2 +- 89 files changed, 5382 insertions(+), 2559 deletions(-) delete mode 100644 docs/dev/bundev.md delete mode 100644 docs/dev/cra.md delete mode 100644 docs/dev/css.md delete mode 100644 docs/dev/discord.md delete mode 100644 docs/rfcs/README.md delete mode 100644 src/ArenaAllocator.zig create mode 100644 src/bake/bun-framework-react/client.tsx create mode 100644 src/bake/bun-framework-react/index.ts create mode 100644 src/bake/bun-framework-react/server.tsx create mode 100644 src/bake/bun-framework-react/ssr.tsx delete mode 100644 src/bake/bun-framework-rsc/client.tsx delete mode 100644 src/bake/bun-framework-rsc/index.ts delete mode 100644 src/bake/bun-framework-rsc/server.tsx delete mode 100644 src/bake/bun-framework-rsc/ssr.tsx create mode 100644 src/gen_classes_lib.zig create mode 100644 test/bake/framework-router.test.ts diff --git a/cmake/CompilerFlags.cmake b/cmake/CompilerFlags.cmake index bf8cf576ab..31d738134a 100644 --- a/cmake/CompilerFlags.cmake +++ b/cmake/CompilerFlags.cmake @@ -265,7 +265,7 @@ if(ENABLE_LTO) endif() # --- Remapping --- -if(UNIX) +if(UNIX AND CI) register_compiler_flags( DESCRIPTION "Remap source files" -ffile-prefix-map=${CWD}=. diff --git a/cmake/targets/BuildBun.cmake b/cmake/targets/BuildBun.cmake index 68e462447d..b95b9d030d 100644 --- a/cmake/targets/BuildBun.cmake +++ b/cmake/targets/BuildBun.cmake @@ -528,6 +528,7 @@ register_command( -Dcanary=${CANARY_REVISION} -Dcodegen_path=${CODEGEN_PATH} -Dcodegen_embed=$,true,false> + --prominent-compile-errors ${ZIG_FLAGS_BUN} ARTIFACTS ${BUN_ZIG_OUTPUT} diff --git a/docs/dev/bundev.md b/docs/dev/bundev.md deleted file mode 100644 index baccf7658a..0000000000 --- a/docs/dev/bundev.md +++ /dev/null @@ -1,11 +0,0 @@ -- pages -- auto-bundle dependencies -- pages is function that returns a list of pages? -- plugins for svelte and vue -- custom loaders -- HMR -- server endpoints - -```ts -Bun.serve({}); -``` diff --git a/docs/dev/cra.md b/docs/dev/cra.md deleted file mode 100644 index 8eb8687150..0000000000 --- a/docs/dev/cra.md +++ /dev/null @@ -1,31 +0,0 @@ -To create a new React app: - -```bash -$ bun create react ./app -$ cd app -$ bun dev # start dev server -``` - -To use an existing React app: - -```bash -$ bun add -d react-refresh # install React Fast Refresh -$ bun bun ./src/index.js # generate a bundle for your entry point(s) -$ bun dev # start the dev server -``` - -From there, Bun relies on the filesystem for mapping dev server paths to source files. All URL paths are relative to the project root (where `package.json` is located). - -Here are examples of routing source code file paths: - -| Dev Server URL | File Path (relative to cwd) | -| -------------------------- | --------------------------- | -| /src/components/Button.tsx | src/components/Button.tsx | -| /src/index.tsx | src/index.tsx | -| /pages/index.js | pages/index.js | - -You do not need to include file extensions in `import` paths. CommonJS-style import paths without the file extension work. - -You can override the public directory by passing `--public-dir="path-to-folder"`. - -If no directory is specified and `./public/` doesn’t exist, Bun will try `./static/`. If `./static/` does not exist, but won’t serve from a public directory. If you pass `--public-dir=./` Bun will serve from the current directory, but it will check the current directory last instead of first. diff --git a/docs/dev/css.md b/docs/dev/css.md deleted file mode 100644 index 53ebc6c066..0000000000 --- a/docs/dev/css.md +++ /dev/null @@ -1,77 +0,0 @@ -## With `bun dev` - -When importing CSS in JavaScript-like loaders, CSS is treated special. - -By default, Bun will transform a statement like this: - -```js -import "../styles/global.css"; -``` - -### When `platform` is `browser` - -```js -globalThis.document?.dispatchEvent( - new CustomEvent("onimportcss", { - detail: "http://localhost:3000/styles/globals.css", - }), -); -``` - -An event handler for turning that into a `` is automatically registered when HMR is enabled. That event handler can be turned off either in a framework’s `package.json` or by setting `globalThis["Bun_disableCSSImports"] = true;` in client-side code. Additionally, you can get a list of every .css file imported this way via `globalThis["__BUN"].allImportedStyles`. - -### When `platform` is `bun` - -```js -//@import url("http://localhost:3000/styles/globals.css"); -``` - -Additionally, Bun exposes an API for SSR/SSG that returns a flat list of URLs to css files imported. That function is `Bun.getImportedStyles()`. - -```ts -// This specifically is for "framework" in package.json when loaded via `bun dev` -// This API needs to be changed somewhat to work more generally with Bun.js -// Initially, you could only use Bun.js through `bun dev` -// and this API was created at that time -addEventListener("fetch", async (event: FetchEvent) => { - let route = Bun.match(event); - const App = await import("pages/_app"); - - // This returns all .css files that were imported in the line above. - // It’s recursive, so any file that imports a CSS file will be included. - const appStylesheets = bun.getImportedStyles(); - - // ...rest of code -}); -``` - -This is useful for preventing flash of unstyled content. - -## With `bun bun` - -Bun bundles `.css` files imported via `@import` into a single file. It doesn’t auto-prefix or minify CSS today. Multiple `.css` files imported in one JavaScript file will _not_ be bundled into one file. You’ll have to import those from a `.css` file. - -This input: - -```css -@import url("./hi.css"); -@import url("./hello.css"); -@import url("./yo.css"); -``` - -Becomes: - -```css -/* hi.css */ -/* ...contents of hi.css */ -/* hello.css */ -/* ...contents of hello.css */ -/* yo.css */ -/* ...contents of yo.css */ -``` - -## CSS runtime - -To support hot CSS reloading, Bun inserts `@supports` annotations into CSS that tag which files a stylesheet is composed of. Browsers ignore this, so it doesn’t impact styles. - -By default, Bun’s runtime code automatically listens to `onimportcss` and will insert the `event.detail` into a `` if there is no existing `link` tag with that stylesheet. That’s how Bun’s equivalent of `style-loader` works. diff --git a/docs/dev/discord.md b/docs/dev/discord.md deleted file mode 100644 index d3e9c5a2b7..0000000000 --- a/docs/dev/discord.md +++ /dev/null @@ -1,26 +0,0 @@ -## Creating a Discord bot with Bun - -Discord bots perform actions in response to _application commands_. There are 3 types of commands accessible in different interfaces: the chat input, a message's context menu (top-right menu or right-clicking in a message), and a user's context menu (right-clicking on a user). - -To get started you can use the interactions template: - -```bash -bun create discord-interactions my-interactions-bot -cd my-interactions-bot -``` - -If you don't have a Discord bot/application yet, you can create one [here (https://discord.com/developers/applications/me)](https://discord.com/developers/applications/me). - -Invite bot to your server by visiting `https://discord.com/api/oauth2/authorize?client_id=&scope=bot%20applications.commands` - -Afterwards you will need to get your bot's token, public key, and application id from the application page and put them into `.env.example` file - -Then you can run the http server that will handle your interactions: - -```bash -$ bun install -$ mv .env.example .env -$ bun run.js # listening on port 1337 -``` - -Discord does not accept an insecure HTTP server, so you will need to provide an SSL certificate or put the interactions server behind a secure reverse proxy. For development, you can use ngrok/cloudflare tunnel to expose local ports as secure URL. diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md deleted file mode 100644 index 65ef33ead8..0000000000 --- a/docs/rfcs/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# RFCs - -| Number | Name | Issue | -| ------ | ---- | ----- | diff --git a/src/ArenaAllocator.zig b/src/ArenaAllocator.zig deleted file mode 100644 index 4c62038cab..0000000000 --- a/src/ArenaAllocator.zig +++ /dev/null @@ -1,248 +0,0 @@ -const std = @import("std"); -const bun = @import("root").bun; -const assert = bun.assert; -const mem = std.mem; -const Allocator = std.mem.Allocator; - -/// This allocator takes an existing allocator, wraps it, and provides an interface -/// where you can allocate without freeing, and then free it all together. -pub const ArenaAllocator = struct { - child_allocator: Allocator, - state: State, - - /// Inner state of ArenaAllocator. Can be stored rather than the entire ArenaAllocator - /// as a memory-saving optimization. - pub const State = struct { - buffer_list: std.SinglyLinkedList(usize) = .{}, - end_index: usize = 0, - - pub fn promote(self: State, child_allocator: Allocator) ArenaAllocator { - return .{ - .child_allocator = child_allocator, - .state = self, - }; - } - }; - - pub fn allocator(self: *ArenaAllocator) Allocator { - return .{ - .ptr = self, - .vtable = &.{ - .alloc = alloc, - .resize = resize, - .free = free, - }, - }; - } - - const BufNode = std.SinglyLinkedList(usize).Node; - - pub fn init(child_allocator: Allocator) ArenaAllocator { - return (State{}).promote(child_allocator); - } - - pub fn deinit(self: ArenaAllocator) void { - // NOTE: When changing this, make sure `reset()` is adjusted accordingly! - - var it = self.state.buffer_list.first; - while (it) |node| { - // this has to occur before the free because the free frees node - const next_it = node.next; - const align_bits = std.math.log2_int(usize, @alignOf(BufNode)); - const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data]; - self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress()); - it = next_it; - } - } - - pub const ResetMode = union(enum) { - /// Releases all allocated memory in the arena. - free_all, - /// This will pre-heat the arena for future allocations by allocating a - /// large enough buffer for all previously done allocations. - /// Preheating will speed up the allocation process by invoking the backing allocator - /// less often than before. If `reset()` is used in a loop, this means that after the - /// biggest operation, no memory allocations are performed anymore. - retain_capacity, - /// This is the same as `retain_capacity`, but the memory will be shrunk to - /// this value if it exceeds the limit. - retain_with_limit: usize, - }; - /// Queries the current memory use of this arena. - /// This will **not** include the storage required for internal keeping. - pub fn queryCapacity(self: ArenaAllocator) usize { - var size: usize = 0; - var it = self.state.buffer_list.first; - while (it) |node| : (it = node.next) { - // Compute the actually allocated size excluding the - // linked list node. - size += node.data - @sizeOf(BufNode); - } - return size; - } - /// Resets the arena allocator and frees all allocated memory. - /// - /// `mode` defines how the currently allocated memory is handled. - /// See the variant documentation for `ResetMode` for the effects of each mode. - /// - /// The function will return whether the reset operation was successful or not. - /// If the reallocation failed `false` is returned. The arena will still be fully - /// functional in that case, all memory is released. Future allocations just might - /// be slower. - /// - /// NOTE: If `mode` is `free_mode`, the function will always return `true`. - pub fn reset(self: *ArenaAllocator, mode: ResetMode) bool { - // Some words on the implementation: - // The reset function can be implemented with two basic approaches: - // - Counting how much bytes were allocated since the last reset, and storing that - // information in State. This will make reset fast and alloc only a teeny tiny bit - // slower. - // - Counting how much bytes were allocated by iterating the chunk linked list. This - // will make reset slower, but alloc() keeps the same speed when reset() as if reset() - // would not exist. - // - // The second variant was chosen for implementation, as with more and more calls to reset(), - // the function will get faster and faster. At one point, the complexity of the function - // will drop to amortized O(1), as we're only ever having a single chunk that will not be - // reallocated, and we're not even touching the backing allocator anymore. - // - // Thus, only the first hand full of calls to reset() will actually need to iterate the linked - // list, all future calls are just taking the first node, and only resetting the `end_index` - // value. - const requested_capacity = switch (mode) { - .retain_capacity => self.queryCapacity(), - .retain_with_limit => |limit| @min(limit, self.queryCapacity()), - .free_all => 0, - }; - if (requested_capacity == 0) { - // just reset when we don't have anything to reallocate - self.deinit(); - self.state = State{}; - return true; - } - const total_size = requested_capacity + @sizeOf(BufNode); - const align_bits = std.math.log2_int(usize, @alignOf(BufNode)); - // Free all nodes except for the last one - var it = self.state.buffer_list.first; - const maybe_first_node = while (it) |node| { - // this has to occur before the free because the free frees node - const next_it = node.next; - if (next_it == null) - break node; - const alloc_buf = @as([*]u8, @ptrCast(node))[0..node.data]; - self.child_allocator.rawFree(alloc_buf, align_bits, @returnAddress()); - it = next_it; - } else null; - assert(maybe_first_node == null or maybe_first_node.?.next == null); - // reset the state before we try resizing the buffers, so we definitely have reset the arena to 0. - self.state.end_index = 0; - if (maybe_first_node) |first_node| { - self.state.buffer_list.first = first_node; - // perfect, no need to invoke the child_allocator - if (first_node.data == total_size) - return true; - const first_alloc_buf = @as([*]u8, @ptrCast(first_node))[0..first_node.data]; - if (self.child_allocator.rawResize(first_alloc_buf, align_bits, total_size, @returnAddress())) { - // successful resize - first_node.data = total_size; - } else { - // manual realloc - const new_ptr = self.child_allocator.rawAlloc(total_size, align_bits, @returnAddress()) orelse { - // we failed to preheat the arena properly, signal this to the user. - return false; - }; - self.child_allocator.rawFree(first_alloc_buf, align_bits, @returnAddress()); - const node: *BufNode = @ptrCast(@alignCast(new_ptr)); - node.* = .{ .data = total_size }; - self.state.buffer_list.first = node; - } - } - return true; - } - - fn createNode(self: *ArenaAllocator, prev_len: usize, minimum_size: usize) ?*BufNode { - const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16); - const big_enough_len = prev_len + actual_min_size; - const len = big_enough_len + big_enough_len / 2; - const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode)); - const ptr = self.child_allocator.rawAlloc(len, log2_align, @returnAddress()) orelse - return null; - const buf_node: *BufNode = @ptrCast(@alignCast(ptr)); - buf_node.* = .{ .data = len }; - self.state.buffer_list.prepend(buf_node); - self.state.end_index = 0; - return buf_node; - } - - fn alloc(ctx: *anyopaque, n: usize, log2_ptr_align: u8, ra: usize) ?[*]u8 { - const self: *ArenaAllocator = @ptrCast(@alignCast(ctx)); - _ = ra; - - const ptr_align = @as(usize, 1) << @as(Allocator.Log2Align, @intCast(log2_ptr_align)); - var cur_node = if (self.state.buffer_list.first) |first_node| - first_node - else - (self.createNode(0, n + ptr_align) orelse return null); - while (true) { - const cur_alloc_buf = @as([*]u8, @ptrCast(cur_node))[0..cur_node.data]; - const cur_buf = cur_alloc_buf[@sizeOf(BufNode)..]; - const addr = @intFromPtr(cur_buf.ptr) + self.state.end_index; - const adjusted_addr = mem.alignForward(usize, addr, ptr_align); - const adjusted_index = self.state.end_index + (adjusted_addr - addr); - const new_end_index = adjusted_index + n; - - if (new_end_index <= cur_buf.len) { - const result = cur_buf[adjusted_index..new_end_index]; - self.state.end_index = new_end_index; - return result.ptr; - } - - const bigger_buf_size = @sizeOf(BufNode) + new_end_index; - const log2_align = comptime std.math.log2_int(usize, @alignOf(BufNode)); - if (self.child_allocator.rawResize(cur_alloc_buf, log2_align, bigger_buf_size, @returnAddress())) { - cur_node.data = bigger_buf_size; - } else { - // Allocate a new node if that's not possible - cur_node = self.createNode(cur_buf.len, n + ptr_align) orelse return null; - } - } - } - - fn resize(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, new_len: usize, ret_addr: usize) bool { - const self: *ArenaAllocator = @ptrCast(@alignCast(ctx)); - _ = log2_buf_align; - _ = ret_addr; - - const cur_node = self.state.buffer_list.first orelse return false; - const cur_buf = @as([*]u8, @ptrCast(cur_node))[@sizeOf(BufNode)..cur_node.data]; - if (@intFromPtr(cur_buf.ptr) + self.state.end_index != @intFromPtr(buf.ptr) + buf.len) { - // It's not the most recent allocation, so it cannot be expanded, - // but it's fine if they want to make it smaller. - return new_len <= buf.len; - } - - if (buf.len >= new_len) { - self.state.end_index -= buf.len - new_len; - return true; - } else if (cur_buf.len - self.state.end_index >= new_len - buf.len) { - self.state.end_index += new_len - buf.len; - return true; - } else { - return false; - } - } - - fn free(ctx: *anyopaque, buf: []u8, log2_buf_align: u8, ret_addr: usize) void { - _ = log2_buf_align; - _ = ret_addr; - - const self: *ArenaAllocator = @ptrCast(@alignCast(ctx)); - - const cur_node = self.state.buffer_list.first orelse return; - const cur_buf = @as([*]u8, @ptrCast(cur_node))[@sizeOf(BufNode)..cur_node.data]; - - if (@intFromPtr(cur_buf.ptr) + self.state.end_index == @intFromPtr(buf.ptr) + buf.len) { - self.state.end_index -= buf.len; - } - } -}; diff --git a/src/bake/BakeGlobalObject.cpp b/src/bake/BakeGlobalObject.cpp index 14b8809fc8..44ee0e1854 100644 --- a/src/bake/BakeGlobalObject.cpp +++ b/src/bake/BakeGlobalObject.cpp @@ -2,38 +2,50 @@ #include "JSNextTickQueue.h" #include "JavaScriptCore/GlobalObjectMethodTable.h" #include "JavaScriptCore/JSInternalPromise.h" -#include "ProcessIdentifier.h" #include "headers-handwritten.h" +#include "JavaScriptCore/JSModuleLoader.h" +#include "JavaScriptCore/Completion.h" + +extern "C" BunString BakeProdResolve(JSC::JSGlobalObject*, BunString a, BunString b); namespace Bake { -extern "C" void BakeInitProcessIdentifier() -{ - // assert is on main thread - WebCore::Process::identifier(); -} - JSC::JSInternalPromise* -bakeModuleLoaderImportModule(JSC::JSGlobalObject* jsGlobalObject, - JSC::JSModuleLoader*, JSC::JSString* moduleNameValue, +bakeModuleLoaderImportModule(JSC::JSGlobalObject* global, + JSC::JSModuleLoader* moduleLoader, JSC::JSString* moduleNameValue, JSC::JSValue parameters, const JSC::SourceOrigin& sourceOrigin) { - // TODO: forward this to the runtime? - JSC::VM& vm = jsGlobalObject->vm(); - WTF::String keyString = moduleNameValue->getString(jsGlobalObject); - auto err = JSC::createTypeError( - jsGlobalObject, - WTF::makeString( - "Dynamic import to '"_s, keyString, - "' should have been replaced with a hook into the module runtime"_s)); - auto* promise = JSC::JSInternalPromise::create( - vm, jsGlobalObject->internalPromiseStructure()); - promise->reject(jsGlobalObject, err); - return promise; -} + WTF::String keyString = moduleNameValue->getString(global); + if (keyString.startsWith("bake:/"_s)) { + JSC::VM& vm = global->vm(); + return JSC::importModule(global, JSC::Identifier::fromString(vm, keyString), + JSC::jsUndefined(), parameters, JSC::jsUndefined()); + } -extern "C" BunString BakeProdResolve(JSC::JSGlobalObject*, BunString a, BunString b); + if (!sourceOrigin.isNull() && sourceOrigin.string().startsWith("bake:/"_s)) { + JSC::VM& vm = global->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + WTF::String refererString = sourceOrigin.string(); + WTF::String keyString = moduleNameValue->getString(global); + + if (!keyString) { + auto promise = JSC::JSInternalPromise::create(vm, global->internalPromiseStructure()); + promise->reject(global, JSC::createError(global, "import() requires a string"_s)); + return promise; + } + + BunString result = BakeProdResolve(global, Bun::toString(refererString), Bun::toString(keyString)); + RETURN_IF_EXCEPTION(scope, nullptr); + + return JSC::importModule(global, JSC::Identifier::fromString(vm, result.toWTFString()), + JSC::jsUndefined(), parameters, JSC::jsUndefined()); + } + + // Use Zig::GlobalObject's function + return jsCast(global)->moduleLoaderImportModule(global, moduleLoader, moduleNameValue, parameters, sourceOrigin); +} JSC::Identifier bakeModuleLoaderResolve(JSC::JSGlobalObject* jsGlobal, JSC::JSModuleLoader* loader, JSC::JSValue key, @@ -43,19 +55,21 @@ JSC::Identifier bakeModuleLoaderResolve(JSC::JSGlobalObject* jsGlobal, JSC::VM& vm = global->vm(); auto scope = DECLARE_THROW_SCOPE(vm); - if (global->isProduction()) { - WTF::String keyString = key.toWTFString(global); + ASSERT(referrer.isString()); + WTF::String refererString = jsCast(referrer)->getString(global); + + WTF::String keyString = key.toWTFString(global); + RETURN_IF_EXCEPTION(scope, vm.propertyNames->emptyIdentifier); + + if (refererString.startsWith("bake:/"_s) || (refererString == "."_s && keyString.startsWith("bake:/"_s))) { + BunString result = BakeProdResolve(global, Bun::toString(referrer.getString(global)), Bun::toString(keyString)); RETURN_IF_EXCEPTION(scope, vm.propertyNames->emptyIdentifier); - ASSERT(referrer.isString()); - auto refererString = jsCast(referrer)->value(global); - - BunString result = BakeProdResolve(global, Bun::toString(referrer.getString(global)), Bun::toString(keyString)); return JSC::Identifier::fromString(vm, result.toWTFString(BunString::ZeroCopy)); - } else { - JSC::throwTypeError(global, scope, "External imports are not allowed in Bun Bake's dev server. This is a bug in Bun's bundler."_s); - return vm.propertyNames->emptyIdentifier; } + + // Use Zig::GlobalObject's function + return Zig::GlobalObject::moduleLoaderResolve(jsGlobal, loader, key, referrer, origin); } #define INHERIT_HOOK_METHOD(name) \ @@ -100,12 +114,12 @@ void GlobalObject::finishCreation(JSC::VM& vm) ASSERT(inherits(info())); } +struct BunVirtualMachine; extern "C" BunVirtualMachine* Bun__getVM(); // A lot of this function is taken from 'Zig__GlobalObject__create' // TODO: remove this entire method -extern "C" GlobalObject* BakeCreateDevGlobal(DevServer* owner, - void* console) +extern "C" GlobalObject* BakeCreateProdGlobal(void* console) { JSC::VM& vm = JSC::VM::create(JSC::HeapType::Large).leakRef(); vm.heap.acquireAccess(); @@ -119,7 +133,6 @@ extern "C" GlobalObject* BakeCreateDevGlobal(DevServer* owner, if (!global) BUN_PANIC("Failed to create BakeGlobalObject"); - global->m_devServer = owner; global->m_bunVM = bunVM; JSC::gcProtect(global); @@ -142,25 +155,4 @@ extern "C" GlobalObject* BakeCreateDevGlobal(DevServer* owner, return global; } -extern "C" GlobalObject* BakeCreateProdGlobal(JSC::VM* vm, void* console) -{ - JSC::JSLockHolder locker(vm); - BunVirtualMachine* bunVM = Bun__getVM(); - - JSC::Structure* structure = GlobalObject::createStructure(*vm); - GlobalObject* global = GlobalObject::create(*vm, structure, &GlobalObject::s_globalObjectMethodTable); - if (!global) - BUN_PANIC("Failed to create BakeGlobalObject"); - - global->m_devServer = nullptr; - global->m_bunVM = bunVM; - - JSC::gcProtect(global); - - global->setConsole(console); - global->setStackTraceLimit(10); // Node.js defaults to 10 - - return global; -} - }; // namespace Bake diff --git a/src/bake/BakeGlobalObject.h b/src/bake/BakeGlobalObject.h index 406a509ec1..af2b3490f9 100644 --- a/src/bake/BakeGlobalObject.h +++ b/src/bake/BakeGlobalObject.h @@ -4,17 +4,10 @@ namespace Bake { -struct DevServer; // DevServer.zig -struct Route; // DevServer.zig -struct BunVirtualMachine; - class GlobalObject : public Zig::GlobalObject { public: using Base = Zig::GlobalObject; - /// Null if in production - DevServer* m_devServer; - template static JSC::GCClient::IsoSubspace* subspaceFor(JSC::VM& vm) { if constexpr (mode == JSC::SubspaceAccess::Concurrently) @@ -31,16 +24,10 @@ public: static const JSC::GlobalObjectMethodTable s_globalObjectMethodTable; static GlobalObject* create(JSC::VM& vm, JSC::Structure* structure, const JSC::GlobalObjectMethodTable* methodTable); - ALWAYS_INLINE bool isProduction() const { return !m_devServer; } - void finishCreation(JSC::VM& vm); GlobalObject(JSC::VM& vm, JSC::Structure* structure, const JSC::GlobalObjectMethodTable* methodTable) : Zig::GlobalObject(vm, structure, methodTable) { } }; -// Zig API -extern "C" void KitInitProcessIdentifier(); -extern "C" GlobalObject* KitCreateDevGlobal(DevServer* owner, void* console); - }; // namespace Kit diff --git a/src/bake/BakeProduction.cpp b/src/bake/BakeProduction.cpp index 726a8ea258..887dbb565e 100644 --- a/src/bake/BakeProduction.cpp +++ b/src/bake/BakeProduction.cpp @@ -6,25 +6,33 @@ namespace Bake { -extern "C" JSC::JSPromise* BakeRenderRoutesForProd( +extern "C" JSC::JSPromise* BakeRenderRoutesForProdStatic( JSC::JSGlobalObject* global, - BunString outbase, - JSC::JSValue renderStaticCallback, + BunString outBase, + JSC::JSValue allServerFiles, + JSC::JSValue renderStatic, JSC::JSValue clientEntryUrl, + JSC::JSValue pattern, JSC::JSValue files, - JSC::JSValue patterns, + JSC::JSValue typeAndFlags, + JSC::JSValue sourceRouteFiles, + JSC::JSValue paramInformation, JSC::JSValue styles) { JSC::VM& vm = global->vm(); - JSC::JSFunction* cb = JSC::JSFunction::create(vm, global, WebCore::bakeRenderRoutesForProdCodeGenerator(vm), global); + JSC::JSFunction* cb = JSC::JSFunction::create(vm, global, WebCore::bakeRenderRoutesForProdStaticCodeGenerator(vm), global); JSC::CallData callData = JSC::getCallData(cb); JSC::MarkedArgumentBuffer args; - args.append(JSC::jsString(vm, outbase.toWTFString())); - args.append(renderStaticCallback); + args.append(JSC::jsString(vm, outBase.toWTFString())); + args.append(allServerFiles); + args.append(renderStatic); args.append(clientEntryUrl); + args.append(pattern); args.append(files); - args.append(patterns); + args.append(typeAndFlags); + args.append(sourceRouteFiles); + args.append(paramInformation); args.append(styles); NakedPtr returnedException = nullptr; diff --git a/src/bake/BakeSourceProvider.cpp b/src/bake/BakeSourceProvider.cpp index b821d13670..cf7ef839ab 100644 --- a/src/bake/BakeSourceProvider.cpp +++ b/src/bake/BakeSourceProvider.cpp @@ -8,33 +8,40 @@ #include "JavaScriptCore/JSLock.h" #include "JavaScriptCore/JSMap.h" #include "JavaScriptCore/JSModuleLoader.h" +#include "JavaScriptCore/JSModuleRecord.h" #include "JavaScriptCore/JSString.h" #include "JavaScriptCore/JSModuleNamespaceObject.h" +#include "ImportMetaObject.h" namespace Bake { -extern "C" LoadServerCodeResult BakeLoadInitialServerCode(GlobalObject* global, BunString source) { +extern "C" JSC::EncodedJSValue BakeLoadInitialServerCode(GlobalObject* global, BunString source, bool separateSSRGraph) { JSC::VM& vm = global->vm(); auto scope = DECLARE_THROW_SCOPE(vm); - String string = "bake://server.js"_s; - JSC::JSString* key = JSC::jsString(vm, string); + String string = "bake://server-runtime.js"_s; JSC::SourceOrigin origin = JSC::SourceOrigin(WTF::URL(string)); JSC::SourceCode sourceCode = JSC::SourceCode(DevSourceProvider::create( source.toWTFString(), origin, WTFMove(string), WTF::TextPosition(), - JSC::SourceProviderSourceType::Module + JSC::SourceProviderSourceType::Program )); - global->moduleLoader()->provideFetch(global, key, sourceCode); - RETURN_IF_EXCEPTION(scope, {}); - - JSC::JSInternalPromise* internalPromise = global->moduleLoader()->loadAndEvaluateModule(global, key, JSC::jsUndefined(), JSC::jsUndefined()); - RETURN_IF_EXCEPTION(scope, {}); + JSC::JSValue fnValue = vm.interpreter.executeProgram(sourceCode, global, global); + RETURN_IF_EXCEPTION(scope, JSC::JSValue::encode({})); - return { internalPromise, key }; + RELEASE_ASSERT(fnValue); + + JSC::JSFunction* fn = jsCast(fnValue); + JSC::CallData callData = JSC::getCallData(fn); + + JSC::MarkedArgumentBuffer args; + args.append(JSC::jsBoolean(separateSSRGraph)); // separateSSRGraph + args.append(Zig::ImportMetaObject::create(global, "bake://server-runtime.js"_s)); // importMeta + + return JSC::JSValue::encode(JSC::call(global, fn, callData, JSC::jsUndefined(), args)); } extern "C" JSC::JSInternalPromise* BakeLoadModuleByKey(GlobalObject* global, JSC::JSString* key) { diff --git a/src/bake/BakeSourceProvider.h b/src/bake/BakeSourceProvider.h index 191dd927f7..2d821fc401 100644 --- a/src/bake/BakeSourceProvider.h +++ b/src/bake/BakeSourceProvider.h @@ -6,11 +6,6 @@ namespace Bake { -struct LoadServerCodeResult { - JSC::JSInternalPromise* promise; - JSC::JSString* key; -}; - class DevSourceProvider final : public JSC::StringSourceProvider { public: static Ref create( diff --git a/src/bake/DevServer.zig b/src/bake/DevServer.zig index c07599f638..fe5e658d45 100644 --- a/src/bake/DevServer.zig +++ b/src/bake/DevServer.zig @@ -1,7 +1,11 @@ -//! Instance of the development server. Controls an event loop, web server, -//! bundling state, filesystem watcher, and JavaScript VM instance. +//! Instance of the development server. Attaches to an instance of `Bun.serve`, +//! controlling bundler, routing, and hot module reloading. //! -//! All work is cached in-memory. +//! Reprocessing files that did not change is banned; by having perfect +//! incremental tracking over the project, editing a file's contents (asides +//! adjusting imports) must always rebundle only that one file. +//! +//! All work is held in-memory, using manually managed data-oriented design. //! //! TODO: Currently does not have a `deinit()`, as it was assumed to be alive for //! the remainder of this process' lifespan. Later, it will be required to fully @@ -11,14 +15,12 @@ pub const debug = bun.Output.Scoped(.Bake, false); pub const igLog = bun.Output.scoped(.IncrementalGraph, false); pub const Options = struct { - allocator: ?Allocator = null, // defaults to a named heap - cwd: []u8, - routes: []Route, + root: []const u8, framework: bake.Framework, - listen_config: uws.AppListenConfig = .{ .port = 3000 }, dump_sources: ?[]const u8 = if (Environment.isDebug) ".bake-debug" else null, + dump_state_on_crash: bool = bun.FeatureFlags.bake_debugging_features, verbose_watcher: bool = false, - // TODO: make it required to inherit a js VM + vm: *VirtualMachine, }; // The fields `client_graph`, `server_graph`, and `directory_watchers` all @@ -29,28 +31,48 @@ pub const Options = struct { /// Used for all server-wide allocations. In debug, this shows up in /// a separate named heap. Thread-safe. allocator: Allocator, -/// Project root directory. For the HMR runtime, its -/// module IDs are strings relative to this. -cwd: []const u8, +/// Absolute path to project root directory. For the HMR +/// runtime, its module IDs are strings relative to this. +root: []const u8, /// Hex string generated by hashing the framework config and bun revision. /// Emebedding in client bundles and sent when the HMR Socket is opened; /// When the value mismatches the page is forcibly reloaded. configuration_hash_key: [16]u8, - -// UWS App -app: *App, -routes: []Route, -address: struct { - port: u16, - hostname: [*:0]const u8, -}, -listener: ?*App.ListenSocket, - -// Server Runtime -server_global: *DevGlobalObject, +/// The virtual machine (global object) to execute code in. vm: *VirtualMachine, +/// May be `null` if not attached to an HTTP server yet. +server: ?bun.JSC.API.AnyServer, +/// Contains the tree of routes. This structure contains FileIndex +router: FrameworkRouter, +/// Every navigatable route has bundling state here. +route_bundles: ArrayListUnmanaged(RouteBundle), +/// All access into IncrementalGraph is guarded by a DebugThreadLock. This is +/// only a debug assertion as contention to this is always a bug; If a bundle is +/// active and a file is changed, that change is placed into the next bundle. +graph_safety_lock: bun.DebugThreadLock, +client_graph: IncrementalGraph(.client), +server_graph: IncrementalGraph(.server), +/// State populated during bundling and hot updates. Often cleared +incremental_result: IncrementalResult, +/// CSS files are accessible via `/_bun/css/.css` +/// Value is bundled code owned by `dev.allocator` +css_files: AutoArrayHashMapUnmanaged(u64, []const u8), +/// JS files are accessible via `/_bun/client/route..js` +/// These are randomly generated to avoid possible browser caching of old assets. +route_js_payloads: AutoArrayHashMapUnmanaged(u64, Route.Index), +// /// Assets are accessible via `/_bun/asset/` +// assets: bun.StringArrayHashMapUnmanaged(u64, Asset), +/// All bundling failures are stored until a file is saved and rebuilt. +/// They are stored in the wire format the HMR runtime expects so that +/// serialization only happens once. +bundling_failures: std.ArrayHashMapUnmanaged( + SerializedFailure, + void, + SerializedFailure.ArrayHashContextViaOwner, + false, +) = .{}, -// These values are handles to the functions in server_exports. +// These values are handles to the functions in `hmr-runtime-server.ts`. // For type definitions, see `./bake.private.d.ts` server_fetch_function_callback: JSC.Strong, server_register_update_callback: JSC.Strong, @@ -67,79 +89,70 @@ watch_events: [2]HotReloadTask.Aligned, watch_state: std.atomic.Value(u32), watch_current: u1 = 0, -// Bundling +/// Number of bundles that have been executed. This is currently not read, but +/// will be used later to determine when to invoke graph garbage collection. generation: usize = 0, +/// Displayed in the HMR success indicator bundles_since_last_error: usize = 0, -/// All access into IncrementalGraph is guarded by this. This is only -/// a debug assertion since there is no actual contention. -graph_safety_lock: bun.DebugThreadLock, -client_graph: IncrementalGraph(.client), -server_graph: IncrementalGraph(.server), -/// CSS files are accessible via `/_bun/css/.css` -/// Value is bundled code. -css_files: AutoArrayHashMapUnmanaged(u64, []const u8), -// /// Assets are accessible via `/_bun/asset/` -// assets: bun.StringArrayHashMapUnmanaged(u64, Asset), -/// All bundling failures are stored until a file is saved and rebuilt. -/// They are stored in the wire format the HMR runtime expects so that -/// serialization only happens once. -bundling_failures: std.ArrayHashMapUnmanaged( - SerializedFailure, - void, - SerializedFailure.ArrayHashContextViaOwner, - false, -) = .{}, -/// Quickly retrieve a route's index from the entry point file. -route_lookup: AutoArrayHashMapUnmanaged(IncrementalGraph(.server).FileIndex, Route.Index), -/// State populated during bundling. Often cleared -incremental_result: IncrementalResult, + +/// Quickly retrieve a route's index from the entry point file. These are +/// populated as the routes are discovered. The route may not be bundled or +/// navigatable, in the case a layout's index is looked up. +route_lookup: AutoArrayHashMapUnmanaged(IncrementalGraph(.server).FileIndex, RouteIndexAndRecurseFlag), + framework: bake.Framework, // Each logical graph gets its own bundler configuration server_bundler: Bundler, client_bundler: Bundler, ssr_bundler: Bundler, + +// TODO: This being shared state is likely causing a crash /// Stored and reused for bundling tasks log: Log, // Debugging dump_dir: ?std.fs.Dir, +/// Reference count to number of active sockets with the visualizer enabled. emit_visualizer_events: u32, +has_pre_crash_handler: bool, pub const internal_prefix = "/_bun"; pub const client_prefix = internal_prefix ++ "/client"; pub const asset_prefix = internal_prefix ++ "/asset"; pub const css_prefix = internal_prefix ++ "/css"; -pub const Route = struct { - pub const Index = bun.GenericIndex(u30, Route); +pub const RouteBundle = struct { + pub const Index = bun.GenericIndex(u30, RouteBundle); - // Config - pattern: [:0]const u8, - entry_point: []const u8, + route: Route.Index, - server_state: State = .unqueued, - /// Cached to avoid looking up by filename in `server_graph` - server_file: IncrementalGraph(.server).FileIndex.Optional = .none, + server_state: State, + + /// Used to communicate over WebSocket the pattern. The HMR client contains code + /// to match this against the URL bar to determine if a reloading route applies + /// or not. + full_pattern: []const u8, /// Generated lazily when the client JS is requested (HTTP GET /_bun/client/*.js), /// which is only needed when a hard-reload is performed. /// /// Freed when a client module updates. - client_bundle: ?[]const u8 = null, + client_bundle: ?[]const u8, /// Contain the list of serialized failures. Hashmap allows for /// efficient lookup and removal of failing files. /// When state == .evaluation_failure, this is popualted with that error. - evaluate_failure: ?SerializedFailure = null, + evaluate_failure: ?SerializedFailure, - /// Cached to avoid re-creating the string every request - module_name_string: JSC.Strong = .{}, - /// Cached to avoid re-creating the string every request - client_bundle_url_value: JSC.Strong = .{}, - /// Cached to avoid re-creating the array every request - css_file_array: JSC.Strong = .{}, + // TODO: micro-opt: use a singular strong - /// Assigned in DevServer.init - dev: *DevServer = undefined, - client_bundled_url: []u8 = undefined, + /// Cached to avoid re-creating the array every request. + /// Invalidated when a layout is added or removed from this route. + cached_module_list: JSC.Strong, + /// Cached to avoid re-creating the string every request. + /// Invalidated when any client file associated with the route is updated. + cached_client_bundle_url: JSC.Strong, + /// Cached to avoid re-creating the array every request. + /// Invalidated when the list of CSS files changes. + cached_css_file_array: JSC.Strong, /// A union is not used so that `bundler_failure_logs` can re-use memory, as /// this state frequently changes between `loaded` and the failure variants. @@ -147,8 +160,11 @@ pub const Route = struct { /// In development mode, routes are lazily built. This state implies a /// build of this route has never been run. It is possible to bundle the /// route entry point and still have an unqueued route if another route - /// imports this one. + /// imports this one. This state is implied if `FrameworkRouter.Route` + /// has no bundle index assigned. unqueued, + /// A bundle associated with this route is happening + bundling, /// This route was flagged for bundling failures. There are edge cases /// where a route can be disconnected from its failures, so the route /// imports has to be traced to discover if possible failures still @@ -162,51 +178,48 @@ pub const Route = struct { }; }; -const Asset = union(enum) { - /// File contents are allocated with `dev.allocator` - /// The slice is mirrored in `dev.client_graph.bundled_files`, so freeing this slice is not required. - css: []const u8, - /// A file path relative to cwd, owned by `dev.allocator` - file_path: []const u8, +pub const DeferredRequest = struct { + next: ?*DeferredRequest, + bundle: RouteBundle.Index, + data: Data, + + const Data = union(enum) { + server_handler: bun.JSC.API.SavedRequest, + /// onJsRequestWithBundle + js_payload: *Response, + + const Tag = @typeInfo(Data).Union.tag_type.?; + }; }; /// DevServer is stored on the heap, storing its allocator. +// TODO: change the error set to JSOrMemoryError!*DevServer pub fn init(options: Options) !*DevServer { - const allocator = options.allocator orelse bun.default_allocator; + const allocator = bun.default_allocator; bun.analytics.Features.kit_dev +|= 1; - if (JSC.VirtualMachine.VMHolder.vm != null) - @panic("Cannot initialize bake.DevServer on a thread with an active JSC.VirtualMachine"); - const dump_dir = if (options.dump_sources) |dir| - std.fs.cwd().makeOpenPath(dir, .{}) catch |err| dir: { - bun.handleErrorReturnTrace(err, @errorReturnTrace()); - Output.warn("Could not open directory for dumping sources: {}", .{err}); - break :dir null; - } - else - null; - - const app = App.create(.{}) orelse { - Output.prettyErrorln("Failed to create app", .{}); - return error.AppInitialization; - }; + var dump_dir = if (bun.FeatureFlags.bake_debugging_features) + if (options.dump_sources) |dir| + std.fs.cwd().makeOpenPath(dir, .{}) catch |err| dir: { + bun.handleErrorReturnTrace(err, @errorReturnTrace()); + Output.warn("Could not open directory for dumping sources: {}", .{err}); + break :dir null; + } + else + null; + errdefer if (bun.FeatureFlags.bake_debugging_features) if (dump_dir) |*dir| dir.close(); const separate_ssr_graph = if (options.framework.server_components) |sc| sc.separate_ssr_graph else false; const dev = bun.create(allocator, DevServer, .{ .allocator = allocator, - .cwd = options.cwd, - .app = app, - .routes = options.routes, - .address = .{ - .port = @intCast(options.listen_config.port), - .hostname = options.listen_config.host orelse "localhost", - }, + .root = options.root, + .vm = options.vm, + .server = null, .directory_watchers = DirectoryWatchStore.empty, .server_fetch_function_callback = .{}, .server_register_update_callback = .{}, - .listener = null, .generation = 0, .graph_safety_lock = .{}, .log = Log.init(allocator), @@ -215,7 +228,9 @@ pub fn init(options: Options) !*DevServer { .watch_state = .{ .raw = 0 }, .watch_current = 0, .emit_visualizer_events = 0, + .has_pre_crash_handler = options.dump_state_on_crash, .css_files = .{}, + .route_js_payloads = .{}, // .assets = .{}, .client_graph = IncrementalGraph(.client).empty, @@ -227,13 +242,13 @@ pub fn init(options: Options) !*DevServer { .client_bundler = undefined, .ssr_bundler = undefined, - .server_global = undefined, - .vm = undefined, - .bun_watcher = undefined, .watch_events = undefined, .configuration_hash_key = undefined, + + .router = undefined, + .route_bundles = .{}, }); errdefer allocator.destroy(dev); @@ -241,7 +256,10 @@ pub fn init(options: Options) !*DevServer { assert(dev.client_graph.owner() == dev); assert(dev.directory_watchers.owner() == dev); - const fs = try bun.fs.FileSystem.init(options.cwd); + dev.graph_safety_lock.lock(); + defer dev.graph_safety_lock.unlock(); + + const fs = try bun.fs.FileSystem.init(options.root); dev.bun_watcher = try Watcher.init(DevServer, dev, fs, bun.default_allocator); errdefer dev.bun_watcher.deinit(false); @@ -264,25 +282,16 @@ pub fn init(options: Options) !*DevServer { dev.ssr_bundler.options.dev_server = dev; } - dev.framework = dev.framework.resolve( - &dev.server_bundler.resolver, - &dev.client_bundler.resolver, - ) catch { - // bun i react@experimental react-dom@experimental react-server-dom-webpack@experimental react-refresh@experimental + dev.framework = dev.framework.resolve(&dev.server_bundler.resolver, &dev.client_bundler.resolver) catch { Output.errGeneric("Failed to resolve all imports required by the framework", .{}); return error.FrameworkInitialization; }; - dev.vm = VirtualMachine.initKit(.{ - .allocator = bun.default_allocator, - .args = std.mem.zeroes(bun.Schema.Api.TransformOptions), - }) catch |err| - Output.panic("Failed to create Global object: {}", .{err}); - dev.server_global = c.BakeCreateDevGlobal(dev, dev.vm.console); - dev.vm.global = dev.server_global.js(); - dev.vm.regular_event_loop.global = dev.vm.global; - dev.vm.jsc = dev.vm.global.vm(); - dev.vm.event_loop.ensureWaker(); + errdefer dev.route_lookup.clearAndFree(allocator); + // errdefer dev.client_graph.deinit(allocator); + // errdefer dev.server_graph.deinit(allocator); + + dev.vm.global = @ptrCast(dev.vm.global); dev.configuration_hash_key = hash_key: { var hash = std.hash.Wyhash.init(128); @@ -296,8 +305,10 @@ pub fn init(options: Options) !*DevServer { hash.update(bun.Environment.git_sha_short); } - hash.update(dev.framework.entry_client); - hash.update(dev.framework.entry_server); + // TODO: hash router types + // hash.update(dev.framework.entry_client); + // hash.update(dev.framework.entry_server); + if (dev.framework.server_components) |sc| { bun.writeAnyToHasher(&hash, true); bun.writeAnyToHasher(&hash, sc.separate_ssr_graph); @@ -325,20 +336,106 @@ pub fn init(options: Options) !*DevServer { break :hash_key std.fmt.bytesToHex(std.mem.asBytes(&hash.final()), .lower); }; - var has_fallback = false; + // Add react fast refresh if needed. This is the first file on the client side, + // as it will be referred to by index. + if (dev.framework.react_fast_refresh) |rfr| { + assert(try dev.client_graph.insertStale(rfr.import_source, false) == IncrementalGraph(.client).react_refresh_index); + } - for (options.routes, 0..) |*route, i| { - app.any(route.pattern, *Route, route, onServerRequest); + try dev.initServerRuntime(); - route.dev = dev; - route.client_bundled_url = std.fmt.allocPrint( - allocator, - client_prefix ++ "/{d}.js", - .{i}, - ) catch bun.outOfMemory(); + // Initialize the router + dev.router = router: { + var types = try std.ArrayListUnmanaged(FrameworkRouter.Type).initCapacity(allocator, options.framework.file_system_router_types.len); + errdefer types.deinit(allocator); - if (bun.strings.eqlComptime(route.pattern, "/*")) - has_fallback = true; + for (options.framework.file_system_router_types, 0..) |fsr, i| { + const joined_root = bun.path.joinAbs(dev.root, .auto, fsr.root); + const entry = dev.server_bundler.resolver.readDirInfoIgnoreError(joined_root) orelse + continue; + + const server_file = try dev.server_graph.insertStaleExtra(fsr.entry_server, false, true); + + try types.append(allocator, .{ + .abs_root = bun.strings.withoutTrailingSlash(entry.abs_path), + .prefix = fsr.prefix, + .ignore_underscores = fsr.ignore_underscores, + .ignore_dirs = fsr.ignore_dirs, + .extensions = fsr.extensions, + .style = fsr.style, + .server_file = toOpaqueFileId(.server, server_file), + .client_file = if (fsr.entry_client) |client| + toOpaqueFileId(.client, try dev.client_graph.insertStale(client, false)).toOptional() + else + .none, + .server_file_string = .{}, + }); + + try dev.route_lookup.put(allocator, server_file, .{ + .route_index = FrameworkRouter.Route.Index.init(@intCast(i)), + .should_recurse_when_visiting = true, + }); + } + + break :router try FrameworkRouter.initEmpty(types.items, allocator); + }; + + // TODO: move pre-bundling to be one tick after server startup. + // this way the line saying the server is ready shows quicker + try dev.scanInitialRoutes(); + + if (bun.FeatureFlags.bake_debugging_features and options.dump_state_on_crash) + try bun.crash_handler.appendPreCrashHandler(DevServer, dev, dumpStateDueToCrash); + + return dev; +} + +fn initServerRuntime(dev: *DevServer) !void { + const runtime = bun.String.static(bun.bake.getHmrRuntime(.server)); + + const interface = c.BakeLoadInitialServerCode( + @ptrCast(dev.vm.global), + runtime, + if (dev.framework.server_components) |sc| sc.separate_ssr_graph else false, + ) catch |err| { + dev.vm.printErrorLikeObjectToConsole(dev.vm.global.takeException(err)); + @panic("Server runtime failed to start. The above error is always a bug in Bun"); + }; + + if (!interface.isObject()) + @panic("Internal assertion failure: expected interface from HMR runtime to be an object"); + const fetch_function: JSValue = interface.get(dev.vm.global, "handleRequest") orelse + @panic("Internal assertion failure: expected interface from HMR runtime to contain handleRequest"); + bun.assert(fetch_function.isCallable(dev.vm.jsc)); + dev.server_fetch_function_callback = JSC.Strong.create(fetch_function, dev.vm.global); + const register_update = interface.get(dev.vm.global, "registerUpdate") orelse + @panic("Internal assertion failure: expected interface from HMR runtime to contain registerUpdate"); + dev.server_register_update_callback = JSC.Strong.create(register_update, dev.vm.global); + + fetch_function.ensureStillAlive(); + register_update.ensureStillAlive(); +} + +/// Deferred one tick so that the server can be up faster +fn scanInitialRoutes(dev: *DevServer) !void { + try dev.router.scanAll( + dev.allocator, + &dev.server_bundler.resolver, + FrameworkRouter.InsertionContext.wrap(DevServer, dev), + ); + + try dev.server_graph.ensureStaleBitCapacity(true); + try dev.client_graph.ensureStaleBitCapacity(true); +} + +pub fn attachRoutes(dev: *DevServer, server: anytype) !void { + dev.server = bun.JSC.API.AnyServer.from(server); + const app = server.app.?; + + // For this to work, the route handlers need to be augmented to use the comptime + // SSL parameter. It's worth considering removing the SSL boolean. + if (@TypeOf(app) == *uws.NewApp(true)) { + bun.todoPanic(@src(), "DevServer does not support SSL yet", .{}); } app.get(client_prefix ++ "/:route", *DevServer, dev, onJsRequest); @@ -355,133 +452,31 @@ pub fn init(options: Options) !*DevServer { app.get(internal_prefix ++ "/incremental_visualizer", *DevServer, dev, onIncrementalVisualizer); - if (!has_fallback) - app.any("/*", void, {}, onFallbackRoute); - - // Some indices at the start of the graph are reserved for framework files. - { - dev.graph_safety_lock.lock(); - defer dev.graph_safety_lock.unlock(); - - assert(try dev.client_graph.insertStale(dev.framework.entry_client, false) == IncrementalGraph(.client).framework_entry_point_index); - assert(try dev.server_graph.insertStale(dev.framework.entry_server, false) == IncrementalGraph(.server).framework_entry_point_index); - - if (dev.framework.react_fast_refresh) |rfr| { - assert(try dev.client_graph.insertStale(rfr.import_source, false) == IncrementalGraph(.client).react_refresh_index); - } - - try dev.client_graph.ensureStaleBitCapacity(true); - try dev.server_graph.ensureStaleBitCapacity(true); - - const client_files = dev.client_graph.bundled_files.values(); - client_files[IncrementalGraph(.client).framework_entry_point_index.get()].flags.is_special_framework_file = true; - } - - // Pre-bundle the framework code - { - // Since this will enter JavaScript to load code, ensure we have a lock. - const lock = dev.vm.jsc.getAPILock(); - defer lock.release(); - - dev.bundle(&.{ - BakeEntryPoint.init(dev.framework.entry_server, .server), - BakeEntryPoint.init(dev.framework.entry_client, .client), - }) catch |err| { - _ = &err; // autofix - bun.todoPanic(@src(), "handle error", .{}); - }; - } - - app.listenWithConfig(*DevServer, dev, onListen, options.listen_config); - - return dev; + app.any("/*", *DevServer, dev, onRequest); } -fn deinit(dev: *DevServer) void { +pub fn deinit(dev: *DevServer) void { const allocator = dev.allocator; + if (dev.has_pre_crash_handler) + bun.crash_handler.removePreCrashHandler(dev); allocator.destroy(dev); - bun.todoPanic(@src(), "bake.DevServer.deinit()"); -} - -pub fn runLoopForever(dev: *DevServer) noreturn { - const lock = dev.vm.jsc.getAPILock(); - defer lock.release(); - - while (true) { - dev.vm.tick(); - dev.vm.eventLoop().autoTickActive(); - } -} - -// uws handlers - -fn onListen(ctx: *DevServer, maybe_listen: ?*App.ListenSocket) void { - const listen: *App.ListenSocket = maybe_listen orelse { - bun.todoPanic(@src(), "handle listen failure", .{}); - }; - - ctx.listener = listen; - ctx.address.port = @intCast(listen.getLocalPort()); - - Output.prettyErrorln("--\\> http://{s}:{d}\n", .{ - bun.span(ctx.address.hostname), - ctx.address.port, - }); - Output.flush(); + bun.todoPanic(@src(), "bake.DevServer.deinit()", .{}); } fn onJsRequest(dev: *DevServer, req: *Request, resp: *Response) void { - const route = route: { + const route_bundle = route: { const route_id = req.parameter(0); if (!bun.strings.hasSuffixComptime(route_id, ".js")) return req.setYield(true); - const i = std.fmt.parseInt(u16, route_id[0 .. route_id.len - 3], 10) catch + if (!bun.strings.hasPrefixComptime(route_id, "route.")) return req.setYield(true); - if (i >= dev.routes.len) + const i = parseHexToInt(u64, route_id["route.".len .. route_id.len - ".js".len]) orelse + return req.setYield(true); + break :route dev.route_js_payloads.get(i) orelse return req.setYield(true); - break :route &dev.routes[i]; }; - const js_source = route.client_bundle orelse code: { - if (route.server_state == .unqueued) { - dev.bundleRouteFirstTime(route); - } - - switch (route.server_state) { - .unqueued => bun.assertWithLocation(false, @src()), - .possible_bundling_failures => { - if (dev.bundling_failures.count() > 0) { - resp.corked(sendSerializedFailures, .{ - dev, - resp, - dev.bundling_failures.keys(), - .bundler, - }); - return; - } else { - route.server_state = .loaded; - } - }, - .evaluation_failure => { - resp.corked(sendSerializedFailures, .{ - dev, - resp, - &.{route.evaluate_failure orelse @panic("missing error")}, - .evaluation, - }); - return; - }, - .loaded => {}, - } - - // TODO: there can be stale files in this if you request an asset after - // a watch but before the bundle task starts. - - const out = dev.generateClientBundle(route) catch bun.outOfMemory(); - route.client_bundle = out; - break :code out; - }; - sendTextFile(js_source, MimeType.javascript.value, resp); + dev.ensureRouteIsBundled(route_bundle, .js_payload, req, resp) catch bun.outOfMemory(); } fn onAssetRequest(dev: *DevServer, req: *Request, resp: *Response) void { @@ -515,6 +510,12 @@ fn onCssRequest(dev: *DevServer, req: *Request, resp: *Response) void { sendTextFile(css, MimeType.css.value, resp); } +fn parseHexToInt(comptime T: type, slice: []const u8) ?T { + var out: [@sizeOf(T)]u8 = undefined; + assert((std.fmt.hexToBytes(&out, slice) catch return null).len == @sizeOf(T)); + return @bitCast(out); +} + fn onIncrementalVisualizer(_: *DevServer, _: *Request, resp: *Response) void { resp.corked(onIncrementalVisualizerCorked, .{resp}); } @@ -528,38 +529,65 @@ fn onIncrementalVisualizerCorked(resp: *Response) void { resp.end(code, false); } -/// `route.server_state` must be `.unenqueued` -fn bundleRouteFirstTime(dev: *DevServer, route: *Route) void { - if (Environment.allow_assert) switch (route.server_state) { - .unqueued => {}, - .possible_bundling_failures => unreachable, // should watch affected files and bundle on save - .evaluation_failure => unreachable, // bundling again wont fix this issue - .loaded => unreachable, // should not be bundling since it already passed - }; +fn ensureRouteIsBundled( + dev: *DevServer, + route_index: Route.Index, + kind: DeferredRequest.Data.Tag, + req: *Request, + resp: *Response, +) bun.OOM!void { + const bundle_index = if (dev.router.routePtr(route_index).bundle.unwrap()) |bundle_index| + bundle_index + else + try dev.insertRouteBundle(route_index); - if (dev.bundle(&.{ - BakeEntryPoint.route( - route.entry_point, - Route.Index.init(@intCast(bun.indexOfPointerInSlice(Route, dev.routes, route))), - ), - })) |_| { - route.server_state = .loaded; - } else |err| switch (err) { - error.OutOfMemory => bun.outOfMemory(), - error.BuildFailed => assert(route.server_state == .possible_bundling_failures), - error.ServerLoadFailed => route.server_state = .evaluation_failure, + switch (dev.routeBundlePtr(bundle_index).server_state) { + .unqueued => { + const server_file_names = dev.server_graph.bundled_files.keys(); + const client_file_names = dev.client_graph.bundled_files.keys(); + + var sfa = std.heap.stackFallback(4096, dev.allocator); + const temp_alloc = sfa.get(); + + var entry_points = std.ArrayList(BakeEntryPoint).init(temp_alloc); + defer entry_points.deinit(); + + // Build a list of all files that have not yet been bundled. + var route = dev.router.routePtr(route_index); + const router_type = dev.router.typePtr(route.type); + try dev.appendOpaqueEntryPoint(server_file_names, &entry_points, .server, router_type.server_file); + try dev.appendOpaqueEntryPoint(client_file_names, &entry_points, .client, router_type.client_file); + try dev.appendOpaqueEntryPoint(server_file_names, &entry_points, .server, route.file_page); + try dev.appendOpaqueEntryPoint(server_file_names, &entry_points, .server, route.file_layout); + while (route.parent.unwrap()) |parent_index| { + route = dev.router.routePtr(parent_index); + try dev.appendOpaqueEntryPoint(server_file_names, &entry_points, .server, route.file_layout); + } + + if (entry_points.items.len == 0) { + @panic("TODO: trace graph for possible errors, so DevServer knows what state this should go to"); + } + + const route_bundle = dev.routeBundlePtr(bundle_index); + if (dev.bundle(entry_points.items)) |_| { + route_bundle.server_state = .loaded; + } else |err| switch (err) { + error.OutOfMemory => bun.outOfMemory(), + error.BuildFailed => assert(route_bundle.server_state == .possible_bundling_failures), + error.ServerLoadFailed => route_bundle.server_state = .evaluation_failure, + } + }, + .bundling => { + const prepared = dev.server.?.DebugHTTPServer.prepareJsRequestContext(req, resp) orelse + return; + _ = prepared; + @panic("TODO: Async Bundler"); + }, + else => {}, } -} - -fn onServerRequest(route: *Route, req: *Request, resp: *Response) void { - const dev = route.dev; - - if (route.server_state == .unqueued) { - dev.bundleRouteFirstTime(route); - } - - switch (route.server_state) { - .unqueued => bun.assertWithLocation(false, @src()), + switch (dev.routeBundlePtr(bundle_index).server_state) { + .unqueued => unreachable, + .bundling => @panic("TODO: Async Bundler"), .possible_bundling_failures => { // TODO: perform a graph trace to find just the errors that are needed if (dev.bundling_failures.count() > 0) { @@ -571,14 +599,14 @@ fn onServerRequest(route: *Route, req: *Request, resp: *Response) void { }); return; } else { - route.server_state = .loaded; + dev.routeBundlePtr(bundle_index).server_state = .loaded; } }, .evaluation_failure => { resp.corked(sendSerializedFailures, .{ dev, resp, - (&(route.evaluate_failure orelse @panic("missing error")))[0..1], + (&(dev.routeBundlePtr(bundle_index).evaluate_failure orelse @panic("missing error")))[0..1], .evaluation, }); return; @@ -586,101 +614,94 @@ fn onServerRequest(route: *Route, req: *Request, resp: *Response) void { .loaded => {}, } - // TODO: this does not move the body, reuse memory, and many other things - // that server.zig does. - const url_bun_string = bun.String.init(req.url()); - defer url_bun_string.deref(); - - const headers = JSC.FetchHeaders.createFromUWS(req); - const request_object = JSC.WebCore.Request.init( - url_bun_string, - headers, - dev.vm.initRequestBodyValue(.Null) catch bun.outOfMemory(), - bun.http.Method.which(req.method()) orelse .GET, - ).new(); - - const js_request = request_object.toJS(dev.server_global.js()); - - const global = dev.server_global.js(); + switch (kind) { + .server_handler => dev.onRequestWithBundle(bundle_index, .{ .stack = req }, resp), + .js_payload => dev.onJsRequestWithBundle(bundle_index, resp), + } +} +fn onRequestWithBundle( + dev: *DevServer, + route_bundle_index: RouteBundle.Index, + req: bun.JSC.API.SavedRequest.Union, + resp: *Response, +) void { const server_request_callback = dev.server_fetch_function_callback.get() orelse unreachable; // did not bundle - var result = server_request_callback.call( - global, - .undefined, - &.{ - // req - js_request, - // routeModuleId - route.module_name_string.get() orelse str: { - const js = bun.String.createUTF8( - bun.path.relative(dev.cwd, route.entry_point), - ).toJS(dev.server_global.js()); - route.module_name_string = JSC.Strong.create(js, dev.server_global.js()); + const route_bundle = dev.routeBundlePtr(route_bundle_index); + + const router_type = dev.router.typePtr(dev.router.routePtr(route_bundle.route).type); + + dev.server.?.onRequestFromSaved( + req, + resp, + server_request_callback, + 4, + .{ + // routerTypeMain + router_type.server_file_string.get() orelse str: { + const name = dev.server_graph.bundled_files.keys()[fromOpaqueFileId(.server, router_type.server_file).get()]; + const str = bun.String.createUTF8(name); + defer str.deref(); + const js = str.toJS(dev.vm.global); + router_type.server_file_string = JSC.Strong.create(js, dev.vm.global); break :str js; }, + // routeModules + route_bundle.cached_module_list.get() orelse arr: { + const global = dev.vm.global; + const keys = dev.server_graph.bundled_files.keys(); + var n: usize = 1; + var route = dev.router.routePtr(route_bundle.route); + while (true) { + if (route.file_layout != .none) n += 1; + route = dev.router.routePtr(route.parent.unwrap() orelse break); + } + const arr = JSValue.createEmptyArray(global, n); + route = dev.router.routePtr(route_bundle.route); + var route_name = bun.String.createUTF8(dev.relativePath(keys[fromOpaqueFileId(.server, route.file_page.unwrap().?).get()])); + arr.putIndex(global, 0, route_name.transferToJS(global)); + n = 1; + while (true) { + if (route.file_layout.unwrap()) |layout| { + var layout_name = bun.String.createUTF8(dev.relativePath(keys[fromOpaqueFileId(.server, layout).get()])); + arr.putIndex(global, @intCast(n), layout_name.transferToJS(global)); + n += 1; + } + route = dev.router.routePtr(route.parent.unwrap() orelse break); + } + route_bundle.cached_module_list = JSC.Strong.create(arr, global); + break :arr arr; + }, // clientId - route.client_bundle_url_value.get() orelse str: { - const js = bun.String.createUTF8(route.client_bundled_url).toJS(global); - route.client_bundle_url_value = JSC.Strong.create(js, dev.server_global.js()); + route_bundle.cached_client_bundle_url.get() orelse str: { + const id = std.crypto.random.int(u64); + dev.route_js_payloads.put(dev.allocator, id, route_bundle.route) catch bun.outOfMemory(); + const str = bun.String.createFormat(client_prefix ++ "/route.{}.js", .{std.fmt.fmtSliceHexLower(std.mem.asBytes(&id))}) catch bun.outOfMemory(); + defer str.deref(); + const js = str.toJS(dev.vm.global); + route_bundle.cached_client_bundle_url = JSC.Strong.create(js, dev.vm.global); break :str js; }, // styles - route.css_file_array.get() orelse arr: { - const js = dev.generateCssList(route) catch bun.outOfMemory(); - route.css_file_array = JSC.Strong.create(js, dev.server_global.js()); + route_bundle.cached_css_file_array.get() orelse arr: { + const js = dev.generateCssList(route_bundle) catch bun.outOfMemory(); + route_bundle.cached_css_file_array = JSC.Strong.create(js, dev.vm.global); break :arr js; }, }, - ) catch |err| { - const exception = global.takeException(err); - dev.vm.printErrorLikeObjectToConsole(exception); - // const fail = try SerializedFailure.initFromJs(.none, exception); - // defer fail.deinit(); - // dev.sendSerializedFailures(resp, &.{fail}, .runtime); - dev.sendStubErrorMessage(route, resp, exception); - return; + ); +} + +pub fn onJsRequestWithBundle(dev: *DevServer, bundle_index: RouteBundle.Index, resp: *Response) void { + const route_bundle = dev.routeBundlePtr(bundle_index); + const code = route_bundle.client_bundle orelse code: { + const code = dev.generateClientBundle(route_bundle) catch bun.outOfMemory(); + route_bundle.client_bundle = code; + break :code code; }; - - if (result.asAnyPromise()) |promise| { - dev.vm.waitForPromise(promise); - switch (promise.unwrap(dev.vm.jsc, .mark_handled)) { - .pending => unreachable, // was waited for - .fulfilled => |r| result = r, - .rejected => |exception| { - dev.vm.printErrorLikeObjectToConsole(exception); - dev.sendStubErrorMessage(route, resp, exception); - // const fail = try SerializedFailure.initFromJs(.none, e); - // defer fail.deinit(); - // dev.sendSerializedFailures(resp, &.{fail}, .runtime); - return; - }, - } - } - - // TODO: This interface and implementation is very poor. It is fine as - // the runtime currently emulates returning a `new Response` - // - // It probably should use code from `server.zig`, but most importantly it should - // not have a tie to DevServer, but instead be generic with a context structure - // containing just a *uws.App, *JSC.EventLoop, and JSValue response object. - // - // This would allow us to support all of the nice things `new Response` allows - - bun.assert(result.isString()); - const bun_string = result.toBunString(dev.server_global.js()); - defer bun_string.deref(); - if (bun_string.tag == .Dead) { - bun.outOfMemory(); - } - - const utf8 = bun_string.toUTF8(dev.allocator); - defer utf8.deinit(); - - resp.writeStatus("200 OK"); - resp.writeHeader("Content-Type", MimeType.html.value); - resp.end(utf8.slice(), true); // TODO: You should never call res.end(huge buffer) + sendTextFile(code, MimeType.javascript.value, resp); } pub fn onSrcRequest(dev: *DevServer, req: *uws.Request, resp: *App.Response) void { @@ -726,6 +747,15 @@ fn bundle(dev: *DevServer, files: []const BakeEntryPoint) BundleError!void { assert(files.len > 0); + const bundle_file_list = bun.Output.Scoped(.bundle_file_list, false); + + if (bundle_file_list.isVisible()) { + bundle_file_list.log("Start bundle {d} files", .{files.len}); + for (files) |f| { + bundle_file_list.log("- {s} (.{s})", .{ f.path, @tagName(f.graph) }); + } + } + var heap = try ThreadlocalArena.init(); defer heap.deinit(); @@ -804,82 +834,43 @@ fn bundle(dev: *DevServer, files: []const BakeEntryPoint) BundleError!void { const is_first_server_chunk = !dev.server_fetch_function_callback.has(); if (dev.server_graph.current_chunk_len > 0) { - const server_bundle = try dev.server_graph.takeBundle(if (is_first_server_chunk) .initial_response else .hmr_chunk); + const server_bundle = try dev.server_graph.takeBundle( + if (is_first_server_chunk) .initial_response else .hmr_chunk, + "", + ); defer dev.allocator.free(server_bundle); - if (is_first_server_chunk) { - const server_code = c.BakeLoadInitialServerCode(dev.server_global, bun.String.createLatin1(server_bundle)) catch |err| { - dev.vm.printErrorLikeObjectToConsole(dev.server_global.js().takeException(err)); - { - // TODO: document the technical reasons this should not be allowed to fail - bun.todoPanic(@src(), "First Server Load Fails. This should become a bundler bug.", .{}); - } - _ = &err; // autofix - // fail.* = Failure.fromJSServerLoad(dev.server_global.js().takeException(err), dev.server_global.js()); - return error.ServerLoadFailed; - }; - dev.vm.waitForPromise(.{ .internal = server_code.promise }); - - switch (server_code.promise.unwrap(dev.vm.jsc, .mark_handled)) { - .pending => unreachable, // promise is settled - .rejected => |err| { - dev.vm.printErrorLikeObjectToConsole(err); - { - bun.todoPanic(@src(), "First Server Load Fails. This should become a bundler bug.", .{}); - } - _ = &err; // autofix - // fail.* = Failure.fromJSServerLoad(err, dev.server_global.js()); - return error.ServerLoadFailed; - }, - .fulfilled => |v| bun.assert(v == .undefined), - } - - const default_export = c.BakeGetDefaultExportFromModule(dev.server_global.js(), server_code.key.toJS()); - if (!default_export.isObject()) - @panic("Internal assertion failure: expected interface from HMR runtime to be an object"); - const fetch_function: JSValue = default_export.get(dev.server_global.js(), "handleRequest") orelse - @panic("Internal assertion failure: expected interface from HMR runtime to contain handleRequest"); - bun.assert(fetch_function.isCallable(dev.vm.jsc)); - dev.server_fetch_function_callback = JSC.Strong.create(fetch_function, dev.server_global.js()); - const register_update = default_export.get(dev.server_global.js(), "registerUpdate") orelse - @panic("Internal assertion failure: expected interface from HMR runtime to contain registerUpdate"); - dev.server_register_update_callback = JSC.Strong.create(register_update, dev.server_global.js()); - - fetch_function.ensureStillAlive(); - register_update.ensureStillAlive(); - } else { - const server_modules = c.BakeLoadServerHmrPatch(dev.server_global, bun.String.createLatin1(server_bundle)) catch |err| { - // No user code has been evaluated yet, since everything is to - // be wrapped in a function clousure. This means that the likely - // error is going to be a syntax error, or other mistake in the - // bundler. - dev.vm.printErrorLikeObjectToConsole(dev.server_global.js().takeException(err)); - @panic("Error thrown while evaluating server code. This is always a bug in the bundler."); - }; - const errors = dev.server_register_update_callback.get().?.call( - dev.server_global.js(), - dev.server_global.js().toJSValue(), - &.{ - server_modules, - dev.makeArrayForServerComponentsPatch(dev.server_global.js(), dev.incremental_result.client_components_added.items), - dev.makeArrayForServerComponentsPatch(dev.server_global.js(), dev.incremental_result.client_components_removed.items), - }, - ) catch |err| { - // One module replacement error should NOT prevent follow-up - // module replacements to fail. It is the HMR runtime's - // responsibility to collect all module load errors, and - // bubble them up. - dev.vm.printErrorLikeObjectToConsole(dev.server_global.js().takeException(err)); - @panic("Error thrown in Hot-module-replacement code. This is always a bug in the HMR runtime."); - }; - _ = errors; // TODO: - } + const server_modules = c.BakeLoadServerHmrPatch(@ptrCast(dev.vm.global), bun.String.createLatin1(server_bundle)) catch |err| { + // No user code has been evaluated yet, since everything is to + // be wrapped in a function clousure. This means that the likely + // error is going to be a syntax error, or other mistake in the + // bundler. + dev.vm.printErrorLikeObjectToConsole(dev.vm.global.takeException(err)); + @panic("Error thrown while evaluating server code. This is always a bug in the bundler."); + }; + const errors = dev.server_register_update_callback.get().?.call( + dev.vm.global, + dev.vm.global.toJSValue(), + &.{ + server_modules, + dev.makeArrayForServerComponentsPatch(dev.vm.global, dev.incremental_result.client_components_added.items), + dev.makeArrayForServerComponentsPatch(dev.vm.global, dev.incremental_result.client_components_removed.items), + }, + ) catch |err| { + // One module replacement error should NOT prevent follow-up + // module replacements to fail. It is the HMR runtime's + // responsibility to collect all module load errors, and + // bubble them up. + dev.vm.printErrorLikeObjectToConsole(dev.vm.global.takeException(err)); + @panic("Error thrown in Hot-module-replacement code. This is always a bug in the HMR runtime."); + }; + _ = errors; // TODO: } const css_chunks = bundle_result.cssChunks(); if ((dev.client_graph.current_chunk_len > 0 or css_chunks.len > 0) and - dev.app.num_subscribers(HmrSocket.global_topic) > 0) + dev.numSubscribers(HmrSocket.global_topic) > 0) { var sfb2 = std.heap.stackFallback(65536, bun.default_allocator); var payload = std.ArrayList(u8).initCapacity(sfb2.get(), 65536) catch @@ -902,9 +893,9 @@ fn bundle(dev: *DevServer, files: []const BakeEntryPoint) BundleError!void { } if (dev.client_graph.current_chunk_len > 0) - try dev.client_graph.takeBundleToList(.hmr_chunk, &payload); + try dev.client_graph.takeBundleToList(.hmr_chunk, &payload, ""); - _ = dev.app.publish(HmrSocket.global_topic, payload.items, .binary, true); + dev.publish(HmrSocket.global_topic, payload.items, .binary); } if (dev.incremental_result.failures_added.items.len > 0) { @@ -954,15 +945,18 @@ fn indexFailures(dev: *DevServer) !void { } } - for (dev.incremental_result.routes_affected.items) |route_index| { - const route = &dev.routes[route_index.get()]; - route.server_state = .possible_bundling_failures; + { + @panic("TODO: revive"); } + // for (dev.incremental_result.routes_affected.items) |route_index| { + // const route = &dev.routes[route_index.get()]; + // route.server_state = .possible_bundling_failures; + // } - _ = dev.app.publish(HmrSocket.global_topic, payload.items, .binary, false); + dev.publish(HmrSocket.global_topic, payload.items, .binary); } else if (dev.incremental_result.failures_removed.items.len > 0) { if (dev.bundling_failures.count() == 0) { - _ = dev.app.publish(HmrSocket.global_topic, &.{MessageId.errors_cleared.char()}, .binary, false); + dev.publish(HmrSocket.global_topic, &.{MessageId.errors_cleared.char()}, .binary); for (dev.incremental_result.failures_removed.items) |removed| { removed.deinit(); } @@ -979,7 +973,7 @@ fn indexFailures(dev: *DevServer) !void { removed.deinit(); } - _ = dev.app.publish(HmrSocket.global_topic, payload.items, .binary, false); + dev.publish(HmrSocket.global_topic, payload.items, .binary); } } @@ -988,9 +982,9 @@ fn indexFailures(dev: *DevServer) !void { /// Used to generate the entry point. Unlike incremental patches, this always /// contains all needed files for a route. -fn generateClientBundle(dev: *DevServer, route: *Route) bun.OOM![]const u8 { - assert(route.client_bundle == null); - assert(route.server_state == .loaded); // page is unfit to load +fn generateClientBundle(dev: *DevServer, route_bundle: *RouteBundle) bun.OOM![]const u8 { + assert(route_bundle.client_bundle == null); + assert(route_bundle.server_state == .loaded); // page is unfit to load dev.graph_safety_lock.lock(); defer dev.graph_safety_lock.unlock(); @@ -1008,31 +1002,20 @@ fn generateClientBundle(dev: *DevServer, route: *Route) bun.OOM![]const u8 { // Run tracing dev.client_graph.reset(); + try dev.traceAllRouteImports(route_bundle, .{ .find_client_modules = true }); - // Framework entry point is always needed. - try dev.client_graph.traceImports( - IncrementalGraph(.client).framework_entry_point_index, - .{ .find_client_modules = true }, + const client_file = dev.router.typePtr(dev.router.routePtr(route_bundle.route).type).client_file.unwrap() orelse + @panic("No client side entrypoint in client bundle"); + + return dev.client_graph.takeBundle( + .initial_response, + dev.relativePath(dev.client_graph.bundled_files.keys()[fromOpaqueFileId(.client, client_file).get()]), ); - - // If react fast refresh is enabled, it will be imported by the runtime instantly. - if (dev.framework.react_fast_refresh != null) { - try dev.client_graph.traceImports(IncrementalGraph(.client).react_refresh_index, .{ .find_client_modules = true }); - } - - // Trace the route to the client components - try dev.server_graph.traceImports( - route.server_file.unwrap() orelse - Output.panic("File index for route not present", .{}), - .{ .find_client_modules = true }, - ); - - return dev.client_graph.takeBundle(.initial_response); } -fn generateCssList(dev: *DevServer, route: *Route) bun.OOM!JSC.JSValue { - if (!Environment.allow_assert) assert(!route.css_file_array.has()); - assert(route.server_state == .loaded); // page is unfit to load +fn generateCssList(dev: *DevServer, route_bundle: *RouteBundle) bun.OOM!JSC.JSValue { + if (Environment.allow_assert) assert(!route_bundle.cached_css_file_array.has()); + assert(route_bundle.server_state == .loaded); // page is unfit to load dev.graph_safety_lock.lock(); defer dev.graph_safety_lock.unlock(); @@ -1049,36 +1032,48 @@ fn generateCssList(dev: *DevServer, route: *Route) bun.OOM!JSC.JSValue { // Run tracing dev.client_graph.reset(); - - // Framework entry point is allowed to include its own CSS - try dev.client_graph.traceImports( - IncrementalGraph(.client).framework_entry_point_index, - .{ .find_css = true }, - ); - - // Trace the route to the css files - try dev.server_graph.traceImports( - route.server_file.unwrap() orelse - Output.panic("File index for route not present", .{}), - .{ .find_css = true }, - ); + try dev.traceAllRouteImports(route_bundle, .{ .find_css = true }); const names = dev.client_graph.current_css_files.items; - const arr = JSC.JSArray.createEmpty(dev.server_global.js(), names.len); + const arr = JSC.JSArray.createEmpty(dev.vm.global, names.len); for (names, 0..) |item, i| { const str = bun.String.createUTF8(item); defer str.deref(); - arr.putIndex(dev.server_global.js(), @intCast(i), str.toJS(dev.server_global.js())); + arr.putIndex(dev.vm.global, @intCast(i), str.toJS(dev.vm.global)); } return arr; } +fn traceAllRouteImports(dev: *DevServer, route_bundle: *RouteBundle, goal: TraceImportGoal) !void { + var route = dev.router.routePtr(route_bundle.route); + const router_type = dev.router.typePtr(route.type); + + // Both framework entry points are considered + try dev.server_graph.traceImports(fromOpaqueFileId(.server, router_type.server_file), .{ .find_css = true }); + if (router_type.client_file.unwrap()) |id| { + try dev.client_graph.traceImports(fromOpaqueFileId(.client, id), goal); + } + + // The route file is considered + if (route.file_page.unwrap()) |id| { + try dev.server_graph.traceImports(fromOpaqueFileId(.server, id), goal); + } + + // For all parents, the layout is considered + while (true) { + if (route.file_layout.unwrap()) |id| { + try dev.server_graph.traceImports(fromOpaqueFileId(.server, id), goal); + } + route = dev.router.routePtr(route.parent.unwrap() orelse break); + } +} + fn makeArrayForServerComponentsPatch(dev: *DevServer, global: *JSC.JSGlobalObject, items: []const IncrementalGraph(.server).FileIndex) JSValue { if (items.len == 0) return .null; const arr = JSC.JSArray.createEmpty(global, items.len); const names = dev.server_graph.bundled_files.keys(); for (items, 0..) |item, i| { - const str = bun.String.createUTF8(bun.path.relative(dev.cwd, names[item.get()])); + const str = bun.String.createUTF8(dev.relativePath(names[item.get()])); defer str.deref(); arr.putIndex(global, @intCast(i), str.toJS(global)); } @@ -1253,7 +1248,7 @@ pub fn handleParseTaskFailure( ) bun.OOM!void { // Print each error only once Output.prettyErrorln("Errors while bundling '{s}':", .{ - bun.path.relative(dev.cwd, abs_path), + dev.relativePath(abs_path), }); Output.flush(); log.print(Output.errorWriter()) catch {}; @@ -1287,10 +1282,75 @@ pub fn isFileCached(dev: *DevServer, path: []const u8, side: bake.Graph) ?CacheE } } -fn onFallbackRoute(_: void, _: *Request, resp: *Response) void { +fn appendOpaqueEntryPoint( + dev: *DevServer, + file_names: [][]const u8, + entry_points: *std.ArrayList(BakeEntryPoint), + comptime side: bake.Side, + optional_id: anytype, +) !void { + const file = switch (@TypeOf(optional_id)) { + OpaqueFileId.Optional => optional_id.unwrap() orelse return, + OpaqueFileId => optional_id, + else => @compileError("invalid type here"), + }; + + const file_index = fromOpaqueFileId(side, file); + if (switch (side) { + .server => dev.server_graph.stale_files.isSet(file_index.get()), + .client => dev.client_graph.stale_files.isSet(file_index.get()), + }) { + try entry_points.append(.{ + .path = file_names[file_index.get()], + .graph = switch (side) { + .server => .server, + .client => .client, + }, + }); + } +} + +pub fn routeBundlePtr(dev: *DevServer, idx: RouteBundle.Index) *RouteBundle { + return &dev.route_bundles.items[idx.get()]; +} + +fn onRequest(dev: *DevServer, req: *Request, resp: *Response) void { + var params: FrameworkRouter.MatchedParams = undefined; + if (dev.router.matchSlow(req.url(), ¶ms)) |route_index| { + dev.ensureRouteIsBundled(route_index, .server_handler, req, resp) catch bun.outOfMemory(); + return; + } + sendBuiltInNotFound(resp); } +fn insertRouteBundle(dev: *DevServer, route: Route.Index) !RouteBundle.Index { + const full_pattern = full_pattern: { + var buf = bake.PatternBuffer.empty; + var current: *Route = dev.router.routePtr(route); + while (true) { + buf.prependPart(current.part); + current = dev.router.routePtr(current.parent.unwrap() orelse break); + } + break :full_pattern try dev.allocator.dupe(u8, buf.slice()); + }; + errdefer dev.allocator.free(full_pattern); + + try dev.route_bundles.append(dev.allocator, .{ + .route = route, + .server_state = .unqueued, + .full_pattern = full_pattern, + .client_bundle = null, + .evaluate_failure = null, + .cached_module_list = .{}, + .cached_client_bundle_url = .{}, + .cached_css_file_array = .{}, + }); + const bundle_index = RouteBundle.Index.init(@intCast(dev.route_bundles.items.len - 1)); + dev.router.routePtr(route).bundle = bundle_index.toOptional(); + return bundle_index; +} + fn sendTextFile(code: []const u8, content_type: []const u8, resp: *Response) void { if (code.len == 0) { resp.writeStatus("202 No Content"); @@ -1376,7 +1436,7 @@ fn sendBuiltInNotFound(resp: *Response) void { resp.end(message, true); } -fn sendStubErrorMessage(dev: *DevServer, route: *Route, resp: *Response, err: JSValue) void { +fn sendStubErrorMessage(dev: *DevServer, route: *RouteBundle, resp: *Response, err: JSValue) void { var sfb = std.heap.stackFallback(65536, dev.allocator); var a = std.ArrayList(u8).initCapacity(sfb.get(), 65536) catch bun.outOfMemory(); @@ -1389,7 +1449,7 @@ fn sendStubErrorMessage(dev: *DevServer, route: *Route, resp: *Response, err: JS resp.end(a.items, true); // TODO: "You should never call res.end(huge buffer)" } -const FileKind = enum { +const FileKind = enum(u2) { /// Files that failed to bundle or do not exist on disk will appear in the /// graph as "unknown". unknown, @@ -1587,8 +1647,7 @@ pub fn IncrementalGraph(side: bake.Side) type { /// An index into `bundled_files`, `stale_files`, `first_dep`, `first_import`, or `affected_by_trace` /// Top bits cannot be relied on due to `SerializedFailure.Owner.Packed` pub const FileIndex = bun.GenericIndex(u30, File); - pub const framework_entry_point_index = FileIndex.init(0); - pub const react_refresh_index = if (side == .client) FileIndex.init(1); + pub const react_refresh_index = if (side == .client) FileIndex.init(0); /// An index into `edges` const EdgeIndex = bun.GenericIndex(u32, Edge); @@ -1638,8 +1697,8 @@ pub fn IncrementalGraph(side: bake.Side) type { g.current_chunk_len += code.len; // Dump to filesystem if enabled - if (dev.dump_dir) |dump_dir| { - const cwd = dev.cwd; + if (bun.FeatureFlags.bake_debugging_features) if (dev.dump_dir) |dump_dir| { + const cwd = dev.root; var a: bun.PathBuffer = undefined; var b: [bun.MAX_PATH_BYTES * 2]u8 = undefined; const rel_path = bun.path.relativeBufZ(&a, cwd, abs_path); @@ -1653,7 +1712,7 @@ pub fn IncrementalGraph(side: bake.Side) type { bun.handleErrorReturnTrace(err, @errorReturnTrace()); Output.warn("Could not dump bundle: {}", .{err}); }; - } + }; const gop = try g.bundled_files.getOrPut(dev.allocator, abs_path); const file_index = FileIndex.init(@intCast(gop.index)); @@ -1932,11 +1991,6 @@ pub fn IncrementalGraph(side: bake.Side) type { stop_at_boundary, no_stop, }; - const TraceImportGoal = struct { - // gts: *GraphTraceState, - find_css: bool = false, - find_client_modules: bool = false, - }; fn traceDependencies(g: *@This(), file_index: FileIndex, trace_kind: TraceDependencyKind) !void { g.owner().graph_safety_lock.assertLocked(); @@ -2061,16 +2115,10 @@ pub fn IncrementalGraph(side: bake.Side) type { /// Never takes ownership of `abs_path` /// Marks a chunk but without any content. Used to track dependencies to files that don't exist. pub fn insertStale(g: *@This(), abs_path: []const u8, is_ssr_graph: bool) bun.OOM!FileIndex { - return g.insertStaleExtra(abs_path, is_ssr_graph, false, {}); + return g.insertStaleExtra(abs_path, is_ssr_graph, false); } - pub fn insertStaleExtra( - g: *@This(), - abs_path: []const u8, - is_ssr_graph: bool, - comptime is_route: bool, - route_index: if (is_route) Route.Index else void, - ) bun.OOM!FileIndex { + pub fn insertStaleExtra(g: *@This(), abs_path: []const u8, is_ssr_graph: bool, is_route: bool) bun.OOM!FileIndex { g.owner().graph_safety_lock.assertLocked(); debug.log("Insert stale: {s}", .{abs_path}); @@ -2087,18 +2135,10 @@ pub fn IncrementalGraph(side: bake.Side) type { } } - if (is_route) { - g.owner().routes[route_index.get()].server_file = file_index.toOptional(); - } - if (g.stale_files.bit_length > gop.index) { g.stale_files.set(gop.index); } - if (is_route) { - try g.owner().route_lookup.put(g.owner().allocator, file_index, route_index); - } - switch (side) { .client => { gop.value_ptr.* = File.init("", .{ @@ -2221,7 +2261,7 @@ pub fn IncrementalGraph(side: bake.Side) type { }; const failure = try SerializedFailure.initFromLog( fail_owner, - bun.path.relative(dev.cwd, abs_path), + dev.relativePath(abs_path), log.msgs.items, ); const fail_gop = try dev.bundling_failures.getOrPut(dev.allocator, failure); @@ -2232,7 +2272,7 @@ pub fn IncrementalGraph(side: bake.Side) type { } } - pub fn ensureStaleBitCapacity(g: *@This(), val: bool) !void { + pub fn ensureStaleBitCapacity(g: *@This(), are_new_files_stale: bool) !void { try g.stale_files.resize( g.owner().allocator, std.mem.alignForward( @@ -2241,7 +2281,7 @@ pub fn IncrementalGraph(side: bake.Side) type { // allocate 8 in 8 usize chunks std.mem.byte_size_in_bits * @sizeOf(usize) * 8, ), - val, + are_new_files_stale, ); } @@ -2283,17 +2323,26 @@ pub fn IncrementalGraph(side: bake.Side) type { if (side == .client) g.current_css_files.clearRetainingCapacity(); } - pub fn takeBundle(g: *@This(), kind: ChunkKind) ![]const u8 { + pub fn takeBundle( + g: *@This(), + kind: ChunkKind, + initial_response_entry_point: []const u8, + ) ![]const u8 { var chunk = std.ArrayList(u8).init(g.owner().allocator); - try g.takeBundleToList(kind, &chunk); + try g.takeBundleToList(kind, &chunk, initial_response_entry_point); bun.assert(chunk.items.len == chunk.capacity); return chunk.items; } - pub fn takeBundleToList(g: *@This(), kind: ChunkKind, list: *std.ArrayList(u8)) !void { + pub fn takeBundleToList( + g: *@This(), + kind: ChunkKind, + list: *std.ArrayList(u8), + initial_response_entry_point: []const u8, + ) !void { g.owner().graph_safety_lock.assertLocked(); // initial bundle needs at least the entry point - // hot updates shouldnt be emitted if there are no chunks + // hot updates shouldn't be emitted if there are no chunks assert(g.current_chunk_len > 0); const runtime = switch (kind) { @@ -2314,12 +2363,8 @@ pub fn IncrementalGraph(side: bake.Side) type { .initial_response => { const fw = g.owner().framework; try w.writeAll("}, {\n main: "); - const entry = switch (side) { - .server => fw.entry_server, - .client => fw.entry_client, - }; try bun.js_printer.writeJSONString( - bun.path.relative(g.owner().cwd, entry), + g.owner().relativePath(initial_response_entry_point), @TypeOf(w), w, .utf8, @@ -2332,7 +2377,7 @@ pub fn IncrementalGraph(side: bake.Side) type { if (fw.react_fast_refresh) |rfr| { try w.writeAll(",\n refresh: "); try bun.js_printer.writeJSONString( - bun.path.relative(g.owner().cwd, rfr.import_source), + g.owner().relativePath(rfr.import_source), @TypeOf(w), w, .utf8, @@ -2375,7 +2420,7 @@ pub fn IncrementalGraph(side: bake.Side) type { } list.appendSliceAssumeCapacity(end); - if (g.owner().dump_dir) |dump_dir| { + if (bun.FeatureFlags.bake_debugging_features) if (g.owner().dump_dir) |dump_dir| { const rel_path_escaped = "latest_chunk.js"; dumpBundle(dump_dir, switch (side) { .client => .client, @@ -2384,7 +2429,7 @@ pub fn IncrementalGraph(side: bake.Side) type { bun.handleErrorReturnTrace(err, @errorReturnTrace()); Output.warn("Could not dump bundle: {}", .{err}); }; - } + }; } fn disconnectAndDeleteFile(g: *@This(), file_index: FileIndex) void { @@ -2406,6 +2451,12 @@ pub fn IncrementalGraph(side: bake.Side) type { } } + // TODO: it is infeasible to do this since FrameworkRouter contains file indices + // to the server graph + { + return; + } + g.bundled_files.swapRemoveAt(file_index.get()); // Move out-of-line data from `last` to replace `file_index` @@ -2478,8 +2529,10 @@ pub fn IncrementalGraph(side: bake.Side) type { const IncrementalResult = struct { /// When tracing a file's dependencies via `traceDependencies`, this is - /// populated with the hit routes. Tracing is used for many purposes. - routes_affected: ArrayListUnmanaged(Route.Index), + /// populated with the hit `Route.Index`s. To know what `RouteBundle`s + /// are affected, the route graph must be traced downwards. + /// Tracing is used for multiple purposes. + routes_affected: ArrayListUnmanaged(RouteIndexAndRecurseFlag), // Following three fields are populated during `receiveChunk` @@ -2508,6 +2561,7 @@ const IncrementalResult = struct { failures_added: ArrayListUnmanaged(SerializedFailure), /// Removing files clobbers indices, so removing anything is deferred. + // TODO: remove delete_client_files_later: ArrayListUnmanaged(IncrementalGraph(.client).FileIndex), const empty: IncrementalResult = .{ @@ -2545,6 +2599,12 @@ const GraphTraceState = struct { } }; +const TraceImportGoal = struct { + // gts: *GraphTraceState, + find_css: bool = false, + find_client_modules: bool = false, +}; + fn initGraphTraceState(dev: *const DevServer, sfa: Allocator) !GraphTraceState { const server_bits = try DynamicBitSetUnmanaged.initEmpty(sfa, dev.server_graph.bundled_files.count()); errdefer server_bits.deinit(sfa); @@ -2824,7 +2884,7 @@ pub const SerializedFailure = struct { /// is given to the HMR runtime as an opaque handle. pub const Owner = union(enum) { none, - route: Route.Index, + route: RouteBundle.Index, client: IncrementalGraph(.client).FileIndex, server: IncrementalGraph(.server).FileIndex, @@ -2846,7 +2906,7 @@ pub const SerializedFailure = struct { .none => .none, .client => .{ .client = IncrementalGraph(.client).FileIndex.init(owner.data) }, .server => .{ .server = IncrementalGraph(.server).FileIndex.init(owner.data) }, - .route => .{ .route = Route.Index.init(owner.data) }, + .route => .{ .route = RouteBundle.Index.init(owner.data) }, }; } }; @@ -3056,11 +3116,19 @@ fn dumpBundle(dump_dir: std.fs.Dir, side: bake.Graph, rel_path: []const u8, chun } fn emitVisualizerMessageIfNeeded(dev: *DevServer) !void { + if (!bun.FeatureFlags.bake_debugging_features) return; if (dev.emit_visualizer_events == 0) return; var sfb = std.heap.stackFallback(65536, bun.default_allocator); var payload = try std.ArrayList(u8).initCapacity(sfb.get(), 65536); defer payload.deinit(); + + try dev.writeVisualizerMessage(&payload); + + dev.publish(HmrSocket.visualizer_topic, payload.items, .binary); +} + +fn writeVisualizerMessage(dev: *DevServer, payload: *std.ArrayList(u8)) !void { payload.appendAssumeCapacity(MessageId.visualizer.char()); const w = payload.writer(); @@ -3074,9 +3142,10 @@ fn emitVisualizerMessageIfNeeded(dev: *DevServer) !void { g.bundled_files.values(), 0.., ) |k, v, i| { - try w.writeInt(u32, @intCast(k.len), .little); + const normalized_key = dev.relativePath(k); + try w.writeInt(u32, @intCast(normalized_key.len), .little); if (k.len == 0) continue; - try w.writeAll(k); + try w.writeAll(normalized_key); try w.writeByte(@intFromBool(g.stale_files.isSet(i) or switch (side) { .server => v.failed, .client => v.flags.failed, @@ -3103,8 +3172,6 @@ fn emitVisualizerMessageIfNeeded(dev: *DevServer) !void { try w.writeInt(u32, @intCast(edge.imported.get()), .little); } } - - _ = dev.app.publish(HmrSocket.visualizer_topic, payload.items, .binary, false); } pub fn onWebSocketUpgrade( @@ -3198,7 +3265,7 @@ pub const MessageId = enum(u8) { /// - `u32`: File index of the imported file visualizer = 'v', - pub fn char(id: MessageId) u8 { + pub inline fn char(id: MessageId) u8 { return @intFromEnum(id); } }; @@ -3217,12 +3284,12 @@ const HmrSocket = struct { pub const global_topic = "*"; pub const visualizer_topic = "v"; - pub fn onOpen(dw: *HmrSocket, ws: AnyWebSocket) void { - _ = ws.send(&(.{MessageId.version.char()} ++ dw.dev.configuration_hash_key), .binary, false, true); + pub fn onOpen(s: *HmrSocket, ws: AnyWebSocket) void { + _ = ws.send(&(.{MessageId.version.char()} ++ s.dev.configuration_hash_key), .binary, false, true); _ = ws.subscribe(global_topic); } - pub fn onMessage(dw: *HmrSocket, ws: AnyWebSocket, msg: []const u8, opcode: uws.Opcode) void { + pub fn onMessage(s: *HmrSocket, ws: AnyWebSocket, msg: []const u8, opcode: uws.Opcode) void { _ = opcode; if (msg.len == 0) { @@ -3232,11 +3299,11 @@ const HmrSocket = struct { switch (@as(IncomingMessageId, @enumFromInt(msg[0]))) { .visualizer => { - if (!dw.emit_visualizer_events) { - dw.emit_visualizer_events = true; - dw.dev.emit_visualizer_events += 1; + if (!s.emit_visualizer_events) { + s.emit_visualizer_events = true; + s.dev.emit_visualizer_events += 1; _ = ws.subscribe(visualizer_topic); - dw.dev.emitVisualizerMessageIfNeeded() catch bun.outOfMemory(); + s.dev.emitVisualizerMessageIfNeeded() catch bun.outOfMemory(); } }, else => { @@ -3245,73 +3312,36 @@ const HmrSocket = struct { } } - pub fn onClose(dw: *HmrSocket, ws: AnyWebSocket, exit_code: i32, message: []const u8) void { + pub fn onClose(s: *HmrSocket, ws: AnyWebSocket, exit_code: i32, message: []const u8) void { _ = ws; _ = exit_code; _ = message; - if (dw.emit_visualizer_events) { - dw.dev.emit_visualizer_events -= 1; + if (s.emit_visualizer_events) { + s.dev.emit_visualizer_events -= 1; } - defer dw.dev.allocator.destroy(dw); + defer s.dev.allocator.destroy(s); } }; -/// Bake uses a special global object extending Zig::GlobalObject -pub const DevGlobalObject = opaque { - /// Safe downcast to use other Bun APIs - pub fn js(ptr: *DevGlobalObject) *JSC.JSGlobalObject { - return @ptrCast(ptr); - } - - pub fn vm(ptr: *DevGlobalObject) *JSC.VM { - return ptr.js().vm(); - } -}; - -pub const BakeSourceProvider = opaque {}; - const c = struct { - // BakeDevGlobalObject.cpp - extern fn BakeCreateDevGlobal(owner: *DevServer, console: *JSC.ConsoleObject) *DevGlobalObject; - // BakeSourceProvider.cpp extern fn BakeGetDefaultExportFromModule(global: *JSC.JSGlobalObject, module: JSValue) JSValue; - const LoadServerCodeResult = struct { - promise: *JSInternalPromise, - key: *JSC.JSString, - }; - - fn BakeLoadServerHmrPatch(global: *DevGlobalObject, code: bun.String) !JSValue { - const f = @extern(*const fn (*DevGlobalObject, bun.String) callconv(.C) JSValue, .{ - .name = "BakeLoadServerHmrPatch", - }); - const result = f(global, code); - if (result == .zero) { - if (Environment.allow_assert) assert(global.js().hasException()); - return error.JSError; - } - return result; + fn BakeLoadServerHmrPatch(global: *JSC.JSGlobalObject, code: bun.String) !JSValue { + const f = @extern( + *const fn (*JSC.JSGlobalObject, bun.String) callconv(.C) JSValue.MaybeException, + .{ .name = "BakeLoadServerHmrPatch" }, + ); + return f(global, code).unwrap(); } - fn BakeLoadInitialServerCode(global: *DevGlobalObject, code: bun.String) bun.JSError!LoadServerCodeResult { - const Return = extern struct { - promise: ?*JSInternalPromise, - key: *JSC.JSString, - }; - const f = @extern(*const fn (*DevGlobalObject, bun.String) callconv(.C) Return, .{ + fn BakeLoadInitialServerCode(global: *JSC.JSGlobalObject, code: bun.String, separate_ssr_graph: bool) bun.JSError!JSValue { + const f = @extern(*const fn (*JSC.JSGlobalObject, bun.String, bool) callconv(.C) JSValue.MaybeException, .{ .name = "BakeLoadInitialServerCode", }); - const result = f(global, code); - return .{ - .promise = result.promise orelse { - if (Environment.allow_assert) assert(global.js().hasException()); - return error.JSError; - }, - .key = result.key, - }; + return f(global, code, separate_ssr_graph).unwrap(); } }; @@ -3331,7 +3361,7 @@ pub fn reload(dev: *DevServer, reload_task: *HotReloadTask) bun.OOM!void { @panic("timers unsupported"); var sfb = std.heap.stackFallback(4096, bun.default_allocator); - const temp_alloc = sfb.get(); + var temp_alloc = sfb.get(); // pre-allocate a few files worth of strings. it is unlikely but supported // to change more than 8 files in the same bundling round. @@ -3352,15 +3382,6 @@ pub fn reload(dev: *DevServer, reload_task: *HotReloadTask) bun.OOM!void { return; } - const reload_file_list = bun.Output.Scoped(.reload_file_list, false); - - if (reload_file_list.isVisible()) { - reload_file_list.log("Hot update hits {d} files", .{files.items.len}); - for (files.items) |f| { - reload_file_list.log("- {s} (.{s})", .{ f.path, @tagName(f.graph) }); - } - } - dev.incremental_result.reset(); defer { // Remove files last to start, to avoid issues where removing a file @@ -3387,23 +3408,48 @@ pub fn reload(dev: *DevServer, reload_task: *HotReloadTask) bun.OOM!void { // This list of routes affected excludes client code. This means changing // a client component wont count as a route to trigger a reload on. + // + // A second trace is required to determine what routes had changed bundles, + // since changing a layout affects all child routes. Additionally, routes + // that do not have a bundle will not be cleared (as there is nothing to + // clear for those) if (dev.incremental_result.routes_affected.items.len > 0) { + // re-use some earlier stack memory + files.clearAndFree(); + sfb = std.heap.stackFallback(4096, bun.default_allocator); + temp_alloc = sfb.get(); + + // A bit-set is used to avoid duplicate entries. This is not a problem + // with `dev.incremental_result.routes_affected` + var second_trace_result = try DynamicBitSetUnmanaged.initEmpty(temp_alloc, dev.route_bundles.items.len); + for (dev.incremental_result.routes_affected.items) |request| { + const route = dev.router.routePtr(request.route_index); + if (route.bundle.unwrap()) |id| second_trace_result.set(id.get()); + if (request.should_recurse_when_visiting) { + markAllRouteChildren(&dev.router, &second_trace_result, request.route_index); + } + } + var sfb2 = std.heap.stackFallback(65536, bun.default_allocator); var payload = std.ArrayList(u8).initCapacity(sfb2.get(), 65536) catch unreachable; // enough space defer payload.deinit(); payload.appendAssumeCapacity(MessageId.route_update.char()); const w = payload.writer(); - try w.writeInt(u32, @intCast(dev.incremental_result.routes_affected.items.len), .little); + const count = second_trace_result.count(); + assert(count > 0); + try w.writeInt(u32, @intCast(count), .little); - for (dev.incremental_result.routes_affected.items) |route| { - try w.writeInt(u32, route.get(), .little); - const pattern = dev.routes[route.get()].pattern; + var it = second_trace_result.iterator(.{ .kind = .set }); + while (it.next()) |bundled_route_index| { + try w.writeInt(u32, @intCast(bundled_route_index), .little); + const pattern = dev.route_bundles.items[bundled_route_index].full_pattern; try w.writeInt(u32, @intCast(pattern.len), .little); try w.writeAll(pattern); } - _ = dev.app.publish(HmrSocket.global_topic, payload.items, .binary, true); + // Notify + dev.publish(HmrSocket.global_topic, payload.items, .binary); } // When client component roots get updated, the `client_components_affected` @@ -3423,13 +3469,14 @@ pub fn reload(dev: *DevServer, reload_task: *HotReloadTask) bun.OOM!void { try dev.server_graph.traceDependencies(index, .no_stop); } - for (dev.incremental_result.routes_affected.items) |route| { - // Free old bundles - if (dev.routes[route.get()].client_bundle) |old| { - dev.allocator.free(old); - } - dev.routes[route.get()].client_bundle = null; - } + // TODO: + // for (dev.incremental_result.routes_affected.items) |route| { + // // Free old bundles + // if (dev.routes[route.get()].client_bundle) |old| { + // dev.allocator.free(old); + // } + // dev.routes[route.get()].client_bundle = null; + // } } // TODO: improve this visual feedback @@ -3446,7 +3493,7 @@ pub fn reload(dev: *DevServer, reload_task: *HotReloadTask) bun.OOM!void { Output.prettyError("[x{d}] ", .{dev.bundles_since_last_error}); } - Output.prettyError("Reloaded in {d}ms: {s}", .{ @divFloor(timer.read(), std.time.ns_per_ms), bun.path.relative(dev.cwd, changed_file_paths[0]) }); + Output.prettyError("Reloaded in {d}ms: {s}", .{ @divFloor(timer.read(), std.time.ns_per_ms), dev.relativePath(changed_file_paths[0]) }); if (changed_file_paths.len > 1) { Output.prettyError(" + {d} more", .{files.items.len - 1}); } @@ -3455,6 +3502,16 @@ pub fn reload(dev: *DevServer, reload_task: *HotReloadTask) bun.OOM!void { } else {} } +fn markAllRouteChildren(router: *FrameworkRouter, bits: *DynamicBitSetUnmanaged, route_index: Route.Index) void { + var next = router.routePtr(route_index).first_child.unwrap(); + while (next) |child_index| { + const route = router.routePtr(child_index); + if (route.bundle.unwrap()) |index| bits.set(index.get()); + markAllRouteChildren(router, bits, child_index); + next = route.next_sibling.unwrap(); + } +} + pub const HotReloadTask = struct { /// Align to cache lines to reduce contention. const Aligned = struct { aligned: HotReloadTask align(std.atomic.cache_line) }; @@ -3585,7 +3642,8 @@ pub fn onFileUpdate(dev: *DevServer, events: []Watcher.Event, changed_files: []? }, .directory => { // bust the directory cache since this directory has changed - _ = dev.server_bundler.resolver.bustDirCache(file_path); + // TODO: correctly solve https://github.com/oven-sh/bun/issues/14913 + _ = dev.server_bundler.resolver.bustDirCache(bun.strings.withoutTrailingSlash(file_path)); // if a directory watch exists for resolution // failures, check those now. @@ -3636,6 +3694,107 @@ pub fn onWatchError(_: *DevServer, err: bun.sys.Error) void { } } +pub fn publish(dev: *DevServer, topic: []const u8, message: []const u8, opcode: uws.Opcode) void { + if (dev.server) |s| _ = s.publish(topic, message, opcode, false); +} + +pub fn numSubscribers(dev: *DevServer, topic: []const u8) u32 { + return if (dev.server) |s| s.numSubscribers(topic) else 0; +} + +const SafeFileId = packed struct(u32) { + side: bake.Side, + index: u30, + unused: enum(u1) { unused = 0 } = .unused, +}; + +/// Interface function for FrameworkRouter +pub fn getFileIdForRouter(dev: *DevServer, abs_path: []const u8, associated_route: Route.Index, file_kind: Route.FileKind) !OpaqueFileId { + const index = try dev.server_graph.insertStaleExtra(abs_path, false, true); + try dev.route_lookup.put(dev.allocator, index, .{ + .route_index = associated_route, + .should_recurse_when_visiting = file_kind == .layout, + }); + return toOpaqueFileId(.server, index); +} + +fn toOpaqueFileId(comptime side: bake.Side, index: IncrementalGraph(side).FileIndex) OpaqueFileId { + if (Environment.allow_assert) { + return OpaqueFileId.init(@bitCast(SafeFileId{ + .side = side, + .index = index.get(), + })); + } + + return OpaqueFileId.init(index.get()); +} + +fn fromOpaqueFileId(comptime side: bake.Side, id: OpaqueFileId) IncrementalGraph(side).FileIndex { + if (Environment.allow_assert) { + const safe: SafeFileId = @bitCast(id.get()); + assert(side == safe.side); + return IncrementalGraph(side).FileIndex.init(safe.index); + } + return IncrementalGraph(side).FileIndex.init(@intCast(id.get())); +} + +fn relativePath(dev: *const DevServer, path: []const u8) []const u8 { + // TODO: windows slash normalization + bun.assert(dev.root[dev.root.len - 1] != '/'); + if (path.len >= dev.root.len + 1 and + path[dev.root.len] == '/' and + bun.strings.startsWith(path, dev.root)) + { + return path[dev.root.len + 1 ..]; + } + return bun.path.relative(dev.root, path); +} + +fn dumpStateDueToCrash(dev: *DevServer) !void { + comptime assert(bun.FeatureFlags.bake_debugging_features); + + // being conservative about how much stuff is put on the stack. + var filepath_buf: [@min(4096, bun.MAX_PATH_BYTES)]u8 = undefined; + const filepath = std.fmt.bufPrintZ(&filepath_buf, "incremental-graph-crash-dump.{d}.html", .{std.time.timestamp()}) catch "incremental-graph-crash-dump.html"; + const file = std.fs.cwd().createFileZ(filepath, .{}) catch |err| { + bun.handleErrorReturnTrace(err, @errorReturnTrace()); + Output.warn("Could not open directory for dumping sources: {}", .{err}); + return; + }; + defer file.close(); + + const start, const end = comptime brk: { + const visualizer = @embedFile("incremental_visualizer.html"); + const i = (std.mem.indexOf(u8, visualizer, ""); + } catch { + // The chunk cannot be embedded as a UTF-8 string in the script tag. + // No data should have been written yet, so a base64 fallback can be used. + const base64 = btoa(String.fromCodePoint(...chunk)); + controller.write(`Uint8Array.from(atob(\"${base64}\"),m=>m.codePointAt(0))`); + } +} + +/** + * Attempts to combine RSC chunks together to minimize the number of chunks the + * client processes. + */ +function writeManyFlightScriptData( + chunks: Uint8Array[], + decoder: TextDecoder, + controller: { write: (str: string) => void }, +) { + if (chunks.length === 1) return writeSingleFlightScriptData(chunks[0], decoder, controller); + + let i = 0; + try { + // Combine all chunks into a single string if possible. + for (; i < chunks.length; i++) { + // `decode()` will throw on invalid UTF-8 sequences. + const str = toSingleQuote(decoder.decode(chunks[i], { stream: true })); + if (i === 0) controller.write("'"); + controller.write(str); + } + controller.write("')"); + } catch { + // The chunk cannot be embedded as a UTF-8 string in the script tag. + // Since this is rare, just make the rest of the chunks base64. + if (i > 0) controller.write("');__bun_f.push("); + controller.write('Uint8Array.from(atob("'); + for (; i < chunks.length; i++) { + const chunk = chunks[i]; + const base64 = btoa(String.fromCodePoint(...chunk)); + controller.write(base64.slice(1, -1)); + } + controller.write('"),m=>m.codePointAt(0))'); + } +} + +// Instead of using `JSON.stringify`, this uses a single quote variant of it, since +// the RSC payload includes a ton of " characters. This is slower, but an easy +// component to move into native code. +function toSingleQuote(str: string): string { + return ( + str // Escape single quotes, backslashes, and newlines + .replace(/\\/g, "\\\\") + .replace(/'/g, "\\'") + .replace(/\n/g, "\\n") + // Escape closing script tags and HTML comments in JS content. + .replace(/