Implement napi_create_external and napi_get_value_external

src/bun.js/bindings/napi.cpp

@@ -50,6 +50,7 @@
 #include "../modules/ObjectModule.h"
 
 #include "JavaScriptCore/JSSourceCode.h"
+#include "napi_external.h"
 
 // #include <iostream>
 using namespace JSC;
@@ -1433,3 +1434,49 @@ extern "C" napi_status napi_get_property_names(napi_env env, napi_value object,
 
     return napi_ok;
 }
+
+extern "C" napi_status napi_create_external(napi_env env, void* data,
+    napi_finalize finalize_cb,
+    void* finalize_hint,
+    napi_value* result)
+{
+    if (UNLIKELY(result == nullptr)) {
+        return napi_invalid_arg;
+    }
+
+    Zig::GlobalObject* globalObject = toJS(env);
+    JSC::VM& vm = globalObject->vm();
+
+    auto* structure = Bun::NapiExternal::createStructure(vm, globalObject, globalObject->objectPrototype());
+    JSValue value = JSValue(Bun::NapiExternal::create(vm, structure, data, finalize_hint, finalize_cb));
+    JSC::EnsureStillAliveScope ensureStillAlive(value);
+    *result = toNapi(value);
+    return napi_ok;
+}
+
+extern "C" napi_status napi_get_value_external(napi_env env, napi_value value,
+    void** result)
+{
+    if (UNLIKELY(result == nullptr)) {
+        return napi_invalid_arg;
+    }
+
+    Zig::GlobalObject* globalObject = toJS(env);
+    JSC::VM& vm = globalObject->vm();
+
+    auto scope = DECLARE_CATCH_SCOPE(vm);
+    JSC::JSValue jsValue = JSC::JSValue::decode(reinterpret_cast<JSC::EncodedJSValue>(value));
+    JSC::EnsureStillAliveScope ensureStillAlive(jsValue);
+
+    if (!jsValue || !jsValue.isObject()) {
+        return napi_invalid_arg;
+    }
+
+    JSC::JSObject* object = jsValue.getObject();
+    if (!object->inherits<Bun::NapiExternal>()) {
+        return napi_invalid_arg;
+    }
+
+    *result = jsCast<Bun::NapiExternal*>(object)->value();
+    return napi_ok;
+}

src/bun.js/bindings/napi_external.cpp

@@ -1,50 +1,20 @@
+#include "napi_external.h"
+#include "napi.h"
 
+namespace Bun {
 
-// #pragma once
+NapiExternal::~NapiExternal()
+{
+    if (finalizer) {
+        finalizer(toNapi(globalObject()), m_value, m_finalizerHint);
+    }
+}
 
-// #include "root.h"
+void NapiExternal::destroy(JSC::JSCell* cell)
+{
+    jsCast<NapiExternal*>(cell)->~NapiExternal();
+}
 
-// #include "BunBuiltinNames.h"
-// #include "BunClientData.h"
+const ClassInfo NapiExternal::s_info = { "External"_s, &Base::s_info, nullptr, nullptr, CREATE_METHOD_TABLE(NapiExternal) };
 
-// namespace Zig {
-
-// using namespace JSC;
-
-// class NapiExternal : public JSC::JSNonFinalObject {
-//     using Base = JSC::JSNonFinalObject;
-
-// public:
-//     NapiExternal(JSC::VM& vm, JSC::Structure* structure)
-//         : Base(vm, structure)
-//     {
-//     }
-
-//     DECLARE_INFO;
-
-//     static constexpr unsigned StructureFlags = Base::StructureFlags;
-
-//     template<typename CellType, SubspaceAccess> static GCClient::IsoSubspace* subspaceFor(VM& vm)
-//     {
-//         return &vm.plainObjectSpace();
-//     }
-
-//     static JSC::Structure* createStructure(JSC::VM& vm, JSC::JSGlobalObject* globalObject,
-//         JSC::JSValue prototype)
-//     {
-//         return JSC::Structure::create(vm, globalObject, prototype,
-//             JSC::TypeInfo(JSC::ObjectType, StructureFlags), info());
-//     }
-
-//     static NapiExternal* create(JSC::VM& vm, JSC::Structure* structure)
-//     {
-//         NapiExternal* accessor = new (NotNull, JSC::allocateCell<NapiExternal>(vm)) NapiExternal(vm, structure);
-//         accessor->finishCreation(vm);
-//         return accessor;
-//     }
-
-//     void finishCreation(JSC::VM& vm);
-//     void* m_value;
-// };
-
-// } // namespace Zig
+}

src/bun.js/bindings/napi_external.h

@@ -6,13 +6,15 @@
 
 #include "BunBuiltinNames.h"
 #include "BunClientData.h"
+#include "node_api.h"
 
-namespace Zig {
+namespace Bun {
 
 using namespace JSC;
+using namespace WebCore;
 
-class NapiExternal : public JSC::JSNonFinalObject {
-    using Base = JSC::JSNonFinalObject;
+class NapiExternal : public JSC::JSDestructibleObject {
+    using Base = JSC::JSDestructibleObject;
 
 public:
     NapiExternal(JSC::VM& vm, JSC::Structure* structure)
@@ -20,45 +22,54 @@ public:
     {
     }
 
-    DECLARE_INFO;
+    DECLARE_EXPORT_INFO;
 
-    ~NapiExternal()
+    static constexpr unsigned StructureFlags = Base::StructureFlags;
+
+    template<typename, JSC::SubspaceAccess mode> static JSC::GCClient::IsoSubspace* subspaceFor(JSC::VM& vm)
     {
-        if (m_value) {
-            delete m_value;
-        }
+        if constexpr (mode == JSC::SubspaceAccess::Concurrently)
+            return nullptr;
 
-        static constexpr unsigned StructureFlags = Base::StructureFlags;
+        return WebCore::subspaceForImpl<NapiExternal, UseCustomHeapCellType::No>(
+            vm,
+            [](auto& spaces) { return spaces.m_clientSubspaceForNapiExternal.get(); },
+            [](auto& spaces, auto&& space) { spaces.m_clientSubspaceForNapiExternal = WTFMove(space); },
+            [](auto& spaces) { return spaces.m_subspaceForNapiExternal.get(); },
+            [](auto& spaces, auto&& space) { spaces.m_subspaceForNapiExternal = WTFMove(space); });
+    }
 
-        template<typename, SubspaceAccess mode> static JSC::GCClient::IsoSubspace* subspaceFor(JSC::VM & vm)
-        {
-            if constexpr (mode == JSC::SubspaceAccess::Concurrently)
-                return nullptr;
-            return WebCore::subspaceForImpl<JSNapiExternal, WebCore::UseCustomHeapCellType::No>(
-                vm,
-                [](auto& spaces) { return spaces.m_clientSubspaceForNapiExternal.get(); },
-                [](auto& spaces, auto&& space) { spaces.m_clientSubspaceForNapiExternal = WTFMove(space); },
-                [](auto& spaces) { return spaces.m_subspaceForNapiExternal.get(); },
-                [](auto& spaces, auto&& space) { spaces.m_subspaceForNapiExternal = WTFMove(space); });
-        }
+    ~NapiExternal();
 
-        static JSC::Structure* createStructure(JSC::VM & vm, JSC::JSGlobalObject * globalObject,
-            JSC::JSValue prototype)
-        {
-            return JSC::Structure::create(vm, globalObject, prototype,
-                JSC::TypeInfo(JSC::ObjectType, StructureFlags), info());
-        }
+    static JSC::Structure* createStructure(JSC::VM& vm, JSC::JSGlobalObject* globalObject,
+        JSC::JSValue prototype)
+    {
+        return JSC::Structure::create(vm, globalObject, prototype,
+            JSC::TypeInfo(JSC::ObjectType, StructureFlags), info());
+    }
 
-        static NapiExternal* create(JSC::VM & vm, JSC::Structure * structure)
-        {
-            NapiExternal* accessor = new (NotNull, JSC::allocateCell<NapiExternal>(vm)) NapiExternal(vm, structure);
-            accessor->finishCreation(vm);
-            return accessor;
-        }
+    static NapiExternal* create(JSC::VM& vm, JSC::Structure* structure, void* value, void* finalizer_hint, napi_finalize finalizer)
+    {
+        NapiExternal* accessor = new (NotNull, JSC::allocateCell<NapiExternal>(vm)) NapiExternal(vm, structure);
+        accessor->finishCreation(vm, value, finalizer_hint, finalizer);
+        return accessor;
+    }
 
-        void finishCreation(JSC::VM & vm);
-        void* m_value;
-        void* finalizer_context;
-    };
+    void finishCreation(JSC::VM& vm, void* value, void* finalizer_hint, napi_finalize finalizer)
+    {
+        Base::finishCreation(vm);
+        m_value = value;
+        m_finalizerHint = finalizer_hint;
+        this->finalizer = finalizer;
+    }
+
+    static void destroy(JSC::JSCell* cell);
+
+    void* value() const { return m_value; }
+
+    void* m_value;
+    void* m_finalizerHint;
+    napi_finalize finalizer;
+};
 
 } // namespace Zig

src/bun.js/bindings/webcore/DOMClientIsoSubspaces.h

@@ -31,6 +31,7 @@ public:
     std::unique_ptr<GCClient::IsoSubspace> m_clientSubspaceForPendingVirtualModuleResult;
     std::unique_ptr<GCClient::IsoSubspace> m_clientSubspaceForOnigurumaRegExp;
     std::unique_ptr<GCClient::IsoSubspace> m_clientSubspaceForCallSite;
+    std::unique_ptr<GCClient::IsoSubspace> m_clientSubspaceForNapiExternal;
 #include "ZigGeneratedClasses+DOMClientIsoSubspaces.h"
     /* --- bun --- */
 

src/bun.js/bindings/webcore/DOMIsoSubspaces.h

@@ -31,6 +31,7 @@ public:
     std::unique_ptr<IsoSubspace> m_subspaceForPendingVirtualModuleResult;
     std::unique_ptr<IsoSubspace> m_subspaceForOnigurumaRegExp;
     std::unique_ptr<IsoSubspace> m_subspaceForCallSite;
+    std::unique_ptr<IsoSubspace> m_subspaceForNapiExternal;
 #include "ZigGeneratedClasses+DOMIsoSubspaces.h"
     /*-- BUN --*/
 

src/napi/napi.zig

@@ -1506,6 +1506,8 @@ pub fn fixDeadCodeElimination() void {
     std.mem.doNotOptimizeAway(&napi_add_async_cleanup_hook);
     std.mem.doNotOptimizeAway(&napi_remove_async_cleanup_hook);
     std.mem.doNotOptimizeAway(&napi_add_finalizer);
+    std.mem.doNotOptimizeAway(&napi_create_external);
+    std.mem.doNotOptimizeAway(&napi_get_value_external);
 
     std.mem.doNotOptimizeAway(&@import("../bun.js/node/buffer.zig").BufferVectorized.fill);
 }
@@ -1600,5 +1602,7 @@ comptime {
         _ = napi_remove_async_cleanup_hook;
         _ = @import("../bun.js/node/buffer.zig").BufferVectorized.fill;
         _ = napi_add_finalizer;
+        _ = napi_create_external;
+        _ = napi_get_value_external;
     }
 }

src/options.zig

@@ -1402,6 +1402,10 @@ pub const BundleOptions = struct {
                 // If we're not doing SSR, we want all the import paths to be absolute
                 opts.import_path_format = if (opts.import_path_format == .absolute_url) .absolute_url else .absolute_path;
                 opts.env.behavior = .load_all;
+                if (transform.extension_order.len == 0) {
+                    // we must also support require'ing .node files
+                    opts.extension_order = Defaults.ExtensionOrder ++ &[_][]const u8{".node"};
+                }
             },
             else => {},
         }

src/symbols.dyn

@@ -126,4 +126,6 @@
     _napi_unwrap;
     _napi_wrap;
     _napi_remove_wrap;
+    _napi_create_external;
+    _napi_get_value_external;
 };

src/symbols.txt

@@ -125,3 +125,5 @@ _napi_unref_threadsafe_function
 _napi_unwrap
 _napi_wrap
 _napi_remove_wrap
+_napi_create_external
+_napi_get_value_external

test/bun.js/napi-test.c

@@ -0,0 +1,3 @@
+#include "../../src/napi/node_api.h"
+
+void

test/bun.js/third-party/napi_create_external/napi-create-external.test.ts

@@ -0,0 +1,199 @@
+import { test, it, describe, expect } from "bun:test";
+import * as _ from "lodash";
+
+function rebase(str, inBase, outBase) {
+  const mapBase = (b) => (b === 2 ? 32 : b === 16 ? 8 : null);
+  const stride = mapBase(inBase);
+  const pad = mapBase(outBase);
+  if (!stride) throw new Error(`Bad inBase ${inBase}`);
+  if (!pad) throw new Error(`Bad outBase ${outBase}`);
+  if (str.length % stride) throw new Error(`Bad string length ${str.length}`);
+  const out = [];
+  for (let i = 0; i < str.length; i += stride)
+    out.push(
+      parseInt(str.slice(i, i + stride), inBase)
+        .toString(outBase)
+        .padStart(pad, "0"),
+    );
+  return out.join("");
+}
+
+function expectDeepEqual(a, b) {
+  expect(JSON.stringify(a)).toBe(JSON.stringify(b));
+}
+class HashMaker {
+  constructor(length) {
+    this.length = length;
+    this._dist = {};
+  }
+  length: number;
+  _dist: any;
+
+  binToHex(binHash) {
+    if (binHash.length !== this.length)
+      throw new Error(
+        `Hash length mismatch ${this.length} != ${binHash.length}`,
+      );
+    return rebase(binHash, 2, 16);
+  }
+
+  makeBits() {
+    const bits = [];
+    for (let i = 0; i < this.length; i++) bits.push(i);
+    return _.shuffle(bits);
+  }
+
+  makeRandom() {
+    const bits = [];
+    for (let i = 0; i < this.length; i++)
+      bits.push(Math.random() < 0.5 ? 1 : 0);
+    return bits;
+  }
+
+  get keySet() {
+    return (this._set = this._set || new Set(this.data));
+  }
+
+  randomKey() {
+    while (true) {
+      const hash = this.binToHex(this.makeRandom().join(""));
+      if (!this.keySet.has(hash)) return hash;
+    }
+  }
+
+  get data() {
+    return (this._data =
+      this._data ||
+      (() => {
+        const bits = this.makeBits();
+        const base = this.makeRandom();
+        const data = [];
+        for (let stride = 0; bits.length; stride++) {
+          const flip = bits.splice(0, stride);
+          for (const bit of flip) base[bit] = 1 - base[bit];
+          data.push(this.binToHex(base.join("")));
+        }
+        return data;
+      })());
+  }
+
+  get random() {
+    const d = this.data;
+    return d[Math.floor(Math.random() * d.length)];
+  }
+
+  distance(a, b) {
+    const bitCount = (n) => {
+      n = n - ((n >> 1) & 0x55555555);
+      n = (n & 0x33333333) + ((n >> 2) & 0x33333333);
+      return (((n + (n >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
+    };
+
+    if (a === b) return 0;
+    if (a > b) return this.distance(b, a);
+    const hash = a + "-" + b;
+    return (this._dist[hash] =
+      this._dist[hash] ||
+      (() => {
+        let dist = 0;
+        for (let i = 0; i < a.length; i += 8) {
+          const va = parseInt(a.slice(i, i + 8), 16);
+          const vb = parseInt(b.slice(i, i + 8), 16);
+          dist += bitCount(va ^ vb);
+        }
+        return dist;
+      })());
+  }
+
+  query(baseKey, maxDist) {
+    const out = [];
+    for (const key of this.data) {
+      const distance = this.distance(key, baseKey);
+      if (distance <= maxDist) out.push({ key, distance });
+    }
+    return out.sort((a, b) => a.distance - b.distance);
+  }
+}
+
+const treeClass = require("bktree-fast/native");
+
+for (let keyLen = 64; keyLen <= 512; keyLen += 64) {
+  const hm = new HashMaker(keyLen);
+  describe(`Key length: ${keyLen}`, () => {
+    it("should compute distance", () => {
+      const tree = new treeClass(keyLen);
+      for (const a of hm.data)
+        for (const b of hm.data)
+          expect(tree.distance(a, b)).toBe(hm.distance(a, b));
+    });
+
+    it("should know which keys it has", () => {
+      const tree = new treeClass(keyLen).add(hm.data);
+      expectDeepEqual(
+        hm.data.map((hash) => tree.has(hash)),
+        hm.data.map(() => true),
+      );
+      // Not interested in the hash
+      for (const hash of hm.data) expect(tree.has(hm.randomKey())).toBe(false);
+    });
+
+    it("should know the tree size", () => {
+      const tree = new treeClass(keyLen, { foo: 1 });
+      expect(tree.size).toBe(0);
+      tree.add(hm.data);
+      expect(tree.size).toBe(hm.data.length);
+      tree.add(hm.data);
+      expect(tree.size).toBe(hm.data.length);
+    });
+
+    it("should walk the tree", () => {
+      const tree = new treeClass(keyLen).add(hm.data);
+      const got = [];
+      tree.walk((hash, depth) => got.push(hash));
+      expectDeepEqual(got.sort(), hm.data.slice(0).sort());
+    });
+
+    it("should query", () => {
+      Bun.gc(true);
+      ((treeClass, expectDeepEqual) => {
+        const tree = new treeClass(keyLen).add(hm.data);
+
+        for (let dist = 0; dist <= hm.length; dist++) {
+          for (const baseKey of [hm.random, hm.data[0]]) {
+            const baseKey = hm.random;
+            const got = [];
+            tree.query(baseKey, dist, (key, distance) =>
+              got.push({ key, distance }),
+            );
+            const want = hm.query(baseKey, dist);
+            expectDeepEqual(
+              got.sort((a, b) => a.distance - b.distance),
+              want,
+            );
+            expectDeepEqual(tree.find(baseKey, dist), want);
+          }
+        }
+      })(treeClass, expectDeepEqual);
+      Bun.gc(true);
+    });
+  });
+}
+
+describe("Misc functions", () => {
+  it("should pad keys", () => {
+    const tree = new treeClass(64);
+    expect(tree.padKey("1")).toBe("0000000000000001");
+    tree.add(["1", "2", "3"]);
+
+    const got = [];
+    tree.query("2", 3, (hash, distance) => got.push({ hash, distance }));
+    const res = got.sort((a, b) => a.distance - b.distance);
+    const want = [
+      { hash: "0000000000000002", distance: 0 },
+      { hash: "0000000000000003", distance: 1 },
+      { hash: "0000000000000001", distance: 2 },
+    ];
+
+    expectDeepEqual(res, want);
+  });
+});

test/bun.js/third-party/napi_create_external/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "napi-create-external-test",
+  "version": "1.0.0",
+  "description": "Test for napi_create_external",
+  "dependencies": {
+    "bktree-fast": "0.0.7",
+    "lodash": "^4.17.21"
+  },
+  "scripts": {
+    "postinstall": "cd node_modules/bktree-fast && node-gyp configure"
+  }
+}