Compare commits

...

1 Commits

Author SHA1 Message Date
Dylan Conway
dbacff1409 experiment 2026-02-01 15:44:01 -08:00
52 changed files with 10646 additions and 118 deletions

View File

@@ -902,6 +902,19 @@ target_include_directories(${bun} PRIVATE
${NODEJS_HEADERS_PATH}/include/node
)
# --- Python ---
set(PYTHON_ROOT /Users/dylan/code/bun/vendor/cpython/install)
set(PYTHON_VERSION_MAJOR 3)
set(PYTHON_VERSION_MINOR 13)
set(PYTHON_VERSION "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}")
target_include_directories(${bun} PRIVATE
${PYTHON_ROOT}/include/python${PYTHON_VERSION}
)
# Pass PYTHON_ROOT to C++ so BunPython.cpp can use it for runtime paths
target_compile_definitions(${bun} PRIVATE
PYTHON_ROOT="${PYTHON_ROOT}"
)
if(NOT WIN32)
target_include_directories(${bun} PRIVATE ${CWD}/src/bun.js/bindings/libuv)
endif()
@@ -1314,6 +1327,19 @@ if(APPLE)
target_compile_definitions(${bun} PRIVATE U_DISABLE_RENAMING=1)
endif()
# --- Python ---
# Link against shared Python library so extension modules can find symbols
if(APPLE)
target_link_libraries(${bun} PRIVATE
"${PYTHON_ROOT}/lib/libpython${PYTHON_VERSION}.dylib"
"-framework CoreFoundation"
)
else()
target_link_libraries(${bun} PRIVATE
"${PYTHON_ROOT}/lib/libpython${PYTHON_VERSION}.so"
)
endif()
if(USE_STATIC_SQLITE)
target_compile_definitions(${bun} PRIVATE LAZY_LOAD_SQLITE=0)
else()

View File

@@ -345,6 +345,7 @@ pub const api = struct {
yaml = 19,
json5 = 20,
md = 21,
py = 22,
_,
pub fn jsonStringify(self: @This(), writer: anytype) !void {

View File

@@ -58,6 +58,7 @@ pub fn trackResolutionFailure(store: *DirectoryWatchStore, import_source: []cons
.sqlite,
.sqlite_embedded,
.md,
.py,
=> bun.debugAssert(false),
}

View File

@@ -370,6 +370,78 @@ pub const HardcodedModule = enum {
.{ "bun:internal-for-testing", .{ .path = "bun:internal-for-testing" } },
.{ "ffi", .{ .path = "bun:ffi" } },
// Python builtin modules
entry("python:this"),
entry("python:builtins"),
entry("python:pathlib"),
entry("python:os"),
entry("python:json"),
entry("python:sys"),
entry("python:re"),
entry("python:math"),
entry("python:datetime"),
entry("python:collections"),
entry("python:itertools"),
entry("python:functools"),
entry("python:random"),
entry("python:hashlib"),
entry("python:base64"),
entry("python:urllib"),
entry("python:http"),
entry("python:io"),
entry("python:struct"),
entry("python:copy"),
entry("python:pickle"),
entry("python:csv"),
entry("python:sqlite3"),
entry("python:subprocess"),
entry("python:threading"),
entry("python:multiprocessing"),
entry("python:asyncio"),
entry("python:typing"),
entry("python:dataclasses"),
entry("python:enum"),
entry("python:abc"),
entry("python:contextlib"),
entry("python:logging"),
entry("python:argparse"),
entry("python:shutil"),
entry("python:glob"),
entry("python:fnmatch"),
entry("python:tempfile"),
entry("python:gzip"),
entry("python:zipfile"),
entry("python:tarfile"),
entry("python:uuid"),
entry("python:socket"),
entry("python:ssl"),
entry("python:email"),
entry("python:html"),
entry("python:xml"),
entry("python:configparser"),
entry("python:inspect"),
entry("python:traceback"),
entry("python:warnings"),
entry("python:time"),
entry("python:calendar"),
entry("python:string"),
entry("python:textwrap"),
entry("python:difflib"),
entry("python:pprint"),
entry("python:statistics"),
entry("python:decimal"),
entry("python:fractions"),
entry("python:operator"),
entry("python:heapq"),
entry("python:bisect"),
entry("python:array"),
entry("python:weakref"),
entry("python:types"),
entry("python:codecs"),
entry("python:unicodedata"),
entry("python:secrets"),
entry("python:hmac"),
// Thirdparty packages we override
.{ "@vercel/fetch", .{ .path = "@vercel/fetch" } },
.{ "isomorphic-fetch", .{ .path = "isomorphic-fetch" } },

View File

@@ -735,6 +735,18 @@ pub fn transpileSourceCode(
};
},
.py => {
// Return the file path with .python tag - C++ will run Python
// and create JSPyObject wrappers for exports
return ResolvedSource{
.allocator = null,
.source_code = bun.String.cloneUTF8(path.text),
.specifier = input_specifier,
.source_url = input_specifier.createIfDifferent(path.text),
.tag = .python,
};
},
else => {
if (flags.disableTranspiling()) {
return ResolvedSource{
@@ -828,17 +840,32 @@ pub export fn Bun__resolveAndFetchBuiltinModule(
var log = logger.Log.init(jsc_vm.transpiler.allocator);
defer log.deinit();
const alias = HardcodedModule.Alias.bun_aliases.getWithEql(specifier.*, bun.String.eqlComptime) orelse
return false;
const hardcoded = HardcodedModule.map.get(alias.path) orelse {
bun.debugAssert(false);
return false;
};
ret.* = .ok(
getHardcodedModule(jsc_vm, specifier.*, hardcoded) orelse
return false,
);
return true;
// Check hardcoded aliases first
if (HardcodedModule.Alias.bun_aliases.getWithEql(specifier.*, bun.String.eqlComptime)) |alias| {
const hardcoded = HardcodedModule.map.get(alias.path) orelse {
bun.debugAssert(false);
return false;
};
ret.* = .ok(
getHardcodedModule(jsc_vm, specifier.*, hardcoded) orelse
return false,
);
return true;
}
// Handle any python: prefixed module (for submodule imports like python:matplotlib.pyplot)
if (specifier.hasPrefixComptime("python:")) {
ret.* = .ok(.{
.allocator = null,
.source_code = specifier.dupeRef(),
.specifier = specifier.dupeRef(),
.source_url = specifier.dupeRef(),
.tag = .python_builtin,
});
return true;
}
return false;
}
pub export fn Bun__fetchBuiltinModule(
@@ -1222,6 +1249,43 @@ pub fn fetchBuiltinModule(jsc_vm: *VirtualMachine, specifier: bun.String) !?Reso
}
}
// Handle python: prefix for Python builtin modules
if (specifier.hasPrefixComptime("python:")) {
// Pass the full specifier (python:pathlib) - C++ will strip the prefix
return .{
.allocator = null,
.source_code = specifier.dupeRef(),
.specifier = specifier.dupeRef(),
.source_url = specifier.dupeRef(),
.tag = .python_builtin,
};
}
// Check if this is a Python package in .venv/lib/python{version}/site-packages/
// This allows `import numpy from "numpy"` to work for installed Python packages
const specifier_utf8 = specifier.toUTF8(bun.default_allocator);
defer specifier_utf8.deinit();
const spec_slice = specifier_utf8.slice();
// Only check for bare specifiers (not paths)
if (spec_slice.len > 0 and spec_slice[0] != '.' and spec_slice[0] != '/') {
// Check if package exists in .venv/lib/python{version}/site-packages/
var path_buf: bun.PathBuffer = undefined;
const venv_path = std.fmt.bufPrint(&path_buf, pypi.venv_site_packages ++ "/{s}", .{spec_slice}) catch return null;
// Check if directory exists (Python package) or .py file exists
if (bun.sys.directoryExistsAt(bun.FD.cwd(), venv_path).unwrap() catch false) {
// Return as python_builtin - the module loader will import it via Python
return .{
.allocator = null,
.source_code = specifier.dupeRef(),
.specifier = specifier.dupeRef(),
.source_url = specifier.dupeRef(),
.tag = .python_builtin,
};
}
}
return null;
}
@@ -1363,6 +1427,7 @@ const dumpSourceString = @import("./RuntimeTranspilerStore.zig").dumpSourceStrin
const setBreakPointOnFirstLine = @import("./RuntimeTranspilerStore.zig").setBreakPointOnFirstLine;
const bun = @import("bun");
const pypi = bun.install.PyPI;
const Environment = bun.Environment;
const MutableString = bun.MutableString;
const Output = bun.Output;

View File

@@ -1808,6 +1808,12 @@ pub fn resolveMaybeNeedsTrailingSlash(
return;
}
// Handle any python: prefixed module (allows submodule imports like python:matplotlib.pyplot)
if (bun.strings.hasPrefixComptime(specifier_utf8.slice(), "python:")) {
res.* = ErrorableString.ok(specifier);
return;
}
const old_log = jsc_vm.log;
// the logger can end up being called on another thread, it must not use threadlocal Heap Allocator
var log = logger.Log.init(bun.default_allocator);
@@ -1821,6 +1827,58 @@ pub fn resolveMaybeNeedsTrailingSlash(
jsc_vm.transpiler.resolver.log = old_log;
}
jsc_vm._resolve(&result, specifier_utf8.slice(), normalizeSource(source_utf8.slice()), is_esm, is_a_file_path) catch |err_| {
// Check if this is a Python package in .venv (fallback after node_modules)
// Only check for bare specifiers (not paths)
const spec_slice = specifier_utf8.slice();
if (spec_slice.len > 0 and spec_slice[0] != '.' and spec_slice[0] != '/') {
// Handle submodule imports like "matplotlib/pyplot" -> "python:matplotlib.pyplot"
// Extract the base package name (before any /)
const slash_idx = bun.strings.indexOfChar(spec_slice, '/');
const base_package = if (slash_idx) |idx| spec_slice[0..idx] else spec_slice;
// Check if package exists in .venv/lib/python{version}/site-packages/
// Normalize package name: Python uses underscores in module names, pip uses hyphens
var normalized_name_buf: [256]u8 = undefined;
var normalized_name = normalized_name_buf[0..@min(base_package.len, normalized_name_buf.len)];
for (base_package, 0..) |c, i| {
if (i >= normalized_name.len) break;
normalized_name[i] = if (c == '-') '_' else c;
}
var path_buf: bun.PathBuffer = undefined;
if (std.fmt.bufPrint(&path_buf, pypi.venv_site_packages ++ "/{s}", .{normalized_name})) |venv_path| {
// Check if directory exists (Python package directory)
const is_dir = bun.sys.directoryExistsAt(bun.FD.cwd(), venv_path).unwrap() catch false;
// Also check for single-file packages like typing_extensions.py, six.py
var py_path_buf: bun.PathBuffer = undefined;
const py_path = std.fmt.bufPrint(&py_path_buf, pypi.venv_site_packages ++ "/{s}.py", .{normalized_name}) catch null;
const is_py_file = if (py_path) |p| brk: {
break :brk switch (bun.sys.existsAtType(bun.FD.cwd(), p)) {
.result => |t| t == .file,
.err => false,
};
} else false;
if (is_dir or is_py_file) {
// Add python: prefix so fetchBuiltinModule handles it
// Normalize hyphens to underscores for Python module names
// Keep slashes - BunPython.cpp will convert them to dots
var module_buf: [512]u8 = undefined;
var module_name = std.ArrayList(u8).initBuffer(&module_buf);
module_name.appendSliceAssumeCapacity("python:");
// Append normalized spec_slice (hyphens -> underscores)
for (spec_slice) |c| {
module_name.appendAssumeCapacity(if (c == '-') '_' else c);
}
res.* = ErrorableString.ok(bun.String.createAtomASCII(module_name.items));
return;
}
} else |_| {}
}
var err = err_;
const msg: logger.Msg = brk: {
const msgs: []logger.Msg = log.msgs.items;
@@ -3784,3 +3842,5 @@ const ServerEntryPoint = bun.transpiler.EntryPoints.ServerEntryPoint;
const webcore = bun.webcore;
const Body = webcore.Body;
const pypi = @import("../install/pypi.zig");

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,24 @@
#pragma once
#include "root.h"
#include <JavaScriptCore/SyntheticModuleRecord.h>
#include <Python.h>
namespace Bun::Python {
// Generate module source code for importing Python files as ES modules
// If isMainEntry is true, __name__ will be "__main__", otherwise it's derived from the filename
JSC::SyntheticSourceProvider::SyntheticSourceGenerator
generatePythonModuleSourceCode(JSC::JSGlobalObject* globalObject, const WTF::String& filePath, bool isMainEntry);
// Generate module source code for importing Python builtin modules (e.g., "python:pathlib")
JSC::SyntheticSourceProvider::SyntheticSourceGenerator
generatePythonBuiltinModuleSourceCode(JSC::JSGlobalObject* globalObject, const WTF::String& moduleName);
JSC::JSValue toJS(JSC::JSGlobalObject* globalObject, PyObject* value);
PyObject* fromJS(JSC::JSGlobalObject* globalObject, JSC::JSValue value);
// Ensure Python is initialized
void ensurePythonInitialized();
} // namespace Bun::Python

View File

@@ -0,0 +1,634 @@
#include "JSPyObject.h"
#include "BunPython.h"
#include "ZigGlobalObject.h"
#include "BunClientData.h"
#include <JavaScriptCore/ObjectConstructor.h>
#include <JavaScriptCore/FunctionPrototype.h>
#include <JavaScriptCore/JSFunction.h>
#include <wtf/text/WTFString.h>
namespace Bun {
using namespace JSC;
// Forward declaration for toString
static JSC_DECLARE_HOST_FUNCTION(jsPyObjectToString);
// Forward declaration for call
static JSC_DECLARE_HOST_FUNCTION(jsPyObjectCall);
// Forward declaration for iterator
static JSC_DECLARE_HOST_FUNCTION(jsPyObjectIterator);
// Forward declaration for iterator next
static JSC_DECLARE_HOST_FUNCTION(jsPyIteratorNext);
const ClassInfo JSPyObject::s_info = { "PythonValue"_s, &Base::s_info, nullptr, nullptr, CREATE_METHOD_TABLE(JSPyObject) };
template<typename Visitor>
void JSPyObject::visitChildrenImpl(JSCell* cell, Visitor& visitor)
{
JSPyObject* thisObject = jsCast<JSPyObject*>(cell);
ASSERT_GC_OBJECT_INHERITS(thisObject, info());
Base::visitChildren(thisObject, visitor);
}
DEFINE_VISIT_CHILDREN(JSPyObject);
void JSPyObject::finishCreation(VM& vm)
{
Base::finishCreation(vm);
ASSERT(inherits(info()));
}
JSC::GCClient::IsoSubspace* JSPyObject::subspaceForImpl(JSC::VM& vm)
{
return WebCore::subspaceForImpl<JSPyObject, WebCore::UseCustomHeapCellType::No>(
vm,
[](auto& spaces) { return spaces.m_clientSubspaceForPyObject.get(); },
[](auto& spaces, auto&& space) { spaces.m_clientSubspaceForPyObject = std::forward<decltype(space)>(space); },
[](auto& spaces) { return spaces.m_subspaceForPyObject.get(); },
[](auto& spaces, auto&& space) { spaces.m_subspaceForPyObject = std::forward<decltype(space)>(space); });
}
// Property access - proxy to Python's getattr
bool JSPyObject::getOwnPropertySlot(JSObject* object, JSGlobalObject* globalObject, PropertyName propertyName, PropertySlot& slot)
{
JSPyObject* thisObject = jsCast<JSPyObject*>(object);
VM& vm = globalObject->vm();
// Handle special JS properties
if (propertyName == vm.propertyNames->toStringTagSymbol) {
slot.setValue(object, static_cast<unsigned>(PropertyAttribute::DontEnum | PropertyAttribute::ReadOnly), jsString(vm, String("PythonValue"_s)));
return true;
}
// Handle toString
if (propertyName == vm.propertyNames->toString) {
slot.setValue(object, static_cast<unsigned>(PropertyAttribute::DontEnum),
JSFunction::create(vm, globalObject, 0, "toString"_s, jsPyObjectToString, ImplementationVisibility::Public));
return true;
}
// Handle nodejs.util.inspect.custom for console.log
if (propertyName == Identifier::fromUid(vm.symbolRegistry().symbolForKey("nodejs.util.inspect.custom"_s))) {
slot.setValue(object, static_cast<unsigned>(PropertyAttribute::DontEnum),
JSFunction::create(vm, globalObject, 0, "inspect"_s, jsPyObjectToString, ImplementationVisibility::Public));
return true;
}
// Handle Symbol.iterator for Python iterables
if (propertyName == vm.propertyNames->iteratorSymbol) {
// Check if this Python object is iterable
if (PyIter_Check(thisObject->m_pyObject) || PyObject_HasAttrString(thisObject->m_pyObject, "__iter__")) {
slot.setValue(object, static_cast<unsigned>(PropertyAttribute::DontEnum),
JSFunction::create(vm, globalObject, 0, "[Symbol.iterator]"_s, jsPyObjectIterator, ImplementationVisibility::Public));
return true;
}
}
// Handle length property for Python sequences (needed for Array.prototype methods)
if (propertyName == vm.propertyNames->length) {
if (PySequence_Check(thisObject->m_pyObject) && !PyUnicode_Check(thisObject->m_pyObject)) {
Py_ssize_t len = PySequence_Size(thisObject->m_pyObject);
if (len >= 0) {
slot.setValue(object, static_cast<unsigned>(PropertyAttribute::DontEnum | PropertyAttribute::ReadOnly), jsNumber(len));
return true;
}
PyErr_Clear();
}
}
// Convert property name to Python string
auto* nameString = propertyName.publicName();
if (!nameString) {
return Base::getOwnPropertySlot(object, globalObject, propertyName, slot);
}
auto nameUTF8 = nameString->utf8();
PyObject* pyName = PyUnicode_FromStringAndSize(nameUTF8.data(), nameUTF8.length());
if (!pyName) {
PyErr_Clear();
return false;
}
// First try attribute access (for regular objects)
PyObject* attr = PyObject_GetAttr(thisObject->m_pyObject, pyName);
if (!attr) {
PyErr_Clear();
// If attribute access fails, try item access (for dicts/mappings)
if (PyMapping_Check(thisObject->m_pyObject)) {
attr = PyObject_GetItem(thisObject->m_pyObject, pyName);
if (!attr) {
PyErr_Clear();
}
}
}
Py_DECREF(pyName);
if (!attr) {
return false;
}
JSValue jsAttr = Python::toJS(globalObject, attr);
Py_DECREF(attr);
slot.setValue(object, static_cast<unsigned>(PropertyAttribute::None), jsAttr);
return true;
}
bool JSPyObject::getOwnPropertySlotByIndex(JSObject* object, JSGlobalObject* globalObject, unsigned index, PropertySlot& slot)
{
JSPyObject* thisObject = jsCast<JSPyObject*>(object);
PyObject* item = PySequence_GetItem(thisObject->m_pyObject, static_cast<Py_ssize_t>(index));
if (!item) {
PyErr_Clear();
return false;
}
JSValue jsItem = Python::toJS(globalObject, item);
Py_DECREF(item);
slot.setValue(object, static_cast<unsigned>(PropertyAttribute::None), jsItem);
return true;
}
void JSPyObject::getOwnPropertyNames(JSObject* object, JSGlobalObject* globalObject, PropertyNameArrayBuilder& propertyNames, DontEnumPropertiesMode mode)
{
JSPyObject* thisObject = jsCast<JSPyObject*>(object);
VM& vm = globalObject->vm();
// Get dir() of the object
PyObject* dir = PyObject_Dir(thisObject->m_pyObject);
if (!dir) {
PyErr_Clear();
return;
}
Py_ssize_t len = PyList_Size(dir);
for (Py_ssize_t i = 0; i < len; i++) {
PyObject* name = PyList_GetItem(dir, i); // borrowed reference
if (PyUnicode_Check(name)) {
const char* nameStr = PyUnicode_AsUTF8(name);
if (nameStr && nameStr[0] != '_') { // Skip private/dunder
propertyNames.add(Identifier::fromString(vm, String::fromUTF8(nameStr)));
}
}
}
Py_DECREF(dir);
}
// Helper to convert JSValue to PyObject
static PyObject* jsValueToPyObject(JSGlobalObject* globalObject, JSValue value)
{
if (value.isNull() || value.isUndefined()) {
Py_INCREF(Py_None);
return Py_None;
}
if (value.isBoolean()) {
PyObject* result = value.asBoolean() ? Py_True : Py_False;
Py_INCREF(result);
return result;
}
if (value.isNumber()) {
double num = value.asNumber();
constexpr double maxSafeInt = 9007199254740992.0;
if (std::floor(num) == num && num >= -maxSafeInt && num <= maxSafeInt) {
return PyLong_FromLongLong(static_cast<long long>(num));
}
return PyFloat_FromDouble(num);
}
if (value.isString()) {
auto str = value.toWTFString(globalObject);
auto utf8 = str.utf8();
return PyUnicode_FromStringAndSize(utf8.data(), utf8.length());
}
if (auto* pyVal = jsDynamicCast<JSPyObject*>(value)) {
PyObject* obj = pyVal->pyObject();
Py_INCREF(obj);
return obj;
}
// For other JS objects, return None for now
Py_INCREF(Py_None);
return Py_None;
}
bool JSPyObject::put(JSCell* cell, JSGlobalObject* globalObject, PropertyName propertyName, JSValue value, PutPropertySlot& slot)
{
JSPyObject* thisObject = jsCast<JSPyObject*>(cell);
auto* nameString = propertyName.publicName();
if (!nameString) {
return false;
}
auto nameUTF8 = nameString->utf8();
PyObject* pyName = PyUnicode_FromStringAndSize(nameUTF8.data(), nameUTF8.length());
if (!pyName) {
PyErr_Clear();
return false;
}
PyObject* pyValue = jsValueToPyObject(globalObject, value);
if (!pyValue) {
Py_DECREF(pyName);
PyErr_Clear();
return false;
}
int result = -1;
// For dicts/mappings, use item assignment
if (PyDict_Check(thisObject->m_pyObject)) {
result = PyDict_SetItem(thisObject->m_pyObject, pyName, pyValue);
} else if (PyMapping_Check(thisObject->m_pyObject)) {
result = PyObject_SetItem(thisObject->m_pyObject, pyName, pyValue);
} else {
// For other objects, try attribute assignment
result = PyObject_SetAttr(thisObject->m_pyObject, pyName, pyValue);
}
Py_DECREF(pyName);
Py_DECREF(pyValue);
if (result < 0) {
PyErr_Clear();
return false;
}
return true;
}
bool JSPyObject::putByIndex(JSCell* cell, JSGlobalObject* globalObject, unsigned index, JSValue value, bool)
{
JSPyObject* thisObject = jsCast<JSPyObject*>(cell);
if (!PySequence_Check(thisObject->m_pyObject)) {
return false;
}
PyObject* pyValue = jsValueToPyObject(globalObject, value);
if (!pyValue) {
PyErr_Clear();
return false;
}
// Get current length
Py_ssize_t length = PySequence_Size(thisObject->m_pyObject);
if (length < 0) {
PyErr_Clear();
Py_DECREF(pyValue);
return false;
}
int result;
if (static_cast<Py_ssize_t>(index) >= length) {
// Index is beyond current length - we need to extend the list
if (PyList_Check(thisObject->m_pyObject)) {
// For lists, extend with None values up to the index, then set
PyObject* list = thisObject->m_pyObject;
for (Py_ssize_t i = length; i < static_cast<Py_ssize_t>(index); i++) {
if (PyList_Append(list, Py_None) < 0) {
PyErr_Clear();
Py_DECREF(pyValue);
return false;
}
}
result = PyList_Append(list, pyValue);
} else {
// For other sequences, try insert or set item
result = PySequence_SetItem(thisObject->m_pyObject, static_cast<Py_ssize_t>(index), pyValue);
}
} else {
result = PySequence_SetItem(thisObject->m_pyObject, static_cast<Py_ssize_t>(index), pyValue);
}
Py_DECREF(pyValue);
if (result < 0) {
PyErr_Clear();
return false;
}
return true;
}
// toString - returns Python's str() representation
JSC_DEFINE_HOST_FUNCTION(jsPyObjectToString, (JSGlobalObject * globalObject, CallFrame* callFrame))
{
VM& vm = globalObject->vm();
auto scope = DECLARE_THROW_SCOPE(vm);
JSValue thisValue = callFrame->thisValue();
JSPyObject* thisObject = jsDynamicCast<JSPyObject*>(thisValue);
if (!thisObject) {
return JSValue::encode(jsString(vm, String("[object PythonValue]"_s)));
}
PyObject* str = PyObject_Str(thisObject->pyObject());
if (!str) {
PyErr_Clear();
return JSValue::encode(jsString(vm, String("[object PythonValue]"_s)));
}
const char* utf8 = PyUnicode_AsUTF8(str);
if (!utf8) {
Py_DECREF(str);
PyErr_Clear();
return JSValue::encode(jsString(vm, String("[object PythonValue]"_s)));
}
JSValue result = jsString(vm, WTF::String::fromUTF8(utf8));
Py_DECREF(str);
return JSValue::encode(result);
}
// Iterator next - called from the JS iterator's next() method
JSC_DEFINE_HOST_FUNCTION(jsPyIteratorNext, (JSGlobalObject * globalObject, CallFrame* callFrame))
{
VM& vm = globalObject->vm();
auto scope = DECLARE_THROW_SCOPE(vm);
// Get the Python iterator from the thisValue (which should be the iterator wrapper object)
JSValue thisValue = callFrame->thisValue();
JSObject* thisObject = thisValue.toObject(globalObject);
RETURN_IF_EXCEPTION(scope, {});
// Get the stored Python iterator
JSValue pyIterValue = thisObject->getDirect(vm, Identifier::fromString(vm, "_pyIter"_s));
if (!pyIterValue) {
return JSValue::encode(constructEmptyObject(globalObject));
}
JSPyObject* pyIter = jsDynamicCast<JSPyObject*>(pyIterValue);
if (!pyIter) {
return JSValue::encode(constructEmptyObject(globalObject));
}
// Call Python's next() on the iterator
PyObject* nextItem = PyIter_Next(pyIter->pyObject());
// Create the result object { value, done }
JSObject* result = constructEmptyObject(globalObject);
if (nextItem) {
// Got an item
result->putDirect(vm, Identifier::fromString(vm, "value"_s), Python::toJS(globalObject, nextItem));
result->putDirect(vm, Identifier::fromString(vm, "done"_s), jsBoolean(false));
Py_DECREF(nextItem);
} else {
// Check if it's StopIteration or an error
if (PyErr_Occurred()) {
if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
PyErr_Clear();
} else {
// Real error - propagate it
PyErr_Print();
PyErr_Clear();
throwTypeError(globalObject, scope, "Python iterator error"_s);
return {};
}
}
// Iterator exhausted
result->putDirect(vm, Identifier::fromString(vm, "value"_s), jsUndefined());
result->putDirect(vm, Identifier::fromString(vm, "done"_s), jsBoolean(true));
}
return JSValue::encode(result);
}
// Symbol.iterator - returns a JS iterator that wraps Python iteration
JSC_DEFINE_HOST_FUNCTION(jsPyObjectIterator, (JSGlobalObject * globalObject, CallFrame* callFrame))
{
VM& vm = globalObject->vm();
auto scope = DECLARE_THROW_SCOPE(vm);
JSValue thisValue = callFrame->thisValue();
JSPyObject* thisObject = jsDynamicCast<JSPyObject*>(thisValue);
if (!thisObject) {
throwTypeError(globalObject, scope, "Not a Python object"_s);
return {};
}
// Get a Python iterator for this object
PyObject* pyIter = PyObject_GetIter(thisObject->pyObject());
if (!pyIter) {
PyErr_Clear();
throwTypeError(globalObject, scope, "Python object is not iterable"_s);
return {};
}
// Create a JS iterator object
JSObject* jsIter = constructEmptyObject(globalObject);
// Store the Python iterator (as JSPyObject) on the JS iterator object
auto* zigGlobalObject = jsCast<Zig::GlobalObject*>(globalObject);
Structure* structure = zigGlobalObject->m_JSPyObjectStructure.get();
if (!structure) {
structure = JSPyObject::createStructure(vm, globalObject, globalObject->objectPrototype());
zigGlobalObject->m_JSPyObjectStructure.set(vm, zigGlobalObject, structure);
}
JSPyObject* wrappedIter = JSPyObject::create(vm, globalObject, structure, pyIter);
Py_DECREF(pyIter); // JSPyObject takes ownership
jsIter->putDirect(vm, Identifier::fromString(vm, "_pyIter"_s), wrappedIter);
// Add the next() method
jsIter->putDirect(vm, Identifier::fromString(vm, "next"_s),
JSFunction::create(vm, globalObject, 0, "next"_s, jsPyIteratorNext, ImplementationVisibility::Public));
return JSValue::encode(jsIter);
}
// Helper to check if a JSValue is a plain object (not array, not wrapped Python object)
static bool isPlainJSObject(JSGlobalObject* globalObject, JSValue value)
{
if (!value.isObject())
return false;
JSObject* obj = value.getObject();
// Not a plain object if it's a JSPyObject (wrapped Python object)
if (jsDynamicCast<JSPyObject*>(obj))
return false;
// Not a plain object if it's an array
if (isJSArray(obj))
return false;
// Not a plain object if it's a function
if (obj->isCallable())
return false;
// Check if it's a plain Object (not a special type like Date, Map, etc.)
// We consider it kwargs-eligible if its prototype is Object.prototype or null
JSValue proto = obj->getPrototype(globalObject);
return proto.isNull() || proto == globalObject->objectPrototype();
}
// Get the expected positional argument count for a Python callable
// Returns -1 if we can't determine (e.g., built-in functions)
static int getExpectedArgCount(PyObject* callable)
{
PyObject* codeObj = nullptr;
// For regular functions, get __code__
if (PyFunction_Check(callable)) {
codeObj = PyFunction_GET_CODE(callable);
}
// For methods, get the underlying function's __code__
else if (PyMethod_Check(callable)) {
PyObject* func = PyMethod_GET_FUNCTION(callable);
if (PyFunction_Check(func)) {
codeObj = PyFunction_GET_CODE(func);
}
}
// Try getting __code__ attribute for other callables (like lambdas assigned to variables)
else if (PyObject_HasAttrString(callable, "__code__")) {
codeObj = PyObject_GetAttrString(callable, "__code__");
if (codeObj) {
PyObject* argCountObj = PyObject_GetAttrString(codeObj, "co_argcount");
Py_DECREF(codeObj);
if (argCountObj) {
int count = static_cast<int>(PyLong_AsLong(argCountObj));
Py_DECREF(argCountObj);
return count;
}
}
PyErr_Clear();
return -1;
}
if (!codeObj) {
return -1;
}
// Get co_argcount from the code object
PyCodeObject* code = reinterpret_cast<PyCodeObject*>(codeObj);
return code->co_argcount;
}
// Call Python function from JS
JSC_DEFINE_HOST_FUNCTION(jsPyObjectCall, (JSGlobalObject * globalObject, CallFrame* callFrame))
{
VM& vm = globalObject->vm();
auto scope = DECLARE_THROW_SCOPE(vm);
JSPyObject* thisObject = jsDynamicCast<JSPyObject*>(callFrame->jsCallee());
if (!thisObject) {
throwTypeError(globalObject, scope, "Not a Python callable"_s);
return {};
}
PyObject* pyFunc = thisObject->pyObject();
if (!PyCallable_Check(pyFunc)) {
throwTypeError(globalObject, scope, "Python object is not callable"_s);
return {};
}
// Convert all arguments as positional args
// TODO: Support kwargs via a special marker like $kwargs from "bun:python"
size_t argCount = callFrame->argumentCount();
// Check if the Python function expects fewer arguments than provided
// If so, trim the argument list to match (allows flexible callback signatures)
int expectedArgs = getExpectedArgCount(pyFunc);
if (expectedArgs >= 0 && static_cast<size_t>(expectedArgs) < argCount) {
argCount = static_cast<size_t>(expectedArgs);
}
PyObject* kwargs = nullptr;
// Convert JS arguments to Python tuple
PyObject* args = PyTuple_New(static_cast<Py_ssize_t>(argCount));
if (!args) {
Py_XDECREF(kwargs);
throwOutOfMemoryError(globalObject, scope);
return {};
}
for (size_t i = 0; i < argCount; i++) {
JSValue jsArg = callFrame->uncheckedArgument(i);
PyObject* pyArg = nullptr;
// Check if it's already a wrapped Python object first
if (auto* pyVal = jsDynamicCast<JSPyObject*>(jsArg)) {
// Unwrap JSPyObject back to PyObject
pyArg = pyVal->pyObject();
Py_INCREF(pyArg);
} else {
// Convert JS value to Python using the standard conversion
// This handles primitives, arrays (as list), and objects (as dict)
pyArg = Python::fromJS(globalObject, jsArg);
}
if (!pyArg) {
Py_DECREF(args);
Py_XDECREF(kwargs);
throwTypeError(globalObject, scope, "Failed to convert argument to Python"_s);
return {};
}
PyTuple_SET_ITEM(args, i, pyArg); // steals reference
}
// Call the Python function with args and optional kwargs
PyObject* result = PyObject_Call(pyFunc, args, kwargs);
Py_DECREF(args);
Py_XDECREF(kwargs);
if (!result) {
// Get Python exception info
PyObject *type, *value, *traceback;
PyErr_Fetch(&type, &value, &traceback);
PyErr_NormalizeException(&type, &value, &traceback);
WTF::String errorMessage = "Python error"_s;
if (value) {
PyObject* str = PyObject_Str(value);
if (str) {
const char* errStr = PyUnicode_AsUTF8(str);
if (errStr) {
errorMessage = WTF::String::fromUTF8(errStr);
}
Py_DECREF(str);
}
}
Py_XDECREF(type);
Py_XDECREF(value);
Py_XDECREF(traceback);
throwTypeError(globalObject, scope, errorMessage);
return {};
}
JSValue jsResult = Python::toJS(globalObject, result);
Py_DECREF(result);
return JSValue::encode(jsResult);
}
CallData JSPyObject::getCallData(JSCell* cell)
{
JSPyObject* thisObject = jsCast<JSPyObject*>(cell);
CallData callData;
// Only allow direct calls for non-type callables (functions, lambdas, etc.)
// Python types (classes) should require `new`, like JS classes
if (thisObject->isCallable() && !PyType_Check(thisObject->m_pyObject)) {
callData.type = CallData::Type::Native;
callData.native.function = jsPyObjectCall;
}
return callData;
}
// For Python, constructing and calling are the same thing
// This allows `new Counter()` to work for Python classes
CallData JSPyObject::getConstructData(JSCell* cell)
{
JSPyObject* thisObject = jsCast<JSPyObject*>(cell);
CallData constructData;
if (thisObject->isCallable()) {
constructData.type = CallData::Type::Native;
constructData.native.function = jsPyObjectCall;
}
return constructData;
}
} // namespace Bun

View File

@@ -0,0 +1,86 @@
#pragma once
#include "root.h"
#include <Python.h>
namespace Bun {
using namespace JSC;
// JSPyObject wraps a PyObject* and proxies property access, calls, etc. to Python.
// When created, it increments the Python refcount; when finalized by GC, it decrements it.
class JSPyObject : public JSC::JSDestructibleObject {
using Base = JSC::JSDestructibleObject;
public:
JSPyObject(JSC::VM& vm, JSC::Structure* structure, PyObject* pyObject)
: Base(vm, structure)
, m_pyObject(pyObject)
{
// Prevent Python from freeing this object while we hold it
Py_INCREF(m_pyObject);
}
DECLARE_INFO;
DECLARE_VISIT_CHILDREN;
static constexpr unsigned StructureFlags = Base::StructureFlags | OverridesGetOwnPropertySlot | OverridesGetOwnPropertyNames | OverridesPut | OverridesGetCallData | InterceptsGetOwnPropertySlotByIndexEvenWhenLengthIsNotZero;
template<typename, JSC::SubspaceAccess mode>
static JSC::GCClient::IsoSubspace* subspaceFor(JSC::VM& vm)
{
if constexpr (mode == JSC::SubspaceAccess::Concurrently)
return nullptr;
return subspaceForImpl(vm);
}
static JSC::GCClient::IsoSubspace* subspaceForImpl(JSC::VM& vm);
static JSC::Structure* createStructure(JSC::VM& vm, JSC::JSGlobalObject* globalObject, JSC::JSValue prototype)
{
return JSC::Structure::create(vm, globalObject, prototype,
JSC::TypeInfo(JSC::ObjectType, StructureFlags), info());
}
static JSPyObject* create(JSC::VM& vm, JSC::JSGlobalObject* globalObject, JSC::Structure* structure, PyObject* pyObject)
{
JSPyObject* value = new (NotNull, JSC::allocateCell<JSPyObject>(vm)) JSPyObject(vm, structure, pyObject);
value->finishCreation(vm);
return value;
}
void finishCreation(JSC::VM& vm);
static void destroy(JSCell* thisObject)
{
JSPyObject* value = static_cast<JSPyObject*>(thisObject);
// Release Python reference
Py_DECREF(value->m_pyObject);
value->~JSPyObject();
}
// Property access - proxy to Python's __getattr__
static bool getOwnPropertySlot(JSObject*, JSGlobalObject*, PropertyName, PropertySlot&);
static bool getOwnPropertySlotByIndex(JSObject*, JSGlobalObject*, unsigned, PropertySlot&);
static void getOwnPropertyNames(JSObject*, JSGlobalObject*, PropertyNameArrayBuilder&, DontEnumPropertiesMode);
// Property set - proxy to Python's __setattr__
static bool put(JSCell*, JSGlobalObject*, PropertyName, JSValue, PutPropertySlot&);
static bool putByIndex(JSCell*, JSGlobalObject*, unsigned, JSValue, bool);
// If callable, proxy to Python's __call__
static CallData getCallData(JSCell*);
// If callable, also make constructible (for Python classes)
static CallData getConstructData(JSCell*);
// Get the wrapped PyObject
PyObject* pyObject() const { return m_pyObject; }
// Helper to check if Python object is callable
bool isCallable() const { return PyCallable_Check(m_pyObject); }
private:
PyObject* m_pyObject;
};
} // namespace Bun

View File

@@ -40,6 +40,7 @@
#include "JSCommonJSExtensions.h"
#include "BunProcess.h"
#include "BunPython.h"
namespace Bun {
using namespace JSC;
@@ -65,6 +66,7 @@ public:
};
extern "C" BunLoaderType Bun__getDefaultLoader(JSC::JSGlobalObject*, BunString* specifier);
extern "C" JSC::EncodedJSValue BunObject_getter_main(JSC::JSGlobalObject*);
static JSC::JSInternalPromise* rejectedInternalPromise(JSC::JSGlobalObject* globalObject, JSC::JSValue value)
{
@@ -977,6 +979,15 @@ static JSValue fetchESMSourceCode(
auto&& provider = Zig::SourceProvider::create(globalObject, res->result.value, JSC::SourceProviderSourceType::Module, true);
RELEASE_AND_RETURN(scope, rejectOrResolve(JSSourceCode::create(vm, JSC::SourceCode(provider))));
}
case SyntheticModuleType::PythonBuiltin: {
// Python builtin module - import from Python's standard library
WTF::String moduleName = res->result.value.source_code.toWTFString(BunString::NonNull);
auto function = Python::generatePythonBuiltinModuleSourceCode(globalObject, moduleName);
auto source = JSC::SourceCode(
JSC::SyntheticSourceProvider::create(WTF::move(function),
JSC::SourceOrigin(), WTF::move(moduleKey)));
RELEASE_AND_RETURN(scope, rejectOrResolve(JSSourceCode::create(vm, WTF::move(source))));
}
#define CASE(str, name) \
case (SyntheticModuleType::name): { \
@@ -1103,6 +1114,21 @@ static JSValue fetchESMSourceCode(
JSC::SourceOrigin(), specifier->toWTFString(BunString::ZeroCopy)));
JSC::ensureStillAliveHere(value);
RELEASE_AND_RETURN(scope, rejectOrResolve(JSSourceCode::create(globalObject->vm(), WTF::move(source))));
} else if (res->result.value.tag == SyntheticModuleType::Python) {
// Python module - run Python file and wrap exports as JSPyObject
WTF::String filePath = res->result.value.source_code.toWTFString(BunString::NonNull);
// Check if this is the main entry point by comparing against Bun.main
bool isMainEntry = false;
JSValue mainValue = JSValue::decode(BunObject_getter_main(globalObject));
if (mainValue.isString()) {
WTF::String mainPath = mainValue.toWTFString(globalObject);
isMainEntry = (filePath == mainPath);
}
auto function = Python::generatePythonModuleSourceCode(globalObject, filePath, isMainEntry);
auto source = JSC::SourceCode(
JSC::SyntheticSourceProvider::create(WTF::move(function),
JSC::SourceOrigin(), specifier->toWTFString(BunString::ZeroCopy)));
RELEASE_AND_RETURN(scope, rejectOrResolve(JSSourceCode::create(globalObject->vm(), WTF::move(source))));
}
RELEASE_AND_RETURN(scope, rejectOrResolve(JSC::JSSourceCode::create(vm, JSC::SourceCode(Zig::SourceProvider::create(globalObject, res->result.value)))));

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,53 @@
#pragma once
#include "root.h"
#include "Python.h"
namespace Bun {
using namespace JSC;
// Base wrapper for JS values in Python - used for functions and other non-container types
struct PyJSValueObject {
PyObject_HEAD
JSValue jsValue;
JSGlobalObject* globalObject;
static PyJSValueObject* New();
static PyJSValueObject* NewDict(JSGlobalObject* globalObject, JSValue value);
static PyJSValueObject* NewList(JSGlobalObject* globalObject, JSValue value);
static void initType();
};
// Dict subclass wrapper - makes isinstance(obj, dict) return True
// Uses same memory layout as PyJSValueObject but with dict as base type
struct PyJSDictObject {
PyDictObject dict; // Must be first - inherits from dict
JSValue jsValue;
JSGlobalObject* globalObject;
};
// List subclass wrapper - makes isinstance(obj, list) return True
struct PyJSListObject {
PyListObject list; // Must be first - inherits from list
JSValue jsValue;
JSGlobalObject* globalObject;
};
// Bound method wrapper - preserves 'this' context when accessing methods on JS objects
// When you do `obj.method()` in Python, we need to call method with `this` = obj
struct PyJSBoundMethod {
PyObject_HEAD
JSValue function; // The JS function
JSValue thisObject; // The object the function was accessed from
JSGlobalObject* globalObject;
static PyJSBoundMethod* New(JSGlobalObject* globalObject, JSValue function, JSValue thisObject);
static void initType();
};
// Try to unwrap a PyObject that wraps a JSValue back to the underlying JSValue
// Returns empty JSValue if the object is not a PyJSValueObject, PyJSDictObject, or PyJSListObject
JSValue tryUnwrapJSValue(PyObject* obj);
} // namespace Bun

View File

@@ -302,7 +302,7 @@ extern "C" void JSCInitialize(const char* envp[], size_t envc, void (*onCrash)(c
JSC::Options::useJITCage() = false;
JSC::Options::useShadowRealm() = true;
JSC::Options::useV8DateParser() = true;
JSC::Options::useMathSumPreciseMethod() = true;
// JSC::Options::useMathSumPreciseMethod() = true;
JSC::Options::evalMode() = evalMode;
JSC::Options::heapGrowthSteepnessFactor() = 1.0;
JSC::Options::heapGrowthMaxIncrease() = 2.0;

View File

@@ -638,7 +638,11 @@ public:
V(public, LazyPropertyOfGlobalObject<Symbol>, m_nodeVMDontContextify) \
V(public, LazyPropertyOfGlobalObject<Symbol>, m_nodeVMUseMainContextDefaultLoader) \
V(public, LazyPropertyOfGlobalObject<JSFunction>, m_ipcSerializeFunction) \
V(public, LazyPropertyOfGlobalObject<JSFunction>, m_ipcParseHandleFunction)
V(public, LazyPropertyOfGlobalObject<JSFunction>, m_ipcParseHandleFunction) \
\
/* Python integration */ \
V(public, WriteBarrier<Structure>, m_JSPyObjectStructure) \
V(public, WriteBarrier<Structure>, m_JSPyArrayStructure)
#define DECLARE_GLOBALOBJECT_GC_MEMBER(visibility, T, name) \
visibility: \

View File

@@ -50,14 +50,15 @@
macro(JSPrinter.printWithSourceMap, 46) \
macro(ModuleResolver.resolve, 47) \
macro(PackageInstaller.install, 48) \
macro(PackageManifest.Serializer.loadByFile, 49) \
macro(PackageManifest.Serializer.save, 50) \
macro(RuntimeTranspilerCache.fromFile, 51) \
macro(RuntimeTranspilerCache.save, 52) \
macro(RuntimeTranspilerCache.toFile, 53) \
macro(StandaloneModuleGraph.serialize, 54) \
macro(Symbols.followAll, 55) \
macro(TestCommand.printCodeCoverageLCov, 56) \
macro(TestCommand.printCodeCoverageLCovAndText, 57) \
macro(TestCommand.printCodeCoverageText, 58) \
macro(PackageInstaller.installPythonPackage, 49) \
macro(PackageManifest.Serializer.loadByFile, 50) \
macro(PackageManifest.Serializer.save, 51) \
macro(RuntimeTranspilerCache.fromFile, 52) \
macro(RuntimeTranspilerCache.save, 53) \
macro(RuntimeTranspilerCache.toFile, 54) \
macro(StandaloneModuleGraph.serialize, 55) \
macro(Symbols.followAll, 56) \
macro(TestCommand.printCodeCoverageLCov, 57) \
macro(TestCommand.printCodeCoverageLCovAndText, 58) \
macro(TestCommand.printCodeCoverageText, 59) \
// end

View File

@@ -954,5 +954,6 @@ public:
std::unique_ptr<GCClient::IsoSubspace> m_clientSubspaceForJSConnectionsList;
std::unique_ptr<GCClient::IsoSubspace> m_clientSubspaceForJSHTTPParser;
std::unique_ptr<GCClient::IsoSubspace> m_clientSubspaceForPyObject;
};
} // namespace WebCore

View File

@@ -957,6 +957,7 @@ public:
std::unique_ptr<IsoSubspace> m_subspaceForJSConnectionsList;
std::unique_ptr<IsoSubspace> m_subspaceForJSHTTPParser;
std::unique_ptr<IsoSubspace> m_subspaceForPyObject;
};
} // namespace WebCore

View File

@@ -510,7 +510,7 @@ pub const LinkerContext = struct {
const loader = loaders[record.source_index.get()];
switch (loader) {
.jsx, .js, .ts, .tsx, .napi, .sqlite, .json, .jsonc, .json5, .yaml, .html, .sqlite_embedded, .md => {
.jsx, .js, .ts, .tsx, .napi, .sqlite, .json, .jsonc, .json5, .yaml, .html, .sqlite_embedded, .md, .py => {
log.addErrorFmt(
source,
record.range.loc,

View File

@@ -606,7 +606,7 @@ fn getAST(
return ast;
},
// TODO:
.dataurl, .base64, .bunsh => {
.dataurl, .base64, .bunsh, .py => {
return try getEmptyAST(log, transpiler, opts, allocator, source, E.String);
},
.file, .wasm => {

View File

@@ -424,6 +424,10 @@ pub const ResolvedSourceTag = enum(u32) {
export_default_object = 9,
/// Signal upwards that the matching value in 'require.extensions' should be used.
common_js_custom_extension = 10,
/// Python module - execute via embedded Python interpreter
python = 11,
/// Python builtin module - import a module from Python's standard library
python_builtin = 12,
// Built in modules are loaded through InternalModuleRegistry by numerical ID.
// In this enum are represented as \`(1 << 9) & id\`
@@ -454,6 +458,8 @@ writeIfNotChanged(
ExportsObject = 8,
ExportDefaultObject = 9,
CommonJSCustomExtension = 10,
Python = 11,
PythonBuiltin = 12,
// Built in modules are loaded through InternalModuleRegistry by numerical ID.
// In this enum are represented as \`(1 << 9) & id\`
InternalModuleRegistryFlag = 1 << 9,

View File

@@ -49,6 +49,7 @@ pub const PerfEvent = enum(i32) {
@"JSPrinter.printWithSourceMap",
@"ModuleResolver.resolve",
@"PackageInstaller.install",
@"PackageInstaller.installPythonPackage",
@"PackageManifest.Serializer.loadByFile",
@"PackageManifest.Serializer.save",
@"RuntimeTranspilerCache.fromFile",

View File

@@ -17,6 +17,9 @@ callback: union(Task.Tag) {
git_clone: void,
git_checkout: void,
local_tarball: void,
pypi_manifest: struct {
name: strings.StringOrTinyString,
},
},
/// Key in patchedDependencies in package.json
apply_patch_task: ?*PatchTask = null,
@@ -244,6 +247,71 @@ pub fn getCompletionCallback(this: *NetworkTask) HTTP.HTTPClientResult.Callback
return HTTP.HTTPClientResult.Callback.New(*NetworkTask, notify).init(this);
}
/// Configure the network task to fetch a PyPI manifest
pub fn forPyPIManifest(
this: *NetworkTask,
name: string,
version: ?string,
allocator: std.mem.Allocator,
) ForManifestError!void {
// PyPI JSON API URL: https://pypi.org/pypi/{package}/json
// or with version: https://pypi.org/pypi/{package}/{version}/json
const pypi_base = "https://pypi.org/pypi/";
// Build URL: base + name + [/version] + /json
const version_len = if (version) |v| v.len + 1 else 0; // +1 for leading slash
const url_len = pypi_base.len + name.len + version_len + "/json".len;
const url_buf = try allocator.alloc(u8, url_len);
var pos: usize = 0;
@memcpy(url_buf[pos..][0..pypi_base.len], pypi_base);
pos += pypi_base.len;
@memcpy(url_buf[pos..][0..name.len], name);
pos += name.len;
if (version) |v| {
url_buf[pos] = '/';
pos += 1;
@memcpy(url_buf[pos..][0..v.len], v);
pos += v.len;
}
@memcpy(url_buf[pos..][0.."/json".len], "/json");
this.url_buf = url_buf;
// Simple headers - just Accept: application/json
var header_builder = HeaderBuilder{};
header_builder.count("Accept", "application/json");
try header_builder.allocate(allocator);
header_builder.append("Accept", "application/json");
this.response_buffer = try MutableString.init(allocator, 0);
this.allocator = allocator;
const url = URL.parse(this.url_buf);
this.unsafe_http_client = AsyncHTTP.init(allocator, .GET, url, header_builder.entries, header_builder.content.ptr.?[0..header_builder.content.len], &this.response_buffer, "", this.getCompletionCallback(), HTTP.FetchRedirect.follow, .{
.http_proxy = this.package_manager.httpProxy(url),
});
this.unsafe_http_client.client.flags.reject_unauthorized = this.package_manager.tlsRejectUnauthorized();
if (PackageManager.verbose_install) {
this.unsafe_http_client.client.verbose = .headers;
}
this.callback = .{
.pypi_manifest = .{
.name = try strings.StringOrTinyString.initAppendIfNeeded(name, *FileSystem.FilenameStore, FileSystem.FilenameStore.instance),
},
};
if (PackageManager.verbose_install) {
this.unsafe_http_client.verbose = .headers;
this.unsafe_http_client.client.verbose = .headers;
}
}
pub fn schedule(this: *NetworkTask, batch: *ThreadPool.Batch) void {
this.unsafe_http_client.schedule(this.allocator, batch);
}

View File

@@ -1463,6 +1463,157 @@ pub const PackageInstall = struct {
// TODO: linux io_uring
return this.installWithCopyfile(destination_dir);
}
/// Install a Python package from a wheel cache to site-packages.
/// Unlike npm packages, wheel contents (package dirs + dist-info) are copied directly
/// to site-packages, not wrapped in a subdirectory.
pub fn installPythonPackage(this: *@This(), site_packages_dir: std.fs.Dir, method_: Method) Result {
const tracer = bun.perf.trace("PackageInstaller.installPythonPackage");
defer tracer.end();
// Open the cache directory containing the extracted wheel
var cached_wheel_dir = bun.openDir(this.cache_dir, this.cache_dir_subpath) catch |err| {
return Result.fail(err, .opening_cache_dir, @errorReturnTrace());
};
defer cached_wheel_dir.close();
// Save original values
const original_cache_dir = this.cache_dir;
const original_cache_subpath = this.cache_dir_subpath;
const original_dest_subpath = this.destination_dir_subpath;
defer {
this.cache_dir = original_cache_dir;
this.cache_dir_subpath = original_cache_subpath;
this.destination_dir_subpath = original_dest_subpath;
}
// Set cache_dir to the wheel directory
this.cache_dir = cached_wheel_dir;
// Iterate through all entries in the wheel cache and copy each entry
var iter = cached_wheel_dir.iterate();
while (iter.next() catch |err| {
return Result.fail(err, .opening_cache_dir, @errorReturnTrace());
}) |entry| {
if (entry.kind == .directory) {
// Build null-terminated subdir name for both source and dest
if (entry.name.len >= this.destination_dir_subpath_buf.len) continue;
@memcpy(this.destination_dir_subpath_buf[0..entry.name.len], entry.name);
this.destination_dir_subpath_buf[entry.name.len] = 0;
const subdir_name_z: [:0]u8 = this.destination_dir_subpath_buf[0..entry.name.len :0];
// Set paths to point to this subdirectory
this.cache_dir_subpath = subdir_name_z;
this.destination_dir_subpath = subdir_name_z;
// Use the existing install method which handles clonefile/hardlink/copy fallback
const result = this.install(false, site_packages_dir, method_, .pypi);
if (result != .success) {
return result;
}
} else if (entry.kind == .file) {
// Copy individual files at the wheel root (e.g., typing_extensions.py, six.py)
// Build null-terminated filename
if (entry.name.len >= this.destination_dir_subpath_buf.len) continue;
@memcpy(this.destination_dir_subpath_buf[0..entry.name.len], entry.name);
this.destination_dir_subpath_buf[entry.name.len] = 0;
const filename_z: [:0]const u8 = this.destination_dir_subpath_buf[0..entry.name.len :0];
// Copy the file from cache to site-packages
if (comptime Environment.isMac) {
// Try clonefile first, then fall back to fcopyfile
switch (bun.c.clonefileat(
cached_wheel_dir.fd,
filename_z,
site_packages_dir.fd,
filename_z,
0,
)) {
0 => continue,
else => |errno| switch (std.posix.errno(errno)) {
.EXIST => continue,
else => {
// Fall back to fcopyfile
var in_file = bun.sys.openat(.fromStdDir(cached_wheel_dir), filename_z, bun.O.RDONLY, 0).unwrap() catch |open_err| {
return Result.fail(open_err, .copyfile, @errorReturnTrace());
};
defer in_file.close();
var out_file = site_packages_dir.createFile(entry.name, .{}) catch |create_err| {
return Result.fail(create_err, .copyfile, @errorReturnTrace());
};
defer out_file.close();
switch (bun.sys.fcopyfile(in_file, .fromStdFile(out_file), std.posix.system.COPYFILE{ .DATA = true })) {
.result => continue,
.err => |copy_err| switch (copy_err.getErrno()) {
.EXIST => continue,
else => return Result.fail(copy_err.toZigErr(), .copyfile, @errorReturnTrace()),
},
}
},
},
}
} else if (comptime Environment.isLinux) {
// Try hardlink first, then fall back to copy
switch (bun.sys.linkat(
.fromStdDir(cached_wheel_dir),
filename_z,
.fromStdDir(site_packages_dir),
filename_z,
)) {
.result => continue,
.err => |err| switch (err.getErrno()) {
.EXIST => continue,
else => {
// Fall back to copy
var in_file = bun.sys.openat(.fromStdDir(cached_wheel_dir), filename_z, bun.O.RDONLY, 0).unwrap() catch |open_err| {
return Result.fail(open_err, .copyfile, @errorReturnTrace());
};
defer in_file.close();
var out_file = site_packages_dir.createFile(entry.name, .{}) catch |create_err| {
return Result.fail(create_err, .copyfile, @errorReturnTrace());
};
defer out_file.close();
var copy_state: bun.CopyFileState = .{};
bun.copyFileWithState(in_file, .fromStdFile(out_file), &copy_state).unwrap() catch |copy_err| {
return Result.fail(copy_err, .copyfile, @errorReturnTrace());
};
},
},
}
} else if (comptime Environment.isWindows) {
// Use Windows CopyFileW
var src_buf: bun.WPathBuffer = undefined;
var dst_buf: bun.WPathBuffer = undefined;
const src_path = bun.strings.toWPathNormalized(&src_buf, filename_z);
const dst_path = bun.strings.toWPathNormalized(&dst_buf, filename_z);
src_buf[src_path.len] = 0;
dst_buf[dst_path.len] = 0;
if (bun.windows.CopyFileExW(
src_buf[0..src_path.len :0].ptr,
dst_buf[0..dst_path.len :0].ptr,
null,
null,
null,
0,
) == 0) {
const win_err = bun.windows.Win32Error.get();
if (win_err != .ERROR_FILE_EXISTS) {
return Result.fail(win_err.toSystemErrno().?.toZigErr(), .copyfile, @errorReturnTrace());
}
}
}
}
}
return .success;
}
};
const string = []const u8;

View File

@@ -10,6 +10,8 @@ pub const PackageInstaller = struct {
skip_delete: bool,
force_install: bool,
root_node_modules_folder: std.fs.Dir,
/// .venv/lib/python{version}/site-packages/ directory for Python packages
site_packages_folder: ?std.fs.Dir,
summary: *PackageInstall.Summary,
options: *const PackageManager.Options,
metas: []const Lockfile.Package.Meta,
@@ -942,6 +944,10 @@ pub const PackageInstaller = struct {
installer.cache_dir = directory;
}
},
.pypi => {
installer.cache_dir_subpath = this.manager.cachedTarballFolderName(resolution.value.pypi.url, patch_contents_hash);
installer.cache_dir = this.manager.getCacheDirectory();
},
else => {
if (comptime Environment.allow_assert) {
@panic("Internal assertion failure: unexpected resolution tag");
@@ -1110,6 +1116,13 @@ pub const PackageInstaller = struct {
const install_result: PackageInstall.Result = switch (resolution.tag) {
.symlink, .workspace => installer.installFromLink(this.skip_delete, destination_dir),
.pypi => result: {
// Python packages are installed to .venv/lib/python{version}/site-packages/
const site_packages = this.site_packages_folder orelse {
break :result .fail(error.FileNotFound, .opening_cache_dir, null);
};
break :result installer.installPythonPackage(site_packages, installer.getInstallMethod());
},
else => result: {
if (resolution.tag == .root or (resolution.tag == .folder and !this.lockfile.isWorkspaceTreeId(this.current_tree_id))) {
// This is a transitive folder dependency. It is installed with a single symlink to the target folder/file,
@@ -1530,6 +1543,7 @@ const PackageInstall = install.PackageInstall;
const PackageNameHash = install.PackageNameHash;
const PatchTask = install.PatchTask;
const PostinstallOptimizer = install.PostinstallOptimizer;
const pypi = install.PyPI;
const Resolution = install.Resolution;
const Task = install.Task;
const TaskCallbackContext = install.TaskCallbackContext;

View File

@@ -50,6 +50,7 @@ task_batch: ThreadPool.Batch = .{},
task_queue: TaskDependencyQueue = .{},
manifests: PackageManifestMap = .{},
pypi_manifests: PyPIManifestMap = .{},
folders: FolderResolution.Map = .{},
git_repositories: RepositoryMap = .{},
@@ -1196,6 +1197,7 @@ pub const enqueueGitForCheckout = enqueue.enqueueGitForCheckout;
pub const enqueueNetworkTask = enqueue.enqueueNetworkTask;
pub const enqueuePackageForDownload = enqueue.enqueuePackageForDownload;
pub const enqueueParseNPMPackage = enqueue.enqueueParseNPMPackage;
pub const enqueueParsePyPIPackage = enqueue.enqueueParsePyPIPackage;
pub const enqueuePatchTask = enqueue.enqueuePatchTask;
pub const enqueuePatchTaskPre = enqueue.enqueuePatchTaskPre;
pub const enqueueTarballForDownload = enqueue.enqueueTarballForDownload;
@@ -1313,6 +1315,8 @@ const PackageID = bun.install.PackageID;
const PackageManager = bun.install.PackageManager;
const PackageManifestMap = bun.install.PackageManifestMap;
const PackageNameAndVersionHash = bun.install.PackageNameAndVersionHash;
const PyPI = bun.install.PyPI;
const PyPIManifestMap = std.HashMapUnmanaged(PackageNameHash, PyPI.PackageManifest, IdentityContext(PackageNameHash), 80);
const PackageNameHash = bun.install.PackageNameHash;
const PatchTask = bun.install.PatchTask;
const PostinstallOptimizer = bun.install.PostinstallOptimizer;

View File

@@ -407,7 +407,7 @@ pub fn edit(
var i: usize = 0;
loop: while (i < updates.len) {
var request = &updates.*[i];
inline for ([_]string{ "dependencies", "devDependencies", "optionalDependencies", "peerDependencies" }) |list| {
inline for ([_]string{ "dependencies", "devDependencies", "optionalDependencies", "peerDependencies", "pythonDependencies" }) |list| {
if (current_package_json.asProperty(list)) |query| {
if (query.expr.data == .e_object) {
const name = request.getName();
@@ -421,7 +421,7 @@ pub fn edit(
const version_literal = try value.expr.asStringCloned(allocator) orelse break :add_packages_to_update;
var tag = Dependency.Version.Tag.infer(version_literal);
if (tag != .npm and tag != .dist_tag) break :add_packages_to_update;
if (tag != .npm and tag != .dist_tag and tag != .pypi) break :add_packages_to_update;
const entry = bun.handleOom(manager.updating_packages.getOrPut(allocator, name));

View File

@@ -230,6 +230,29 @@ pub fn enqueueParseNPMPackage(
return &task.threadpool_task;
}
pub fn enqueueParsePyPIPackage(
this: *PackageManager,
task_id: Task.Id,
name: strings.StringOrTinyString,
network_task: *NetworkTask,
) *ThreadPool.Task {
var task = this.preallocated_resolve_tasks.get();
task.* = Task{
.package_manager = this,
.log = logger.Log.init(this.allocator),
.tag = Task.Tag.pypi_manifest,
.request = .{
.pypi_manifest = .{
.network = network_task,
.name = name,
},
},
.id = task_id,
.data = undefined,
};
return &task.threadpool_task;
}
pub fn enqueuePackageForDownload(
this: *PackageManager,
name: []const u8,
@@ -447,7 +470,7 @@ pub fn enqueueDependencyWithMainAndSuccessFn(
var name = dependency.realname();
var name_hash = switch (dependency.version.tag) {
.dist_tag, .git, .github, .npm, .tarball, .workspace => String.Builder.stringHash(this.lockfile.str(&name)),
.dist_tag, .git, .github, .npm, .tarball, .workspace, .pypi => String.Builder.stringHash(this.lockfile.str(&name)),
else => dependency.name_hash,
};
@@ -1154,6 +1177,103 @@ pub fn enqueueDependencyWithMainAndSuccessFn(
},
}
},
.pypi => {
// PyPI package - fetch manifest from PyPI JSON API
const name_str = this.lockfile.str(&name);
const task_id = Task.Id.forPyPIManifest(name_str);
if (comptime Environment.allow_assert) bun.assert(task_id.get() != 0);
if (comptime Environment.allow_assert)
debug(
"enqueueDependency({d}, {s}, {s}, {s}) = pypi task {d}",
.{
id,
@tagName(version.tag),
this.lockfile.str(&name),
this.lockfile.str(&version.literal),
task_id,
},
);
// Check if we already have the manifest - if so, resolve immediately
if (this.pypi_manifests.contains(name_hash)) {
// Manifest already downloaded, resolve the package now
const resolve_result = try getOrPutResolvedPackage(
this,
name_hash,
name,
dependency,
version,
dependency.behavior,
id,
resolution,
install_peer,
successFn,
);
if (resolve_result) |result| {
// Queue transitive dependencies (just like npm does)
if (result.is_first_time and result.package.dependencies.len > 0) {
try this.lockfile.scratch.dependency_list_queue.writeItem(result.package.dependencies);
}
if (result.task) |task| {
switch (task) {
.network_task => |network_task| this.enqueueNetworkTask(network_task),
.patch_task => |patch_task| this.enqueuePatchTask(patch_task),
}
}
}
return;
}
if (!this.hasCreatedNetworkTask(task_id, dependency.behavior.isRequired())) {
if (PackageManager.verbose_install) {
Output.prettyErrorln("Enqueue PyPI package manifest for download: {s}", .{name_str});
}
var network_task = this.getNetworkTask();
network_task.* = .{
.package_manager = this,
.callback = undefined,
.task_id = task_id,
.allocator = this.allocator,
};
// Get version string from literal if specified (e.g., "2.3.5" from "pypi:numpy@2.3.5")
const version_str: ?[]const u8 = if (version.literal.len() > 0)
this.lockfile.str(&version.literal)
else
null;
network_task.forPyPIManifest(
name_str,
version_str,
this.allocator,
) catch |err| {
if (dependency.behavior.isRequired()) {
this.log.addErrorFmt(
null,
logger.Loc.Empty,
this.allocator,
"Failed to create PyPI manifest request for {s}: {s}",
.{ name_str, @errorName(err) },
) catch unreachable;
}
return;
};
this.enqueueNetworkTask(network_task);
}
var manifest_entry_parse = this.task_queue.getOrPutContext(this.allocator, task_id, .{}) catch return;
if (!manifest_entry_parse.found_existing) {
manifest_entry_parse.value_ptr.* = TaskCallbackList{};
}
const callback_tag = comptime if (successFn == assignRootResolution) "root_dependency" else "dependency";
manifest_entry_parse.value_ptr.append(this.allocator, @unionInit(TaskCallbackContext, callback_tag, id)) catch return;
},
else => {},
}
}
@@ -1841,6 +1961,210 @@ fn getOrPutResolvedPackage(
}
},
.pypi => {
// Look up the PyPI manifest
const name_str = this.lockfile.str(&name);
debug("getOrPutResolvedPackage .pypi: {s}", .{name_str});
const manifest = this.pypi_manifests.getPtr(name_hash) orelse {
debug(" no manifest found for pypi package {s}", .{name_str});
return null;
};
// Find the best wheel for the current platform
const best_wheel = manifest.findBestWheel(PyPI.PlatformTarget.current()) orelse {
if (behavior.isPeer()) {
return null;
}
// No compatible wheel found
this.log.addErrorFmt(
null,
logger.Loc.Empty,
this.allocator,
"No compatible wheel found for PyPI package <b>{s}<r>",
.{name_str},
) catch unreachable;
return null;
};
const wheel_url = best_wheel.url.slice(manifest.string_buf);
const version_str = manifest.latestVersion();
debug(" found wheel for {s}: {s} (version {s})", .{ name_str, wheel_url, version_str });
// Parse the version string into a Semver.Version
// First normalize Python version (PEP 440) to strip .postN, .devN suffixes
var normalized_version_buf: [64]u8 = undefined;
const normalized_version = PyPI.DependencySpecifier.normalizeVersion(version_str, &normalized_version_buf);
const sliced_version = Semver.SlicedString.init(normalized_version, normalized_version);
const parsed_version = Semver.Version.parse(sliced_version);
if (!parsed_version.valid) {
this.log.addErrorFmt(
null,
logger.Loc.Empty,
this.allocator,
"Invalid version <b>{s}<r> for PyPI package <b>{s}<r>",
.{ version_str, name_str },
) catch unreachable;
return null;
}
const resolved_version = parsed_version.version.min();
// Check if there's already a package with this name and version for PyPI
if (this.lockfile.package_index.get(name_hash)) |index| {
switch (index) {
.id => |existing_id| {
const existing_resolution = this.lockfile.packages.items(.resolution)[existing_id];
if (existing_resolution.tag == .pypi and existing_resolution.value.pypi.version.eql(resolved_version)) {
successFn(this, dependency_id, existing_id);
return .{
.package = this.lockfile.packages.get(existing_id),
.is_first_time = false,
};
}
},
.ids => |list| {
for (list.items) |existing_id| {
const existing_resolution = this.lockfile.packages.items(.resolution)[existing_id];
if (existing_resolution.tag == .pypi and existing_resolution.value.pypi.version.eql(resolved_version)) {
successFn(this, dependency_id, existing_id);
return .{
.package = this.lockfile.packages.get(existing_id),
.is_first_time = false,
};
}
}
},
}
}
if (behavior.isPeer() and !install_peer) {
return null;
}
// Create a new package entry
var package = Lockfile.Package{};
// Build strings for the package
// Use manifest.name() which points to manifest's own buffer, not lockfile's buffer.
// This is important because string_builder.allocate() may resize lockfile's buffer.
const manifest_name = manifest.name();
var string_builder = this.lockfile.stringBuilder();
string_builder.count(manifest_name);
resolved_version.count(manifest.string_buf, *Lockfile.StringBuilder, &string_builder);
string_builder.count(wheel_url);
// Count transitive dependencies from requires_dist
var dep_iter = manifest.iterDependencies(PyPI.PlatformTarget.current());
const total_dependencies_count: u32 = @intCast(dep_iter.count());
if (PackageManager.verbose_install) {
Output.prettyErrorln("PyPI package {s} has {d} transitive dependencies", .{ manifest_name, total_dependencies_count });
}
// Count strings for each dependency name (normalized)
dep_iter = manifest.iterDependencies(PyPI.PlatformTarget.current());
while (dep_iter.next()) |spec| {
var dep_name_buf: [256]u8 = undefined;
const normalized = PyPI.DependencySpecifier.normalizeName(spec.name, &dep_name_buf);
string_builder.count(normalized);
}
try string_builder.allocate();
defer string_builder.clamp();
const name_string = string_builder.append(ExternalString, manifest_name);
package.name = name_string.value;
package.name_hash = name_string.hash;
// Store version and URL in resolution
package.resolution = Resolution.init(.{
.pypi = .{
.version = resolved_version.append(manifest.string_buf, *Lockfile.StringBuilder, &string_builder),
.url = string_builder.append(String, wheel_url),
},
});
// PyPI packages don't have install scripts by default
package.scripts.filled = true;
package.meta.setHasInstallScript(false);
// Parse and store transitive dependencies from requires_dist
if (total_dependencies_count > 0) {
var dependencies_list = &this.lockfile.buffers.dependencies;
var resolutions_list = &this.lockfile.buffers.resolutions;
try dependencies_list.ensureUnusedCapacity(this.allocator, total_dependencies_count);
try resolutions_list.ensureUnusedCapacity(this.allocator, total_dependencies_count);
package.dependencies.off = @intCast(dependencies_list.items.len);
package.dependencies.len = total_dependencies_count;
package.resolutions.off = package.dependencies.off;
package.resolutions.len = package.dependencies.len;
dep_iter = manifest.iterDependencies(PyPI.PlatformTarget.current());
while (dep_iter.next()) |spec| {
var dep_name_buf: [256]u8 = undefined;
const normalized = PyPI.DependencySpecifier.normalizeName(spec.name, &dep_name_buf);
const dep_name = string_builder.append(ExternalString, normalized);
if (PackageManager.verbose_install) {
Output.prettyErrorln(" Adding PyPI dependency: {s}", .{normalized});
}
dependencies_list.appendAssumeCapacity(.{
.name = dep_name.value,
.name_hash = dep_name.hash,
.behavior = .{ .python = true, .prod = true },
.version = .{ .tag = .pypi, .value = .{ .pypi = .{ .name = dep_name.value } } },
});
resolutions_list.appendAssumeCapacity(invalid_package_id);
}
}
// Append the package to the lockfile
package = try this.lockfile.appendPackage(package);
if (comptime Environment.allow_assert) bun.assert(package.meta.id != invalid_package_id);
debug(" created pypi package {s} with id {d}", .{ manifest_name, package.meta.id });
defer successFn(this, dependency_id, package.meta.id);
// Check if wheel is already cached
var name_and_version_hash: ?u64 = null;
var patchfile_hash: ?u64 = null;
return switch (this.determinePreinstallState(
package,
this.lockfile,
&name_and_version_hash,
&patchfile_hash,
)) {
.done => .{ .package = package, .is_first_time = true },
.extract => extract: {
// Skip wheel download when prefetch_resolved_tarballs is disabled
if (!this.options.do.prefetch_resolved_tarballs) {
break :extract .{ .package = package, .is_first_time = true };
}
const task_id = Task.Id.forTarball(wheel_url);
break :extract .{
.package = package,
.is_first_time = true,
.task = .{
.network_task = try this.generateNetworkTaskForTarball(
task_id,
wheel_url,
dependency.behavior.isRequired(),
dependency_id,
package,
null,
.no_authorization,
) orelse unreachable,
},
};
},
else => .{ .package = package, .is_first_time = true },
};
},
else => return null,
}
}
@@ -1876,6 +2200,7 @@ const strings = bun.strings;
const Semver = bun.Semver;
const String = Semver.String;
const ExternalString = Semver.ExternalString;
const Fs = bun.fs;
const FileSystem = Fs.FileSystem;
@@ -1887,6 +2212,7 @@ const ExtractTarball = bun.install.ExtractTarball;
const Features = bun.install.Features;
const FolderResolution = bun.install.FolderResolution;
const Npm = bun.install.Npm;
const PyPI = bun.install.PyPI;
const PackageID = bun.install.PackageID;
const PackageNameHash = bun.install.PackageNameHash;
const PatchTask = bun.install.PatchTask;

View File

@@ -116,6 +116,7 @@ pub fn determinePreinstallState(
.npm => manager.cachedNPMPackageFolderName(lockfile.str(&pkg.name), pkg.resolution.value.npm.version, patch_hash),
.local_tarball => manager.cachedTarballFolderName(pkg.resolution.value.local_tarball, patch_hash),
.remote_tarball => manager.cachedTarballFolderName(pkg.resolution.value.remote_tarball, patch_hash),
.pypi => manager.cachedTarballFolderName(pkg.resolution.value.pypi.url, patch_hash),
else => "",
};

View File

@@ -22,11 +22,13 @@ dry_run: bool = false,
link_workspace_packages: bool = true,
remote_package_features: Features = .{
.optional_dependencies = true,
.python_dependencies = true,
},
local_package_features: Features = .{
.optional_dependencies = true,
.dev_dependencies = true,
.workspaces = true,
.python_dependencies = true,
},
patch_features: union(enum) {
nothing: struct {},

View File

@@ -141,7 +141,22 @@ fn parseWithError(
var value = input;
var alias: ?string = null;
if (!Dependency.isTarball(input) and strings.isNPMPackageName(input)) {
var is_pypi = false;
// Check for pypi: prefix first - extract name and version
if (strings.hasPrefixComptime(input, "pypi:")) {
is_pypi = true;
const after_prefix = input["pypi:".len..];
// Find @ separator between package name and version
if (strings.indexOfChar(after_prefix, '@')) |at| {
alias = after_prefix[0..at]; // "cowsay"
value = after_prefix[at + 1 ..]; // "^6.1"
} else {
// No version specified, just package name
alias = after_prefix;
value = input[input.len..]; // Empty slice at end of input (must be within input for SlicedString)
}
} else if (!Dependency.isTarball(input) and strings.isNPMPackageName(input)) {
alias = input;
value = input[input.len..];
} else if (input.len > 1) {
@@ -160,7 +175,7 @@ fn parseWithError(
if (alias) |name| String.init(input, name) else placeholder,
if (alias) |name| String.Builder.stringHash(name) else null,
value,
null,
if (is_pypi) .pypi else null,
&SlicedString.init(input, value),
log,
pm,
@@ -214,7 +229,14 @@ fn parseWithError(
.version = version,
.version_buf = input,
};
if (alias) |name| {
if (version.tag == .pypi) {
// For pypi packages, get the name from the parsed version
const pypi_name = version.value.pypi.name.slice(input);
// Set is_aliased so getName() returns the name, not the literal
request.is_aliased = true;
request.name = allocator.dupe(u8, pypi_name) catch unreachable;
request.name_hash = String.Builder.stringHash(pypi_name);
} else if (alias) |name| {
request.is_aliased = true;
request.name = allocator.dupe(u8, name) catch unreachable;
request.name_hash = String.Builder.stringHash(name);

View File

@@ -500,6 +500,104 @@ pub fn runTasks(
manager.task_batch.push(ThreadPool.Batch.from(manager.enqueueExtractNPMPackage(extract, task)));
},
.pypi_manifest => |*manifest_req| {
const name = manifest_req.name;
if (log_level.showProgress()) {
if (!has_updated_this_run) {
manager.setNodeName(manager.downloads_node.?, name.slice(), ProgressStrings.download_emoji, true);
has_updated_this_run = true;
}
}
if (!has_network_error and task.response.metadata == null) {
has_network_error = true;
const min = manager.options.min_simultaneous_requests;
const max = AsyncHTTP.max_simultaneous_requests.load(.monotonic);
if (max > min) {
AsyncHTTP.max_simultaneous_requests.store(@max(min, max / 2), .monotonic);
}
}
// Handle retry-able errors.
if (task.response.metadata == null or task.response.metadata.?.response.status_code > 499) {
const err = task.response.fail orelse error.HTTPError;
if (task.retried < manager.options.max_retry_count) {
task.retried += 1;
manager.enqueueNetworkTask(task);
if (manager.options.log_level.isVerbose()) {
manager.log.addWarningFmt(
null,
logger.Loc.Empty,
manager.allocator,
"{s} downloading PyPI package manifest <b>{s}<r>. Retry {d}/{d}...",
.{ bun.span(@errorName(err)), name.slice(), task.retried, manager.options.max_retry_count },
) catch unreachable;
}
continue;
}
}
const metadata = task.response.metadata orelse {
// Handle non-retry-able errors.
const err = task.response.fail orelse error.HTTPError;
if (manager.isNetworkTaskRequired(task.task_id)) {
manager.log.addErrorFmt(
null,
logger.Loc.Empty,
manager.allocator,
"{s} downloading PyPI package manifest <b>{s}<r>",
.{ @errorName(err), name.slice() },
) catch |e| bun.handleOom(e);
} else {
manager.log.addWarningFmt(
null,
logger.Loc.Empty,
manager.allocator,
"{s} downloading PyPI package manifest <b>{s}<r>",
.{ @errorName(err), name.slice() },
) catch |e| bun.handleOom(e);
}
continue;
};
const response = metadata.response;
if (response.status_code > 399) {
if (manager.isNetworkTaskRequired(task.task_id)) {
manager.log.addErrorFmt(
null,
logger.Loc.Empty,
manager.allocator,
"<r><red><b>GET<r><red> {s}<d> - {d}<r>",
.{ metadata.url, response.status_code },
) catch |err| bun.handleOom(err);
} else {
manager.log.addWarningFmt(
null,
logger.Loc.Empty,
manager.allocator,
"<r><yellow><b>GET<r><yellow> {s}<d> - {d}<r>",
.{ metadata.url, response.status_code },
) catch |err| bun.handleOom(err);
}
continue;
}
if (log_level.isVerbose()) {
Output.prettyError(" ", .{});
Output.printElapsed(@as(f64, @floatFromInt(task.unsafe_http_client.elapsed)) / std.time.ns_per_ms);
Output.prettyError("\n<d>Downloaded <r><green>{s}<r> PyPI manifest\n", .{name.slice()});
Output.flush();
}
// Enqueue parsing task
manager.task_batch.push(ThreadPool.Batch.from(manager.enqueueParsePyPIPackage(task.task_id, name, task)));
},
else => unreachable,
}
}
@@ -886,6 +984,62 @@ pub fn runTasks(
}
}
},
.pypi_manifest => {
defer manager.preallocated_network_tasks.put(task.request.pypi_manifest.network);
if (task.status == .fail) {
const name = task.request.pypi_manifest.name;
const err = task.err orelse error.Failed;
manager.log.addErrorFmt(
null,
logger.Loc.Empty,
manager.allocator,
"{s} parsing PyPI package manifest for <b>{s}<r>",
.{
@errorName(err),
name.slice(),
},
) catch |e| bun.handleOom(e);
continue;
}
const manifest = &task.data.pypi_manifest;
const name = task.request.pypi_manifest.name.slice();
const name_hash = String.Builder.stringHash(name);
if (log_level.isVerbose()) {
Output.prettyErrorln("<d>PyPI manifest parsed for {s}, version {s}<r>", .{
manifest.name(),
manifest.latestVersion(),
});
}
// Store the PyPI manifest for later resolution
try manager.pypi_manifests.put(bun.default_allocator, name_hash, manifest.*);
if (@hasField(@TypeOf(callbacks), "manifests_only") and callbacks.manifests_only) {
continue;
}
const dependency_list_entry = manager.task_queue.getEntry(task.id).?;
const dependency_list = dependency_list_entry.value_ptr.*;
dependency_list_entry.value_ptr.* = .{};
if (log_level.isVerbose()) {
Output.prettyErrorln("<d>Processing {d} PyPI dependencies<r>", .{dependency_list.items.len});
}
try manager.processDependencyList(dependency_list, Ctx, extract_ctx, callbacks, install_peer);
if (log_level.showProgress()) {
if (!has_updated_this_run) {
manager.setNodeName(manager.downloads_node.?, name, ProgressStrings.download_emoji, true);
has_updated_this_run = true;
}
}
},
}
}
}
@@ -1103,6 +1257,9 @@ const FileSystem = Fs.FileSystem;
const HTTP = bun.http;
const AsyncHTTP = HTTP.AsyncHTTP;
const Semver = bun.Semver;
const String = Semver.String;
const DependencyID = bun.install.DependencyID;
const Features = bun.install.Features;
const NetworkTask = bun.install.NetworkTask;

View File

@@ -137,7 +137,7 @@ fn updatePackageJSONAndInstallWithManagerWithUpdates(
// if we're removing, they don't have to specify where it is installed in the dependencies list
// they can even put it multiple times and we will just remove all of them
for (updates.*) |request| {
inline for ([_]string{ "dependencies", "devDependencies", "optionalDependencies", "peerDependencies" }) |list| {
inline for ([_]string{ "dependencies", "devDependencies", "optionalDependencies", "peerDependencies", "pythonDependencies" }) |list| {
if (current_package_json.root.asProperty(list)) |query| {
if (query.expr.data == .e_object) {
var dependencies = query.expr.data.e_object.properties.slice();
@@ -186,16 +186,47 @@ fn updatePackageJSONAndInstallWithManagerWithUpdates(
// update will not exceed the current dependency range if it exists
if (updates.len != 0) {
try PackageJSONEditor.edit(
manager,
updates,
&current_package_json.root,
dependency_list,
.{
.exact_versions = manager.options.enable.exact_versions,
.before_install = true,
},
);
// Separate pypi packages from npm packages
var npm_updates = UpdateRequest.Array{};
var pypi_updates = UpdateRequest.Array{};
for (updates.*) |update| {
if (update.version.tag == .pypi) {
pypi_updates.append(manager.allocator, update) catch bun.outOfMemory();
} else {
npm_updates.append(manager.allocator, update) catch bun.outOfMemory();
}
}
// Edit npm packages in the appropriate dependency list
if (npm_updates.items.len > 0) {
var npm_slice = npm_updates.items;
try PackageJSONEditor.edit(
manager,
&npm_slice,
&current_package_json.root,
dependency_list,
.{
.exact_versions = manager.options.enable.exact_versions,
.before_install = true,
},
);
}
// Edit pypi packages in pythonDependencies
if (pypi_updates.items.len > 0) {
var pypi_slice = pypi_updates.items;
try PackageJSONEditor.edit(
manager,
&pypi_slice,
&current_package_json.root,
"pythonDependencies",
.{
.exact_versions = manager.options.enable.exact_versions,
.before_install = true,
},
);
}
} else if (subcommand == .update) {
try PackageJSONEditor.editUpdateNoArgs(
manager,

View File

@@ -66,6 +66,13 @@ pub const Id = enum(u64) {
hasher.update(resolved);
return @enumFromInt(@as(u64, 5 << 61) | @as(u64, @as(u61, @truncate(hasher.final()))));
}
pub fn forPyPIManifest(name: string) Id {
var hasher = bun.Wyhash11.init(0);
hasher.update("pypi-manifest:");
hasher.update(name);
return @enumFromInt(hasher.final());
}
};
pub fn callback(task: *ThreadPool.Task) void {
@@ -288,6 +295,53 @@ pub fn callback(task: *ThreadPool.Task) void {
this.data = .{ .extract = result };
this.status = Status.success;
},
.pypi_manifest => {
const allocator = bun.default_allocator;
var manifest_req = &this.request.pypi_manifest;
const body = &manifest_req.network.response_buffer;
defer body.deinit();
// Check for HTTP errors
if (manifest_req.network.response.metadata) |metadata| {
const status = metadata.response.status_code;
if (status >= 400) {
this.log.addErrorFmt(null, logger.Loc.Empty, allocator, "PyPI error: {d} - GET {s}", .{
status,
manifest_req.name.slice(),
}) catch unreachable;
this.status = Status.fail;
this.data = .{ .pypi_manifest = .{} };
return;
}
}
// Parse the PyPI JSON response
const package_manifest = PyPI.PackageManifest.parse(
allocator,
&this.log,
body.slice(),
manifest_req.name.slice(),
) catch |err| {
bun.handleErrorReturnTrace(err, @errorReturnTrace());
this.err = err;
this.status = Status.fail;
this.data = .{ .pypi_manifest = .{} };
return;
};
if (package_manifest) |result| {
this.status = Status.success;
this.data = .{ .pypi_manifest = result };
} else {
this.log.addErrorFmt(null, logger.Loc.Empty, allocator, "Failed to parse PyPI manifest for {s}", .{
manifest_req.name.slice(),
}) catch unreachable;
this.status = Status.fail;
this.data = .{ .pypi_manifest = .{} };
}
},
}
}
@@ -312,6 +366,7 @@ pub const Tag = enum(u3) {
git_clone = 2,
git_checkout = 3,
local_tarball = 4,
pypi_manifest = 5,
};
pub const Status = enum {
@@ -325,6 +380,7 @@ pub const Data = union {
extract: ExtractData,
git_clone: bun.FileDescriptor,
git_checkout: ExtractData,
pypi_manifest: PyPI.PackageManifest,
};
pub const Request = union {
@@ -357,6 +413,10 @@ pub const Request = union {
local_tarball: struct {
tarball: ExtractTarball,
},
pypi_manifest: struct {
name: strings.StringOrTinyString,
network: *NetworkTask,
},
};
const string = []const u8;
@@ -369,6 +429,7 @@ const ExtractData = install.ExtractData;
const ExtractTarball = install.ExtractTarball;
const NetworkTask = install.NetworkTask;
const Npm = install.Npm;
const PyPI = install.PyPI;
const PackageID = install.PackageID;
const PackageManager = install.PackageManager;
const PatchTask = install.PatchTask;

View File

@@ -110,6 +110,7 @@ pub inline fn realname(this: *const Dependency) String {
.github => this.version.value.github.package_name,
.npm => this.version.value.npm.name,
.tarball => this.version.value.tarball.package_name,
.pypi => this.version.value.pypi.name,
else => this.name,
};
}
@@ -329,6 +330,9 @@ pub const Version = struct {
},
}
},
.pypi => {
object.put(globalThis, "name", try dep.value.pypi.name.toJS(buf, globalThis));
},
else => {
return globalThis.throwTODO("Unsupported dependency type");
},
@@ -428,6 +432,7 @@ pub const Version = struct {
.tarball => lhs.value.tarball.eql(rhs.value.tarball, lhs_buf, rhs_buf),
.symlink => lhs.value.symlink.eql(rhs.value.symlink, lhs_buf, rhs_buf),
.workspace => lhs.value.workspace.eql(rhs.value.workspace, lhs_buf, rhs_buf),
.pypi => lhs.value.pypi.eql(rhs.value.pypi, lhs_buf, rhs_buf),
else => true,
};
}
@@ -463,6 +468,9 @@ pub const Version = struct {
catalog = 9,
/// PyPI package (Python Package Index)
pypi = 10,
pub const map = bun.ComptimeStringMap(Tag, .{
.{ "npm", .npm },
.{ "dist_tag", .dist_tag },
@@ -473,6 +481,7 @@ pub const Version = struct {
.{ "git", .git },
.{ "github", .github },
.{ "catalog", .catalog },
.{ "pypi", .pypi },
});
pub const fromJS = map.fromJS;
@@ -741,6 +750,8 @@ pub const Version = struct {
// TODO(dylan-conway): apply .patch files on packages. In the future this could
// return `Tag.git` or `Tag.npm`.
if (strings.hasPrefixComptime(dependency, "patch:")) return .npm;
// pypi:package@version - Python package from PyPI
if (strings.hasPrefixComptime(dependency, "pypi:")) return .pypi;
},
else => {},
}
@@ -818,6 +829,16 @@ pub const Version = struct {
}
};
/// PyPI package information (Python Package Index)
pub const PypiInfo = struct {
/// Package name (normalized for PyPI lookups)
name: String,
fn eql(this: PypiInfo, that: PypiInfo, this_buf: []const u8, that_buf: []const u8) bool {
return this.name.eql(that.name, this_buf, that_buf);
}
};
pub const Value = union {
uninitialized: void,
@@ -836,6 +857,9 @@ pub const Version = struct {
// dep version without 'catalog:' protocol
// empty string == default catalog
catalog: String,
/// PyPI package
pypi: PypiInfo,
};
};
@@ -1250,6 +1274,15 @@ pub fn parseWithTag(
.literal = sliced.value(),
};
},
.pypi => {
// For PyPI dependencies, the name comes from the alias (dependency key)
// and the version specifier is in the dependency string
return .{
.value = .{ .pypi = .{ .name = alias } },
.tag = .pypi,
.literal = sliced.value(),
};
},
}
}
@@ -1311,7 +1344,8 @@ pub fn fromJS(globalThis: *jsc.JSGlobalObject, callframe: *jsc.CallFrame) bun.JS
}
pub const Behavior = packed struct(u8) {
_unused_1: u1 = 0,
/// Python dependency from pythonDependencies
python: bool = false,
prod: bool = false,
optional: bool = false,
dev: bool = false,
@@ -1349,6 +1383,10 @@ pub const Behavior = packed struct(u8) {
return this.bundled;
}
pub inline fn isPython(this: Behavior) bool {
return this.python;
}
pub inline fn eq(lhs: Behavior, rhs: Behavior) bool {
return @as(u8, @bitCast(lhs)) == @as(u8, @bitCast(rhs));
}
@@ -1422,7 +1460,8 @@ pub const Behavior = packed struct(u8) {
(features.optional_dependencies and this.isOptional()) or
(features.dev_dependencies and this.isDev()) or
(features.peer_dependencies and this.isPeer()) or
(features.workspaces and this.isWorkspace());
(features.workspaces and this.isWorkspace()) or
(features.python_dependencies and this.isPython());
}
comptime {

View File

@@ -167,62 +167,79 @@ fn extract(this: *const ExtractTarball, log: *logger.Log, tgz_bytes: []const u8)
zlib_pool.data.reset();
defer Npm.Registry.BodyPool.release(zlib_pool);
var esimated_output_size: usize = 0;
const time_started_for_verbose_logs: u64 = if (PackageManager.verbose_install) bun.getRoughTickCount(.allow_mocked_time).ns() else 0;
{
// Last 4 bytes of a gzip-compressed file are the uncompressed size.
if (tgz_bytes.len > 16) {
// If the file claims to be larger than 16 bytes and smaller than 64 MB, we'll preallocate the buffer.
// If it's larger than that, we'll do it incrementally. We want to avoid OOMing.
const last_4_bytes: u32 = @bitCast(tgz_bytes[tgz_bytes.len - 4 ..][0..4].*);
if (last_4_bytes > 16 and last_4_bytes < 64 * 1024 * 1024) {
// It's okay if this fails. We will just allocate as we go and that will error if we run out of memory.
esimated_output_size = last_4_bytes;
if (zlib_pool.data.list.capacity == 0) {
zlib_pool.data.list.ensureTotalCapacityPrecise(zlib_pool.data.allocator, last_4_bytes) catch {};
} else {
zlib_pool.data.ensureUnusedCapacity(last_4_bytes) catch {};
// Wheels (PyPI packages) are ZIP files, not gzipped tarballs
const is_wheel = this.resolution.tag == .pypi;
// Data to extract from - either decompressed tar data or raw wheel data
var extract_data: []const u8 = undefined;
if (is_wheel) {
// Wheels are ZIP files - no decompression needed
extract_data = tgz_bytes;
if (PackageManager.verbose_install) {
Output.prettyErrorln("[{s}] Extract wheel {s}<r> ({f})", .{ name, tmpname, bun.fmt.size(tgz_bytes.len, .{}) });
}
} else {
var esimated_output_size: usize = 0;
{
// Last 4 bytes of a gzip-compressed file are the uncompressed size.
if (tgz_bytes.len > 16) {
// If the file claims to be larger than 16 bytes and smaller than 64 MB, we'll preallocate the buffer.
// If it's larger than that, we'll do it incrementally. We want to avoid OOMing.
const last_4_bytes: u32 = @bitCast(tgz_bytes[tgz_bytes.len - 4 ..][0..4].*);
if (last_4_bytes > 16 and last_4_bytes < 64 * 1024 * 1024) {
// It's okay if this fails. We will just allocate as we go and that will error if we run out of memory.
esimated_output_size = last_4_bytes;
if (zlib_pool.data.list.capacity == 0) {
zlib_pool.data.list.ensureTotalCapacityPrecise(zlib_pool.data.allocator, last_4_bytes) catch {};
} else {
zlib_pool.data.ensureUnusedCapacity(last_4_bytes) catch {};
}
}
}
}
}
var needs_to_decompress = true;
if (bun.FeatureFlags.isLibdeflateEnabled() and zlib_pool.data.list.capacity > 16 and esimated_output_size > 0) use_libdeflate: {
const decompressor = bun.libdeflate.Decompressor.alloc() orelse break :use_libdeflate;
defer decompressor.deinit();
var needs_to_decompress = true;
if (bun.FeatureFlags.isLibdeflateEnabled() and zlib_pool.data.list.capacity > 16 and esimated_output_size > 0) use_libdeflate: {
const decompressor = bun.libdeflate.Decompressor.alloc() orelse break :use_libdeflate;
defer decompressor.deinit();
const result = decompressor.gzip(tgz_bytes, zlib_pool.data.list.allocatedSlice());
const result = decompressor.gzip(tgz_bytes, zlib_pool.data.list.allocatedSlice());
if (result.status == .success) {
zlib_pool.data.list.items.len = result.written;
needs_to_decompress = false;
if (result.status == .success) {
zlib_pool.data.list.items.len = result.written;
needs_to_decompress = false;
}
// If libdeflate fails for any reason, fallback to zlib.
}
// If libdeflate fails for any reason, fallback to zlib.
}
if (needs_to_decompress) {
zlib_pool.data.list.clearRetainingCapacity();
var zlib_entry = try Zlib.ZlibReaderArrayList.init(tgz_bytes, &zlib_pool.data.list, default_allocator);
zlib_entry.readAll(true) catch |err| {
log.addErrorFmt(
null,
logger.Loc.Empty,
bun.default_allocator,
"{s} decompressing \"{s}\" to \"{f}\"",
.{ @errorName(err), name, bun.fmt.fmtPath(u8, tmpname, .{}) },
) catch unreachable;
return error.InstallFailed;
};
}
if (needs_to_decompress) {
zlib_pool.data.list.clearRetainingCapacity();
var zlib_entry = try Zlib.ZlibReaderArrayList.init(tgz_bytes, &zlib_pool.data.list, default_allocator);
zlib_entry.readAll(true) catch |err| {
log.addErrorFmt(
null,
logger.Loc.Empty,
bun.default_allocator,
"{s} decompressing \"{s}\" to \"{f}\"",
.{ @errorName(err), name, bun.fmt.fmtPath(u8, tmpname, .{}) },
) catch unreachable;
return error.InstallFailed;
};
}
if (PackageManager.verbose_install) {
const decompressing_ended_at: u64 = bun.getRoughTickCount(.allow_mocked_time).ns();
const elapsed = decompressing_ended_at - time_started_for_verbose_logs;
Output.prettyErrorln("[{s}] Extract {s}<r> (decompressed {f} tgz file in {D})", .{ name, tmpname, bun.fmt.size(tgz_bytes.len, .{}), elapsed });
}
if (PackageManager.verbose_install) {
const decompressing_ended_at: u64 = bun.getRoughTickCount(.allow_mocked_time).ns();
const elapsed = decompressing_ended_at - time_started_for_verbose_logs;
Output.prettyErrorln("[{s}] Extract {s}<r> (decompressed {f} tgz file in {D})", .{ name, tmpname, bun.fmt.size(tgz_bytes.len, .{}), elapsed });
extract_data = zlib_pool.data.list.items;
}
switch (this.resolution.tag) {
@@ -240,7 +257,7 @@ fn extract(this: *const ExtractTarball, log: *logger.Log, tgz_bytes: []const u8)
switch (PackageManager.verbose_install) {
inline else => |verbose_log| _ = try Archiver.extractToDir(
zlib_pool.data.list.items,
extract_data,
extract_destination,
null,
*DirnameReader,
@@ -264,9 +281,24 @@ fn extract(this: *const ExtractTarball, log: *logger.Log, tgz_bytes: []const u8)
};
}
},
.pypi => switch (PackageManager.verbose_install) {
// Wheels are ZIP files with no root directory to skip
inline else => |verbose_log| _ = try Archiver.extractToDir(
extract_data,
extract_destination,
null,
void,
{},
.{
.log = verbose_log,
.depth_to_skip = 0,
.format = .zip,
},
),
},
else => switch (PackageManager.verbose_install) {
inline else => |verbose_log| _ = try Archiver.extractToDir(
zlib_pool.data.list.items,
extract_data,
extract_destination,
null,
void,
@@ -292,6 +324,7 @@ fn extract(this: *const ExtractTarball, log: *logger.Log, tgz_bytes: []const u8)
.npm => this.package_manager.cachedNPMPackageFolderNamePrint(&folder_name_buf, name, this.resolution.value.npm.version, null),
.github => PackageManager.cachedGitHubFolderNamePrint(&folder_name_buf, resolved, null),
.local_tarball, .remote_tarball => PackageManager.cachedTarballFolderNamePrint(&folder_name_buf, this.url.slice(), null),
.pypi => PackageManager.cachedTarballFolderNamePrint(&folder_name_buf, this.package_manager.lockfile.str(&this.resolution.value.pypi.url), null),
else => unreachable,
};
if (folder_name.len == 0 or (folder_name.len == 1 and folder_name[0] == '/')) @panic("Tried to delete root and stopped it");

View File

@@ -81,6 +81,17 @@ pub fn installHoistedPackages(
skip_delete = false;
}
// Create .venv/lib/python{version}/site-packages/ for Python packages
const site_packages_folder: ?std.fs.Dir = brk: {
// Create directory structure using version from pypi constants
bun.sys.mkdir(".venv", 0o755).unwrap() catch {};
bun.sys.mkdir(".venv/lib", 0o755).unwrap() catch {};
bun.sys.mkdir(pypi.venv_lib_dir, 0o755).unwrap() catch {};
bun.sys.mkdir(pypi.venv_site_packages, 0o755).unwrap() catch {};
break :brk bun.openDir(cwd.stdDir(), pypi.venv_site_packages) catch null;
};
var summary = PackageInstall.Summary{};
{
@@ -147,6 +158,7 @@ pub fn installHoistedPackages(
.metas = parts.items(.meta),
.bins = parts.items(.bin),
.root_node_modules_folder = node_modules_folder,
.site_packages_folder = site_packages_folder,
.names = parts.items(.name),
.pkg_name_hashes = parts.items(.name_hash),
.resolutions = parts.items(.resolution),
@@ -371,6 +383,7 @@ const Bin = install.Bin;
const Lockfile = install.Lockfile;
const PackageID = install.PackageID;
const PackageInstall = install.PackageInstall;
const pypi = @import("pypi.zig");
const PackageManager = install.PackageManager;
const ProgressStrings = PackageManager.ProgressStrings;

View File

@@ -141,6 +141,7 @@ pub const Features = struct {
trusted_dependencies: bool = false,
workspaces: bool = false,
patched_dependencies: bool = false,
python_dependencies: bool = false,
check_for_duplicate_dependencies: bool = false,
@@ -162,6 +163,7 @@ pub const Features = struct {
.trusted_dependencies = true,
.patched_dependencies = true,
.workspaces = true,
.python_dependencies = true,
};
pub const folder = Features{
@@ -243,6 +245,8 @@ pub const PackageManifestError = error{
pub const ExtractTarball = @import("./extract_tarball.zig");
pub const NetworkTask = @import("./NetworkTask.zig");
pub const Npm = @import("./npm.zig");
pub const Pep440 = @import("./pep440.zig");
pub const PyPI = @import("./pypi.zig");
pub const PackageManager = @import("./PackageManager.zig");
pub const PackageManifestMap = @import("./PackageManifestMap.zig");
pub const Task = @import("./PackageManagerTask.zig");

View File

@@ -911,6 +911,7 @@ pub fn installIsolatedPackages(
.github,
.local_tarball,
.remote_tarball,
.pypi,
=> |pkg_res_tag| {
const patch_info = try installer.packagePatchInfo(pkg_name, pkg_name_hash, &pkg_res);
@@ -959,6 +960,7 @@ pub fn installIsolatedPackages(
.github => manager.cachedGitHubFolderName(&pkg_res.value.github, patch_info.contentsHash()),
.local_tarball => manager.cachedTarballFolderName(pkg_res.value.local_tarball, patch_info.contentsHash()),
.remote_tarball => manager.cachedTarballFolderName(pkg_res.value.remote_tarball, patch_info.contentsHash()),
.pypi => manager.cachedTarballFolderName(pkg_res.value.pypi.url, patch_info.contentsHash()),
else => comptime unreachable,
});
@@ -1113,6 +1115,32 @@ pub fn installIsolatedPackages(
},
};
},
.pypi => {
manager.enqueueTarballForDownload(
dep_id,
pkg_id,
pkg_res.value.pypi.url.slice(string_buf),
ctx,
patch_info.nameAndVersionHash(),
) catch |err| switch (err) {
error.OutOfMemory => bun.outOfMemory(),
error.InvalidURL => {
Output.err(err, "failed to enqueue pypi tarball for download: {s}@{f}", .{
pkg_name.slice(string_buf),
pkg_res.fmt(string_buf, .auto),
});
Output.flush();
if (manager.options.enable.fail_early) {
Global.exit(1);
}
// .monotonic is okay because an error means the task isn't
// running on another thread.
entry_steps[entry_id.get()].store(.done, .monotonic);
installer.onTaskComplete(entry_id, .fail);
continue;
},
};
},
else => comptime unreachable,
}
},

View File

@@ -156,6 +156,7 @@ pub const Installer = struct {
.root,
.workspace,
.symlink,
.pypi,
=> {},
_ => {},
@@ -488,6 +489,56 @@ pub const Installer = struct {
};
},
.pypi => {
// Python packages are installed to .venv/lib/python{version}/site-packages/
const cache_dir_subpath = manager.cachedTarballFolderName(pkg_res.value.pypi.url, null);
// Create .venv directory structure if it doesn't exist
_ = bun.sys.mkdir(".venv", 0o755);
_ = bun.sys.mkdir(".venv/lib", 0o755);
_ = bun.sys.mkdir(pypi.venv_lib_dir, 0o755);
_ = bun.sys.mkdir(pypi.venv_site_packages, 0o755);
const cache_dir, const cache_dir_path = manager.getCacheDirectoryAndAbsPath();
defer cache_dir_path.deinit();
// Open the wheel cache directory
const cached_wheel_dir = switch (bun.openDirForIteration(cache_dir, cache_dir_subpath)) {
.result => |fd| fd,
.err => |err| return .failure(.{ .link_package = err }),
};
defer cached_wheel_dir.close();
// Build source path (wheel cache directory)
var src_path: bun.AbsPath(.{ .sep = .auto, .unit = .os }) = .fromLongPath(cache_dir_path.slice());
defer src_path.deinit();
src_path.appendJoin(@as([]const u8, cache_dir_subpath));
// Destination is site-packages
var dest_path: bun.RelPath(.{ .sep = .auto, .unit = .os }) = .init();
defer dest_path.deinit();
dest_path.append(pypi.venv_site_packages);
// Copy entire wheel contents to site-packages
var hardlinker: Hardlinker = try .init(
cached_wheel_dir,
src_path,
dest_path,
&.{},
);
defer hardlinker.deinit();
switch (try hardlinker.link()) {
.result => {},
.err => |err| return .failure(.{ .link_package = err }),
}
// Python packages don't need symlinks, binaries, or scripts - skip to done
// Set the step to .done since we're skipping directly to it
this.installer.store.entries.items(.step)[this.entry_id.get()].store(.done, .release);
continue :next_step .done;
},
.folder, .root => {
const path = switch (pkg_res.tag) {
.folder => pkg_res.value.folder.slice(string_buf),
@@ -883,6 +934,7 @@ pub const Installer = struct {
.folder,
.symlink,
.single_file_module,
.pypi,
=> {},
_ => {},
@@ -1612,6 +1664,7 @@ const PackageInstall = install.PackageInstall;
const PackageManager = install.PackageManager;
const PackageNameHash = install.PackageNameHash;
const PostinstallOptimizer = install.PostinstallOptimizer;
const pypi = @import("../pypi.zig");
const Resolution = install.Resolution;
const Store = install.Store;
const TruncatedPackageNameHash = install.TruncatedPackageNameHash;

View File

@@ -55,6 +55,7 @@ pub fn Package(comptime SemverIntType: type) type {
pub const optional = DependencyGroup{ .prop = "optionalDependencies", .field = "optional_dependencies", .behavior = .{ .optional = true } };
pub const peer = DependencyGroup{ .prop = "peerDependencies", .field = "peer_dependencies", .behavior = .{ .peer = true } };
pub const workspaces = DependencyGroup{ .prop = "workspaces", .field = "workspaces", .behavior = .{ .workspace = true } };
pub const python = DependencyGroup{ .prop = "pythonDependencies", .field = "python_dependencies", .behavior = .{ .python = true } };
};
pub inline fn isDisabled(this: *const @This(), cpu: Npm.Architecture, os: Npm.OperatingSystem) bool {
@@ -998,9 +999,12 @@ pub fn Package(comptime SemverIntType: type) type {
key_loc: logger.Loc,
value_loc: logger.Loc,
) !?Dependency {
// For python dependencies, always use .pypi tag
const effective_tag: ?Dependency.Version.Tag = comptime if (group.behavior.python) .pypi else tag;
const external_version = brk: {
if (comptime Environment.isWindows) {
switch (tag orelse Dependency.Version.Tag.infer(version)) {
switch (effective_tag orelse Dependency.Version.Tag.infer(version)) {
.workspace, .folder, .symlink, .tarball => {
if (String.canInline(version)) {
var copy = string_builder.append(String, version);
@@ -1028,7 +1032,7 @@ pub fn Package(comptime SemverIntType: type) type {
external_alias.value,
external_alias.hash,
sliced.slice,
tag,
effective_tag,
&sliced,
log,
pm,
@@ -1062,12 +1066,12 @@ pub fn Package(comptime SemverIntType: type) type {
var workspace_path: ?String = null;
var workspace_version = workspace_ver;
if (comptime tag == null) {
if (comptime effective_tag == null) {
workspace_path = lockfile.workspace_paths.get(name_hash);
workspace_version = lockfile.workspace_versions.get(name_hash);
}
if (comptime tag != null) {
if (comptime effective_tag != null) {
bun.assert(dependency_version.tag != .npm and dependency_version.tag != .dist_tag);
}
@@ -1378,7 +1382,8 @@ pub fn Package(comptime SemverIntType: type) type {
@as(usize, @intFromBool(features.dependencies)) +
@as(usize, @intFromBool(features.dev_dependencies)) +
@as(usize, @intFromBool(features.optional_dependencies)) +
@as(usize, @intFromBool(features.peer_dependencies))
@as(usize, @intFromBool(features.peer_dependencies)) +
@as(usize, @intFromBool(features.python_dependencies))
]DependencyGroup = undefined;
var out_group_i: usize = 0;
@@ -1406,6 +1411,11 @@ pub fn Package(comptime SemverIntType: type) type {
out_group_i += 1;
}
if (features.python_dependencies) {
out_groups[out_group_i] = DependencyGroup.python;
out_group_i += 1;
}
break :brk out_groups;
};
@@ -2145,6 +2155,7 @@ pub fn Package(comptime SemverIntType: type) type {
.workspace => .init(.{ .workspace = old.resolution.value.workspace }),
.remote_tarball => .init(.{ .remote_tarball = old.resolution.value.remote_tarball }),
.single_file_module => .init(.{ .single_file_module = old.resolution.value.single_file_module }),
.pypi => .init(.{ .pypi = old.resolution.value.pypi.migrate() }),
else => .init(.{ .uninitialized = {} }),
},
};

View File

@@ -406,7 +406,7 @@ pub const Stringifier = struct {
const res = pkg_resolutions[pkg_id];
switch (res.tag) {
.root, .npm, .folder, .local_tarball, .github, .git, .symlink, .workspace, .remote_tarball => {},
.root, .npm, .folder, .local_tarball, .github, .git, .symlink, .workspace, .remote_tarball, .pypi => {},
.uninitialized => continue,
// should not be possible, just being safe
.single_file_module => continue,
@@ -648,6 +648,35 @@ pub const Stringifier = struct {
repo.resolved.fmtJson(buf, .{}),
});
},
.pypi => {
// Format: ["name@version", "wheel_url", { info }]
try writer.print("[\"{f}@{f}\", ", .{
pkg_name.fmtJson(buf, .{ .quote = false }),
res.value.pypi.version.fmt(buf),
});
// Write the wheel URL
try writer.print("\"{s}\", ", .{
res.value.pypi.url.slice(buf),
});
try writePackageInfoObject(
writer,
dep.behavior,
deps_buf,
pkg_deps_sort_buf.items,
&pkg_meta,
&pkg_bin,
buf,
&optional_peers_buf,
extern_strings,
&pkg_map,
relative_path,
&path_buf,
);
try writer.writeByte(']');
},
else => unreachable,
}
}

View File

@@ -110,6 +110,15 @@ fn jsonStringifyDependency(this: *const Lockfile, w: anytype, dep_id: Dependency
try w.objectField("version");
try w.print("\"catalog:{f}\"", .{info.fmtJson(sb, .{ .quote = false })});
},
.pypi => {
try w.beginObject();
defer w.endObject() catch {};
const info = dep.version.value.pypi;
try w.objectField("name");
try w.write(info.name.slice(sb));
},
}
try w.objectField("package_id");

View File

@@ -954,6 +954,8 @@ pub fn migrateNPMLockfile(
},
});
},
// npm does not support PyPI packages
.pypi => return error.InvalidNPMLockfile,
};
};
debug("-> {f}", .{res.fmtForDebug(string_buf.bytes.items)});

696
src/install/pep440.zig Normal file
View File

@@ -0,0 +1,696 @@
//! PEP 440 Version Parsing and Comparison
//!
//! Implements version parsing and range matching according to PEP 440:
//! https://peps.python.org/pep-0440/
//!
//! Version format: [N!]N(.N)*[{a|b|rc}N][.postN][.devN][+local]
//! Examples: 1.0, 2.0.0, 1.0a1, 1.0b2, 1.0rc1, 1.0.post1, 1.0.dev1, 1.0+local
//!
//! Specifier operators: ==, !=, <=, >=, <, >, ~=, ===
const Pep440 = @This();
const std = @import("std");
/// PEP 440 Version
/// Stores version components for comparison
pub const Version = struct {
/// Epoch (rarely used, default 0)
epoch: u32 = 0,
/// Release segments (e.g., [1, 2, 3] for "1.2.3")
/// We store up to 4 segments inline for common cases
major: u32 = 0,
minor: u32 = 0,
micro: u32 = 0,
extra: u32 = 0,
/// Number of release segments (1-4 for inline, 0 means unset)
segment_count: u8 = 0,
/// Pre-release type
pre_type: PreType = .none,
/// Pre-release number (e.g., 1 for "a1")
pre_num: u32 = 0,
/// Post-release number (0 means no post-release)
post: u32 = 0,
/// Whether .post was explicitly specified
has_post: bool = false,
/// Dev release number (0 means no dev release)
dev: u32 = 0,
/// Whether .dev was explicitly specified
has_dev: bool = false,
pub const PreType = enum(u8) {
none = 0,
dev = 1, // .devN (lowest precedence in pre-releases)
alpha = 2, // aN or alphaN
beta = 3, // bN or betaN
rc = 4, // rcN or cN
final = 5, // no pre-release suffix (highest)
};
/// Compare two versions
/// Returns: .lt, .eq, or .gt
pub fn order(self: Version, other: Version) std.math.Order {
// Compare epoch first
if (self.epoch != other.epoch) {
return std.math.order(self.epoch, other.epoch);
}
// Compare release segments
const self_segments = [4]u32{ self.major, self.minor, self.micro, self.extra };
const other_segments = [4]u32{ other.major, other.minor, other.micro, other.extra };
const max_segments = @max(self.segment_count, other.segment_count);
var i: usize = 0;
while (i < max_segments) : (i += 1) {
const self_seg = if (i < self.segment_count) self_segments[i] else 0;
const other_seg = if (i < other.segment_count) other_segments[i] else 0;
if (self_seg != other_seg) {
return std.math.order(self_seg, other_seg);
}
}
// Compare pre-release (none/final > rc > beta > alpha > dev)
// But dev without pre-release is LESS than final
const self_pre = self.effectivePreType();
const other_pre = other.effectivePreType();
if (@intFromEnum(self_pre) != @intFromEnum(other_pre)) {
return std.math.order(@intFromEnum(self_pre), @intFromEnum(other_pre));
}
// Same pre-type, compare pre-number
if (self_pre != .none and self_pre != .final) {
if (self.pre_num != other.pre_num) {
return std.math.order(self.pre_num, other.pre_num);
}
}
// Compare post-release
if (self.has_post != other.has_post) {
return if (self.has_post) .gt else .lt;
}
if (self.has_post and self.post != other.post) {
return std.math.order(self.post, other.post);
}
// Compare dev release (has_dev means it's a dev version, which is less than non-dev)
if (self.has_dev != other.has_dev) {
return if (self.has_dev) .lt else .gt;
}
if (self.has_dev and self.dev != other.dev) {
return std.math.order(self.dev, other.dev);
}
return .eq;
}
fn effectivePreType(self: Version) PreType {
if (self.pre_type != .none) return self.pre_type;
// If no pre-release suffix, it's a final release
return .final;
}
pub fn eql(self: Version, other: Version) bool {
return self.order(other) == .eq;
}
/// Parse a PEP 440 version string
pub fn parse(input: []const u8) ?Version {
var result = Version{};
var remaining = input;
// Skip leading 'v' or 'V' if present (common but not in spec)
if (remaining.len > 0 and (remaining[0] == 'v' or remaining[0] == 'V')) {
remaining = remaining[1..];
}
// Parse epoch (N!)
if (std.mem.indexOfScalar(u8, remaining, '!')) |bang_idx| {
result.epoch = std.fmt.parseInt(u32, remaining[0..bang_idx], 10) catch return null;
remaining = remaining[bang_idx + 1 ..];
}
// Parse release segments (N.N.N...)
var segment_idx: u8 = 0;
while (remaining.len > 0 and segment_idx < 4) {
// Find end of this segment
var seg_end: usize = 0;
while (seg_end < remaining.len and remaining[seg_end] >= '0' and remaining[seg_end] <= '9') {
seg_end += 1;
}
if (seg_end == 0) break; // No more digits
const segment = std.fmt.parseInt(u32, remaining[0..seg_end], 10) catch return null;
switch (segment_idx) {
0 => result.major = segment,
1 => result.minor = segment,
2 => result.micro = segment,
3 => result.extra = segment,
else => {},
}
segment_idx += 1;
result.segment_count = segment_idx;
remaining = remaining[seg_end..];
// Check for dot separator
if (remaining.len > 0 and remaining[0] == '.') {
// Peek ahead - if next char is a digit, continue parsing segments
if (remaining.len > 1 and remaining[1] >= '0' and remaining[1] <= '9') {
remaining = remaining[1..];
continue;
}
}
break;
}
if (result.segment_count == 0) return null;
// Parse pre-release, post-release, dev, local
while (remaining.len > 0) {
// Skip separator (., -, _)
if (remaining[0] == '.' or remaining[0] == '-' or remaining[0] == '_') {
remaining = remaining[1..];
if (remaining.len == 0) break;
}
// Local version (+...)
if (remaining[0] == '+') {
// We don't store local version for comparison purposes
break;
}
// Pre-release: a, alpha, b, beta, c, rc, preview, pre
if (parsePreRelease(remaining)) |pre_result| {
result.pre_type = pre_result.pre_type;
result.pre_num = pre_result.pre_num;
remaining = pre_result.remaining;
continue;
}
// Post-release: post, rev, r
if (parsePostRelease(remaining)) |post_result| {
result.has_post = true;
result.post = post_result.post;
remaining = post_result.remaining;
continue;
}
// Dev release: dev
if (parseDevRelease(remaining)) |dev_result| {
result.has_dev = true;
result.dev = dev_result.dev;
remaining = dev_result.remaining;
continue;
}
// Unknown suffix, stop parsing
break;
}
return result;
}
const PreResult = struct {
pre_type: PreType,
pre_num: u32,
remaining: []const u8,
};
fn parsePreRelease(input: []const u8) ?PreResult {
const prefixes = [_]struct { prefix: []const u8, pre_type: PreType }{
.{ .prefix = "alpha", .pre_type = .alpha },
.{ .prefix = "beta", .pre_type = .beta },
.{ .prefix = "preview", .pre_type = .rc },
.{ .prefix = "pre", .pre_type = .rc },
.{ .prefix = "rc", .pre_type = .rc },
.{ .prefix = "a", .pre_type = .alpha },
.{ .prefix = "b", .pre_type = .beta },
.{ .prefix = "c", .pre_type = .rc },
};
for (prefixes) |p| {
if (startsWithIgnoreCase(input, p.prefix)) {
var remaining = input[p.prefix.len..];
// Skip optional separator
if (remaining.len > 0 and (remaining[0] == '.' or remaining[0] == '-' or remaining[0] == '_')) {
remaining = remaining[1..];
}
// Parse number
var num_end: usize = 0;
while (num_end < remaining.len and remaining[num_end] >= '0' and remaining[num_end] <= '9') {
num_end += 1;
}
const num = if (num_end > 0)
std.fmt.parseInt(u32, remaining[0..num_end], 10) catch 0
else
0;
return .{
.pre_type = p.pre_type,
.pre_num = num,
.remaining = remaining[num_end..],
};
}
}
return null;
}
const PostResult = struct {
post: u32,
remaining: []const u8,
};
fn parsePostRelease(input: []const u8) ?PostResult {
const prefixes = [_][]const u8{ "post", "rev", "r" };
for (prefixes) |prefix| {
if (startsWithIgnoreCase(input, prefix)) {
var remaining = input[prefix.len..];
// Skip optional separator
if (remaining.len > 0 and (remaining[0] == '.' or remaining[0] == '-' or remaining[0] == '_')) {
remaining = remaining[1..];
}
// Parse number
var num_end: usize = 0;
while (num_end < remaining.len and remaining[num_end] >= '0' and remaining[num_end] <= '9') {
num_end += 1;
}
const num = if (num_end > 0)
std.fmt.parseInt(u32, remaining[0..num_end], 10) catch 0
else
0;
return .{
.post = num,
.remaining = remaining[num_end..],
};
}
}
return null;
}
const DevResult = struct {
dev: u32,
remaining: []const u8,
};
fn parseDevRelease(input: []const u8) ?DevResult {
if (startsWithIgnoreCase(input, "dev")) {
var remaining = input[3..];
// Skip optional separator
if (remaining.len > 0 and (remaining[0] == '.' or remaining[0] == '-' or remaining[0] == '_')) {
remaining = remaining[1..];
}
// Parse number
var num_end: usize = 0;
while (num_end < remaining.len and remaining[num_end] >= '0' and remaining[num_end] <= '9') {
num_end += 1;
}
const num = if (num_end > 0)
std.fmt.parseInt(u32, remaining[0..num_end], 10) catch 0
else
0;
return .{
.dev = num,
.remaining = remaining[num_end..],
};
}
return null;
}
fn startsWithIgnoreCase(haystack: []const u8, needle: []const u8) bool {
if (haystack.len < needle.len) return false;
for (haystack[0..needle.len], needle) |h, n| {
if (std.ascii.toLower(h) != std.ascii.toLower(n)) return false;
}
return true;
}
pub fn format(self: Version, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void {
if (self.epoch != 0) {
try writer.print("{d}!", .{self.epoch});
}
try writer.print("{d}", .{self.major});
if (self.segment_count >= 2) try writer.print(".{d}", .{self.minor});
if (self.segment_count >= 3) try writer.print(".{d}", .{self.micro});
if (self.segment_count >= 4) try writer.print(".{d}", .{self.extra});
switch (self.pre_type) {
.alpha => try writer.print("a{d}", .{self.pre_num}),
.beta => try writer.print("b{d}", .{self.pre_num}),
.rc => try writer.print("rc{d}", .{self.pre_num}),
.dev => try writer.print(".dev{d}", .{self.dev}),
.none, .final => {},
}
if (self.has_post) {
try writer.print(".post{d}", .{self.post});
}
if (self.has_dev and self.pre_type != .dev) {
try writer.print(".dev{d}", .{self.dev});
}
}
};
/// Comparison operator
pub const Op = enum(u8) {
unset = 0,
/// == (exact match, or prefix match with .*)
eql = 1,
/// !=
neq = 2,
/// <
lt = 3,
/// <=
lte = 4,
/// >
gt = 5,
/// >=
gte = 6,
/// ~= (compatible release)
compat = 7,
/// === (arbitrary equality, string match)
arbitrary = 8,
};
/// A single version specifier (e.g., ">=1.0" or "!=1.5.0")
pub const Specifier = struct {
op: Op = .unset,
version: Version = .{},
/// For == with wildcard (e.g., ==1.0.*)
/// 0 = no wildcard, 1 = major.*, 2 = major.minor.*, etc.
wildcard_segments: u8 = 0,
/// Check if a version satisfies this specifier
pub fn satisfies(self: Specifier, version: Version) bool {
if (self.op == .unset) return true;
const cmp = version.order(self.version);
return switch (self.op) {
.unset => true,
.eql => if (self.wildcard_segments > 0)
self.wildcardMatch(version)
else
cmp == .eq,
.neq => if (self.wildcard_segments > 0)
!self.wildcardMatch(version)
else
cmp != .eq,
.lt => cmp == .lt,
.lte => cmp == .lt or cmp == .eq,
.gt => cmp == .gt,
.gte => cmp == .gt or cmp == .eq,
.compat => self.compatibleMatch(version),
.arbitrary => false, // Not supported, would need string comparison
};
}
fn wildcardMatch(self: Specifier, version: Version) bool {
// Match up to wildcard_segments
const self_segs = [4]u32{ self.version.major, self.version.minor, self.version.micro, self.version.extra };
const other_segs = [4]u32{ version.major, version.minor, version.micro, version.extra };
var i: usize = 0;
while (i < self.wildcard_segments) : (i += 1) {
if (self_segs[i] != other_segs[i]) return false;
}
return true;
}
fn compatibleMatch(self: Specifier, version: Version) bool {
// ~=X.Y is equivalent to >=X.Y,<(X+1).0
// ~=X.Y.Z is equivalent to >=X.Y.Z,<X.(Y+1).0
const cmp = version.order(self.version);
if (cmp == .lt) return false;
// Check upper bound
var upper = self.version;
if (self.version.segment_count >= 2) {
// Increment the second-to-last segment
if (self.version.segment_count == 2) {
upper.major += 1;
upper.minor = 0;
} else if (self.version.segment_count == 3) {
upper.minor += 1;
upper.micro = 0;
} else {
upper.micro += 1;
upper.extra = 0;
}
} else {
// Single segment version, no upper bound restriction
return true;
}
return version.order(upper) == .lt;
}
};
/// A version range consisting of multiple specifiers (AND'd together)
/// e.g., ">=1.0,<2.0,!=1.5.0"
pub const Range = struct {
/// Specifiers are AND'd together (all must match)
/// Stored inline for common case (up to 4 specifiers)
specs: [4]Specifier = [_]Specifier{.{}} ** 4,
count: u8 = 0,
/// Parse a version range string
/// e.g., ">=1.0,<2.0" or "~=1.4.2" or ">=1.0,!=1.5.0"
pub fn parse(input: []const u8) ?Range {
var result = Range{};
var remaining = std.mem.trim(u8, input, " \t\n\r");
while (remaining.len > 0 and result.count < 4) {
// Skip whitespace and commas
while (remaining.len > 0 and (remaining[0] == ',' or remaining[0] == ' ')) {
remaining = remaining[1..];
}
if (remaining.len == 0) break;
// Parse operator
var spec = Specifier{};
if (std.mem.startsWith(u8, remaining, "===")) {
spec.op = .arbitrary;
remaining = remaining[3..];
} else if (std.mem.startsWith(u8, remaining, "==")) {
spec.op = .eql;
remaining = remaining[2..];
} else if (std.mem.startsWith(u8, remaining, "!=")) {
spec.op = .neq;
remaining = remaining[2..];
} else if (std.mem.startsWith(u8, remaining, "~=")) {
spec.op = .compat;
remaining = remaining[2..];
} else if (std.mem.startsWith(u8, remaining, "<=")) {
spec.op = .lte;
remaining = remaining[2..];
} else if (std.mem.startsWith(u8, remaining, ">=")) {
spec.op = .gte;
remaining = remaining[2..];
} else if (std.mem.startsWith(u8, remaining, "<")) {
spec.op = .lt;
remaining = remaining[1..];
} else if (std.mem.startsWith(u8, remaining, ">")) {
spec.op = .gt;
remaining = remaining[1..];
} else {
// No operator means implicit ==
spec.op = .eql;
}
// Skip whitespace after operator
remaining = std.mem.trim(u8, remaining, " \t\n\r");
// Find end of version (comma or end of string)
var ver_end: usize = 0;
while (ver_end < remaining.len and remaining[ver_end] != ',') {
ver_end += 1;
}
var ver_str = std.mem.trim(u8, remaining[0..ver_end], " \t\n\r");
// Check for wildcard (e.g., ==1.0.*)
if (ver_str.len > 2 and std.mem.endsWith(u8, ver_str, ".*")) {
// Count segments before .*
var seg_count: u8 = 1;
for (ver_str[0 .. ver_str.len - 2]) |c| {
if (c == '.') seg_count += 1;
}
spec.wildcard_segments = seg_count;
ver_str = ver_str[0 .. ver_str.len - 2];
}
// Parse version
if (Version.parse(ver_str)) |v| {
spec.version = v;
} else {
return null;
}
result.specs[result.count] = spec;
result.count += 1;
remaining = remaining[ver_end..];
}
return if (result.count > 0) result else null;
}
/// Check if a version satisfies all specifiers in this range
pub fn satisfies(self: Range, version: Version) bool {
if (self.count == 0) return true;
for (self.specs[0..self.count]) |spec| {
if (!spec.satisfies(version)) return false;
}
return true;
}
/// Check if this is a "match any" range (empty or *)
pub fn isAny(self: Range) bool {
return self.count == 0;
}
};
// ============================================================================
// Tests
// ============================================================================
test "Version.parse basic" {
const v1 = Version.parse("1.0").?;
try std.testing.expectEqual(@as(u32, 1), v1.major);
try std.testing.expectEqual(@as(u32, 0), v1.minor);
try std.testing.expectEqual(@as(u8, 2), v1.segment_count);
const v2 = Version.parse("1.2.3").?;
try std.testing.expectEqual(@as(u32, 1), v2.major);
try std.testing.expectEqual(@as(u32, 2), v2.minor);
try std.testing.expectEqual(@as(u32, 3), v2.micro);
try std.testing.expectEqual(@as(u8, 3), v2.segment_count);
const v3 = Version.parse("2.0.0.1").?;
try std.testing.expectEqual(@as(u32, 2), v3.major);
try std.testing.expectEqual(@as(u32, 0), v3.minor);
try std.testing.expectEqual(@as(u32, 0), v3.micro);
try std.testing.expectEqual(@as(u32, 1), v3.extra);
try std.testing.expectEqual(@as(u8, 4), v3.segment_count);
}
test "Version.parse pre-release" {
const v1 = Version.parse("1.0a1").?;
try std.testing.expectEqual(Version.PreType.alpha, v1.pre_type);
try std.testing.expectEqual(@as(u32, 1), v1.pre_num);
const v2 = Version.parse("1.0b2").?;
try std.testing.expectEqual(Version.PreType.beta, v2.pre_type);
try std.testing.expectEqual(@as(u32, 2), v2.pre_num);
const v3 = Version.parse("1.0rc1").?;
try std.testing.expectEqual(Version.PreType.rc, v3.pre_type);
try std.testing.expectEqual(@as(u32, 1), v3.pre_num);
const v4 = Version.parse("1.0.alpha.2").?;
try std.testing.expectEqual(Version.PreType.alpha, v4.pre_type);
try std.testing.expectEqual(@as(u32, 2), v4.pre_num);
}
test "Version.parse post and dev" {
const v1 = Version.parse("1.0.post1").?;
try std.testing.expect(v1.has_post);
try std.testing.expectEqual(@as(u32, 1), v1.post);
const v2 = Version.parse("1.0.dev1").?;
try std.testing.expect(v2.has_dev);
try std.testing.expectEqual(@as(u32, 1), v2.dev);
const v3 = Version.parse("1.0a1.post2.dev3").?;
try std.testing.expectEqual(Version.PreType.alpha, v3.pre_type);
try std.testing.expectEqual(@as(u32, 1), v3.pre_num);
try std.testing.expect(v3.has_post);
try std.testing.expectEqual(@as(u32, 2), v3.post);
try std.testing.expect(v3.has_dev);
try std.testing.expectEqual(@as(u32, 3), v3.dev);
}
test "Version.parse epoch" {
const v1 = Version.parse("1!2.0").?;
try std.testing.expectEqual(@as(u32, 1), v1.epoch);
try std.testing.expectEqual(@as(u32, 2), v1.major);
try std.testing.expectEqual(@as(u32, 0), v1.minor);
}
test "Version.order" {
const v1 = Version.parse("1.0").?;
const v2 = Version.parse("2.0").?;
try std.testing.expectEqual(std.math.Order.lt, v1.order(v2));
try std.testing.expectEqual(std.math.Order.gt, v2.order(v1));
try std.testing.expectEqual(std.math.Order.eq, v1.order(v1));
// Pre-release < final
const v3 = Version.parse("1.0a1").?;
const v4 = Version.parse("1.0").?;
try std.testing.expectEqual(std.math.Order.lt, v3.order(v4));
// alpha < beta < rc
const va = Version.parse("1.0a1").?;
const vb = Version.parse("1.0b1").?;
const vrc = Version.parse("1.0rc1").?;
try std.testing.expectEqual(std.math.Order.lt, va.order(vb));
try std.testing.expectEqual(std.math.Order.lt, vb.order(vrc));
// dev < final
const vdev = Version.parse("1.0.dev1").?;
const vfinal = Version.parse("1.0").?;
try std.testing.expectEqual(std.math.Order.lt, vdev.order(vfinal));
// post > final
const vpost = Version.parse("1.0.post1").?;
try std.testing.expectEqual(std.math.Order.gt, vpost.order(vfinal));
}
test "Range.parse and satisfies" {
// Simple >= range
const r1 = Range.parse(">=1.0").?;
try std.testing.expect(r1.satisfies(Version.parse("1.0").?));
try std.testing.expect(r1.satisfies(Version.parse("2.0").?));
try std.testing.expect(!r1.satisfies(Version.parse("0.9").?));
// Combined range
const r2 = Range.parse(">=1.0,<2.0").?;
try std.testing.expect(r2.satisfies(Version.parse("1.0").?));
try std.testing.expect(r2.satisfies(Version.parse("1.5").?));
try std.testing.expect(!r2.satisfies(Version.parse("2.0").?));
try std.testing.expect(!r2.satisfies(Version.parse("0.5").?));
// Exclusion
const r3 = Range.parse(">=1.0,!=1.5.0").?;
try std.testing.expect(r3.satisfies(Version.parse("1.0").?));
try std.testing.expect(r3.satisfies(Version.parse("1.4").?));
try std.testing.expect(!r3.satisfies(Version.parse("1.5.0").?));
try std.testing.expect(r3.satisfies(Version.parse("1.6").?));
// Compatible release
const r4 = Range.parse("~=1.4.2").?;
try std.testing.expect(r4.satisfies(Version.parse("1.4.2").?));
try std.testing.expect(r4.satisfies(Version.parse("1.4.5").?));
try std.testing.expect(!r4.satisfies(Version.parse("1.5.0").?));
try std.testing.expect(!r4.satisfies(Version.parse("1.4.1").?));
}
test "Range wildcard" {
const r1 = Range.parse("==1.0.*").?;
try std.testing.expect(r1.satisfies(Version.parse("1.0.0").?));
try std.testing.expect(r1.satisfies(Version.parse("1.0.5").?));
try std.testing.expect(!r1.satisfies(Version.parse("1.1.0").?));
}

994
src/install/pypi.zig Normal file
View File

@@ -0,0 +1,994 @@
//! PyPI (Python Package Index) client and wheel selection
//!
//! This module handles:
//! - Parsing PyPI JSON API responses (https://pypi.org/pypi/{package}/json)
//! - Selecting the best wheel for the current platform
//! - Parsing PEP 440 version specifiers from requires_dist
const PyPI = @This();
/// Python version constants - must match the version Bun is linked against
/// These are used for wheel compatibility checking and venv path construction
pub const python_version_major = 3;
pub const python_version_minor = 13;
pub const python_version_string = std.fmt.comptimePrint("{d}.{d}", .{ python_version_major, python_version_minor });
/// Virtual environment paths for Python packages
/// Structure: .venv/lib/python{major}.{minor}/site-packages/
pub const venv_lib_dir = ".venv/lib/python" ++ python_version_string;
pub const venv_site_packages = venv_lib_dir ++ "/site-packages";
const std = @import("std");
const bun = @import("bun");
const strings = bun.strings;
const String = bun.Semver.String;
const Allocator = std.mem.Allocator;
const logger = bun.logger;
const JSON = bun.json;
const Environment = bun.Environment;
const OOM = bun.OOM;
const default_allocator = bun.default_allocator;
const initializeStore = @import("./install.zig").initializeMiniStore;
/// Platform target for wheel compatibility checking
pub const PlatformTarget = struct {
os: Os,
arch: Arch,
/// Python version (e.g., 3.12 = { .major = 3, .minor = 12 })
python_version: PythonVersion,
pub const Os = enum {
macos,
linux,
windows,
unknown,
};
pub const Arch = enum {
x86_64,
aarch64,
unknown,
};
pub const PythonVersion = struct {
major: u8 = 3,
minor: u8 = 12,
pub fn format(self: PythonVersion, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void {
try writer.print("{d}.{d}", .{ self.major, self.minor });
}
};
/// Detect current platform from compile-time target
pub fn current() PlatformTarget {
return .{
.os = comptime if (Environment.isMac)
Os.macos
else if (Environment.isLinux)
Os.linux
else if (Environment.isWindows)
Os.windows
else
Os.unknown,
.arch = comptime if (Environment.isAarch64)
Arch.aarch64
else if (Environment.isX64)
Arch.x86_64
else
Arch.unknown,
// Use the Python version constants defined at module level
.python_version = .{ .major = python_version_major, .minor = python_version_minor },
};
}
/// Check if a platform tag is compatible with this target
pub fn isPlatformCompatible(self: PlatformTarget, platform_tag: []const u8) bool {
// "any" is always compatible
if (strings.eqlComptime(platform_tag, "any")) return true;
// Check OS-specific tags
switch (self.os) {
.macos => {
// macOS tags: macosx_X_Y_arch, macosx_X_Y_universal, macosx_X_Y_universal2
if (strings.hasPrefixComptime(platform_tag, "macosx_")) {
// Check architecture suffix
if (self.arch == .aarch64) {
return strings.hasSuffixComptime(platform_tag, "_arm64") or
strings.hasSuffixComptime(platform_tag, "_universal2") or
strings.hasSuffixComptime(platform_tag, "_universal");
} else if (self.arch == .x86_64) {
return strings.hasSuffixComptime(platform_tag, "_x86_64") or
strings.hasSuffixComptime(platform_tag, "_universal2") or
strings.hasSuffixComptime(platform_tag, "_universal") or
strings.hasSuffixComptime(platform_tag, "_intel");
}
}
},
.linux => {
// Linux tags: linux_x86_64, manylinux1_x86_64, manylinux2010_x86_64,
// manylinux2014_x86_64, manylinux_2_17_x86_64, musllinux_1_1_x86_64
const has_arch_suffix = if (self.arch == .aarch64)
strings.hasSuffixComptime(platform_tag, "_aarch64")
else
strings.hasSuffixComptime(platform_tag, "_x86_64");
if (has_arch_suffix) {
if (strings.hasPrefixComptime(platform_tag, "linux_") or
strings.hasPrefixComptime(platform_tag, "manylinux") or
strings.hasPrefixComptime(platform_tag, "musllinux"))
{
return true;
}
}
},
.windows => {
// Windows tags: win32, win_amd64, win_arm64
if (self.arch == .x86_64) {
return strings.eqlComptime(platform_tag, "win_amd64") or
strings.eqlComptime(platform_tag, "win32");
} else if (self.arch == .aarch64) {
return strings.eqlComptime(platform_tag, "win_arm64");
}
},
.unknown => {},
}
return false;
}
/// Check if a Python version tag is compatible
pub fn isPythonCompatible(self: PlatformTarget, python_tag: []const u8) bool {
// "py3" matches any Python 3.x
if (strings.eqlComptime(python_tag, "py3")) return self.python_version.major == 3;
if (strings.eqlComptime(python_tag, "py2.py3")) return true;
if (strings.eqlComptime(python_tag, "py2")) return self.python_version.major == 2;
// "cpXY" matches CPython X.Y specifically (compiled extensions require exact match)
if (strings.hasPrefixComptime(python_tag, "cp")) {
const version_part = python_tag[2..];
if (version_part.len >= 2) {
const major = std.fmt.parseInt(u8, version_part[0..1], 10) catch return false;
const minor = std.fmt.parseInt(u8, version_part[1..], 10) catch return false;
return self.python_version.major == major and self.python_version.minor == minor;
}
}
// "pyXY" matches Python X.Y or higher minor versions
if (strings.hasPrefixComptime(python_tag, "py")) {
const version_part = python_tag[2..];
if (version_part.len >= 2) {
const major = std.fmt.parseInt(u8, version_part[0..1], 10) catch return false;
const minor = std.fmt.parseInt(u8, version_part[1..], 10) catch return false;
return self.python_version.major == major and self.python_version.minor >= minor;
}
}
return false;
}
/// Check if an ABI tag is compatible
pub fn isAbiCompatible(self: PlatformTarget, abi_tag: []const u8) bool {
// "none" means no ABI dependency (pure Python or uses stable ABI)
if (strings.eqlComptime(abi_tag, "none")) return true;
// "abi3" is the stable ABI, compatible with Python 3.2+
if (strings.eqlComptime(abi_tag, "abi3")) return self.python_version.major == 3 and self.python_version.minor >= 2;
// "cpXY" or "cpXYm" matches specific CPython ABI
if (strings.hasPrefixComptime(abi_tag, "cp")) {
var version_part = abi_tag[2..];
// Remove trailing 'm' if present (legacy ABI marker)
if (version_part.len > 0 and version_part[version_part.len - 1] == 'm') {
version_part = version_part[0 .. version_part.len - 1];
}
if (version_part.len >= 2) {
const major = std.fmt.parseInt(u8, version_part[0..1], 10) catch return false;
const minor = std.fmt.parseInt(u8, version_part[1..], 10) catch return false;
return self.python_version.major == major and self.python_version.minor == minor;
}
}
return false;
}
};
/// Parsed wheel filename components
/// Format: {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl
pub const WheelTag = struct {
python: []const u8,
abi: []const u8,
platform: []const u8,
/// Parse wheel tags from a wheel filename
/// Returns null if not a valid wheel filename
pub fn parse(filename: []const u8) ?WheelTag {
// Must end with .whl
if (!strings.hasSuffixComptime(filename, ".whl")) return null;
const name_without_ext = filename[0 .. filename.len - 4];
// Split by '-' and get the last 3 components (python-abi-platform)
var parts: [8][]const u8 = undefined;
var part_count: usize = 0;
var iter = std.mem.splitScalar(u8, name_without_ext, '-');
while (iter.next()) |part| {
if (part_count >= 8) return null; // Too many parts
parts[part_count] = part;
part_count += 1;
}
// Minimum: name-version-python-abi-platform = 5 parts
if (part_count < 5) return null;
return .{
.platform = parts[part_count - 1],
.abi = parts[part_count - 2],
.python = parts[part_count - 3],
};
}
/// Calculate a compatibility score (higher is better)
/// Returns null if not compatible
pub fn compatibilityScore(self: WheelTag, target: PlatformTarget) ?u32 {
// Check basic compatibility first
if (!target.isPythonCompatible(self.python)) return null;
if (!target.isAbiCompatible(self.abi)) return null;
if (!target.isPlatformCompatible(self.platform)) return null;
var score: u32 = 100;
// Prefer platform-specific wheels over "any"
if (!strings.eqlComptime(self.platform, "any")) {
score += 50;
}
// Prefer specific Python version over generic "py3"
if (strings.hasPrefixComptime(self.python, "cp")) {
score += 30;
}
// Prefer specific ABI over "none" or "abi3"
if (!strings.eqlComptime(self.abi, "none") and !strings.eqlComptime(self.abi, "abi3")) {
score += 20;
}
// Prefer newer manylinux versions
if (strings.hasPrefixComptime(self.platform, "manylinux_2_")) {
score += 10;
} else if (strings.hasPrefixComptime(self.platform, "manylinux2014")) {
score += 8;
} else if (strings.hasPrefixComptime(self.platform, "manylinux2010")) {
score += 5;
}
return score;
}
};
/// A file (wheel or source distribution) from PyPI
pub const File = struct {
filename: String,
url: String,
sha256: String,
python_version: String,
requires_python: String,
packagetype: PackageType,
size: u64,
pub const PackageType = enum(u8) {
bdist_wheel = 0,
sdist = 1,
bdist_egg = 2,
other = 3,
pub fn fromString(s: []const u8) PackageType {
if (strings.eqlComptime(s, "bdist_wheel")) return .bdist_wheel;
if (strings.eqlComptime(s, "sdist")) return .sdist;
if (strings.eqlComptime(s, "bdist_egg")) return .bdist_egg;
return .other;
}
};
/// Check if this file is a wheel
pub fn isWheel(self: File, buf: []const u8) bool {
return self.packagetype == .bdist_wheel or
strings.hasSuffixComptime(self.filename.slice(buf), ".whl");
}
/// Get wheel tags for this file (only valid for wheels)
pub fn wheelTag(self: File, buf: []const u8) ?WheelTag {
return WheelTag.parse(self.filename.slice(buf));
}
};
/// Parsed PyPI package manifest
pub const PackageManifest = struct {
pkg: Package = .{},
string_buf: []const u8 = "",
files: []const File = &.{},
pub const Package = struct {
name: String = .{},
latest_version: String = .{},
requires_python: String = .{},
requires_dist_off: u32 = 0,
requires_dist_len: u32 = 0,
};
pub fn name(self: *const PackageManifest) []const u8 {
return self.pkg.name.slice(self.string_buf);
}
pub fn latestVersion(self: *const PackageManifest) []const u8 {
return self.pkg.latest_version.slice(self.string_buf);
}
/// Get the requires_dist (dependencies) as a slice of the string buffer
pub fn requiresDist(self: *const PackageManifest) []const u8 {
if (self.pkg.requires_dist_len == 0) return "";
return self.string_buf[self.pkg.requires_dist_off..][0..self.pkg.requires_dist_len];
}
/// Iterator over applicable dependencies (filtered by platform/python version)
pub const DependencyIterator = struct {
remaining: []const u8,
target: PlatformTarget,
pub fn next(self: *DependencyIterator) ?DependencySpecifier {
while (self.remaining.len > 0) {
// Find next newline
const end = strings.indexOfChar(self.remaining, '\n') orelse self.remaining.len;
const line = strings.trim(self.remaining[0..end], &strings.whitespace_chars);
self.remaining = if (end < self.remaining.len) self.remaining[end + 1 ..] else "";
if (line.len == 0) continue;
if (DependencySpecifier.parse(line)) |spec| {
if (spec.name.len > 0 and spec.isApplicable(self.target)) {
return spec;
}
}
}
return null;
}
/// Count the number of applicable dependencies
pub fn count(self: *DependencyIterator) usize {
var n: usize = 0;
var iter = self.*;
while (iter.next()) |_| {
n += 1;
}
return n;
}
};
/// Get an iterator over applicable dependencies
pub fn iterDependencies(self: *const PackageManifest, target: PlatformTarget) DependencyIterator {
return .{
.remaining = self.requiresDist(),
.target = target,
};
}
/// Find the best wheel for the given target platform
/// Returns null if no compatible wheel is found
pub fn findBestWheel(self: *const PackageManifest, target: PlatformTarget) ?*const File {
var best_file: ?*const File = null;
var best_score: u32 = 0;
for (self.files) |*file| {
if (!file.isWheel(self.string_buf)) continue;
if (file.wheelTag(self.string_buf)) |tag| {
if (tag.compatibilityScore(target)) |score| {
if (score > best_score) {
best_score = score;
best_file = file;
}
}
}
}
return best_file;
}
/// Parse a PyPI JSON API response
pub fn parse(
allocator: Allocator,
log: *logger.Log,
json_buffer: []const u8,
expected_name: []const u8,
) OOM!?PackageManifest {
const source = &logger.Source.initPathString(expected_name, json_buffer);
initializeStore();
defer bun.ast.Stmt.Data.Store.memory_allocator.?.pop();
var arena = bun.ArenaAllocator.init(allocator);
defer arena.deinit();
const json = JSON.parseUTF8(
source,
log,
arena.allocator(),
) catch {
return null;
};
// Check for error response
if (json.asProperty("message")) |msg| {
if (msg.expr.asString(allocator)) |err_msg| {
log.addErrorFmt(source, logger.Loc.Empty, allocator, "PyPI error: {s}", .{err_msg}) catch {};
return null;
}
}
var result: PackageManifest = .{
.pkg = .{
.name = .{},
.latest_version = .{},
.requires_python = .{},
.requires_dist_off = 0,
.requires_dist_len = 0,
},
.string_buf = &.{},
.files = &.{},
};
var string_pool = String.Builder.StringPool.init(default_allocator);
defer string_pool.deinit();
var string_builder = String.Builder{
.string_pool = string_pool,
};
// Count strings needed
const info = json.asProperty("info") orelse return null;
// Name
if (info.expr.asProperty("name")) |name_prop| {
if (name_prop.expr.asString(allocator)) |n| {
string_builder.count(n);
}
}
// Version
if (info.expr.asProperty("version")) |version_prop| {
if (version_prop.expr.asString(allocator)) |v| {
string_builder.count(v);
}
}
// requires_python
if (info.expr.asProperty("requires_python")) |rp| {
if (rp.expr.asString(allocator)) |rp_str| {
string_builder.count(rp_str);
}
}
// requires_dist (dependencies)
var requires_dist_total_len: usize = 0;
if (info.expr.asProperty("requires_dist")) |rd| {
if (rd.expr.data == .e_array) {
for (rd.expr.data.e_array.slice()) |item| {
if (item.asString(allocator)) |dep| {
requires_dist_total_len += dep.len + 1; // +1 for newline separator
}
}
}
}
if (requires_dist_total_len > 0) {
string_builder.cap += requires_dist_total_len;
}
// Count files from "urls" (files for latest version)
var file_count: usize = 0;
if (json.asProperty("urls")) |urls| {
if (urls.expr.data == .e_array) {
for (urls.expr.data.e_array.slice()) |file_obj| {
if (file_obj.data != .e_object) continue;
file_count += 1;
if (file_obj.asProperty("filename")) |f| {
if (f.expr.asString(allocator)) |filename| {
string_builder.count(filename);
}
}
if (file_obj.asProperty("url")) |u| {
if (u.expr.asString(allocator)) |url| {
string_builder.count(url);
}
}
if (file_obj.asProperty("digests")) |d| {
if (d.expr.asProperty("sha256")) |sha| {
if (sha.expr.asString(allocator)) |sha_str| {
string_builder.count(sha_str);
}
}
}
if (file_obj.asProperty("python_version")) |pv| {
if (pv.expr.asString(allocator)) |pv_str| {
string_builder.count(pv_str);
}
}
if (file_obj.asProperty("requires_python")) |rp| {
if (rp.expr.asString(allocator)) |rp_str| {
string_builder.count(rp_str);
}
}
}
}
}
// Allocate
try string_builder.allocate(default_allocator);
errdefer if (string_builder.ptr) |ptr| default_allocator.free(ptr[0..string_builder.cap]);
const files = try default_allocator.alloc(File, file_count);
errdefer default_allocator.free(files);
// Second pass: populate data
if (info.expr.asProperty("name")) |name_prop| {
if (name_prop.expr.asString(allocator)) |n| {
result.pkg.name = string_builder.append(String, n);
}
}
if (info.expr.asProperty("version")) |version_prop| {
if (version_prop.expr.asString(allocator)) |v| {
result.pkg.latest_version = string_builder.append(String, v);
}
}
if (info.expr.asProperty("requires_python")) |rp| {
if (rp.expr.asString(allocator)) |rp_str| {
result.pkg.requires_python = string_builder.append(String, rp_str);
}
}
// requires_dist - write directly to the buffer
if (info.expr.asProperty("requires_dist")) |rd| {
if (rd.expr.data == .e_array) {
result.pkg.requires_dist_off = @intCast(string_builder.len);
const buf_slice = string_builder.ptr.?[string_builder.len..string_builder.cap];
var write_pos: usize = 0;
for (rd.expr.data.e_array.slice()) |item| {
if (item.asString(allocator)) |dep| {
@memcpy(buf_slice[write_pos..][0..dep.len], dep);
write_pos += dep.len;
buf_slice[write_pos] = '\n';
write_pos += 1;
}
}
string_builder.len += write_pos;
result.pkg.requires_dist_len = @intCast(write_pos);
}
}
// Populate files
var file_idx: usize = 0;
if (json.asProperty("urls")) |urls| {
if (urls.expr.data == .e_array) {
for (urls.expr.data.e_array.slice()) |file_obj| {
if (file_obj.data != .e_object) continue;
if (file_idx >= file_count) break;
var file = File{
.filename = .{},
.url = .{},
.sha256 = .{},
.python_version = .{},
.requires_python = .{},
.packagetype = .other,
.size = 0,
};
if (file_obj.asProperty("filename")) |f| {
if (f.expr.asString(allocator)) |filename| {
file.filename = string_builder.append(String, filename);
}
}
if (file_obj.asProperty("url")) |u| {
if (u.expr.asString(allocator)) |url| {
file.url = string_builder.append(String, url);
}
}
if (file_obj.asProperty("digests")) |d| {
if (d.expr.asProperty("sha256")) |sha| {
if (sha.expr.asString(allocator)) |sha_str| {
file.sha256 = string_builder.append(String, sha_str);
}
}
}
if (file_obj.asProperty("python_version")) |pv| {
if (pv.expr.asString(allocator)) |pv_str| {
file.python_version = string_builder.append(String, pv_str);
}
}
if (file_obj.asProperty("requires_python")) |rp| {
if (rp.expr.asString(allocator)) |rp_str| {
file.requires_python = string_builder.append(String, rp_str);
}
}
if (file_obj.asProperty("packagetype")) |pt| {
if (pt.expr.asString(allocator)) |pt_str| {
file.packagetype = File.PackageType.fromString(pt_str);
}
}
if (file_obj.asProperty("size")) |sz| {
if (sz.expr.data == .e_number) {
file.size = @intFromFloat(sz.expr.data.e_number.value);
}
}
files[file_idx] = file;
file_idx += 1;
}
}
}
result.string_buf = string_builder.allocatedSlice();
result.files = files[0..file_idx];
return result;
}
pub fn deinit(self: *PackageManifest) void {
if (self.string_buf.len > 0) {
default_allocator.free(self.string_buf);
}
if (self.files.len > 0) {
default_allocator.free(self.files);
}
self.* = .{
.pkg = .{
.name = .{},
.latest_version = .{},
.requires_python = .{},
.requires_dist_off = 0,
.requires_dist_len = 0,
},
.string_buf = &.{},
.files = &.{},
};
}
};
/// Parse a PEP 440 dependency specifier from requires_dist
/// Format: "package_name (>=1.0,<2.0) ; extra == 'dev'"
pub const DependencySpecifier = struct {
name: []const u8,
version_spec: []const u8,
extras: []const u8,
markers: []const u8,
pub fn parse(spec: []const u8) ?DependencySpecifier {
var result = DependencySpecifier{
.name = "",
.version_spec = "",
.extras = "",
.markers = "",
};
var remaining = strings.trim(spec, &strings.whitespace_chars);
// Find the end of the package name (first space, [, (, or ;)
var name_end: usize = 0;
for (remaining, 0..) |c, i| {
if (c == ' ' or c == '[' or c == '(' or c == ';' or c == '<' or c == '>' or c == '=' or c == '!' or c == '~') {
name_end = i;
break;
}
} else {
// Entire string is the package name
result.name = remaining;
return result;
}
result.name = remaining[0..name_end];
remaining = remaining[name_end..];
remaining = strings.trim(remaining, &strings.whitespace_chars);
// Check for extras [extra1,extra2]
if (remaining.len > 0 and remaining[0] == '[') {
if (strings.indexOfChar(remaining, ']')) |end| {
result.extras = remaining[1..end];
remaining = remaining[end + 1 ..];
remaining = strings.trim(remaining, &strings.whitespace_chars);
}
}
// Check for version specifier (>=1.0,<2.0) or just >=1.0
if (remaining.len > 0) {
if (remaining[0] == '(') {
if (strings.indexOfChar(remaining, ')')) |end| {
result.version_spec = remaining[1..end];
remaining = remaining[end + 1 ..];
remaining = strings.trim(remaining, &strings.whitespace_chars);
}
} else if (remaining[0] == '>' or remaining[0] == '<' or remaining[0] == '=' or remaining[0] == '!' or remaining[0] == '~') {
// Version spec without parens - find the end (space or ;)
var spec_end: usize = remaining.len;
for (remaining, 0..) |c, i| {
if (c == ' ' or c == ';') {
spec_end = i;
break;
}
}
result.version_spec = remaining[0..spec_end];
remaining = remaining[spec_end..];
remaining = strings.trim(remaining, &strings.whitespace_chars);
}
}
// Check for environment markers ; python_version >= "3.8"
if (remaining.len > 0 and remaining[0] == ';') {
result.markers = strings.trim(remaining[1..], &strings.whitespace_chars);
}
return result;
}
/// Check if this dependency should be included for the given Python version.
/// Returns false for dependencies that:
/// - Require extras (e.g., "extra == 'socks'")
/// - Have unsatisfied Python version markers
/// - Have unsatisfied platform markers (platform_system, sys_platform)
pub fn isApplicable(self: DependencySpecifier, target: PlatformTarget) bool {
if (self.markers.len == 0) return true;
// Skip dependencies that require extras (e.g., "; extra == 'socks'")
// These are optional dependencies that the user must explicitly request
if (strings.containsComptime(self.markers, "extra")) return false;
// Parse python_version markers
// Common formats: python_version >= "3.8", python_version < "3.10"
// For now, we're permissive - include unless we can definitively exclude
if (strings.containsComptime(self.markers, "python_version")) {
// Try to parse simple python_version constraints
// Format: python_version <op> "X.Y"
if (strings.indexOf(self.markers, "python_version")) |idx| {
var marker_remaining = self.markers[idx + "python_version".len ..];
marker_remaining = strings.trim(marker_remaining, &strings.whitespace_chars);
// Parse operator
var op: enum { lt, lte, gt, gte, eq, neq } = .gte;
if (strings.hasPrefixComptime(marker_remaining, ">=")) {
op = .gte;
marker_remaining = marker_remaining[2..];
} else if (strings.hasPrefixComptime(marker_remaining, "<=")) {
op = .lte;
marker_remaining = marker_remaining[2..];
} else if (strings.hasPrefixComptime(marker_remaining, "==")) {
op = .eq;
marker_remaining = marker_remaining[2..];
} else if (strings.hasPrefixComptime(marker_remaining, "!=")) {
op = .neq;
marker_remaining = marker_remaining[2..];
} else if (strings.hasPrefixComptime(marker_remaining, "<")) {
op = .lt;
marker_remaining = marker_remaining[1..];
} else if (strings.hasPrefixComptime(marker_remaining, ">")) {
op = .gt;
marker_remaining = marker_remaining[1..];
}
marker_remaining = strings.trim(marker_remaining, &strings.whitespace_chars);
// Parse version string (remove quotes)
if (marker_remaining.len > 0 and (marker_remaining[0] == '"' or marker_remaining[0] == '\'')) {
const quote = marker_remaining[0];
marker_remaining = marker_remaining[1..];
if (strings.indexOfChar(marker_remaining, quote)) |end| {
const ver_str = marker_remaining[0..end];
// Parse "X.Y" format
if (strings.indexOfChar(ver_str, '.')) |dot| {
const major = std.fmt.parseInt(u8, ver_str[0..dot], 10) catch return true;
const minor = std.fmt.parseInt(u8, ver_str[dot + 1 ..], 10) catch return true;
// Compare with current Python version
const current = @as(u16, target.python_version.major) * 100 + target.python_version.minor;
const required = @as(u16, major) * 100 + minor;
const version_matches = switch (op) {
.lt => current < required,
.lte => current <= required,
.gt => current > required,
.gte => current >= required,
.eq => current == required,
.neq => current != required,
};
if (!version_matches) return false;
}
}
}
}
}
// Handle platform_system markers (e.g., platform_system == "Linux")
if (strings.containsComptime(self.markers, "platform_system")) {
const current_platform: []const u8 = switch (target.os) {
.macos => "Darwin",
.linux => "Linux",
.windows => "Windows",
.unknown => "",
};
// Check for platform_system == "X" or platform_system != "X"
if (strings.indexOf(self.markers, "platform_system")) |idx| {
var marker_remaining = self.markers[idx + "platform_system".len ..];
marker_remaining = strings.trim(marker_remaining, &strings.whitespace_chars);
var is_negated = false;
if (strings.hasPrefixComptime(marker_remaining, "!=")) {
is_negated = true;
marker_remaining = marker_remaining[2..];
} else if (strings.hasPrefixComptime(marker_remaining, "==")) {
marker_remaining = marker_remaining[2..];
} else {
// Unknown operator, be permissive
return true;
}
marker_remaining = strings.trim(marker_remaining, &strings.whitespace_chars);
// Parse quoted platform string
if (marker_remaining.len > 0 and (marker_remaining[0] == '"' or marker_remaining[0] == '\'')) {
const quote = marker_remaining[0];
marker_remaining = marker_remaining[1..];
if (strings.indexOfChar(marker_remaining, quote)) |end| {
const platform_str = marker_remaining[0..end];
const matches = strings.eql(platform_str, current_platform);
const platform_matches = if (is_negated) !matches else matches;
if (!platform_matches) return false;
}
}
}
}
// Handle sys_platform markers (e.g., sys_platform == "linux")
if (strings.containsComptime(self.markers, "sys_platform")) {
const current_sys_platform: []const u8 = switch (target.os) {
.macos => "darwin",
.linux => "linux",
.windows => "win32",
.unknown => "",
};
if (strings.indexOf(self.markers, "sys_platform")) |idx| {
var marker_remaining = self.markers[idx + "sys_platform".len ..];
marker_remaining = strings.trim(marker_remaining, &strings.whitespace_chars);
var is_negated = false;
if (strings.hasPrefixComptime(marker_remaining, "!=")) {
is_negated = true;
marker_remaining = marker_remaining[2..];
} else if (strings.hasPrefixComptime(marker_remaining, "==")) {
marker_remaining = marker_remaining[2..];
} else {
return true;
}
marker_remaining = strings.trim(marker_remaining, &strings.whitespace_chars);
if (marker_remaining.len > 0 and (marker_remaining[0] == '"' or marker_remaining[0] == '\'')) {
const quote = marker_remaining[0];
marker_remaining = marker_remaining[1..];
if (strings.indexOfChar(marker_remaining, quote)) |end| {
const platform_str = marker_remaining[0..end];
const matches = strings.eql(platform_str, current_sys_platform);
const platform_matches = if (is_negated) !matches else matches;
if (!platform_matches) return false;
}
}
}
}
// For other markers (implementation, etc.), be permissive
return true;
}
/// Normalize a Python version (PEP 440) to a semver-compatible format.
/// Strips suffixes like .postN, .devN that semver doesn't understand.
/// Returns the normalized version string length.
pub fn normalizeVersion(version: []const u8, buf: []u8) []const u8 {
// Find and strip Python-specific suffixes:
// - .postN (post-releases)
// - .devN (development releases)
// - +local (local version identifier)
var end = version.len;
// Strip local version identifier (+...)
if (strings.indexOfChar(version, '+')) |plus_idx| {
end = plus_idx;
}
// Strip .post, .dev suffixes
const suffixes = [_][]const u8{ ".post", ".dev" };
for (suffixes) |suffix| {
if (strings.indexOf(version[0..end], suffix)) |suffix_idx| {
end = suffix_idx;
break;
}
}
const copy_len = @min(end, buf.len);
@memcpy(buf[0..copy_len], version[0..copy_len]);
return buf[0..copy_len];
}
/// Normalize a PyPI package name according to PEP 503
/// - Lowercase
/// - Replace runs of [-_.] with single -
pub fn normalizeName(name: []const u8, buf: []u8) []const u8 {
var write_idx: usize = 0;
var prev_was_separator = false;
for (name) |c| {
if (write_idx >= buf.len) break;
const is_separator = (c == '-' or c == '_' or c == '.');
if (is_separator) {
if (!prev_was_separator) {
buf[write_idx] = '-';
write_idx += 1;
}
prev_was_separator = true;
} else {
buf[write_idx] = std.ascii.toLower(c);
write_idx += 1;
prev_was_separator = false;
}
}
return buf[0..write_idx];
}
};
test "WheelTag.parse" {
const tag1 = WheelTag.parse("numpy-2.0.0-cp312-cp312-macosx_14_0_arm64.whl");
try std.testing.expect(tag1 != null);
try std.testing.expectEqualStrings("cp312", tag1.?.python);
try std.testing.expectEqualStrings("cp312", tag1.?.abi);
try std.testing.expectEqualStrings("macosx_14_0_arm64", tag1.?.platform);
const tag2 = WheelTag.parse("requests-2.32.0-py3-none-any.whl");
try std.testing.expect(tag2 != null);
try std.testing.expectEqualStrings("py3", tag2.?.python);
try std.testing.expectEqualStrings("none", tag2.?.abi);
try std.testing.expectEqualStrings("any", tag2.?.platform);
// Not a wheel
try std.testing.expect(WheelTag.parse("requests-2.32.0.tar.gz") == null);
}
test "PlatformTarget.isPlatformCompatible" {
const mac_arm = PlatformTarget{
.os = .macos,
.arch = .aarch64,
.python_version = .{ .major = 3, .minor = 12 },
};
try std.testing.expect(mac_arm.isPlatformCompatible("any"));
try std.testing.expect(mac_arm.isPlatformCompatible("macosx_14_0_arm64"));
try std.testing.expect(mac_arm.isPlatformCompatible("macosx_11_0_universal2"));
try std.testing.expect(!mac_arm.isPlatformCompatible("macosx_14_0_x86_64"));
try std.testing.expect(!mac_arm.isPlatformCompatible("linux_x86_64"));
}
test "DependencySpecifier.parse" {
const spec1 = DependencySpecifier.parse("requests>=2.0,<3.0");
try std.testing.expect(spec1 != null);
try std.testing.expectEqualStrings("requests", spec1.?.name);
try std.testing.expectEqualStrings(">=2.0,<3.0", spec1.?.version_spec);
const spec2 = DependencySpecifier.parse("urllib3 (>=1.21.1,<3)");
try std.testing.expect(spec2 != null);
try std.testing.expectEqualStrings("urllib3", spec2.?.name);
try std.testing.expectEqualStrings(">=1.21.1,<3", spec2.?.version_spec);
const spec3 = DependencySpecifier.parse("PySocks!=1.5.7,>=1.5.6 ; extra == 'socks'");
try std.testing.expect(spec3 != null);
try std.testing.expectEqualStrings("PySocks", spec3.?.name);
try std.testing.expectEqualStrings("!=1.5.7,>=1.5.6", spec3.?.version_spec);
try std.testing.expectEqualStrings("extra == 'socks'", spec3.?.markers);
}

View File

@@ -97,6 +97,9 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
// should not happen
.dist_tag => error.UnexpectedResolution,
.uninitialized => error.UnexpectedResolution,
// TODO: handle PyPI resolutions
.pypi => error.UnexpectedResolution,
};
}
@@ -173,6 +176,7 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
.catalog => error.InvalidPnpmLockfile,
.dist_tag => error.InvalidPnpmLockfile,
.uninitialized => error.InvalidPnpmLockfile,
.pypi => error.InvalidPnpmLockfile,
};
}
@@ -211,6 +215,7 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
.single_file_module => builder.count(this.value.single_file_module.slice(buf)),
.git => this.value.git.count(buf, Builder, builder),
.github => this.value.github.count(buf, Builder, builder),
.pypi => this.value.pypi.count(buf, Builder, builder),
else => {},
}
}
@@ -244,6 +249,9 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
.github => Value.init(.{
.github = this.value.github.clone(buf, Builder, builder),
}),
.pypi => Value.init(.{
.pypi = this.value.pypi.clone(buf, Builder, builder),
}),
.root => Value.init(.{ .root = {} }),
.uninitialized => Value.init(.{ .uninitialized = {} }),
else => {
@@ -264,6 +272,7 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
.single_file_module => .init(.{ .single_file_module = this.value.single_file_module }),
.git => .init(.{ .git = this.value.git }),
.github => .init(.{ .github = this.value.github }),
.pypi => .init(.{ .pypi = this.value.pypi }),
.root => .init(.{ .root = {} }),
.uninitialized => .init(.{ .uninitialized = {} }),
else => {
@@ -370,6 +379,7 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
lhs_string_buf,
rhs_string_buf,
),
.pypi => lhs.value.pypi.eql(rhs.value.pypi),
else => unreachable,
};
}
@@ -419,6 +429,7 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
.path_sep = formatter.path_sep,
})}),
.single_file_module => try writer.print("module:{s}", .{value.single_file_module.slice(buf)}),
.pypi => try value.pypi.version.fmt(buf).format(writer),
else => {},
}
}
@@ -442,6 +453,7 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
.workspace => try writer.print("workspace:{s}", .{formatter.resolution.value.workspace.slice(formatter.buf)}),
.symlink => try writer.print("link:{s}", .{formatter.resolution.value.symlink.slice(formatter.buf)}),
.single_file_module => try writer.print("module:{s}", .{formatter.resolution.value.single_file_module.slice(formatter.buf)}),
.pypi => try formatter.resolution.value.pypi.version.fmt(formatter.buf).format(writer),
else => try writer.writeAll("{}"),
}
try writer.writeAll(" }");
@@ -471,6 +483,9 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
/// URL to a tarball.
remote_tarball: String,
/// PyPI package with version and wheel URL
pypi: VersionedURLType(SemverIntType),
single_file_module: String,
pub var zero: Value = @bitCast(std.mem.zeroes([@sizeOf(Value)]u8));
@@ -505,6 +520,9 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
remote_tarball = 80,
/// PyPI package (wheel URL)
pypi = 88,
// This is a placeholder for now.
// But the intent is to eventually support URL imports at the package manager level.
//
@@ -531,7 +549,7 @@ pub fn ResolutionType(comptime SemverIntType: type) type {
}
pub fn canEnqueueInstallTask(this: Tag) bool {
return this == .npm or this == .local_tarball or this == .remote_tarball or this == .git or this == .github;
return this == .npm or this == .local_tarball or this == .remote_tarball or this == .git or this == .github or this == .pypi;
}
};
};

View File

@@ -4650,6 +4650,7 @@ fn NewPrinter(
.sqlite, .sqlite_embedded => p.printWhitespacer(ws(" with { type: \"sqlite\" }")),
.html => p.printWhitespacer(ws(" with { type: \"html\" }")),
.md => p.printWhitespacer(ws(" with { type: \"md\" }")),
.py => p.printWhitespacer(ws(" with { type: \"py\" }")),
};
p.printSemicolonAfterStatement();
@@ -4678,6 +4679,7 @@ fn NewPrinter(
.html => analyze_transpiled_module.ModuleInfo.FetchParameters.hostDefined(bun.handleOom(mi.str("html"))),
.json5 => analyze_transpiled_module.ModuleInfo.FetchParameters.hostDefined(bun.handleOom(mi.str("json5"))),
.md => analyze_transpiled_module.ModuleInfo.FetchParameters.hostDefined(bun.handleOom(mi.str("md"))),
.py => analyze_transpiled_module.ModuleInfo.FetchParameters.hostDefined(bun.handleOom(mi.str("py"))),
} else .none) else .none;
bun.handleOom(mi.requestModule(irp_id, fetch_parameters));

View File

@@ -32,33 +32,38 @@ pub const BufferReadStream = struct {
_ = this.archive.readFree();
}
pub const Format = enum {
tar_gzip,
zip,
};
pub fn openRead(this: *BufferReadStream) Archive.Result {
// lib.archive_read_set_open_callback(this.archive, this.);
// _ = lib.archive_read_set_read_callback(this.archive, archive_read_callback);
// _ = lib.archive_read_set_seek_callback(this.archive, archive_seek_callback);
// _ = lib.archive_read_set_skip_callback(this.archive, archive_skip_callback);
// _ = lib.archive_read_set_close_callback(this.archive, archive_close_callback);
// // lib.archive_read_set_switch_callback(this.archive, this.archive_s);
// _ = lib.archive_read_set_callback_data(this.archive, this);
return this.openReadWithFormat(.tar_gzip);
}
_ = this.archive.readSupportFormatTar();
_ = this.archive.readSupportFormatGnutar();
_ = this.archive.readSupportFilterGzip();
pub fn openReadWithFormat(this: *BufferReadStream, format: Format) Archive.Result {
switch (format) {
.tar_gzip => {
_ = this.archive.readSupportFormatTar();
_ = this.archive.readSupportFormatGnutar();
_ = this.archive.readSupportFilterGzip();
// Ignore zeroed blocks in the archive, which occurs when multiple tar archives
// have been concatenated together.
// Without this option, only the contents of
// the first concatenated archive would be read.
_ = this.archive.readSetOptions("read_concatenated_archives");
// _ = lib.archive_read_support_filter_none(this.archive);
// Ignore zeroed blocks in the archive, which occurs when multiple tar archives
// have been concatenated together.
// Without this option, only the contents of
// the first concatenated archive would be read.
_ = this.archive.readSetOptions("read_concatenated_archives");
},
.zip => {
_ = this.archive.readSupportFormatZip();
_ = this.archive.readSupportFilterNone();
},
}
const rc = this.archive.readOpenMemory(this.buf);
this.reading = @intFromEnum(rc) > -1;
// _ = lib.archive_read_support_compression_all(this.archive);
return rc;
}
@@ -330,6 +335,7 @@ pub const Archiver = struct {
close_handles: bool = true,
log: bool = false,
npm: bool = false,
format: BufferReadStream.Format = .tar_gzip,
};
pub fn extractToDir(
@@ -345,7 +351,7 @@ pub const Archiver = struct {
var stream: BufferReadStream = undefined;
stream.init(file_buffer);
defer stream.deinit();
_ = stream.openRead();
_ = stream.openReadWithFormat(options.format);
const archive = stream.archive;
var count: u32 = 0;
const dir_fd = dir.fd;

View File

@@ -636,6 +636,7 @@ pub const Loader = enum(u8) {
yaml = 18,
json5 = 19,
md = 20,
py = 21,
pub const Optional = enum(u8) {
none = 254,
@@ -736,7 +737,7 @@ pub const Loader = enum(u8) {
pub fn canBeRunByBun(this: Loader) bool {
return switch (this) {
.jsx, .js, .ts, .tsx, .wasm, .bunsh => true,
.jsx, .js, .ts, .tsx, .wasm, .bunsh, .py => true,
else => false,
};
}
@@ -812,6 +813,7 @@ pub const Loader = enum(u8) {
.{ "html", .html },
.{ "md", .md },
.{ "markdown", .md },
.{ "py", .py },
});
pub const api_names = bun.ComptimeStringMap(api.Loader, .{
@@ -841,6 +843,7 @@ pub const Loader = enum(u8) {
.{ "html", .html },
.{ "md", .md },
.{ "markdown", .md },
.{ "py", .py },
});
pub fn fromString(slice_: string) ?Loader {
@@ -880,6 +883,7 @@ pub const Loader = enum(u8) {
.text => .text,
.sqlite_embedded, .sqlite => .sqlite,
.md => .md,
.py => .py,
};
}
@@ -907,6 +911,7 @@ pub const Loader = enum(u8) {
.sqlite => .sqlite,
.sqlite_embedded => .sqlite_embedded,
.md => .md,
.py => .py,
_ => .file,
};
}
@@ -1126,6 +1131,7 @@ const default_loaders_posix = .{
.{ ".html", .html },
.{ ".jsonc", .jsonc },
.{ ".json5", .json5 },
.{ ".py", .py },
};
const default_loaders_win32 = default_loaders_posix ++ .{
.{ ".sh", .bunsh },

View File

@@ -691,6 +691,28 @@ pub const Transpiler = struct {
.dataurl, .base64 => {
Output.panic("TODO: dataurl, base64", .{}); // TODO
},
.py => {
const entry = transpiler.resolver.caches.fs.readFileWithAllocator(
transpiler.allocator,
transpiler.fs,
file_path.text,
resolve_result.dirname_fd,
false,
null,
) catch |err| {
transpiler.log.addErrorFmt(null, .Empty, transpiler.allocator, "{s} reading \"{s}\"", .{ @errorName(err), file_path.pretty }) catch {};
return null;
};
output_file.size = entry.contents.len;
output_file.value = .{
.buffer = .{
.allocator = transpiler.allocator,
.bytes = entry.contents,
},
};
},
.css => {
const alloc = transpiler.allocator;

File diff suppressed because it is too large Load Diff