Revert "Merge pull request #70 from Jarred-Sumner/lithdew/picohttp-mimalloc"

This reverts commit e8925e47b0, reversing
changes made to f82aa92c38.
This commit is contained in:
Jarred Sumner
2021-11-25 17:08:22 -08:00
parent e8925e47b0
commit d19a6199ea
2 changed files with 131 additions and 152 deletions

View File

@@ -1,3 +1,5 @@
SHELL := /bin/bash # Use bash syntax to be consistent
OS_NAME := $(shell uname -s | tr '[:upper:]' '[:lower:]')
ARCH_NAME_RAW := $(shell uname -m)
BUN_AUTO_UPDATER_REPO = Jarred-Sumner/bun-releases-for-updater
@@ -36,11 +38,11 @@ ZIG ?= $(shell which zig || echo -e "error: Missing zig. Please make sure zig is
# We must use the same compiler version for the JavaScriptCore bindings and JavaScriptCore
# If we don't do this, strange memory allocation failures occur.
# This is easier to happen than you'd expect.
CC = $(shell which clang-12 2>/dev/null || which clang)
CXX = $(shell which clang++-12 2>/dev/null || which clang++)
CC = $(shell which clang-12 || which clang)
CXX = $(shell which clang++-12 || which clang++)
# macOS sed is different
SED = $(shell which gsed 2>/dev/null || which sed)
SED = $(shell which gsed || which sed)
DEPS_DIR = $(shell pwd)/src/deps
CPUS ?= $(shell nproc)
@@ -56,7 +58,7 @@ LIBCRYPTO_STATIC_LIB ?= $(LIBCRYPTO_PREFIX_DIR)/lib/libcrypto.a
LIBCRYPTO_INCLUDE_DIR = $(LIBCRYPTO_PREFIX_DIR)/include
ifeq ($(OS_NAME),linux)
LIBCRYPTO_STATIC_LIB = $(DEPS_DIR)/libcrypto.a
LIBCRYPTO_STATIC_LIB =
LIBICONV_PATH = $(DEPS_DIR)/libiconv.a
endif
@@ -217,14 +219,7 @@ MAC_INCLUDE_DIRS := -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/JavaScriptCo
-Isrc/javascript/jsc/WebKit/WebKitBuild/Release/ICU/Headers \
-Isrc/javascript/jsc/WebKit/WebKitBuild/Release/ \
-Isrc/javascript/jsc/bindings/ \
-Isrc/javascript/jsc/WebKit/Source/bmalloc
# LINUX_INCLUDE_DIRS := -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/JavaScriptCore/PrivateHeaders \
# -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/WTF/Headers \
# -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/ICU/Headers \
# -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/ \
# -Isrc/javascript/jsc/bindings/ \
# -Isrc/javascript/jsc/WebKit/Source/bmalloc
-Isrc/javascript/jsc/WebKit/Source/bmalloc
LINUX_INCLUDE_DIRS := -I$(JSC_INCLUDE_DIR) \
-Isrc/javascript/jsc/bindings/
@@ -289,9 +284,11 @@ endif
ARCHIVE_FILES_WITHOUT_LIBCRYPTO = src/deps/zlib/libz.a \
ARCHIVE_FILES_WITHOUT_LIBCRYPTO = src/deps/mimalloc/libmimalloc.a \
src/deps/zlib/libz.a \
src/deps/libarchive.a \
src/deps/libs2n.a \
src/deps/picohttpparser.o \
ARCHIVE_FILES = $(ARCHIVE_FILES_WITHOUT_LIBCRYPTO) src/deps/libcrypto.a
@@ -322,7 +319,9 @@ BUN_LLD_FLAGS = $(OBJ_FILES) \
bun: vendor identifier-cache build-obj bun-link-lld-release bun-codesign-release-local
vendor-without-check: api analytics node-fallbacks runtime_js fallback_decoder bun_error zlib s2n libarchive
vendor-without-check: api analytics node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp zlib s2n libarchive
libarchive:
cd src/deps/libarchive; \
@@ -407,19 +406,23 @@ generate-install-script:
@esbuild --log-level=error --define:BUN_VERSION="\"$(PACKAGE_JSON_VERSION)\"" --define:process.env.NODE_ENV="\"production\"" --platform=node --format=cjs $(PACKAGES_REALPATH)/bun/install.ts > $(PACKAGES_REALPATH)/bun/install.js
fetch:
$(ZIG) build-obj -Drelease-fast misctools/fetch.zig -fcompiler-rt -lc --main-pkg-path . $(PICOHTTP_BUILD_FLAGS) $(MIMALLOC_BUILD_FLAGS) -femit-bin=misctools/fetch.o
cd misctools; $(ZIG) build-obj -Drelease-fast ./fetch.zig -fcompiler-rt -lc --main-pkg-path ../
$(CXX) ./misctools/fetch.o -g -O3 -o ./misctools/fetch $(DEFAULT_LINKER_FLAGS) -lc \
src/deps/mimalloc/libmimalloc.a \
src/deps/zlib/libz.a \
src/deps/libarchive.a \
src/deps/libs2n.a \
src/deps/picohttpparser.o \
$(LIBCRYPTO_STATIC_LIB)
fetch-debug:
$(ZIG) build-obj misctools/fetch.zig -fcompiler-rt -lc --main-pkg-path . $(PICOHTTP_BUILD_FLAGS) $(MIMALLOC_BUILD_FLAGS) -femit-bin=misctools/fetch.o
cd misctools; $(ZIG) build-obj ./fetch.zig -fcompiler-rt -lc --main-pkg-path ../
$(CXX) ./misctools/fetch.o -g -o ./misctools/fetch $(DEFAULT_LINKER_FLAGS) -lc \
src/deps/mimalloc/libmimalloc.a \
src/deps/zlib/libz.a \
src/deps/libarchive.a \
src/deps/libs2n.a \
src/deps/picohttpparser.o \
$(LIBCRYPTO_STATIC_LIB)
s2n-mac:
@@ -675,12 +678,18 @@ clean-bindings:
clean: clean-bindings
rm src/deps/*.a src/deps/*.o
(cd src/deps/mimalloc && make clean) || echo "";
(cd src/deps/libarchive && make clean) || echo "";
(cd src/deps/s2n-tls && make clean) || echo "";
(cd src/deps/picohttp && make clean) || echo "";
(cd src/deps/zlib && make clean) || echo "";
jsc-bindings-mac: $(OBJ_FILES)
mimalloc:
cd src/deps/mimalloc; cmake .; make;
bun-link-lld-debug:
$(CXX) $(BUN_LLD_FLAGS) \
-g \
@@ -720,6 +729,9 @@ sizegen:
$(CXX) src/javascript/jsc/headergen/sizegen.cpp -o $(BUN_TMP_DIR)/sizegen $(CLANG_FLAGS) -O1
$(BUN_TMP_DIR)/sizegen > src/javascript/jsc/bindings/sizes.zig
picohttp:
$(CC) $(MARCH_NATIVE) -O3 -g -fPIE -c src/deps/picohttpparser/picohttpparser.c -Isrc/deps -o src/deps/picohttpparser.o; cd ../../
analytics:
./node_modules/.bin/peechy --schema src/analytics/schema.peechy --zig src/analytics/analytics_schema.zig
$(ZIG) fmt src/analytics/analytics_schema.zig
@@ -809,41 +821,6 @@ endif
endif
PICOHTTP_BUILD_FLAGS = \
--pkg-begin picohttp $(DEPS_DIR)/picohttp.zig --pkg-end \
-I src/deps \
src/deps/picohttpparser.c
CLAP_BUILD_FLAGS = --pkg-begin clap $(DEPS_DIR)/zig-clap/clap.zig --pkg-end
MIMALLOC_BUILD_FLAGS = -cflags -DMI_ALLOC_OVERRIDE
ifeq ($(OS_NAME), darwin)
MIMALLOC_BUILD_FLAGS += -DMI_OSX_ZONE=1
endif
MIMALLOC_BUILD_FLAGS += -- \
-I src/deps/mimalloc/include \
src/deps/mimalloc/src/stats.c \
src/deps/mimalloc/src/random.c \
src/deps/mimalloc/src/os.c \
src/deps/mimalloc/src/bitmap.c \
src/deps/mimalloc/src/arena.c \
src/deps/mimalloc/src/region.c \
src/deps/mimalloc/src/segment.c \
src/deps/mimalloc/src/page.c \
src/deps/mimalloc/src/alloc.c \
src/deps/mimalloc/src/alloc-aligned.c \
src/deps/mimalloc/src/alloc-posix.c \
src/deps/mimalloc/src/heap.c \
src/deps/mimalloc/src/options.c \
src/deps/mimalloc/src/init.c
ifeq ($(OS_NAME), darwin)
MIMALLOC_BUILD_FLAGS += \
src/deps/mimalloc/src/alloc-override-osx.c
endif
build-unit:
@@ -851,9 +828,8 @@ build-unit:
@mkdir -p zig-out/bin
zig test $(realpath $(testpath)) \
$(testfilterflag) \
$(PICOHTTP_BUILD_FLAGS) \
$(CLAP_BUILD_FLAGS) \
$(MIMALLOC_BUILD_FLAGS) \
--pkg-begin picohttp $(DEPS_DIR)/picohttp.zig --pkg-end \
--pkg-begin clap $(DEPS_DIR)/zig-clap/clap.zig --pkg-end \
--main-pkg-path $(shell pwd) \
--test-no-exec \
-fPIC \

199
build.zig
View File

@@ -1,47 +1,22 @@
const std = @import("std");
const resolve_path = @import("./src/resolver/resolve_path.zig");
pub fn addPicoHTTP(step: *std.build.LibExeObjStep, comptime with_obj: bool) void {
const picohttp = step.addPackage(.{
.name = "picohttp",
.path = .{ .path = "src/deps/picohttp.zig" },
});
pub fn addPicoHTTP(step: *std.build.LibExeObjStep) void {
step.addPackagePath("picohttp", "src/deps/picohttp.zig");
step.addIncludeDir("src/deps");
step.addCSourceFile("src/deps/picohttpparser.c", &.{});
}
pub fn addMimalloc(step: *std.build.LibExeObjStep) void {
var source_files = std.ArrayList([]const u8).init(step.builder.allocator);
defer source_files.deinit();
inline for (.{
"src/deps/mimalloc/src/stats.c",
"src/deps/mimalloc/src/random.c",
"src/deps/mimalloc/src/os.c",
"src/deps/mimalloc/src/bitmap.c",
"src/deps/mimalloc/src/arena.c",
"src/deps/mimalloc/src/region.c",
"src/deps/mimalloc/src/segment.c",
"src/deps/mimalloc/src/page.c",
"src/deps/mimalloc/src/alloc.c",
"src/deps/mimalloc/src/alloc-aligned.c",
"src/deps/mimalloc/src/alloc-posix.c",
"src/deps/mimalloc/src/heap.c",
"src/deps/mimalloc/src/options.c",
"src/deps/mimalloc/src/init.c",
}) |source_file| {
source_files.append(source_file) catch unreachable;
if (with_obj) {
step.addObjectFile("src/deps/picohttpparser.o");
}
var source_flags = std.ArrayList([]const u8).init(step.builder.allocator);
defer source_flags.deinit();
// step.add("/Users/jarred/Code/WebKit/WebKitBuild/Release/lib/libWTF.a");
source_flags.append("-DMI_ALLOC_OVERRIDE") catch unreachable;
if (step.target.getOsTag().isDarwin()) {
source_files.append("src/deps/mimalloc/src/alloc-override-osx.c") catch unreachable;
source_flags.append("-DMI_OSX_ZONE=1") catch unreachable;
}
step.addIncludeDir("src/deps/mimalloc/include");
step.addCSourceFiles(source_files.items, source_flags.items);
// ./Tools/Scripts/build-jsc --jsc-only --cmakeargs="-DENABLE_STATIC_JSC=ON"
// set -gx ICU_INCLUDE_DIRS "/usr/local/opt/icu4c/include"
// homebrew-provided icu4c
}
fn panicIfNotFound(comptime filepath: []const u8) []const u8 {
@@ -55,43 +30,54 @@ fn panicIfNotFound(comptime filepath: []const u8) []const u8 {
return filepath;
}
const x64 = "x64";
var x64 = "x64";
pub fn build(b: *std.build.Builder) !void {
// Standard target options allows the person running `zig build` to choose
// what target to build for. Here we do not override the defaults, which
// means any target is allowed, and the default is native. Other options
// for restricting supported target set are available.
var target = b.standardTargetOptions(.{});
// Standard release options allow the person running `zig build` to select
// between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall.
const mode = b.standardReleaseOptions();
var cwd_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
const cwd: []const u8 = b.pathFromRoot(".");
var exe: *std.build.LibExeObjStep = undefined;
var output_dir_buf = std.mem.zeroes([4096]u8);
const bin_label = if (mode == std.builtin.Mode.Debug) "packages/debug-bun-" else "packages/bun-";
var bin_label = if (mode == std.builtin.Mode.Debug) "packages/debug-bun-" else "packages/bun-";
const cpu_arch: std.Target.Cpu.Arch = target.getCpuArch();
var triplet_buf: [64]u8 = undefined;
var os_tagname = @tagName(target.getOs().tag);
var os_tag_name = @tagName(target.getOs().tag);
if (std.mem.eql(u8, os_tag_name, "macos")) {
os_tag_name = "darwin";
if (cpu_arch.isAARCH64()) {
const arch: std.Target.Cpu.Arch = target.getCpuArch();
if (std.mem.eql(u8, os_tagname, "macos")) {
os_tagname = "darwin";
if (arch.isAARCH64()) {
target.os_version_min = std.build.Target.OsVersion{ .semver = .{ .major = 11, .minor = 0, .patch = 0 } };
} else if (cpu_arch.isX86()) {
} else if (arch.isX86()) {
target.os_version_min = std.build.Target.OsVersion{ .semver = .{ .major = 10, .minor = 14, .patch = 0 } };
}
}
var triplet_buf: [64]u8 = undefined;
std.mem.copy(u8, &triplet_buf, os_tag_name);
const os_name = triplet_buf[0..os_tag_name.len];
triplet_buf[os_name.len] = '-';
std.mem.copy(
u8,
&triplet_buf,
os_tagname,
);
var osname = triplet_buf[0..os_tagname.len];
triplet_buf[osname.len] = '-';
std.mem.copy(u8, triplet_buf[os_name.len + 1 ..], @tagName(target.getCpuArch()));
var cpu_arch_name = triplet_buf[os_name.len + 1 ..][0..@tagName(target.getCpuArch()).len];
std.mem.replaceScalar(u8, cpu_arch_name, '_', '-');
if (std.mem.eql(u8, cpu_arch_name, "x86-64")) {
std.mem.copy(u8, cpu_arch_name, "x64");
cpu_arch_name = cpu_arch_name[0..3];
std.mem.copy(u8, triplet_buf[osname.len + 1 ..], @tagName(target.getCpuArch()));
var cpuArchName = triplet_buf[osname.len + 1 ..][0..@tagName(target.getCpuArch()).len];
std.mem.replaceScalar(u8, cpuArchName, '_', '-');
if (std.mem.eql(u8, cpuArchName, "x86-64")) {
std.mem.copy(u8, cpuArchName, "x64");
cpuArchName = cpuArchName[0..3];
}
const triplet = triplet_buf[0 .. os_name.len + cpu_arch_name.len + 1];
var triplet = triplet_buf[0 .. osname.len + cpuArchName.len + 1];
const output_dir_base = try std.fmt.bufPrint(&output_dir_buf, "{s}{s}", .{ bin_label, triplet });
const output_dir = b.pathFromRoot(output_dir_base);
@@ -103,14 +89,22 @@ pub fn build(b: *std.build.Builder) !void {
exe.linkage = .dynamic;
exe.setOutputDir(output_dir);
} else if (target.getCpuArch().isWasm()) {
const lib = b.addExecutable(bun_executable_name, "src/main_wasm.zig");
// exe = b.addExecutable(
// "bun",
// "src/main_wasm.zig",
// );
// exe.is_linking_libc = false;
// exe.is_dynamic = true;
var lib = b.addExecutable(bun_executable_name, "src/main_wasm.zig");
lib.single_threaded = true;
// exe.want_lto = true;
// exe.linkLibrary(lib);
if (mode == std.builtin.Mode.Debug) {
// exception_handling
target.cpu_features_add.addFeature(2);
var features = target.getCpuFeatures();
features.addFeature(2);
target.updateCpuFeatures(&features);
} else {
// lib.strip = true;
}
@@ -126,7 +120,7 @@ pub fn build(b: *std.build.Builder) !void {
lib.setBuildMode(mode);
std.fs.deleteTreeAbsolute(std.fs.path.join(b.allocator, &.{ cwd, lib.getOutputSource().getPath(b) }) catch unreachable) catch {};
var install = b.getInstallStep();
lib.strip = false;
lib.install();
@@ -143,45 +137,45 @@ pub fn build(b: *std.build.Builder) !void {
} else {
exe = b.addExecutable(bun_executable_name, "src/main.zig");
}
// exe.setLibCFile("libc.txt");
exe.linkLibC();
exe.addPackagePath("clap", "src/deps/zig-clap/clap.zig");
// exe.linkLibCpp();
exe.addPackage(.{
.name = "clap",
.path = .{ .path = "src/deps/zig-clap/clap.zig" },
});
exe.setOutputDir(output_dir);
const cwd_dir = std.fs.cwd();
const runtime_hash = read: {
const runtime_out_file = try cwd_dir.openFile("src/runtime.out.js", .{ .read = true });
defer runtime_out_file.close();
break :read std.hash.Wyhash.hash(0, try runtime_out_file.readToEndAlloc(b.allocator, try runtime_out_file.getEndPos()));
};
const runtime_version_file = cwd_dir.createFile("src/runtime.version", .{ .truncate = true }) catch std.debug.panic("Failed to create src/runtime.version", .{});
var cwd_dir = std.fs.cwd();
var runtime_out_file = try std.fs.cwd().openFile("src/runtime.out.js", .{ .read = true });
const runtime_hash = std.hash.Wyhash.hash(
0,
try runtime_out_file.readToEndAlloc(b.allocator, try runtime_out_file.getEndPos()),
);
const runtime_version_file = std.fs.cwd().createFile("src/runtime.version", .{ .truncate = true }) catch std.debug.panic("Failed to create src/runtime.version", .{});
defer runtime_version_file.close();
runtime_version_file.writer().print("{x}", .{runtime_hash}) catch unreachable;
var fallback_out_file = try std.fs.cwd().openFile("src/fallback.out.js", .{ .read = true });
const fallback_hash = std.hash.Wyhash.hash(
0,
try fallback_out_file.readToEndAlloc(b.allocator, try fallback_out_file.getEndPos()),
);
const fallback_hash = read: {
const fallback_out_file = try cwd_dir.openFile("src/fallback.out.js", .{ .read = true });
defer fallback_out_file.close();
break :read std.hash.Wyhash.hash(0, try fallback_out_file.readToEndAlloc(b.allocator, try fallback_out_file.getEndPos()));
};
const fallback_version_file = cwd_dir.createFile("src/fallback.version", .{ .truncate = true }) catch std.debug.panic("Failed to create src/fallback.version", .{});
defer fallback_version_file.close();
const fallback_version_file = std.fs.cwd().createFile("src/fallback.version", .{ .truncate = true }) catch std.debug.panic("Failed to create src/fallback.version", .{});
fallback_version_file.writer().print("{x}", .{fallback_hash}) catch unreachable;
defer fallback_version_file.close();
exe.setTarget(target);
exe.setBuildMode(mode);
b.install_path = output_dir;
const javascript = b.addExecutable("spjs", "src/main_javascript.zig");
const typings_exe = b.addExecutable("typescript-decls", "src/javascript/jsc/typescript.zig");
exe.setMainPkgPath(b.pathFromRoot("."));
var javascript = b.addExecutable("spjs", "src/main_javascript.zig");
var typings_exe = b.addExecutable("typescript-decls", "src/javascript/jsc/typescript.zig");
javascript.setMainPkgPath(b.pathFromRoot("."));
typings_exe.setMainPkgPath(b.pathFromRoot("."));
exe.setMainPkgPath(b.pathFromRoot("."));
// exe.want_lto = true;
if (!target.getCpuArch().isWasm()) {
@@ -231,7 +225,7 @@ pub fn build(b: *std.build.Builder) !void {
// if (target.getOsTag() == .macos) "-DUSE_CF_RETAIN_PTR=1" else "",
// };
const headers_step = b.step("headers-obj", "JSC headers Step #1");
const headers_obj: *std.build.LibExeObjStep = b.addObject("headers", "src/javascript/jsc/bindings/bindings-generator.zig");
var headers_obj: *std.build.LibExeObjStep = b.addObject("headers", "src/javascript/jsc/bindings/bindings-generator.zig");
headers_obj.setMainPkgPath(javascript.main_pkg_path.?);
headers_step.dependOn(&headers_obj.step);
@@ -248,15 +242,17 @@ pub fn build(b: *std.build.Builder) !void {
b.default_step.dependOn(&exe.step);
{
const steps = [_]*std.build.LibExeObjStep{ exe, javascript, typings_exe };
var steps = [_]*std.build.LibExeObjStep{ exe, javascript, typings_exe };
// const single_threaded = b.option(bool, "single-threaded", "Build single-threaded") orelse false;
for (steps) |step| {
for (steps) |step, i| {
step.linkLibC();
step.linkLibCpp();
addPicoHTTP(step);
addMimalloc(step);
addPicoHTTP(
step,
true,
);
step.addObjectFile(panicIfNotFound("src/deps/libJavaScriptCore.a"));
step.addObjectFile(panicIfNotFound("src/deps/libWTF.a"));
@@ -265,6 +261,9 @@ pub fn build(b: *std.build.Builder) !void {
step.addObjectFile(panicIfNotFound("src/deps/libarchive.a"));
step.addObjectFile(panicIfNotFound("src/deps/libs2n.a"));
step.addObjectFile(panicIfNotFound("src/deps/zlib/libz.a"));
step.addObjectFile(panicIfNotFound("src/deps/mimalloc/libmimalloc.a"));
step.addLibPath("src/deps/mimalloc");
step.addIncludeDir("src/deps/mimalloc");
// step.single_threaded = single_threaded;
@@ -302,13 +301,12 @@ pub fn build(b: *std.build.Builder) !void {
{
var obj_step = b.step("obj", "Build Bun as a .o file");
var obj = b.addObject(bun_executable_name, exe.root_src.?.path);
obj.setTarget(target);
obj.setBuildMode(mode);
addPicoHTTP(obj);
addMimalloc(obj);
obj.addPackagePath("clap", "src/deps/zig-clap/clap.zig");
addPicoHTTP(obj, false);
obj.addPackage(.{
.name = "clap",
.path = .{ .path = "src/deps/zig-clap/clap.zig" },
});
{
obj_step.dependOn(&b.addLog(
@@ -324,7 +322,7 @@ pub fn build(b: *std.build.Builder) !void {
obj_step.dependOn(&obj.step);
obj.setOutputDir(output_dir);
obj.setBuildMode(mode);
obj.linkLibC();
obj.linkLibCpp();
@@ -340,10 +338,13 @@ pub fn build(b: *std.build.Builder) !void {
{
headers_obj.setTarget(target);
headers_obj.setBuildMode(mode);
headers_obj.setOutputDir(output_dir);
headers_obj.addPackage(.{
.name = "clap",
.path = .{ .path = "src/deps/zig-clap/clap.zig" },
});
headers_obj.addPackagePath("clap", "src/deps/zig-clap/clap.zig");
headers_obj.setOutputDir(output_dir);
headers_obj.setBuildMode(mode);
headers_obj.linkLibC();
headers_obj.linkLibCpp();
headers_obj.bundle_compiler_rt = true;
@@ -392,3 +393,5 @@ pub fn build(b: *std.build.Builder) !void {
var javascript_cmd = b.step("spjs", "Build standalone JavaScript runtime. Must run \"make jsc\" first.");
javascript_cmd.dependOn(&javascript.step);
}
pub var original_make_fn: ?fn (step: *std.build.Step) anyerror!void = null;