From 3b25c8637bc6d8beb466ed542841d1571abab9a2 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Sat, 28 Oct 2023 04:46:56 -0700 Subject: [PATCH] dx: Fix setup script and contributing docs (#6752) * Updated Dependencies Script * demo * fix submodule hell!!! * lol * attmept 2 * install nasm in ci * setup sh 1 * yeah * better zlib building * codegen stuff * attempt 2 at bun codegen ci * o * deps improvements * generaet part of compile-cpp-only.ps1 * restore these * good enough for Unix * remove libuv submodule lol * pass over docs --- .github/workflows/bun-windows-x64.yml | 159 ++++++++++++++++++++++++++ .gitignore | 7 +- .gitmodules | 22 ++-- CMakeLists.txt | 147 +++++++++++++----------- Dockerfile | 2 + Makefile | 22 +++- docs/project/contributing.md | 87 +++++--------- package.json | 6 +- scripts/all-dependencies.ps1 | 83 ++++++++++++++ scripts/all-dependencies.sh | 1 - scripts/build-base64.ps1 | 19 +-- scripts/build-boringssl.ps1 | 20 ++-- scripts/build-cares.ps1 | 21 ++-- scripts/build-libarchive.ps1 | 18 ++- scripts/build-libuv.ps1 | 45 ++++++-- scripts/build-lolhtml.ps1 | 13 ++- scripts/build-mimalloc.ps1 | 17 +-- scripts/build-zlib.ps1 | 44 ++++--- scripts/build-zstd.ps1 | 15 ++- scripts/clean-dependencies.ps1 | 29 +++++ scripts/cross-compile-codegen.sh | 47 ++++++++ scripts/download-webkit.ps1 | 42 +++++++ scripts/download-webkit.sh | 57 +++++++++ scripts/env.ps1 | 60 ++++++++-- scripts/setup.sh | 94 ++++++++++++++- src/bun.js/WebKit | 2 +- src/codegen/download-webkit.ts | 56 --------- src/codegen/generate-classes.ts | 12 +- src/codegen/replacements.ts | 4 +- src/deps/mimalloc | 2 +- 30 files changed, 842 insertions(+), 311 deletions(-) create mode 100644 .github/workflows/bun-windows-x64.yml create mode 100644 scripts/all-dependencies.ps1 create mode 100644 scripts/clean-dependencies.ps1 create mode 100755 scripts/cross-compile-codegen.sh create mode 100644 scripts/download-webkit.ps1 create mode 100644 scripts/download-webkit.sh delete mode 100644 src/codegen/download-webkit.ts diff --git a/.github/workflows/bun-windows-x64.yml b/.github/workflows/bun-windows-x64.yml new file mode 100644 index 0000000000..c787f7d046 --- /dev/null +++ b/.github/workflows/bun-windows-x64.yml @@ -0,0 +1,159 @@ +# TODO(@paperdave) +# this workflow is a work in progress +name: bun-windows-x64 + +concurrency: + group: bun-windows-x64-${{ github.ref }} + cancel-in-progress: true + +on: + # push: + # branches: [main] + # paths: + # - "src/**/*" + # - "test/**/*" + # - "packages/bun-usockets/src/**/*" + # - "build.zig" + # - "Makefile" + # - "Dockerfile" + # pull_request: + # branches: [main] + # paths: + # - "src/**/*" + # - "test/**/*" + # - "packages/bun-usockets/src/**/*" + # - "build.zig" + # - "Makefile" + # - "Dockerfile" + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # windows-zig: + # name: Windows Zig Object + # runs-on: med-ubuntu + # if: github.repository_owner == 'oven-sh' + # strategy: + # matrix: + # include: + # - cpu: native + # arch: x86_64 + # tag: bun-windows-x64 + # steps: + # - uses: actions/checkout@v4 + # - name: Setup Docker Buildx + # uses: docker/setup-buildx-action@v2 + # id: buildx + # with: + # install: true + + # - name: Login to GitHub Container Registry + # uses: docker/login-action@v2 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + + # - name: Compile Zig Object + # uses: docker/build-push-action@v3 + # if: runner.arch == 'X64' + # with: + # context: . + # push: false + # # This doesnt seem to work + # # cache-from: type=s3,endpoint_url=${{ secrets.CACHE_S3_ENDPOINT }},blobs_prefix=docker_blobs/,manifests_prefix=docker_manifests/,access_key_id=${{ secrets.CACHE_S3_ACCESS_KEY_ID }},secret_access_key=${{ secrets.CACHE_S3_SECRET_ACCESS_KEY }},bucket=bun,region=auto + # # cache-to: type=s3,endpoint_url=${{ secrets.CACHE_S3_ENDPOINT }},blobs_prefix=docker_blobs/,manifests_prefix=docker_manifests/,access_key_id=${{ secrets.CACHE_S3_ACCESS_KEY_ID }},secret_access_key=${{ secrets.CACHE_S3_SECRET_ACCESS_KEY }},bucket=bun,region=auto + # build-args: | + # BUILDARCH=${{ runner.arch == 'X64' && 'amd64' || 'arm64' }} + # BUILD_MACHINE_ARCH=${{ runner.arch == 'X64' && 'x86_64' || 'aarch64' }} + # ARCH=${{ matrix.arch }} + # CPU_TARGET=${{ matrix.cpu }} + # TRIPLET=${{ matrix.arch }}-windows-msvc + # GIT_SHA=${{ github.sha }} + # platforms: linux/${{ runner.arch == 'X64' && 'amd64' || 'arm64' }} + # target: build_release_obj + # outputs: type=local,dest=${{runner.temp}}/release + + # - name: Upload Zig Object + # uses: actions/upload-artifact@v3 + # with: + # name: ${{ matrix.tag }}-zig + # path: ${{runner.temp}}/release/bun-zig.o + + # windows-dependencies: + # strategy: + # matrix: + # include: + # - runner: windows-latest + # arch: amd64 + # tag: bun-windows-x64 + # runs-on: windows-latest + # timeout-minutes: 90 + # steps: + # - uses: actions/checkout@v4 + # - uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d + # with: + # arch: ${{ matrix.arch }} + # - uses: KyleMayes/install-llvm-action@1a3da29f56261a1e1f937ec88f0856a9b8321d7e + # with: + # version: 16.0.6 + # - run: choco install -y ninja + # - name: Build Dependencies + # run: | + # git submodule update --init --recursive --progress --depth=1 --checkout + # Invoke-WebRequest -Uri "https://www.nasm.us/pub/nasm/releasebuilds/2.16.01/win64/nasm-2.16.01-win64.zip" -OutFile nasm.zip + # Expand-Archive nasm.zip (mkdir -Force "nasm") + # $Nasm = (Get-ChildItem "nasm") + # $env:Path += ";${Nasm}" + # $env:BUN_DEPS_OUT_DIR = (mkdir -Force "./bun-deps") + # ./scripts/all-dependencies.ps1 + # - uses: actions/upload-artifact@v3 + # with: + # name: bun-deps/ + # path: bun-windows-x64-deps + + windows-codegen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: | + curl -fsSL https://github.com/oven-sh/bun/releases/download/bun-v1.0.7/bun-linux-x64.zip > bun.zip + unzip bun.zip + export PATH="$PWD/bun-linux-x64:$PATH" + ./scripts/cross-compile-codegen.sh win32 x64 + - uses: actions/upload-artifact@v3 + with: + name: bun-windows-x64-codegen + path: build-codegen-win32-x64/ + + windows-dependencies: + strategy: + matrix: + include: + - runner: windows-latest + arch: amd64 + tag: bun-windows-x64 + runs-on: windows-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d + with: + arch: ${{ matrix.arch }} + - uses: KyleMayes/install-llvm-action@1a3da29f56261a1e1f937ec88f0856a9b8321d7e + with: + version: 16.0.6 + - run: choco install -y ninja + - uses: actions/download-artifact@v2 + with: + name: bun-windows-x64-codegen + path: build + - name: Build Dependencies + run: | + git submodule update --init --recursive --progress --depth=1 --checkout + cd build + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DNO_CODEGEN=1 + - uses: actions/upload-artifact@v3 + with: + name: bun-deps/ + path: bun-windows-x64-deps diff --git a/.gitignore b/.gitignore index ce9b7bc307..a2052aec81 100644 --- a/.gitignore +++ b/.gitignore @@ -110,7 +110,7 @@ misctools/machbench *.big .eslintcache -bun-webkit +/bun-webkit src/deps/c-ares/build src/bun.js/bindings-obj @@ -135,7 +135,6 @@ make-dev-stats.csv .uuid tsconfig.tsbuildinfo -build-release *.lib *.pdb @@ -154,4 +153,6 @@ x64 **/*.dir **/*.pdb -.webkit-cache \ No newline at end of file +/.webkit-cache +/src/deps/libuv +/build-*/ diff --git a/.gitmodules b/.gitmodules index fcb03d8bed..c6ec7654da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,10 +1,3 @@ -[submodule "src/deps/picohttpparser"] -path = src/deps/picohttpparser -url = https://github.com/h2o/picohttpparser.git -ignore = dirty -depth = 1 -shallow = true -fetchRecurseSubmodules = false [submodule "src/javascript/jsc/WebKit"] path = src/bun.js/WebKit url = https://github.com/oven-sh/WebKit.git @@ -13,6 +6,13 @@ depth = 1 update = none shallow = true fetchRecurseSubmodules = false +[submodule "src/deps/picohttpparser"] +path = src/deps/picohttpparser +url = https://github.com/h2o/picohttpparser.git +ignore = dirty +depth = 1 +shallow = true +fetchRecurseSubmodules = false [submodule "src/deps/mimalloc"] path = src/deps/mimalloc url = https://github.com/Jarred-Sumner/mimalloc.git @@ -58,13 +58,21 @@ fetchRecurseSubmodules = false [submodule "src/deps/c-ares"] path = src/deps/c-ares url = https://github.com/c-ares/c-ares.git +ignore = dirty +depth = 1 +shallow = true +fetchRecurseSubmodules = false [submodule "src/deps/zstd"] path = src/deps/zstd url = https://github.com/facebook/zstd.git ignore = dirty +depth = 1 +shallow = true +fetchRecurseSubmodules = false [submodule "src/deps/base64"] path = src/deps/base64 url = https://github.com/aklomp/base64.git ignore = dirty depth = 1 shallow = true +fetchRecurseSubmodules = false diff --git a/CMakeLists.txt b/CMakeLists.txt index a8d3aad56c..a288e367f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_policy(SET CMP0091 NEW) cmake_policy(SET CMP0067 NEW) set(Bun_VERSION "1.0.8") -set(WEBKIT_TAG 67c9ba93323b4a5a95bc4d1efe61166dc6e3e691) +set(WEBKIT_TAG 46689737b98c78e48d9eb6885f0b3e8365d79310) set(BUN_WORKDIR "${CMAKE_CURRENT_BINARY_DIR}") message(STATUS "Configuring Bun ${Bun_VERSION} in ${BUN_WORKDIR}") @@ -136,7 +136,21 @@ if(UNIX) BUN_FIND_LLVM() else() - # On windows it is expected to use MSVC, and until we get a better build configuration, it is a free-for-all + # Windows uses Clang-CL + # TODO: good configuration in this regard. -G Ninja will pick clang-cl if possible, which should be fine for most users. + + find_program( + STRIP + NAMES llvm-strip + PATHS ENV PATH ${PLATFORM_LLVM_SEARCH_PATHS} + DOC "Path to LLVM ${LLVM_VERSION}'s llvm-strip binary" + ) + find_program( + AR + NAMES llvm-ar + PATHS ENV PATH ${PLATFORM_LLVM_SEARCH_PATHS} + DOC "Path to LLVM ${LLVM_VERSION}'s llvm-ar binary" + ) endif() set(CMAKE_COLOR_DIAGNOSTICS ON) @@ -164,10 +178,10 @@ endif() message(STATUS "C++ Compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} at ${CMAKE_CXX_COMPILER}") # --- End LLVM --- -set(DEFAULT_ON_UNLESS_WINDOWS ON) - +set(SHELL "bash") +set(SCRIPT_EXTENSION "sh") if(WIN32) - set(DEFAULT_ON_UNLESS_WINDOWS OFF) + set(SCRIPT_EXTENSION "ps1") endif() set(DEFAULT_ON_UNLESS_APPLE ON) @@ -193,8 +207,9 @@ option(USE_CUSTOM_CARES "Use Bun's recommended version of c-ares" ON) option(USE_CUSTOM_BASE64 "Use Bun's recommended version of libbase64" ON) option(USE_CUSTOM_LOLHTML "Use Bun's recommended version of lolhtml" ON) option(USE_CUSTOM_TINYCC "Use Bun's recommended version of tinycc" ON) -option(USE_CUSTOM_LIBUV "Use Bun's recommended version of libuv (Windows only)" OFF) +option(USE_CUSTOM_LIBUV "Use Bun's recommended version of libuv (Windows only)" ON) option(USE_BASELINE_BUILD "Build Bun for baseline (older) CPUs" OFF) + option(USE_DEBUG_JSC "Enable assertions and use a debug build of JavaScriptCore" OFF) option(USE_UNIFIED_SOURCES "Use unified sources to speed up the build" OFF) @@ -312,23 +327,19 @@ else() endif() if(NOT WEBKIT_DIR) - if(WIN32) - message(FATAL_ERROR "Windows does not have prebuilt webkit yet. Please run release-windows.ps1 and pass the path to the built webkit with -DWEBKIT_DIR") - endif() - set(BUN_WEBKIT_PACKAGE_NAME_SUFFIX "") set(ASSERT_ENABLED "0") if(USE_DEBUG_JSC) set(BUN_WEBKIT_PACKAGE_NAME_SUFFIX "-debug") set(ASSERT_ENABLED "1") - elseif(NOT DEBUG) + elseif(NOT DEBUG AND NOT WIN32) set(BUN_WEBKIT_PACKAGE_NAME_SUFFIX "-lto") set(ASSERT_ENABLED "0") endif() if(WIN32) - set(BUN_WEBKIT_PACKAGE_PLATFORM "win32") + set(BUN_WEBKIT_PACKAGE_PLATFORM "windows") elseif(APPLE) set(BUN_WEBKIT_PACKAGE_PLATFORM "macos") else() @@ -345,11 +356,11 @@ if(NOT WEBKIT_DIR) message(STATUS "Using Pre-built WebKit: ${BUN_WEBKIT_PACKAGE_NAME}") execute_process( - COMMAND ${BUN_EXECUTABLE} - "${CMAKE_CURRENT_SOURCE_DIR}/src/codegen/download-webkit.ts" - "--outdir=${BUN_WORKDIR}/bun-webkit" - "--tag=${WEBKIT_TAG}" - "--package=${BUN_WEBKIT_PACKAGE_NAME}" + COMMAND "${SHELL}" + "${CMAKE_CURRENT_SOURCE_DIR}/scripts/download-webkit.${SCRIPT_EXTENSION}" + "${BUN_WORKDIR}/bun-webkit" + "${WEBKIT_TAG}" + "${BUN_WEBKIT_PACKAGE_NAME}" WORKING_DIRECTORY ${BUN_WORKDIR} ) @@ -500,28 +511,31 @@ add_custom_command( list(APPEND BUN_RAW_SOURCES "${BUN_WORKDIR}/codegen/JSSink.cpp") # --- .lut.h Generator --- +set(BUN_OBJECT_LUT_SOURCES + bun.js/bindings/BunObject.cpp + bun.js/bindings/ZigGlobalObject.lut.txt + bun.js/bindings/JSBuffer.cpp + bun.js/bindings/BunProcess.cpp + bun.js/bindings/ProcessBindingConstants.cpp + bun.js/bindings/ProcessBindingNatives.cpp +) +set(BUN_OBJECT_LUT_OUTPUTS "") +set(BUN_HASH_LUT_GENERATOR "${BUN_CODEGEN_SRC}/create-hash-table.ts") if(NOT BUN_LINK_ONLY) - set(BUN_OBJECT_LUT_SOURCES - bun.js/bindings/BunObject.cpp - bun.js/bindings/ZigGlobalObject.lut.txt - bun.js/bindings/JSBuffer.cpp - bun.js/bindings/BunProcess.cpp - bun.js/bindings/ProcessBindingConstants.cpp - bun.js/bindings/ProcessBindingNatives.cpp - ) - set(BUN_HASH_LUT_GENERATOR "${BUN_CODEGEN_SRC}/create-hash-table.ts") - macro(GENERATE_HASH_LUT _input _output _display_name) - add_custom_command( - OUTPUT ${_output} - MAIN_DEPENDENCY ${BUN_HASH_LUT_GENERATOR} - DEPENDS ${_input} - COMMAND ${BUN_EXECUTABLE} ${BUN_HASH_LUT_GENERATOR} ${_input} ${_output} - VERBATIM - COMMENT "Generating ${_display_name}" - ) + if(NOT NO_CODEGEN) + add_custom_command( + OUTPUT ${_output} + MAIN_DEPENDENCY ${BUN_HASH_LUT_GENERATOR} + DEPENDS ${_input} + COMMAND ${BUN_EXECUTABLE} ${BUN_HASH_LUT_GENERATOR} ${_input} ${_output} + VERBATIM + COMMENT "Generating ${_display_name}" + ) + endif() + list(APPEND BUN_OBJECT_LUT_OUTPUTS "${_output}") - # list(APPEND JavaScriptCore_HEADERS ${_output}) + # list(APPEND Bun_HEADERS ${_output}) WEBKIT_ADD_SOURCE_DEPENDENCIES(${_input} ${_output}) endmacro() @@ -728,18 +742,29 @@ set_target_properties(${bun} PROPERTIES ) if(NOT BUN_CPP_ARCHIVE) - string(REPLACE ";" ".o\n " BUN_OBJECT_LIST "${BUN_SOURCES}.o") - string(REPLACE "${BUN_WORKDIR}/" "CMakeFiles/${bun}.dir/" BUN_OBJECT_LIST "${BUN_OBJECT_LIST}") - string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/" "CMakeFiles/${bun}.dir/" BUN_OBJECT_LIST "${BUN_OBJECT_LIST}") - - write_file("${BUN_WORKDIR}/compile-cpp-only.sh" - "#!/usr/bin/env bash\n" - "set -e\n" - "OBJ_LIST=(\n ${BUN_OBJECT_LIST}\n)\n" - "ninja \${OBJ_LIST[@]}\n" - "${AR} rcvs bun-cpp-objects.a \${OBJ_LIST[@]}\n" - "echo '-> bun-cpp-objects.a'\n" - ) + if(NOT WIN32) + string(REPLACE ";" ".o\n " BUN_OBJECT_LIST "${BUN_SOURCES}.o") + string(REPLACE "${BUN_WORKDIR}/" "CMakeFiles/${bun}.dir/" BUN_OBJECT_LIST "${BUN_OBJECT_LIST}") + string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/" "CMakeFiles/${bun}.dir/" BUN_OBJECT_LIST "${BUN_OBJECT_LIST}") + write_file("${BUN_WORKDIR}/compile-cpp-only.sh" + "#!/usr/bin/env bash\n" + "set -e\n" + "OBJ_LIST=(\n ${BUN_OBJECT_LIST}\n)\n" + "ninja \${OBJ_LIST[@]}\n" + "${AR} rcvs bun-cpp-objects.a \${OBJ_LIST[@]}\n" + "echo '-> bun-cpp-objects.a'\n" + ) + else() + string(REPLACE ";" ".obj\",\n \"" BUN_OBJECT_LIST "\"${BUN_SOURCES}.obj\"") + string(REPLACE "${BUN_WORKDIR}/" "CMakeFiles/${bun}.dir/" BUN_OBJECT_LIST "${BUN_OBJECT_LIST}") + string(REPLACE "${CMAKE_CURRENT_SOURCE_DIR}/" "CMakeFiles/${bun}.dir/" BUN_OBJECT_LIST "${BUN_OBJECT_LIST}") + write_file("${BUN_WORKDIR}/compile-cpp-only.ps1" + "$ErrorActionPreference = \"Stop\"\n" + "OBJ_LIST=@(\n ${BUN_OBJECT_LIST}\n)\n" + "ninja @OBJ_LIST\n" + "Write-Host \"TODO: AR\"\n" + ) + endif() else() set_target_properties(${bun} PROPERTIES LINKER_LANGUAGE CXX) endif() @@ -822,14 +847,14 @@ if(NOT MSVC) -fvisibility-inlines-hidden -fno-rtti -ferror-limit=${ERROR_LIMIT} - -fPIC -fno-omit-frame-pointer ) + if(NOT WIN32) + target_compile_options(${bun} PUBLIC -fPIC) + endif() if(NOT APPLE) - target_link_options(${bun} PUBLIC - -fuse-ld=lld - ) + target_link_options(${bun} PUBLIC -fuse-ld=lld) endif() string(APPEND CMAKE_CXX_FLAGS " -std=c++2a ") @@ -1090,13 +1115,11 @@ endif() if(WIN32) if(USE_CUSTOM_LIBUV) - target_link_libraries(${bun} PRIVATE "${BUN_DEPS_OUT_DIR}/uv.lib") + target_link_libraries(${bun} PRIVATE "${BUN_DEPS_OUT_DIR}/libuv.lib") else() find_package(libuv CONFIG REQUIRED) target_link_libraries(${bun} PRIVATE $,libuv::uv_a,libuv::uv>) endif() - - message(STATUS "Found libuv: ${libuv_LIBRARIES}") endif() if(USE_STATIC_SQLITE) @@ -1128,19 +1151,9 @@ else() target_link_libraries(${bun} PRIVATE "${WEBKIT_LIB_DIR}/JavaScriptCore.lib") if(WIN32) - string(APPEND CMAKE_CXX_FLAGS - " /external:anglebrackets /Gs- /Zi" - ) - string(APPEND CMAKE_FLAGS - " /external:anglebrackets /Gs- /Zi" - ) set_target_properties(${bun} PROPERTIES LINK_FLAGS " /SUBSYSTEM:CONSOLE /STACK:4194304,2097152") endif() - if(DEFINED ENV{VCPKG_ROOT}) - include_directories($ENV{VCPKG_ROOT}/installed/x64-windows/include) - endif() - # include_directories(C:/Users/windo/Build/WebKit/WebKitBuild/WTF/DerivedSources) # include_directories(C:/Users/windo/Build/WebKit/WebKitBuild/WTF/Headers) target_include_directories(${bun} PUBLIC C:/Users/windo/Code/WebKit/WebKitLibraries/win/include) @@ -1151,6 +1164,6 @@ else() target_link_libraries(${bun} PUBLIC winmm ws2_32 bcrypt ntdll kernel32 shell32 shlwapi advapi32 vcruntime ucrt legacy_stdio_definitions) endif() -if(NO_CODEGEN) +if(NO_CODEGEN AND NOT CODEGEN) message(STATUS "NO_CODEGEN is ON, a handful of build steps will not run.") -endif() \ No newline at end of file +endif() diff --git a/Dockerfile b/Dockerfile index 8d2a0a6826..53feecde9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ # environment, or to be used as a base image for other projects. # # You likely want this image instead: https://hub.docker.com/r/oven/bun +# +# TODO: move this file to reduce confusion ARG DEBIAN_FRONTEND=noninteractive ARG GITHUB_WORKSPACE=/build ARG WEBKIT_DIR=${GITHUB_WORKSPACE}/bun-webkit diff --git a/Makefile b/Makefile index b95810342d..e542eb5813 100644 --- a/Makefile +++ b/Makefile @@ -929,6 +929,9 @@ clone-submodules: .PHONY: headers headers: + echo please don't run the headers generator anymore. i don't think it works. + echo if you really need it, run make headers2 +headers2: rm -f /tmp/build-jsc-headers src/bun.js/bindings/headers.zig touch src/bun.js/bindings/headers.zig $(ZIG) build headers-obj @@ -1318,7 +1321,7 @@ release-bindings: $(OBJ_DIR) $(OBJ_FILES) $(WEBCORE_OBJ_FILES) $(SQLITE_OBJ_FILE # Do not add $(DEBUG_DIR) to this list # It will break caching, causing you to have to wait for every .cpp file to rebuild. .PHONY: bindings -bindings: $(DEBUG_OBJ_DIR) $(DEBUG_OBJ_FILES) $(DEBUG_WEBCORE_OBJ_FILES) $(DEBUG_SQLITE_OBJ_FILES) $(DEBUG_NODE_OS_OBJ_FILES) $(DEBUG_BUILTINS_OBJ_FILES) $(DEBUG_IO_FILES) $(DEBUG_MODULES_OBJ_FILES) $(DEBUG_WEBCRYPTO_OBJ_FILES) +bindings-real: $(DEBUG_OBJ_DIR) $(DEBUG_OBJ_FILES) $(DEBUG_WEBCORE_OBJ_FILES) $(DEBUG_SQLITE_OBJ_FILES) $(DEBUG_NODE_OS_OBJ_FILES) $(DEBUG_BUILTINS_OBJ_FILES) $(DEBUG_IO_FILES) $(DEBUG_MODULES_OBJ_FILES) $(DEBUG_WEBCRYPTO_OBJ_FILES) .PHONY: jsc-bindings-mac jsc-bindings-mac: bindings @@ -1905,23 +1908,30 @@ vendor: assert-deps submodule vendor-without-check vendor-dev: assert-deps submodule npm-install-dev vendor-without-npm .PHONY: bun -bun: vendor identifier-cache build-obj bun-link-lld-release bun-codesign-release-local - -.PHONY: static-hash-table -static-hash-table: - bun src/js/_codegen/static-hash-tables.ts +bun: + echo makefile is deprecated - use `cmake` / `bun run build` + echo 'See https://bun.sh/docs/project/contributing for more details' cpp: echo makefile is deprecated - use `cmake` / `bun run build` + echo 'See https://bun.sh/docs/project/contributing for more details' zig: echo makefile is deprecated - use `cmake` / `bun run build` + echo 'See https://bun.sh/docs/project/contributing for more details' dev: echo makefile is deprecated - use `cmake` / `bun run build` + echo 'See https://bun.sh/docs/project/contributing for more details' setup: echo makefile is deprecated - use `cmake` / `bun run build` + echo 'See https://bun.sh/docs/project/contributing for more details' + +bindings: + echo makefile is deprecated - use `cmake` / `bun run build` + echo 'See https://bun.sh/docs/project/contributing for more details' help: echo makefile is deprecated - use `cmake` / `bun run build` + echo 'See https://bun.sh/docs/project/contributing for more details' diff --git a/docs/project/contributing.md b/docs/project/contributing.md index b79ed540fa..c62459b7ad 100644 --- a/docs/project/contributing.md +++ b/docs/project/contributing.md @@ -2,7 +2,7 @@ Configuring a development environment for Bun can take 10-30 minutes depending o If you are using Windows, you must use a WSL environment as Bun does not yet compile on Windows natively. -Before starting, you will need to already have a release build of Bun installed, as we use our bundler to transpile and minify our code. +Before starting, you will need to already have a release build of Bun installed, as we use our bundler to transpile and minify our code, as well as for code generation scripts. {% codetabs %} @@ -105,18 +105,6 @@ $ sudo dnf install cargo ccache cmake git golang libtool ninja-build pkg-config {% /codetabs %} - - ## Install Zig Zig can be installed either with our npm package [`@oven/zig`](https://www.npmjs.com/package/@oven/zig), or by using [zigup](https://github.com/marler8997/zigup). @@ -130,41 +118,48 @@ $ zigup 0.12.0-dev.1297+a9e66ed73 We last updated Zig on **October 26th, 2023** {% /callout %} -## First Build +## Building Bun After cloning the repository, run the following command to run the first build. This may take a while as it will clone submodules and build dependencies. ```bash -$ bash ./scripts/setup.sh +$ bun setup ``` The binary will be located at `./build/bun-debug`. It is recommended to add this to your `$PATH`. To verify the build worked, let's print the version number on the development build of Bun. ```bash $ build/bun-debug --version -bun 1.x.y__dev +x.y.z_debug ``` -To rebuild, you can simply invoke `ninja` +To rebuild, you can invoke `bun run build` ```bash -$ ninja -C build +$ bun run build ``` -Note: `make setup` is just an alias for the following steps: +These two scripts, `setup` and `build`, are aliases to do roughly the following: ```bash -$ ./src/scripts/update-submodules.sh -$ ./src/scripts/all-dependencies.sh -$ cmake . -GNinja -B build -DCMAKE_BUILD_TYPE=Debug -$ ninja -C build +$ ./scripts/setup.sh +$ cmake -S . -G Ninja -B build -DCMAKE_BUILD_TYPE=Debug +$ ninja -C build # 'bun run build' runs just this ``` +Advanced uses can pass CMake flags to customize the build. + ## VSCode VSCode is the recommended IDE for working on Bun, as it has been configured. Once opening, you can run `Extensions: Show Recommended Extensions` to install the recommended extensions for Zig and C++. ZLS is automatically configured. - +You probably won't need to run that one much. ## Modifying ESM modules @@ -205,9 +200,7 @@ Certain modules like `node:fs`, `node:stream`, `bun:sqlite`, and `ws` are implem To build a release build of Bun, run: ```bash -$ mkdir build-release -$ cmake . -GNinja -B build-release -DCMAKE_BUILD_TYPE=Release -$ ninja -C build-release +$ bun run build:release ``` The binary will be located at `./build-release/bun` and `./build-release/bun-profile`. @@ -230,18 +223,13 @@ You'll need a very recent version of Valgrind due to DWARF 5 debug symbols. You $ valgrind --fair-sched=try --track-origins=yes bun-debug ``` -## Updating `WebKit` - -The Bun team will occasionally bump the version of WebKit used in Bun. When this happens, you may see errors in `src/bun.js/bindings` during builds. When you see this, install the latest version of `bun-webkit` and re-compile. - -```bash -$ bun install -$ make cpp -``` - ## Building WebKit locally + Debug mode of JSC -**TODO**: This is out of date. TLDR is pass `-DUSE_DEBUG_JSC=1` or `-DWEBKIT_DIR=...` to cmake +{% callout %} + +**TODO**: This is out of date. TLDR is pass `-DUSE_DEBUG_JSC=1` or `-DWEBKIT_DIR=...` to CMake. it will probably need more fiddling. ask @paperdave if you need this. + +{% /callout %} WebKit is not cloned by default (to save time and disk space). To clone and build WebKit locally, run: @@ -304,29 +292,6 @@ If you see an error when compiling `libarchive`, run this: $ brew install pkg-config ``` - - - - ### macOS `library not found for -lSystem` If you see this error when compiling, run: diff --git a/package.json b/package.json index ad8ea7bb19..88eff0303c 100644 --- a/package.json +++ b/package.json @@ -15,11 +15,9 @@ }, "private": true, "scripts": { - "build": "cmake . -DCMAKE_BUILD_TYPE=Debug -GNinja -Bbuild && ninja -Cbuild", + "setup": "./scripts/setup.sh", + "build": "if [ ! -e build ]; then bun setup; fi && ninja -C build", "build:release": "cmake . -DCMAKE_BUILD_TYPE=Release -GNinja -Bbuild-release && ninja -Cbuild-release", - "build-runtime": "esbuild --target=esnext --bundle src/runtime/index.ts --format=iife --platform=browser --global-name=BUN_RUNTIME > src/runtime.out.js; cat src/runtime.footer.js >> src/runtime.out.js", - "build-fallback": "esbuild --target=esnext --bundle src/fallback.ts --format=iife --platform=browser --minify > src/fallback.out.js", - "postinstall_not_anymore": "bash .scripts/postinstall.sh", "typecheck": "tsc --noEmit && cd test && bun run typecheck", "fmt": "prettier --write --cache './{src,test,bench,packages/{bun-types,bun-inspector-*,bun-vscode,bun-debug-adapter-protocol}}/**/*.{mjs,ts,tsx,js,jsx}'", "lint": "eslint './**/*.d.ts' --cache", diff --git a/scripts/all-dependencies.ps1 b/scripts/all-dependencies.ps1 new file mode 100644 index 0000000000..945f301da0 --- /dev/null +++ b/scripts/all-dependencies.ps1 @@ -0,0 +1,83 @@ +param( + [Alias("f")][switch]$Force = $false +) + +$ErrorActionPreference = 'Stop' + +$DidAnything = $false; + +$BUN_BASE_DIR = if ($env:BUN_BASE_DIR) { $env:BUN_BASE_DIR } else { Join-Path $PSScriptRoot '..' } +$BUN_DEPS_DIR = if ($env:BUN_DEPS_DIR) { $env:BUN_DEPS_DIR } else { Join-Path $BUN_BASE_DIR 'src\deps' } +$BUN_DEPS_OUT_DIR = if ($env:BUN_DEPS_OUT_DIR) { $env:BUN_DEPS_OUT_DIR } else { $BUN_DEPS_DIR } + +function Build-Dependency { + param( + $Script, + [string[]]$Outputs + ) + + $ScriptPath = Join-Path $PSScriptRoot "build-$Script.ps1" + + if (!$Force) { + foreach ($Output in $Outputs) { + $OutputPath = Join-Path $BUN_DEPS_OUT_DIR $Output + if (Test-Path $OutputPath) { + Write-Host "$Script - already built" + return + } + } + } + else { + Remove-Item $Outputs -ErrorAction SilentlyContinue + } + + Write-Host "$Script - Building" + Push-Location $PSScriptRoot + try { + & $ScriptPath + } + catch { + Write-Host "Failed to build $Script" + throw $_ + } + finally { + Pop-Location + } + + $Script:DidAnything = $true +} + +Build-Dependency ` + -Script "base64" ` + -Outputs @("base64.lib") +Build-Dependency ` + -Script "boringssl" ` + -Outputs @("crypto.lib", "ssl.lib", "decrepit.lib") +Build-Dependency ` + -Script "cares" ` + -Outputs @("cares.lib") +Build-Dependency ` + -Script "libarchive" ` + -Outputs @("archive.lib") +Build-Dependency ` + -Script "lolhtml" ` + -Outputs @("lolhtml.lib") +Build-Dependency ` + -Script "mimalloc" ` + -Outputs @("mimalloc.lib") +# Build-Dependency ` +# -Script "tinycc" ` +# -Outputs @("tcc.lib") +Build-Dependency ` + -Script "zlib" ` + -Outputs @("zlib.lib") +Build-Dependency ` + -Script "zstd" ` + -Outputs @("zstd.lib") +Build-Dependency ` + -Script "libuv" ` + -Outputs @("libuv.lib") + +if (!($Script:DidAnything)) { + Write-Host "(run with -Force to rebuild all)" +} \ No newline at end of file diff --git a/scripts/all-dependencies.sh b/scripts/all-dependencies.sh index b87b6ef7fc..bd6a015a22 100755 --- a/scripts/all-dependencies.sh +++ b/scripts/all-dependencies.sh @@ -2,7 +2,6 @@ set -euo pipefail source "$(dirname -- "${BASH_SOURCE[0]}")/env.sh" FORCE= - while getopts "f" opt; do case ${opt} in f ) diff --git a/scripts/build-base64.ps1 b/scripts/build-base64.ps1 index f74c7dcb61..91f5ed08cf 100644 --- a/scripts/build-base64.ps1 +++ b/scripts/build-base64.ps1 @@ -1,11 +1,16 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash . (Join-Path $PSScriptRoot "env.ps1") -$CWD = Get-Location +Push-Location (Join-Path $BUN_DEPS_DIR 'base64') +try { + Set-Location (mkdir -Force build) -Set-Location (Join-Path $BUN_DEPS_DIR 'base64') -cmake $CMAKE_FLAGS . -cmake --build . --clean-first --config Release -Copy-Item **/*.lib $BUN_DEPS_OUT_DIR - -Set-Location $CWD \ No newline at end of file + Run cmake @CMAKE_FLAGS -DBASE64_WERROR=0 .. + Run cmake --build . --clean-first --config Release + + Copy-Item base64.lib $BUN_DEPS_OUT_DIR + Write-Host "-> base64.lib" +} +finally { + Pop-Location +} diff --git a/scripts/build-boringssl.ps1 b/scripts/build-boringssl.ps1 index 77adb94f51..48a82ef0d8 100644 --- a/scripts/build-boringssl.ps1 +++ b/scripts/build-boringssl.ps1 @@ -1,13 +1,15 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash . (Join-Path $PSScriptRoot "env.ps1") -$CWD = Get-Location -$null = mkdir -p $BUN_DEPS_OUT_DIR -Force -Set-Location $BUN_DEPS_DIR/boringssl -cmake $CMAKE_FLAGS . -cmake --build . --target crypto --target ssl --target decrepit --clean-first --config Release -Copy-Item crypto/Release/crypto.lib $BUN_DEPS_OUT_DIR -Copy-Item ssl/Release/ssl.lib $BUN_DEPS_OUT_DIR -Copy-Item decrepit/Release/decrepit.lib $BUN_DEPS_OUT_DIR +Push-Location (Join-Path $BUN_DEPS_DIR 'boringssl') +try { + Set-Location (mkdir -Force build) + + Run cmake @CMAKE_FLAGS .. + Run cmake --build . --target crypto --target ssl --target decrepit --clean-first --config Release -Set-Location $CWD + Copy-Item crypto/crypto.lib $BUN_DEPS_OUT_DIR + Copy-Item ssl/ssl.lib $BUN_DEPS_OUT_DIR + Copy-Item decrepit/decrepit.lib $BUN_DEPS_OUT_DIR + Write-Host "-> crypto.lib, ssl.lib, decrepit.lib" +} finally { Pop-Location } \ No newline at end of file diff --git a/scripts/build-cares.ps1 b/scripts/build-cares.ps1 index 7e4b587bb0..c5d364fde3 100644 --- a/scripts/build-cares.ps1 +++ b/scripts/build-cares.ps1 @@ -1,13 +1,16 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash . (Join-Path $PSScriptRoot "env.ps1") -$CWD = Get-Location -Set-Location (Join-Path $BUN_DEPS_DIR 'c-ares') -Remove-Item -r build -ErrorAction SilentlyContinue -$null = mkdir build -ErrorAction SilentlyContinue -Set-Location build -cmake $CMAKE_FLAGS -DCARES_STATIC=ON -DCARES_SHARED=OFF .. -cmake --build . --clean-first --config Release -Copy-Item ./lib/Release/*.lib $BUN_DEPS_OUT_DIR +Push-Location (Join-Path $BUN_DEPS_DIR 'c-ares') +try { + Set-Location (mkdir -Force build) -Set-Location $CWD \ No newline at end of file + Run cmake @CMAKE_FLAGS -DCARES_STATIC=ON -DCARES_SHARED=OFF .. + Run cmake --build . --clean-first --config Release + + Copy-Item lib\cares.lib $BUN_DEPS_OUT_DIR + Write-Host "-> cares.lib" +} +finally { + Pop-Location +} diff --git a/scripts/build-libarchive.ps1 b/scripts/build-libarchive.ps1 index 5e8f23c4fe..e85b8ba9db 100644 --- a/scripts/build-libarchive.ps1 +++ b/scripts/build-libarchive.ps1 @@ -1,10 +1,16 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash . (Join-Path $PSScriptRoot "env.ps1") -$CWD = Get-Location -Set-Location $BUN_DEPS_DIR/libarchive -cmake -DBUILD_SHARED_LIBS=OFF -DENABLE_TEST=OFF -DENABLE_INSTALL=OFF -DENABLE_WERROR=0 $CMAKE_FLAGS . -cmake --build . --target ALL_BUILD --clean-first --config Release -Copy-Item libarchive/Release/archive.lib $BUN_DEPS_OUT_DIR +Push-Location (Join-Path $BUN_DEPS_DIR 'libarchive') +try { + Set-Location (mkdir -Force build) -Set-Location $CWD + Run cmake @CMAKE_FLAGS -DBUILD_SHARED_LIBS=OFF -DENABLE_TEST=OFF -DENABLE_INSTALL=OFF -DENABLE_WERROR=0 .. + Run cmake --build . --clean-first --config Release + + Copy-Item libarchive\archive.lib $BUN_DEPS_OUT_DIR + Write-Host "-> libarchive.lib" +} +finally { + Pop-Location +} diff --git a/scripts/build-libuv.ps1 b/scripts/build-libuv.ps1 index 9ccda67f6b..9a10121d83 100644 --- a/scripts/build-libuv.ps1 +++ b/scripts/build-libuv.ps1 @@ -2,18 +2,39 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipe . (Join-Path $PSScriptRoot "env.ps1") $CWD = Get-Location -$UVSource = (Join-Path $PSScriptRoot "../build/libuv") -if (!(test-path -PathType container $UVSource)) { - Set-Location (Join-Path $PSScriptRoot "../build") - git clone "https://github.com/libuv/libuv" libuv --depth=1 +$Source = (Join-Path $PSScriptRoot "../src/deps/libuv") +$Commit = "da527d8d2a908b824def74382761566371439003" + +if (!(Test-Path -PathType Container $Source)) { + Write-Host "Cloning libuv: $Commit" + New-Item -ItemType Directory -Force -Path $Source + Push-Location $Source + try { + Run git init + Run git remote add origin "https://github.com/libuv/libuv" + Run git fetch --depth 1 origin $Commit + Run git checkout FETCH_HEAD + } finally { Pop-Location } +} else { + Push-Location $Source + try { + $CurrentCommit = git rev-parse HEAD + if ($CurrentCommit -ne $Commit) { + Write-Host "Updating libuv: $Commit" + Run git fetch --depth 1 origin $Commit + Run git checkout FETCH_HEAD + } + } finally { Pop-Location } } -Set-Location $UVSource -New-Item -ItemType Directory -Force -Path build -Set-Location build -cmake .. $CMAKE_FLAGS "-DCMAKE_C_FLAGS=`"/DWIN32 /D_WINDOWS -Wno-int-conversion`"" -cmake --build . --clean-first --config Release +Push-Location $Source +try { + $null = mkdir build -ErrorAction SilentlyContinue + Set-Location build + + Run cmake .. @CMAKE_FLAGS "-DCMAKE_C_FLAGS=/DWIN32 /D_WINDOWS -Wno-int-conversion" + Run cmake --build . --clean-first --config Release -Copy-Item .\libuv.lib $BUN_DEPS_OUT_DIR - -Set-Location $CWD + Copy-Item libuv.lib $BUN_DEPS_OUT_DIR + Write-Host "-> libuv.lib" +} finally { Pop-Location } diff --git a/scripts/build-lolhtml.ps1 b/scripts/build-lolhtml.ps1 index fa3898922d..e7ee42fa09 100644 --- a/scripts/build-lolhtml.ps1 +++ b/scripts/build-lolhtml.ps1 @@ -1,10 +1,11 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash . (Join-Path $PSScriptRoot "env.ps1") -$CWD = Get-Location -Set-Location $BUN_DEPS_DIR/lol-html/c-api -cargo build --release --target x86_64-pc-windows-msvc -Copy-Item target/x86_64-pc-windows-msvc/release/lolhtml.lib $BUN_DEPS_OUT_DIR -Copy-Item target/x86_64-pc-windows-msvc/release/lolhtml.pdb $BUN_DEPS_OUT_DIR +Push-Location (Join-Path $BUN_DEPS_DIR 'lol-html/c-api') +try { + Run cargo build --release --target x86_64-pc-windows-msvc -Set-Location $CWD \ No newline at end of file + Copy-Item target/x86_64-pc-windows-msvc/release/lolhtml.lib $BUN_DEPS_OUT_DIR + Copy-Item target/x86_64-pc-windows-msvc/release/lolhtml.pdb $BUN_DEPS_OUT_DIR + Write-Host "-> lolhtml.lib" +} finally { Pop-Location } \ No newline at end of file diff --git a/scripts/build-mimalloc.ps1 b/scripts/build-mimalloc.ps1 index 5149ef0d04..a62bc9bcaf 100644 --- a/scripts/build-mimalloc.ps1 +++ b/scripts/build-mimalloc.ps1 @@ -1,9 +1,11 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash . (Join-Path $PSScriptRoot "env.ps1") -$CWD = Get-Location -Set-Location (Join-Path $BUN_DEPS_DIR 'mimalloc') -cmake $CMAKE_FLAGS ` +Push-Location (Join-Path $BUN_DEPS_DIR 'mimalloc') +try { + Set-Location (mkdir -Force build) + + Run cmake .. @CMAKE_FLAGS ` -DMI_SKIP_COLLECT_ON_EXIT=1 ` -DMI_BUILD_SHARED=OFF ` -DMI_BUILD_STATIC=ON ` @@ -13,10 +15,11 @@ cmake $CMAKE_FLAGS ` -DMI_BUILD_OBJECT=ON ` -DMI_USE_CXX=ON ` -DMI_OVERRIDE=OFF ` - -DMI_OSX_ZONE=OFF ` + -DMI_OSX_ZONE=OFF -cmake --build . --clean-first --config Release + Run cmake --build . --clean-first --config Release -Copy-Item **/*.lib $BUN_DEPS_OUT_DIR + Copy-Item mimalloc-static.lib $BUN_DEPS_OUT_DIR/mimalloc.lib -Set-Location $CWD + Write-Host "-> mimalloc.lib" +} finally { Pop-Location } diff --git a/scripts/build-zlib.ps1 b/scripts/build-zlib.ps1 index f3cae80753..e9841a826b 100644 --- a/scripts/build-zlib.ps1 +++ b/scripts/build-zlib.ps1 @@ -1,21 +1,29 @@ -$ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash +$ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash . (Join-Path $PSScriptRoot "env.ps1") -$CWD = Get-Location -# The current pinned commit of zlib in bun is on a fork that doesnt work on Windows, -# so here we use a different repo. There's a possibility this other fork (zlib-ng) has similar -# performance to what we have now (cloudflare/zlib), but need to benchmark first. -$ZlibSource = (Join-Path $PSScriptRoot "../build/zlib-ng") -if (!(test-path -PathType container $ZlibSource)) { - Set-Location (Join-Path $PSScriptRoot "../build") - git clone "https://github.com/zlib-ng/zlib-ng" zlib-ng +Push-Location (Join-Path $BUN_DEPS_DIR 'zlib') +try { + Run git reset --hard + + # TODO: make a patch upstream to change the line + # `#ifdef _MSC_VER` + # to account for clang-cl, which implements `__builtin_ctzl` and `__builtin_expect` + $textToReplace = [regex]::Escape("int __inline __builtin_ctzl(unsigned long mask)") + "[^}]*}" + $fileContent = Get-Content "deflate.h" -Raw + if ($fileContent -match $textToReplace) { + Set-Content -Path "deflate.h" -Value ($fileContent -replace $textToReplace, "") + } + else { + throw "Failed to patch deflate.h" + } + + Set-Location (mkdir -Force build) + + Run cmake .. @CMAKE_FLAGS + Run cmake --build . --clean-first --config Release + + Copy-Item zlib.lib $BUN_DEPS_OUT_DIR + + Write-Host "-> zlib.lib" } - -Set-Location $ZlibSource -New-Item -ItemType Directory -Force -Path build -Set-Location build -cmake .. -DCMAKE_BUILD_TYPE=Release -DZLIB_COMPAT=1 -DWITH_NATIVE_INSTRUCTIONS=1 -DWITH_GTEST=0 -cmake --build . --clean-first --config Release -Copy-Item .\Release\zlibstatic.lib $BUN_DEPS_OUT_DIR - -Set-Location $CWD +finally { Pop-Location } diff --git a/scripts/build-zstd.ps1 b/scripts/build-zstd.ps1 index e5588d514e..701125ffd7 100644 --- a/scripts/build-zstd.ps1 +++ b/scripts/build-zstd.ps1 @@ -1,10 +1,13 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash . (Join-Path $PSScriptRoot "env.ps1") -$CWD = Get-Location -Set-Location (Join-Path $BUN_DEPS_DIR 'zstd\build\cmake') -cmake $CMAKE_FLAGS -DZSTD_BUILD_STATIC=ON -DCMAKE_BUILD_TYPE=Release -cmake --build . --clean-first --config Release -Copy-Item lib\*\**.lib $BUN_DEPS_OUT_DIR +Push-Location (Join-Path $BUN_DEPS_DIR 'zstd') +try { + Remove-Item CMakeCache.txt -ErrorAction SilentlyContinue + + Run cmake -S "build/cmake" @CMAKE_FLAGS -DZSTD_BUILD_STATIC=ON + Run cmake --build . --clean-first --config Release -Set-Location $CWD + Copy-Item lib/zstd_static.lib $BUN_DEPS_OUT_DIR/zstd.lib + Write-Host "-> zstd.lib" +} finally { Pop-Location } \ No newline at end of file diff --git a/scripts/clean-dependencies.ps1 b/scripts/clean-dependencies.ps1 new file mode 100644 index 0000000000..8001ebf128 --- /dev/null +++ b/scripts/clean-dependencies.ps1 @@ -0,0 +1,29 @@ +$ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pipefail' in bash +. (Join-Path $PSScriptRoot "env.ps1") + +function Reset-Submodule { + param ( + $Repository + ) + Push-Location $Repository + try { + Run git reset --hard + Run git clean -fdx + } + finally { + Pop-Location + } +} + +$Deps = Join-Path $PSScriptRoot "../src/deps" + +Reset-Submodule $Deps/base64 +Reset-Submodule $Deps/boringssl +Reset-Submodule $Deps/c-ares +Reset-Submodule $Deps/libarchive +Reset-Submodule $Deps/lol-html +Reset-Submodule $Deps/mimalloc +Reset-Submodule $Deps/picohttpparser +Reset-Submodule $Deps/tinycc +Reset-Submodule $Deps/zlib +Reset-Submodule $Deps/zstd \ No newline at end of file diff --git a/scripts/cross-compile-codegen.sh b/scripts/cross-compile-codegen.sh new file mode 100755 index 0000000000..2fb1d8488b --- /dev/null +++ b/scripts/cross-compile-codegen.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -e + +export TARGET_PLATFORM=${1:-win32} +export TARGET_ARCH=${2:-x64} + +cd "$(dirname "${BASH_SOURCE[0]}")/../" + +OUT=$(realpath build-codegen-${TARGET_PLATFORM}-${TARGET_ARCH}) + +rm -rf "$OUT" +mkdir -p "$OUT" + +bun ./src/codegen/bundle-functions.ts --debug=OFF "$OUT" & + +bun ./src/codegen/bundle-modules.ts --debug=OFF "$OUT" & + +rm -rf "$OUT/tmp_functions" +rm -rf "$OUT/tmp_modules" + +CLASSES=( + ./src/bun.js/*.classes.ts + ./src/bun.js/api/*.classes.ts + ./src/bun.js/test/*.classes.ts + ./src/bun.js/webcore/*.classes.ts + ./src/bun.js/node/*.classes.ts +) +bun "./src/codegen/generate-classes.ts" ${CLASSES[@]} "$OUT/codegen" & + +LUTS=( + ./src/bun.js/bindings/BunObject.cpp + ./src/bun.js/bindings/ZigGlobalObject.lut.txt + ./src/bun.js/bindings/JSBuffer.cpp + ./src/bun.js/bindings/BunProcess.cpp + ./src/bun.js/bindings/ProcessBindingConstants.cpp + ./src/bun.js/bindings/ProcessBindingNatives.cpp +) +for lut in ${LUTS[@]}; do + result=$(basename $lut | sed 's/.lut.txt/.cpp/' | sed 's/.cpp/.h/') + echo bun "./src/codegen/create-hash-table.ts" "$lut" "$OUT/codegen/$result" +done + +wait + +rm -rf "$OUT/tmp"* + +echo "-> `basename "$OUT"`" \ No newline at end of file diff --git a/scripts/download-webkit.ps1 b/scripts/download-webkit.ps1 new file mode 100644 index 0000000000..b25871c076 --- /dev/null +++ b/scripts/download-webkit.ps1 @@ -0,0 +1,42 @@ +param ( + [Parameter(Mandatory)] + [string]$OutDir, + [Parameter(Mandatory)][string]$Tag, + [Parameter(Mandatory)][string]$PackageName +) + +$ErrorActionPreference = "Stop" + +$Url = "https://github.com/oven-sh/WebKit/releases/download/autobuild-$Tag/$PackageName.tar.gz" +$CacheDir = (mkdir -Force (Join-Path $PSScriptRoot "../.webkit-cache")) +$TarPath = Join-Path $CacheDir "$PackageName-$Tag.tar.gz" + +if (Test-Path $OutDir\.tag) { + $CurrentTag = Get-Content -Path (Join-Path $OutDir ".tag") + if ($CurrentTag -eq $Tag) { + return + } +} + +Remove-Item $OutDir -ErrorAction SilentlyContinue -Recurse +$null = mkdir -Force $OutDir +try { + Write-Host "-- Downloading WebKit" + if (!(Test-Path $TarPath)) { + try { + Invoke-WebRequest $Url -OutFile $TarPath + } catch { + Write-Error "Failed to fetch WebKit from: $Url" + throw $_ + } + } + + Push-Location $CacheDir + tar.exe "-xzf" "$PackageName-$Tag.tar.gz" -C (Resolve-Path -Relative $OutDir\..\).replace('\', '/') + Pop-Location + + Set-Content -Path (Join-Path $OutDir ".tag") -Value "$Tag" +} catch { + Remove-Item -Force -ErrorAction SilentlyContinue $OutDir + throw $_ +} \ No newline at end of file diff --git a/scripts/download-webkit.sh b/scripts/download-webkit.sh new file mode 100644 index 0000000000..78ef76e342 --- /dev/null +++ b/scripts/download-webkit.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +OUTDIR="$1" +TAG="$2" +PKG="$3" + +if [ -z "$OUTDIR" ]; then + echo "Missing outdir" + exit 1 +fi +if [ -z "$TAG" ]; then + echo "Missing tag" + exit 1 +fi +if [ -z "$PKG" ]; then + echo "Missing package" + exit 1 +fi + +mkdir -p "$OUTDIR" + +url="https://github.com/oven-sh/WebKit/releases/download/autobuild-$TAG/$PKG.tar.gz" +tar_dir="$(dirname "$0")/../.webkit-cache" +tar="$tar_dir/$PKG-$TAG.tar.gz" + +mkdir -p "$tar_dir" + +# TODO: Remove this block, future builds may not include a package.json +if [ -f "$OUTDIR/package.json" ]; then + read_version=$(grep -o '"version": "[^"]*"' "$OUTDIR/package.json" | sed 's/"version": "\(.*\)"/\1/' 2>/dev/null) + if [ "$read_version" == "0.0.1-$TAG" ]; then + echo "$TAG" > "$OUTDIR/.tag" + exit 0 + fi +fi + +if [ -f "$OUTDIR/.tag" ]; then + read_tag="$(cat "$OUTDIR/.tag")" + if [ "$read_tag" == "$TAG" ]; then + exit 0 + fi +fi + +rm -rf "$OUTDIR" + +if [ ! -f "$tar" ]; then + echo "-- Downloading WebKit" + if ! curl -o "$tar" -L "$url"; then + echo "Failed to download $url" + exit 1 + fi +fi + +tar -xzf "$tar" -C "$(dirname "$OUTDIR")" + +echo "$TAG" > "$OUTDIR/.tag" diff --git a/scripts/env.ps1 b/scripts/env.ps1 index effcc8521c..16d8bbe3c6 100644 --- a/scripts/env.ps1 +++ b/scripts/env.ps1 @@ -4,26 +4,70 @@ $ErrorActionPreference = 'Stop' # Setting strict mode, similar to 'set -euo pip # it sets c compiler and flags $ScriptDir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +if ($env:VSINSTALLDIR -eq $null) { + Write-Host "Loading Visual Studio environment, this may take a second..." + $vsDir = Get-ChildItem -Path "C:\Program Files\Microsoft Visual Studio\2022" -Directory + if ($vsDir -eq $null) { + throw "Visual Studio directory not found." + } + Push-Location $vsDir + try { + . (Join-Path -Path $vsDir.FullName -ChildPath "Common7\Tools\Launch-VsDevShell.ps1") -Arch amd64 -HostArch amd64 + } finally { Pop-Location } +} + +if($Env:VSCMD_ARG_TGT_ARCH -eq "x86") { + # Please do not try to compile Bun for 32 bit. It will not work. I promise. + throw "Visual Studio environment is targetting 32 bit. This configuration is definetly a mistake." +} + $BUN_BASE_DIR = if ($env:BUN_BASE_DIR) { $env:BUN_BASE_DIR } else { Join-Path $ScriptDir '..' } $BUN_DEPS_DIR = if ($env:BUN_DEPS_DIR) { $env:BUN_DEPS_DIR } else { Join-Path $BUN_BASE_DIR 'src\deps' } $BUN_DEPS_OUT_DIR = if ($env:BUN_DEPS_OUT_DIR) { $env:BUN_DEPS_OUT_DIR } else { $BUN_DEPS_DIR } -# this compiler detection could be better $CPUS = if ($env:CPUS) { $env:CPUS } else { (Get-WmiObject -Class Win32_Processor).NumberOfCores } +$CC = "clang-cl" +$CXX = "clang-cl" + $CFLAGS = '/O2' $CXXFLAGS = '/O2' -$Env:CFLAGS = $CFLAGS -$Env:CXXFLAGS = $CXXFLAGS - $CMAKE_FLAGS = @( "-GNinja", - "-DCMAKE_C_COMPILER=clang-cl", - "-DCMAKE_CXX_COMPILER=clang-cl", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_C_COMPILER=$CC", + "-DCMAKE_CXX_COMPILER=$CXX", "-DCMAKE_C_FLAGS=`"$CFLAGS`"", - "-DCMAKE_CXX_FLAGS=`"$CXXFLAGS`"", - "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_CXX_FLAGS=`"$CXXFLAGS`"" ) +$env:CC = "clang-cl" +$env:CXX = "clang-cl" +$env:CFLAGS = $CFLAGS +$env:CXXFLAGS = $CXXFLAGS +$env:CPUS = $CPUS + $null = New-Item -ItemType Directory -Force -Path $BUN_DEPS_OUT_DIR + +function Run() { + # A handy way to run a command, and automatically throw an error if the + # exit code is non-zero. + + if ($args.Count -eq 0) { + throw "Must supply some arguments." + } + + $command = $args[0] + $commandArgs = @() + if ($args.Count -gt 1) { + $commandArgs = $args[1..($args.Count - 1)] + } + + & $command $commandArgs + $result = $LASTEXITCODE + + if ($result -ne 0) { + throw "$command $commandArgs exited with code $result." + } +} diff --git a/scripts/setup.sh b/scripts/setup.sh index 8df663df75..ba6058b88b 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -1,4 +1,60 @@ -cd -- "$(dirname -- "${BASH_SOURCE[0]}")" +#!/usr/bin/env bash +set -e + +C_BOLD="\e[1;1m" +C_GREEN="\e[32m" +C_RED="\e[31m" +C_BLUE="\e[34m" +C_RESET="\e[0m" + +has_exec() { + which "$1" >/dev/null 2>&1 || return 1 +} +fail() { + printf "${C_RED}setup error${C_RESET}: %s\n" "$@" + exit 1 +} + +LLVM_VERSION=16 + +# this compiler detection could be better +# it is copy pasted from ./env.sh +CC=${CC:-$(which clang-16 || which clang || which cc)} +CXX=${CXX:-$(which clang++-16 || which clang++ || which c++)} + +test -n "$CC" || fail "missing LLVM $LLVM_VERSION (could not find clang)" +test -n "$CXX" || fail "missing LLVM $LLVM_VERSION (could not find clang++)" + +for type in CC CXX; do + compiler="${!type}" + $( + "$compiler" --version | grep "clang version ${LLVM_VERSION}." >/dev/null 2>&1 + ) || fail "LLVM ${LLVM_VERSION} is required. Detected $type as '$compiler'" +done + +has_exec "bun" || fail "you need an existing copy of 'bun' in your path to build bun" +has_exec "cmake" || fail "'cmake' is missing" +has_exec "ninja" || fail "'ninja' is missing" +$( + has_exec "rustc" \ + && (test $(cargo --version | awk '{print $2}' | cut -d. -f2) -gt 57) \ + && has_exec "cargo" +) || fail "Rust and Cargo version must be installed (minimum version 1.57)" +has_exec "go" || fail "'go' is missing" + +has_exec "pkg-config" || fail "'pkg-config' is missing" +has_exec "automake" || fail "'automake' is missing" +has_exec "perl" || fail "'perl' is missing" +has_exec "ruby" || fail "'ruby' is missing" + +rm -f .vscode/clang++ +ln -s "$CXX" .vscode/clang++ + +printf "All system dependencies OK\n" +printf "C Compiler for dependencies: ${CC}\n" +printf "C++ Compiler for dependencies: ${CXX}\n" + +cd "$(dirname "${BASH_SOURCE[0]}")" bash ./update-submodules.sh bash ./all-dependencies.sh @@ -6,6 +62,36 @@ cd ../ bun i -mkdir build -cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug -G Ninja -ninja \ No newline at end of file +# TODO(@paperdave): do not use the Makefile please +make runtime_js fallback_decoder bun_error node-fallbacks + +mkdir -p build +rm -f build/CMakeCache.txt +cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug -G Ninja -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" +ninja -C build + +printf "Checking if built bun functions\n" +BUN_VERSION=$(BUN_DEBUG_QUIET_LOGS=1 ./build/bun-debug --version) + +printf "\n" +printf "🎉 ${C_GREEN}${C_BOLD}Development environment setup complete!${C_RESET}\n" +printf "${C_BLUE}bun v${BUN_VERSION} is located at ./build/bun-debug${C_RESET}\n" + +if has_exec bun-debug; then + bun_is_at=$(which bun-debug) + if [ "$(realpath "$bun_is_at")" != "$(realpath "./build/bun-debug")" ]; then + printf "\n" + printf "${C_RED}"'Your $PATH is not configured correctly!\n'"${C_RESET}" + printf "\n" + printf "which bun-debug --> %s\n" "${bun_is_at}" + printf "\n" + printf "You should remove this binary and switch it to ./build:\n" + printf ' export PATH="$PATH:%s"\n' $(realpath "$PWD/build") + fi +else + printf "\n" + printf "You should add ./build to your path:\n" + printf ' export PATH="$PATH:%s"\n' $(realpath "$PWD/build") +fi +printf "\n" +printf "To rebuild bun, run '${C_GREEN}bun run build${C_RESET}'\n\n" diff --git a/src/bun.js/WebKit b/src/bun.js/WebKit index f0cdcc409f..b82708ff8c 160000 --- a/src/bun.js/WebKit +++ b/src/bun.js/WebKit @@ -1 +1 @@ -Subproject commit f0cdcc409fdb7a8dd4718f1d5b2ad9f7e2aa4dfb +Subproject commit b82708ff8c18785f0f8c210f27cef316c7e40fda diff --git a/src/codegen/download-webkit.ts b/src/codegen/download-webkit.ts deleted file mode 100644 index 9e73089374..0000000000 --- a/src/codegen/download-webkit.ts +++ /dev/null @@ -1,56 +0,0 @@ -import fs from "fs"; -import path from "path"; - -let OUTDIR: string | null = null; -let TAG: string | null = null; -let PKG: string | null = null; - -for (const arg of process.argv.slice(2)) { - if (arg.startsWith("--outdir=")) OUTDIR = arg.slice("--outdir=".length); - if (arg.startsWith("--tag=")) TAG = arg.slice("--tag=".length); - if (arg.startsWith("--package=")) PKG = arg.slice("--package=".length); -} - -if (!OUTDIR) { - console.error(`Missing --outdir`); - process.exit(); -} -if (!TAG) { - console.error(`Missing --tag`); - process.exit(); -} -if (!PKG) { - console.error(`Missing --package`); - process.exit(); -} - -fs.mkdirSync(OUTDIR, { recursive: true }); - -const url = `https://github.com/oven-sh/WebKit/releases/download/autobuild-${TAG}/${PKG}.tar.gz`; -const tar_dir = path.join(import.meta.dir, "../../.webkit-cache"); -const tar = path.join(tar_dir, `./${PKG}-${TAG}.tar.gz`); - -fs.mkdirSync(tar_dir, { recursive: true }); - -try { - if (fs.existsSync(OUTDIR + "/package.json")) { - const read = JSON.parse(fs.readFileSync(OUTDIR + "/package.json", "utf-8")); - if (read.version === "0.0.1-" + TAG && read.name === PKG) { - process.exit(); - } - } -} catch {} - -fs.rmSync(OUTDIR, { force: true, recursive: true }); - -if (!fs.existsSync(tar)) { - console.log(`-- Downloading WebKit`); - const res = await fetch(url); - if (!res.ok) { - console.error(`Failed to download ${url}: ${res.status} ${res.statusText}`); - process.exit(); - } - await Bun.write(tar, res); -} - -Bun.spawnSync(["tar", "-xzf", tar], { cwd: path.dirname(OUTDIR) }); diff --git a/src/codegen/generate-classes.ts b/src/codegen/generate-classes.ts index c1d7c9447d..8670986c64 100644 --- a/src/codegen/generate-classes.ts +++ b/src/codegen/generate-classes.ts @@ -1,5 +1,5 @@ // @ts-nocheck -import { resolve } from "path"; +import path from "path"; import type { Field, ClassDefinition } from "./class-definitions"; import { writeIfNotChanged } from "./helpers"; @@ -17,14 +17,6 @@ function toIdentifier(propertyName) { return `Identifier::fromString(vm, ${JSON.stringify(propertyName)}_s)`; } -const directoriesToSearch = [ - resolve(`${import.meta.dir}/../bun.js/`), - resolve(`${import.meta.dir}/../bun.js/api`), - resolve(`${import.meta.dir}/../bun.js/test`), - resolve(`${import.meta.dir}/../bun.js/webcore`), - resolve(`${import.meta.dir}/../bun.js/node`), -]; - function symbolName(typeName, name) { return `${typeName}__${name.replaceAll("@@", "")}`; } @@ -1700,7 +1692,7 @@ pub const StaticCallbackType = fn(*JSC.JSGlobalObject, *JSC.CallFrame) callconv( const classes = []; for (const file of files) { - const result = require(file); + const result = require(path.resolve(file)); if (!(result?.default?.length ?? 0)) continue; console.log("Found", result.default.length, "classes from", file); for (let { name } of result.default) { diff --git a/src/codegen/replacements.ts b/src/codegen/replacements.ts index 3520076466..2d16667949 100644 --- a/src/codegen/replacements.ts +++ b/src/codegen/replacements.ts @@ -93,8 +93,8 @@ export const define: Record = { $streamWaiting: "5", $streamWritable: "6", - "process.platform": JSON.stringify(Bun.env.PROCESS_PLATFORM ?? process.platform), - "process.arch": JSON.stringify(Bun.env.PROCESS_ARCH ?? process.arch), + "process.platform": JSON.stringify(Bun.env.TARGET_PLATFORM ?? process.platform), + "process.arch": JSON.stringify(Bun.env.TARGET_ARCH ?? process.arch), }; // ------------------------------ // diff --git a/src/deps/mimalloc b/src/deps/mimalloc index 7968d42850..011b62a66b 160000 --- a/src/deps/mimalloc +++ b/src/deps/mimalloc @@ -1 +1 @@ -Subproject commit 7968d4285043401bb36573374710d47a4081a063 +Subproject commit 011b62a66bc74bbfda87b32b54d394a544d9a7f0