From 5a108c5027a060dfc518e10011bc0b6f93ccf2e6 Mon Sep 17 00:00:00 2001 From: Ciro Spaciari Date: Fri, 23 Aug 2024 17:08:57 -0700 Subject: [PATCH] fix(fetch) always make sure that abort tracker is cleaned, revise ref count (#13459) --- packages/bun-usockets/src/context.c | 2 +- src/bun.js/bindings/webcore/FetchHeaders.cpp | 24 +- src/bun.js/bindings/webcore/HTTPHeaderMap.cpp | 39 +- src/bun.js/bindings/webcore/HTTPHeaderMap.h | 13 + src/bun.js/webcore/response.zig | 125 ++-- src/http.zig | 95 +-- src/js/builtins/ReadableStream.ts | 8 + src/js/node/http.ts | 2 +- test/js/web/fetch/client-fetch.test.ts | 499 +++++++++++++ test/js/web/fetch/content-length.test.ts | 25 + test/js/web/fetch/cookies.test.ts | 89 +++ test/js/web/fetch/encoding.test.ts | 53 ++ test/js/web/fetch/exiting.test.ts | 28 + test/js/web/fetch/fetch-leak.test.ts | 42 ++ .../fetch/fetch-url-after-redirect.test.ts | 55 ++ test/js/web/fetch/headers-case.test.ts | 26 + test/js/web/fetch/headers.undici.test.ts | 685 ++++++++++++++++++ 17 files changed, 1688 insertions(+), 122 deletions(-) create mode 100644 test/js/web/fetch/client-fetch.test.ts create mode 100644 test/js/web/fetch/content-length.test.ts create mode 100644 test/js/web/fetch/cookies.test.ts create mode 100644 test/js/web/fetch/encoding.test.ts create mode 100644 test/js/web/fetch/exiting.test.ts create mode 100644 test/js/web/fetch/fetch-leak.test.ts create mode 100644 test/js/web/fetch/fetch-url-after-redirect.test.ts create mode 100644 test/js/web/fetch/headers-case.test.ts create mode 100644 test/js/web/fetch/headers.undici.test.ts diff --git a/packages/bun-usockets/src/context.c b/packages/bun-usockets/src/context.c index 1def15457f..a59c80e83a 100644 --- a/packages/bun-usockets/src/context.c +++ b/packages/bun-usockets/src/context.c @@ -24,7 +24,7 @@ #include #endif -#define CONCURRENT_CONNECTIONS 6 +#define CONCURRENT_CONNECTIONS 4 // clang-format off int default_is_low_prio_handler(struct us_socket_t *s) { diff --git a/src/bun.js/bindings/webcore/FetchHeaders.cpp b/src/bun.js/bindings/webcore/FetchHeaders.cpp index cc0cb1356c..127f6d29a1 100644 --- a/src/bun.js/bindings/webcore/FetchHeaders.cpp +++ b/src/bun.js/bindings/webcore/FetchHeaders.cpp @@ -71,10 +71,16 @@ static ExceptionOr appendToHeaderMap(const String& name, const String& val String combinedValue = normalizedValue; HTTPHeaderName headerName; if (findHTTPHeaderName(name, headerName)) { + auto index = headers.indexOf(headerName); if (headerName != HTTPHeaderName::SetCookie) { - if (headers.contains(headerName)) { - combinedValue = makeString(headers.get(headerName), ", "_s, normalizedValue); + if (index.isValid()) { + auto existing = headers.getIndex(index); + if (headerName == HTTPHeaderName::Cookie) { + combinedValue = makeString(existing, "; "_s, normalizedValue); + } else { + combinedValue = makeString(existing, ", "_s, normalizedValue); + } } } @@ -86,22 +92,26 @@ static ExceptionOr appendToHeaderMap(const String& name, const String& val return {}; if (headerName != HTTPHeaderName::SetCookie) { - headers.set(headerName, combinedValue); + if (!headers.setIndex(index, combinedValue)) + headers.set(headerName, combinedValue); } else { headers.add(headerName, normalizedValue); } return {}; } - - if (headers.contains(name)) - combinedValue = makeString(headers.get(name), ", "_s, normalizedValue); + auto index = headers.indexOf(name); + if (index.isValid()) { + combinedValue = makeString(headers.getIndex(index), ", "_s, normalizedValue); + } auto canWriteResult = canWriteHeader(name, normalizedValue, combinedValue, guard); if (canWriteResult.hasException()) return canWriteResult.releaseException(); if (!canWriteResult.releaseReturnValue()) return {}; - headers.set(name, combinedValue); + + if (!headers.setIndex(index, combinedValue)) + headers.set(name, combinedValue); // if (guard == FetchHeaders::Guard::RequestNoCors) // removePrivilegedNoCORSRequestHeaders(headers); diff --git a/src/bun.js/bindings/webcore/HTTPHeaderMap.cpp b/src/bun.js/bindings/webcore/HTTPHeaderMap.cpp index 55beffbe66..d2765a7b8f 100644 --- a/src/bun.js/bindings/webcore/HTTPHeaderMap.cpp +++ b/src/bun.js/bindings/webcore/HTTPHeaderMap.cpp @@ -252,6 +252,30 @@ String HTTPHeaderMap::get(HTTPHeaderName name) const return index != notFound ? m_commonHeaders[index].value : String(); } +HTTPHeaderMap::HeaderIndex HTTPHeaderMap::indexOf(HTTPHeaderName name) const +{ + auto index = m_commonHeaders.findIf([&](auto& header) { + return header.key == name; + }); + return (HeaderIndex) { .index = index, .isCommon = true }; +} + +HTTPHeaderMap::HeaderIndex HTTPHeaderMap::indexOf(const String& name) const +{ + auto index = m_uncommonHeaders.findIf([&](auto& header) { + return equalIgnoringASCIICase(header.key, name); + }); + return (HeaderIndex) { .index = index, .isCommon = false }; +} + +String HTTPHeaderMap::getIndex(HTTPHeaderMap::HeaderIndex index) const +{ + if (index.index == notFound) + return String(); + if (index.isCommon) + return m_commonHeaders[index.index].value; + return m_uncommonHeaders[index.index].value; +} void HTTPHeaderMap::set(HTTPHeaderName name, const String& value) { if (name == HTTPHeaderName::SetCookie) { @@ -269,6 +293,19 @@ void HTTPHeaderMap::set(HTTPHeaderName name, const String& value) m_commonHeaders[index].value = value; } +bool HTTPHeaderMap::setIndex(HTTPHeaderMap::HeaderIndex index, const String& value) +{ + if (!index.isValid()) + return false; + + if (index.isCommon) { + m_commonHeaders[index.index].value = value; + } else { + m_uncommonHeaders[index.index].value = value; + } + return true; +} + bool HTTPHeaderMap::contains(HTTPHeaderName name) const { if (name == HTTPHeaderName::SetCookie) @@ -303,7 +340,7 @@ void HTTPHeaderMap::add(HTTPHeaderName name, const String& value) return header.key == name; }); if (index != notFound) - m_commonHeaders[index].value = makeString(m_commonHeaders[index].value, ", "_s, value); + m_commonHeaders[index].value = makeString(m_commonHeaders[index].value, name == HTTPHeaderName::Cookie ? "; "_s : ", "_s, value); else m_commonHeaders.append(CommonHeader { name, value }); } diff --git a/src/bun.js/bindings/webcore/HTTPHeaderMap.h b/src/bun.js/bindings/webcore/HTTPHeaderMap.h index ca506a0e10..589d2945cf 100644 --- a/src/bun.js/bindings/webcore/HTTPHeaderMap.h +++ b/src/bun.js/bindings/webcore/HTTPHeaderMap.h @@ -48,6 +48,13 @@ public: bool operator==(const CommonHeader &other) const { return key == other.key && value == other.value; } }; + struct HeaderIndex { + size_t index; + bool isCommon; + + bool isValid() const { return index != notFound; } + }; + struct UncommonHeader { String key; String value; @@ -180,8 +187,14 @@ public: WEBCORE_EXPORT void add(const String &name, const String &value); WEBCORE_EXPORT void append(const String &name, const String &value); WEBCORE_EXPORT bool contains(const String &) const; + WEBCORE_EXPORT int64_t indexOf(String &name) const; WEBCORE_EXPORT bool remove(const String &); + WEBCORE_EXPORT String getIndex(HeaderIndex index) const; + WEBCORE_EXPORT bool setIndex(HeaderIndex index, const String &value); + HeaderIndex indexOf(const String &name) const; + HeaderIndex indexOf(HTTPHeaderName name) const; + #if USE(CF) void set(CFStringRef name, const String &value); #ifdef __OBJC__ diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig index f6fc924d11..66998ea094 100644 --- a/src/bun.js/webcore/response.zig +++ b/src/bun.js/webcore/response.zig @@ -911,8 +911,8 @@ pub const Fetch = struct { this.request_headers.buf.deinit(allocator); this.request_headers = Headers{ .allocator = undefined }; - if (this.http != null) { - this.http.?.clearData(); + if (this.http) |http_| { + http_.clearData(); } if (this.metadata != null) { @@ -948,7 +948,10 @@ pub const Fetch = struct { var reporter = this.memory_reporter; const allocator = reporter.allocator(); - if (this.http) |http_| allocator.destroy(http_); + if (this.http) |http_| { + this.http = null; + allocator.destroy(http_); + } allocator.destroy(this); // reporter.assert(); bun.default_allocator.destroy(reporter); @@ -973,20 +976,12 @@ pub const Fetch = struct { pub fn onBodyReceived(this: *FetchTasklet) void { const success = this.result.isSuccess(); const globalThis = this.global_this; - const is_done = !success or !this.result.has_more; // reset the buffer if we are streaming or if we are not waiting for bufferig anymore var buffer_reset = true; defer { if (buffer_reset) { this.scheduled_response_buffer.reset(); } - - this.mutex.unlock(); - if (is_done) { - const vm = globalThis.bunVM(); - this.poll_ref.unref(vm); - this.deref(); - } } if (!success) { @@ -1120,10 +1115,32 @@ pub const Fetch = struct { pub fn onProgressUpdate(this: *FetchTasklet) void { JSC.markBinding(@src()); log("onProgressUpdate", .{}); - defer this.deref(); this.mutex.lock(); this.has_schedule_callback.store(false, .monotonic); + const is_done = !this.result.has_more; + const vm = this.javascript_vm; + // vm is shutting down we cannot touch JS + if (vm.isShuttingDown()) { + this.mutex.unlock(); + if (is_done) { + this.deref(); + } + return; + } + + const globalThis = this.global_this; + defer { + this.mutex.unlock(); + // if we are not done we wait until the next call + if (is_done) { + var poll_ref = this.poll_ref; + this.poll_ref = .{}; + poll_ref.unref(vm); + this.deref(); + } + } + // if we already respond the metadata and still need to process the body if (this.is_waiting_body) { this.onBodyReceived(); return; @@ -1131,33 +1148,14 @@ pub const Fetch = struct { // if we abort because of cert error // we wait the Http Client because we already have the response // we just need to deinit - const globalThis = this.global_this; - if (this.is_waiting_abort) { - // has_more will be false when the request is aborted/finished - if (this.result.has_more) { - this.mutex.unlock(); - return; - } - this.mutex.unlock(); - var poll_ref = this.poll_ref; - const vm = globalThis.bunVM(); - - poll_ref.unref(vm); - this.deref(); return; } const promise_value = this.promise.valueOrEmpty(); - var poll_ref = this.poll_ref; - const vm = globalThis.bunVM(); - if (promise_value.isEmptyOrUndefinedOrNull()) { log("onProgressUpdate: promise_value is null", .{}); this.promise.deinit(); - this.mutex.unlock(); - poll_ref.unref(vm); - this.deref(); return; } @@ -1175,25 +1173,15 @@ pub const Fetch = struct { defer result.deinit(); promise_value.ensureStillAlive(); - promise.reject(globalThis, result.toJS(globalThis)); tracker.didDispatch(globalThis); this.promise.deinit(); - this.mutex.unlock(); - if (this.is_waiting_abort) { - return; - } - // we are already done we can deinit - poll_ref.unref(vm); - this.deref(); return; } // everything ok if (this.metadata == null) { log("onProgressUpdate: metadata is null", .{}); - // cannot continue without metadata - this.mutex.unlock(); return; } } @@ -1204,11 +1192,6 @@ pub const Fetch = struct { log("onProgressUpdate: promise_value is not null", .{}); tracker.didDispatch(globalThis); this.promise.deinit(); - this.mutex.unlock(); - if (!this.is_waiting_body) { - poll_ref.unref(vm); - this.deref(); - } } const success = this.result.isSuccess(); @@ -1230,27 +1213,28 @@ pub const Fetch = struct { task: JSC.AnyTask, pub fn resolve(self: *@This()) void { + // cleanup + defer bun.default_allocator.destroy(self); + defer self.held.deinit(); + defer self.promise.deinit(); + // resolve the promise var prom = self.promise.swap().asAnyPromise().?; - const globalObject = self.globalObject; const res = self.held.swap(); - self.held.deinit(); - self.promise.deinit(); res.ensureStillAlive(); - - bun.default_allocator.destroy(self); - prom.resolve(globalObject, res); + prom.resolve(self.globalObject, res); } pub fn reject(self: *@This()) void { - var prom = self.promise.swap().asAnyPromise().?; - const globalObject = self.globalObject; - const res = self.held.swap(); - self.held.deinit(); - self.promise.deinit(); - res.ensureStillAlive(); + // cleanup + defer bun.default_allocator.destroy(self); + defer self.held.deinit(); + defer self.promise.deinit(); - bun.default_allocator.destroy(self); - prom.reject(globalObject, res); + // reject the promise + var prom = self.promise.swap().asAnyPromise().?; + const res = self.held.swap(); + res.ensureStillAlive(); + prom.reject(self.globalObject, res); } }; var holder = bun.default_allocator.create(Holder) catch bun.outOfMemory(); @@ -1267,7 +1251,7 @@ pub const Fetch = struct { false => JSC.AnyTask.New(Holder, Holder.reject).init(holder), }; - globalThis.bunVM().enqueueTask(JSC.Task.init(&holder.task)); + vm.enqueueTask(JSC.Task.init(&holder.task)); } pub fn checkServerIdentity(this: *FetchTasklet, certificate_info: http.CertificateInfo) bool { @@ -1295,8 +1279,8 @@ pub const Fetch = struct { this.tracker.didCancel(this.global_this); // we need to abort the request - if (this.http != null) { - http.http_thread.scheduleShutdown(this.http.?); + if (this.http) |http_| { + http.http_thread.scheduleShutdown(http_); } this.result.fail = error.ERR_TLS_CERT_ALTNAME_INVALID; return false; @@ -1560,7 +1544,7 @@ pub const Fetch = struct { http_.enableBodyStreaming(); } // we should not keep the process alive if we are ignoring the body - const vm = this.global_this.bunVM(); + const vm = this.javascript_vm; this.poll_ref.unref(vm); // clean any remaining refereces this.readable_stream_ref.deinit(); @@ -1735,8 +1719,8 @@ pub const Fetch = struct { this.signal_store.aborted.store(true, .monotonic); this.tracker.didCancel(this.global_this); - if (this.http != null) { - http.http_thread.scheduleShutdown(this.http.?); + if (this.http) |http_| { + http.http_thread.scheduleShutdown(http_); } } @@ -1781,16 +1765,19 @@ pub const Fetch = struct { node.http.?.schedule(allocator, &batch); node.poll_ref.ref(global.bunVM()); + // increment ref so we can keep it alive until the http client is done + node.ref(); http.http_thread.schedule(batch); return node; } pub fn callback(task: *FetchTasklet, async_http: *http.AsyncHTTP, result: http.HTTPClientResult) void { - task.ref(); - task.mutex.lock(); defer task.mutex.unlock(); + const is_done = !result.has_more; + // we are done with the http client so we can deref our side + defer if (is_done) task.deref(); task.http.?.* = async_http.*; task.http.?.response_buffer = async_http.response_buffer; @@ -1838,7 +1825,6 @@ pub const Fetch = struct { } if (success and result.has_more) { // we are ignoring the body so we should not receive more data, so will only signal when result.has_more = true - task.deref(); return; } } else { @@ -1851,7 +1837,6 @@ pub const Fetch = struct { if (task.has_schedule_callback.cmpxchgStrong(false, true, .acquire, .monotonic)) |has_schedule_callback| { if (has_schedule_callback) { - task.deref(); return; } } diff --git a/src/http.zig b/src/http.zig index 4c01335caa..099b82d446 100644 --- a/src/http.zig +++ b/src/http.zig @@ -531,7 +531,7 @@ fn NewHTTPContext(comptime ssl: bool) type { // if checkServerIdentity returns false, we dont call open this means that the connection was rejected if (!client.checkServerIdentity(comptime ssl, socket, handshake_error)) { client.flags.did_have_handshaking_error = true; - + client.unregisterAbortTracker(); if (!socket.isClosed()) terminateSocket(socket); return; } @@ -791,13 +791,13 @@ pub const HTTPThread = struct { lazy_libdeflater: ?*LibdeflateState = null, + const threadlog = Output.scoped(.HTTPThread, true); + const ShutdownMessage = struct { async_http_id: u32, is_tls: bool, }; - const threadlog = Output.scoped(.HTTPThread, true); - pub const LibdeflateState = struct { decompressor: *bun.libdeflate.Decompressor = undefined, shared_buffer: [512 * 1024]u8 = undefined, @@ -1086,6 +1086,24 @@ pub fn checkServerIdentity( return true; } +fn registerAbortTracker( + client: *HTTPClient, + comptime is_ssl: bool, + socket: NewHTTPContext(is_ssl).HTTPSocket, +) void { + if (client.signals.aborted != null) { + socket_async_http_abort_tracker.put(client.async_http_id, socket.socket) catch unreachable; + } +} + +fn unregisterAbortTracker( + client: *HTTPClient, +) void { + if (client.signals.aborted != null) { + _ = socket_async_http_abort_tracker.swapRemove(client.async_http_id); + } +} + pub fn onOpen( client: *HTTPClient, comptime is_ssl: bool, @@ -1098,9 +1116,7 @@ pub fn onOpen( assert(is_ssl == client.url.isHTTPS()); } } - if (client.signals.aborted != null) { - socket_async_http_abort_tracker.put(client.async_http_id, socket.socket) catch unreachable; - } + client.registerAbortTracker(is_ssl, socket); log("Connected {s} \n", .{client.url.href}); if (client.signals.get(.aborted)) { @@ -1160,6 +1176,12 @@ pub fn onClose( socket: NewHTTPContext(is_ssl).HTTPSocket, ) void { log("Closed {s}\n", .{client.url.href}); + // the socket is closed, we need to unregister the abort tracker + client.unregisterAbortTracker(); + if (client.signals.get(.aborted)) { + client.fail(error.Aborted); + return; + } const in_progress = client.state.stage != .done and client.state.stage != .fail and client.state.flags.is_redirect_pending == false; @@ -1203,19 +1225,15 @@ pub fn onTimeout( ) void { if (client.flags.disable_timeout) return; log("Timeout {s}\n", .{client.url.href}); - defer NewHTTPContext(is_ssl).terminateSocket(socket); - if (client.state.stage != .done and client.state.stage != .fail) { - client.fail(error.Timeout); - } + client.fail(error.Timeout); } pub fn onConnectError( client: *HTTPClient, ) void { log("onConnectError {s}\n", .{client.url.href}); - if (client.state.stage != .done and client.state.stage != .fail) - client.fail(error.ConnectionRefused); + client.fail(error.ConnectionRefused); } pub inline fn getAllocator() std.mem.Allocator { @@ -2429,6 +2447,7 @@ pub fn doRedirect( this.remaining_redirect_count -|= 1; this.flags.redirected = true; assert(this.redirect_type == FetchRedirect.follow); + this.unregisterAbortTracker(); // we need to clean the client reference before closing the socket because we are going to reuse the same ref in a another request if (this.isKeepAlivePossible()) { @@ -2458,9 +2477,6 @@ pub fn doRedirect( tunnel.deinit(); this.proxy_tunnel = null; } - if (this.signals.aborted != null) { - _ = socket_async_http_abort_tracker.swapRemove(this.async_http_id); - } return this.start(.{ .bytes = request_body }, body_out_str); } @@ -2554,7 +2570,7 @@ fn printResponse(response: picohttp.Response) void { pub fn onPreconnect(this: *HTTPClient, comptime is_ssl: bool, socket: NewHTTPContext(is_ssl).HTTPSocket) void { log("onPreconnect({})", .{this.url}); - _ = socket_async_http_abort_tracker.swapRemove(this.async_http_id); + this.unregisterAbortTracker(); const ctx = if (comptime is_ssl) &http_thread.https_context else &http_thread.http_context; ctx.releaseSocket( socket, @@ -2863,13 +2879,11 @@ pub fn onWritable(this: *HTTPClient, comptime is_first_call: bool, comptime is_s } pub fn closeAndFail(this: *HTTPClient, err: anyerror, comptime is_ssl: bool, socket: NewHTTPContext(is_ssl).HTTPSocket) void { - if (this.state.stage != .fail and this.state.stage != .done) { - if (!socket.isClosed()) { - NewHTTPContext(is_ssl).terminateSocket(socket); - } - log("closeAndFail: {s}", .{@errorName(err)}); - this.fail(err); + log("closeAndFail: {s}", .{@errorName(err)}); + if (!socket.isClosed()) { + NewHTTPContext(is_ssl).terminateSocket(socket); } + this.fail(err); } fn startProxySendHeaders(this: *HTTPClient, comptime is_ssl: bool, socket: NewHTTPContext(is_ssl).HTTPSocket) void { @@ -2941,11 +2955,11 @@ inline fn handleShortRead( } pub fn onData(this: *HTTPClient, comptime is_ssl: bool, incoming_data: []const u8, ctx: *NewHTTPContext(is_ssl), socket: NewHTTPContext(is_ssl).HTTPSocket) void { log("onData {}", .{incoming_data.len}); - if (this.signals.get(.aborted)) { this.closeAndAbort(is_ssl, socket); return; } + switch (this.state.response_stage) { .pending, .headers, .proxy_decoded_headers => { var to_read = incoming_data; @@ -3193,21 +3207,20 @@ pub fn closeAndAbort(this: *HTTPClient, comptime is_ssl: bool, socket: NewHTTPCo } fn fail(this: *HTTPClient, err: anyerror) void { - if (this.signals.aborted != null) { - _ = socket_async_http_abort_tracker.swapRemove(this.async_http_id); + this.unregisterAbortTracker(); + if (this.state.stage != .done and this.state.stage != .fail) { + this.state.request_stage = .fail; + this.state.response_stage = .fail; + this.state.fail = err; + this.state.stage = .fail; + + const callback = this.result_callback; + const result = this.toResult(); + this.state.reset(this.allocator); + this.flags.proxy_tunneling = false; + + callback.run(@fieldParentPtr("client", this), result); } - - this.state.request_stage = .fail; - this.state.response_stage = .fail; - this.state.fail = err; - this.state.stage = .fail; - - const callback = this.result_callback; - const result = this.toResult(); - this.state.reset(this.allocator); - this.flags.proxy_tunneling = false; - - callback.run(@fieldParentPtr("client", this), result); } // We have to clone metadata immediately after use @@ -3267,15 +3280,13 @@ pub fn progressUpdate(this: *HTTPClient, comptime is_ssl: bool, ctx: *NewHTTPCon const result = this.toResult(); const is_done = !result.has_more; - if (this.signals.aborted != null and is_done) { - _ = socket_async_http_abort_tracker.swapRemove(this.async_http_id); - } - log("progressUpdate {}", .{is_done}); const callback = this.result_callback; if (is_done) { + this.unregisterAbortTracker(); + if (this.isKeepAlivePossible() and !socket.isClosedOrHasError()) { ctx.releaseSocket( socket, @@ -3406,7 +3417,7 @@ pub fn toResult(this: *HTTPClient) HTTPClientResult { .redirected = this.flags.redirected, .fail = this.state.fail, // check if we are reporting cert errors, do not have a fail state and we are not done - .has_more = this.state.fail == null and !this.state.isDone(), + .has_more = certificate_info != null or (this.state.fail == null and !this.state.isDone()), .body_size = body_size, .certificate_info = null, }; diff --git a/src/js/builtins/ReadableStream.ts b/src/js/builtins/ReadableStream.ts index 16726eeff0..89387e6542 100644 --- a/src/js/builtins/ReadableStream.ts +++ b/src/js/builtins/ReadableStream.ts @@ -112,6 +112,8 @@ export function readableStreamToArray(stream: ReadableStream): Promise { if (underlyingSource !== undefined) { return $readableStreamToTextDirect(stream, underlyingSource); } + if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); return $readableStreamIntoText(stream); } @@ -133,6 +136,7 @@ export function readableStreamToArrayBuffer(stream: ReadableStream) if (underlyingSource !== undefined) { return $readableStreamToArrayBufferDirect(stream, underlyingSource, false); } + if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); var result = Bun.readableStreamToArray(stream); if ($isPromise(result)) { @@ -152,6 +156,7 @@ export function readableStreamToBytes(stream: ReadableStream): Prom if (underlyingSource !== undefined) { return $readableStreamToArrayBufferDirect(stream, underlyingSource, true); } + if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); var result = Bun.readableStreamToArray(stream); if ($isPromise(result)) { @@ -168,6 +173,7 @@ export function readableStreamToFormData( stream: ReadableStream, contentType: string | ArrayBuffer | ArrayBufferView, ): Promise { + if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); return Bun.readableStreamToBlob(stream).then(blob => { return FormData.from(blob, contentType); }); @@ -175,11 +181,13 @@ export function readableStreamToFormData( $linkTimeConstant; export function readableStreamToJSON(stream: ReadableStream): unknown { + if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); return Promise.resolve(Bun.readableStreamToText(stream)).then(globalThis.JSON.parse); } $linkTimeConstant; export function readableStreamToBlob(stream: ReadableStream): Promise { + if ($isReadableStreamLocked(stream)) return Promise.$reject($makeTypeError("ReadableStream is locked")); return Promise.resolve(Bun.readableStreamToArray(stream)).then(array => new Blob(array)); } diff --git a/src/js/node/http.ts b/src/js/node/http.ts index b48018c9a7..4bb8ee17ca 100644 --- a/src/js/node/http.ts +++ b/src/js/node/http.ts @@ -2161,7 +2161,7 @@ function _writeHead(statusCode, reason, obj, response) { } else { // writeHead(statusCode[, headers]) if (!response.statusMessage) response.statusMessage = STATUS_CODES[statusCode] || "unknown"; - obj = reason; + obj ??= reason; } response.statusCode = statusCode; diff --git a/test/js/web/fetch/client-fetch.test.ts b/test/js/web/fetch/client-fetch.test.ts new file mode 100644 index 0000000000..23e3f0bfb0 --- /dev/null +++ b/test/js/web/fetch/client-fetch.test.ts @@ -0,0 +1,499 @@ +/* globals AbortController */ + +import { test, expect, describe } from "bun:test"; +import { once } from "node:events"; +import { createServer } from "node:http"; +import { promisify } from "node:util"; +import { randomFillSync, createHash } from "node:crypto"; +import { gzipSync } from "node:zlib"; + +test("function signature", () => { + expect(fetch.name).toBe("fetch"); + expect(fetch.length).toBe(1); +}); + +test("args validation", async () => { + expect(fetch()).rejects.toThrow(TypeError); + expect(fetch("ftp://unsupported")).rejects.toThrow(TypeError); +}); + +test("request json", async () => { + const obj = { asd: true }; + await using server = createServer((req, res) => { + res.end(JSON.stringify(obj)); + }).listen(0); + await once(server, "listening"); + + const body = await fetch(`http://localhost:${server.address().port}`); + expect(obj).toEqual(await body.json()); +}); + +test("request text", async () => { + const obj = { asd: true }; + await using server = createServer((req, res) => { + res.end(JSON.stringify(obj)); + }).listen(0); + await once(server, "listening"); + + const body = await fetch(`http://localhost:${server.address().port}`); + expect(JSON.stringify(obj)).toEqual(await body.text()); +}); + +test("request arrayBuffer", async () => { + const obj = { asd: true }; + await using server = createServer((req, res) => { + res.end(JSON.stringify(obj)); + }).listen(0); + await once(server, "listening"); + + const body = await fetch(`http://localhost:${server.address().port}`); + expect(Buffer.from(JSON.stringify(obj))).toEqual(Buffer.from(await body.arrayBuffer())); +}); + +test("should set type of blob object to the value of the `Content-Type` header from response", async () => { + const obj = { asd: true }; + await using server = createServer((req, res) => { + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify(obj)); + }).listen(0); + await once(server, "listening"); + + const response = await fetch(`http://localhost:${server.address().port}`); + expect("application/json;charset=utf-8").toBe((await response.blob()).type); +}); + +test("pre aborted with readable request body", async () => { + const server = createServer((req, res) => {}).listen(0); + try { + await once(server, "listening"); + + const ac = new AbortController(); + ac.abort(); + expect( + fetch(`http://localhost:${server.address().port}`, { + signal: ac.signal, + method: "POST", + body: new ReadableStream({ + async cancel(reason) { + expect(reason.name).toBe("AbortError"); + }, + }), + duplex: "half", + }), + ).rejects.toThrow(); + } finally { + server.closeAllConnections(); + } +}); + +test("pre aborted with closed readable request body", async () => { + await using server = createServer((req, res) => {}).listen(0); + await once(server, "listening"); + const ac = new AbortController(); + ac.abort(); + const body = new ReadableStream({ + async start(c) { + expect(true).toBe(true); + c.close(); + }, + async cancel(reason) { + expect.unreachable(); + }, + }); + + expect( + fetch(`http://localhost:${server.address().port}`, { + signal: ac.signal, + method: "POST", + body, + duplex: "half", + }), + ).rejects.toThrow(); +}); + +test("unsupported formData 1", async () => { + await using server = createServer((req, res) => { + res.setHeader("content-type", "asdasdsad"); + res.end(); + }).listen(0); + await once(server, "listening"); + expect(fetch(`http://localhost:${server.address().port}`).then(res => res.formData())).rejects.toThrow(TypeError); +}); + +test("multipart formdata not base64", async () => { + // Construct example form data, with text and blob fields + const formData = new FormData(); + formData.append("field1", "value1"); + const blob = new Blob(["example\ntext file"], { type: "text/plain" }); + formData.append("field2", blob, "file.txt"); + + const tempRes = new Response(formData); + const boundary = tempRes.headers.get("content-type").split("boundary=")[1]; + const formRaw = await tempRes.text(); + + await using server = createServer((req, res) => { + res.setHeader("content-type", "multipart/form-data; boundary=" + boundary); + res.write(formRaw); + res.end(); + }); + const listen = promisify(server.listen.bind(server)); + await listen(0); + const res = await fetch(`http://localhost:${server.address().port}`); + const form = await res.formData(); + expect(form.get("field1")).toBe("value1"); + + const text = await form.get("field2").text(); + expect(text).toBe("example\ntext file"); +}); + +test.todo("multipart formdata base64", async () => { + // Example form data with base64 encoding + const data = randomFillSync(Buffer.alloc(256)); + const formRaw = + "------formdata-bun-0.5786922755719377\r\n" + + 'Content-Disposition: form-data; name="file"; filename="test.txt"\r\n' + + "Content-Type: application/octet-stream\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + data.toString("base64") + + "\r\n" + + "------formdata-bun-0.5786922755719377--"; + + await using server = createServer(async (req, res) => { + res.setHeader("content-type", "multipart/form-data; boundary=----formdata-bun-0.5786922755719377"); + + for (let offset = 0; offset < formRaw.length; ) { + res.write(formRaw.slice(offset, (offset += 2))); + await new Promise(resolve => setTimeout(resolve)); + } + res.end(); + }).listen(0); + await once(server, "listening"); + + const digest = await fetch(`http://localhost:${server.address().port}`) + .then(res => res.formData()) + .then(form => form.get("file").arrayBuffer()) + .then(buffer => createHash("sha256").update(Buffer.from(buffer)).digest("base64")); + expect(createHash("sha256").update(data).digest("base64")).toBe(digest); +}); + +test("multipart fromdata non-ascii filed names", async () => { + const request = new Request("http://localhost", { + method: "POST", + headers: { + "Content-Type": "multipart/form-data; boundary=----formdata-undici-0.6204674738279623", + }, + body: + "------formdata-undici-0.6204674738279623\r\n" + + 'Content-Disposition: form-data; name="fiŝo"\r\n' + + "\r\n" + + "value1\r\n" + + "------formdata-undici-0.6204674738279623--", + }); + + const form = await request.formData(); + expect(form.get("fiŝo")).toBe("value1"); +}); + +test("busboy emit error", async () => { + const formData = new FormData(); + formData.append("field1", "value1"); + + const tempRes = new Response(formData); + const formRaw = await tempRes.text(); + + await using server = createServer((req, res) => { + res.setHeader("content-type", "multipart/form-data; boundary=wrongboundary"); + res.write(formRaw); + res.end(); + }); + + const listen = promisify(server.listen.bind(server)); + await listen(0); + + const res = await fetch(`http://localhost:${server.address().port}`); + expect(res.formData()).rejects.toThrow("FormData parse error missing final boundary"); +}); + +// https://github.com/nodejs/undici/issues/2244 +test("parsing formData preserve full path on files", async () => { + const formData = new FormData(); + formData.append("field1", new File(["foo"], "a/b/c/foo.txt")); + + const tempRes = new Response(formData); + const form = await tempRes.formData(); + + expect(form.get("field1").name).toBe("a/b/c/foo.txt"); +}); + +test("urlencoded formData", async () => { + await using server = createServer((req, res) => { + res.setHeader("content-type", "application/x-www-form-urlencoded"); + res.end("field1=value1&field2=value2"); + }).listen(0); + await once(server, "listening"); + + const formData = await fetch(`http://localhost:${server.address().port}`).then(res => res.formData()); + expect(formData.get("field1")).toBe("value1"); + expect(formData.get("field2")).toBe("value2"); +}); + +test("text with BOM", async () => { + await using server = createServer((req, res) => { + res.setHeader("content-type", "application/x-www-form-urlencoded"); + res.end("\uFEFFtest=\uFEFF"); + }).listen(0); + await once(server, "listening"); + + const text = await fetch(`http://localhost:${server.address().port}`).then(res => res.text()); + expect(text).toBe("test=\uFEFF"); +}); + +test.todo("formData with BOM", async () => { + await using server = createServer((req, res) => { + res.setHeader("content-type", "application/x-www-form-urlencoded"); + res.end("\uFEFFtest=\uFEFF"); + }).listen(0); + await once(server, "listening"); + + const formData = await fetch(`http://localhost:${server.address().port}`).then(res => res.formData()); + expect(formData.get("\uFEFFtest")).toBe("\uFEFF"); +}); + +test("locked blob body", async () => { + await using server = createServer((req, res) => { + res.end(); + }).listen(0); + await once(server, "listening"); + + const res = await fetch(`http://localhost:${server.address().port}`); + const reader = res.body.getReader(); + expect(res.blob()).rejects.toThrow("ReadableStream is locked"); + reader.cancel(); +}); + +test("disturbed blob body", async () => { + await using server = createServer((req, res) => { + res.end(); + }).listen(0); + await once(server, "listening"); + + const res = await fetch(`http://localhost:${server.address().port}`); + await res.blob(); + expect(res.blob()).rejects.toThrow("Body already used"); +}); + +test("redirect with body", async () => { + let count = 0; + await using server = createServer(async (req, res) => { + let body = ""; + for await (const chunk of req) { + body += chunk; + } + expect(body).toBe("asd"); + if (count++ === 0) { + res.setHeader("location", "asd"); + res.statusCode = 302; + res.end(); + } else { + res.end(String(count)); + } + }).listen(0); + await once(server, "listening"); + + const res = await fetch(`http://localhost:${server.address().port}`, { + method: "PUT", + body: "asd", + }); + expect(await res.text()).toBe("2"); +}); + +test("redirect with stream", async () => { + const location = "/asd"; + const body = "hello!"; + await using server = createServer(async (req, res) => { + res.writeHead(302, { location }); + let count = 0; + const l = setInterval(() => { + res.write(body[count++]); + if (count === body.length) { + res.end(); + clearInterval(l); + } + }, 50); + }).listen(0); + + await once(server, "listening"); + + const res = await fetch(`http://localhost:${server.address().port}`, { + redirect: "manual", + }); + expect(res.status).toBe(302); + expect(res.headers.get("location")).toBe(location); + expect(await res.text()).toBe(body); +}); + +test("fail to extract locked body", () => { + const stream = new ReadableStream({}); + const reader = stream.getReader(); + try { + // eslint-disable-next-line + new Response(stream); + } catch (err) { + expect((err as Error).name).toBe("TypeError"); + } + reader.cancel(); +}); + +test("fail to extract locked body", () => { + const stream = new ReadableStream({}); + const reader = stream.getReader(); + try { + // eslint-disable-next-line + new Request("http://asd", { + method: "PUT", + body: stream, + keepalive: true, + }); + } catch (err) { + expect((err as Error).message).toBe("keepalive"); + } + reader.cancel(); +}); + +test("post FormData with Blob", async () => { + const body = new FormData(); + body.append("field1", new Blob(["asd1"])); + + await using server = createServer((req, res) => { + req.pipe(res); + }).listen(0); + await once(server, "listening"); + + const res = await fetch(`http://localhost:${server.address().port}`, { + method: "PUT", + body, + }); + expect(/asd1/.test(await res.text())).toBeTruthy(); +}); + +test("post FormData with File", async () => { + const body = new FormData(); + body.append("field1", new File(["asd1"], "filename123")); + + await using server = createServer((req, res) => { + req.pipe(res); + }).listen(0); + await once(server, "listening"); + + const res = await fetch(`http://localhost:${server.address().port}`, { + method: "PUT", + body, + }); + const result = await res.text(); + expect(/asd1/.test(result)).toBeTrue(); + expect(/filename123/.test(result)).toBeTrue(); +}); + +test("invalid url", async () => { + try { + await fetch("http://invalid"); + } catch (e) { + expect(e.message).toBe("Unable to connect. Is the computer able to access the url?"); + } +}); + +test("do not decode redirect body", async () => { + const obj = { asd: true }; + await using server = createServer((req, res) => { + if (req.url === "/resource") { + res.statusCode = 301; + res.setHeader("location", "/resource/"); + // Some dumb http servers set the content-encoding gzip + // even if there is no response + res.setHeader("content-encoding", "gzip"); + res.end(); + return; + } + res.setHeader("content-encoding", "gzip"); + res.end(gzipSync(JSON.stringify(obj))); + }).listen(0); + await once(server, "listening"); + const body = await fetch(`http://localhost:${server.address().port}/resource`); + expect(JSON.stringify(obj)).toBe(await body.text()); +}); + +test("decode non-redirect body with location header", async () => { + const obj = { asd: true }; + await using server = createServer((req, res) => { + res.statusCode = 201; + res.setHeader("location", "/resource/"); + res.setHeader("content-encoding", "gzip"); + res.end(gzipSync(JSON.stringify(obj))); + }).listen(0); + await once(server, "listening"); + + const body = await fetch(`http://localhost:${server.address().port}/resource`); + expect(JSON.stringify(obj)).toBe(await body.text()); +}); + +test("error on redirect", async () => { + await using server = createServer((req, res) => { + res.statusCode = 302; + res.end(); + }).listen(0); + await once(server, "listening"); + + expect( + fetch(`http://localhost:${server.address().port}`, { + redirect: "error", + }), + ).rejects.toThrow(/UnexpectedRedirect/); +}); + +test("Receiving non-Latin1 headers", async () => { + const ContentDisposition = [ + "inline; filename=rock&roll.png", + "inline; filename=\"rock'n'roll.png\"", + "inline; filename=\"image â\x80\x94 copy (1).png\"; filename*=UTF-8''image%20%E2%80%94%20copy%20(1).png", + "inline; filename=\"_å\x9C\x96ç\x89\x87_ð\x9F\x96¼_image_.png\"; filename*=UTF-8''_%E5%9C%96%E7%89%87_%F0%9F%96%BC_image_.png", + "inline; filename=\"100 % loading&perf.png\"; filename*=UTF-8''100%20%25%20loading%26perf.png", + ]; + + await using server = createServer((req, res) => { + for (let i = 0; i < ContentDisposition.length; i++) { + res.setHeader(`Content-Disposition-${i + 1}`, ContentDisposition[i]); + } + + res.end(); + }).listen(0); + await once(server, "listening"); + + const url = `http://localhost:${server.address().port}`; + const response = await fetch(url, { method: "HEAD" }); + const cdHeaders = [...response.headers].filter(([k]) => k.startsWith("content-disposition")).map(([, v]) => v); + const lengths = cdHeaders.map(h => h.length); + + expect(cdHeaders).toEqual(ContentDisposition); + expect(lengths).toEqual([30, 34, 94, 104, 90]); +}); + +// https://github.com/nodejs/undici/issues/1527 +test("fetching with Request object - issue #1527", async () => { + const server = createServer((req, res) => { + res.end(); + }).listen(0); + try { + await once(server, "listening"); + + const body = JSON.stringify({ foo: "bar" }); + const request = new Request(`http://localhost:${server.address().port}`, { + method: "POST", + body, + }); + + expect(fetch(request)).resolves.pass(); + } finally { + server.closeAllConnections(); + } +}); diff --git a/test/js/web/fetch/content-length.test.ts b/test/js/web/fetch/content-length.test.ts new file mode 100644 index 0000000000..955999e75c --- /dev/null +++ b/test/js/web/fetch/content-length.test.ts @@ -0,0 +1,25 @@ +import { Blob } from "node:buffer"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import { expect, test } from "bun:test"; + +// https://github.com/nodejs/undici/issues/1783 +test("Content-Length is set when using a FormData body with fetch", async () => { + await using server = createServer((req, res) => { + // TODO: check the length's value once the boundary has a fixed length + expect("content-length" in req.headers).toBeTrue(); // request has content-length header + expect(Number.isNaN(Number(req.headers["content-length"]))).toBeFalse(); // content-length is a number + res.end(); + }).listen(0); + + await once(server, "listening"); + + const fd = new FormData(); + fd.set("file", new Blob(["hello world 👋"], { type: "text/plain" }), "readme.md"); + fd.set("string", "some string value"); + + await fetch(`http://localhost:${server.address().port}`, { + method: "POST", + body: fd, + }); +}); diff --git a/test/js/web/fetch/cookies.test.ts b/test/js/web/fetch/cookies.test.ts new file mode 100644 index 0000000000..92491abdbb --- /dev/null +++ b/test/js/web/fetch/cookies.test.ts @@ -0,0 +1,89 @@ +"use strict"; + +import { createServer } from "node:http"; +import { once } from "node:events"; +import { test, expect } from "bun:test"; + +test("Can receive set-cookie headers from a server using fetch - issue #1262", async () => { + await using server = createServer((req, res) => { + res.setHeader("set-cookie", "name=value; Domain=example.com"); + res.end(); + }).listen(0); + + await once(server, "listening"); + + const response = await fetch(`http://localhost:${server.address().port}`); + + expect(response.headers.get("set-cookie")).toBe("name=value; Domain=example.com"); + + const response2 = await fetch(`http://localhost:${server.address().port}`, { + credentials: "include", + }); + + expect(response2.headers.get("set-cookie")).toBe("name=value; Domain=example.com"); +}); + +test("Can send cookies to a server with fetch - issue #1463", async () => { + await using server = createServer((req, res) => { + expect(req.headers.cookie).toBe("value"); + res.end(); + }).listen(0); + + await once(server, "listening"); + + const headersInit = [new Headers([["cookie", "value"]]), { cookie: "value" }, [["cookie", "value"]]]; + + for (const headers of headersInit) { + await fetch(`http://localhost:${server.address().port}`, { headers }); + } +}); + +test("Cookie header is delimited with a semicolon rather than a comma - issue #1905", async () => { + await using server = createServer((req, res) => { + expect(req.headers.cookie).toBe("FOO=lorem-ipsum-dolor-sit-amet; BAR=the-quick-brown-fox"); + res.end(); + }).listen(0); + + await once(server, "listening"); + + await fetch(`http://localhost:${server.address().port}`, { + headers: [ + ["cookie", "FOO=lorem-ipsum-dolor-sit-amet"], + ["cookie", "BAR=the-quick-brown-fox"], + ], + }); +}); + +test.todo("Can receive set-cookie headers from a http2 server using fetch - issue #2885", async t => { + // const server = createSecureServer(pem); + // server.on("stream", async (stream, headers) => { + // stream.respond({ + // "content-type": "text/plain; charset=utf-8", + // "x-method": headers[":method"], + // "set-cookie": "Space=Cat; Secure; HttpOnly", + // ":status": 200, + // }); + // stream.end("test"); + // }); + // server.listen(); + // await once(server, "listening"); + // const client = new Client(`https://localhost:${server.address().port}`, { + // connect: { + // rejectUnauthorized: false, + // }, + // allowH2: true, + // }); + // const response = await fetch( + // `https://localhost:${server.address().port}/`, + // // Needs to be passed to disable the reject unauthorized + // { + // method: "GET", + // dispatcher: client, + // headers: { + // "content-type": "text-plain", + // }, + // }, + // ); + // t.after(closeClientAndServerAsPromise(client, server)); + // assert.deepStrictEqual(response.headers.getSetCookie(), ["Space=Cat; Secure; HttpOnly"]); +}); diff --git a/test/js/web/fetch/encoding.test.ts b/test/js/web/fetch/encoding.test.ts new file mode 100644 index 0000000000..6b24709c41 --- /dev/null +++ b/test/js/web/fetch/encoding.test.ts @@ -0,0 +1,53 @@ +import { test, expect } from "bun:test"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import { createBrotliCompress, createGzip, createDeflate } from "node:zlib"; + +test.todo("content-encoding header is case-iNsENsITIve", async () => { + const contentCodings = "GZiP, bR"; + const text = "Hello, World!"; + + await using server = createServer((req, res) => { + const gzip = createGzip(); + const brotli = createBrotliCompress(); + + res.setHeader("Content-Encoding", contentCodings); + res.setHeader("Content-Type", "text/plain"); + + gzip.pipe(brotli).pipe(res); + + gzip.write(text); + gzip.end(); + }).listen(0); + + await once(server, "listening"); + + const response = await fetch(`http://localhost:${server.address().port}`); + + expect(await response.text()).toBe(text); + expect(response.headers.get("content-encoding")).toBe(contentCodings); +}); + +test.todo("response decompression according to content-encoding should be handled in a correct order", async () => { + const contentCodings = "deflate, gzip"; + const text = "Hello, World!"; + + await using server = createServer((req, res) => { + const gzip = createGzip(); + const deflate = createDeflate(); + + res.setHeader("Content-Encoding", contentCodings); + res.setHeader("Content-Type", "text/plain"); + + deflate.pipe(gzip).pipe(res); + + deflate.write(text); + deflate.end(); + }).listen(0); + + await once(server, "listening"); + + const response = await fetch(`http://localhost:${server.address().port}`); + + expect(await response.text()).toBe(text); +}); diff --git a/test/js/web/fetch/exiting.test.ts b/test/js/web/fetch/exiting.test.ts new file mode 100644 index 0000000000..86e3bc37a4 --- /dev/null +++ b/test/js/web/fetch/exiting.test.ts @@ -0,0 +1,28 @@ +import { test, expect } from "bun:test"; +import { createServer } from "node:http"; +import { once } from "node:events"; +test.todo("abort the request on the other side if the stream is canceled", async () => { + const { promise: abort, resolve: resolveAbort } = Promise.withResolvers(); + await using server = createServer((req, res) => { + res.writeHead(200); + res.write("hello"); + req.on("aborted", resolveAbort); + // Let's not end the response on purpose + }).listen(0); + await once(server, "listening"); + + const url = new URL(`http://127.0.0.1:${server.address().port}`); + + const response = await fetch(url); + + const reader = response.body.getReader(); + + try { + await reader.read(); + } finally { + reader.releaseLock(); + await response.body.cancel(); + } + + await abort; +}); diff --git a/test/js/web/fetch/fetch-leak.test.ts b/test/js/web/fetch/fetch-leak.test.ts new file mode 100644 index 0000000000..b4a04f51cb --- /dev/null +++ b/test/js/web/fetch/fetch-leak.test.ts @@ -0,0 +1,42 @@ +import { once } from "node:events"; +import { createServer } from "node:http"; +import { test, expect } from "bun:test"; +import { gc } from "harness"; + +test("do not leak", async () => { + await using server = createServer((req, res) => { + res.end(); + }).listen(0); + await once(server, "listening"); + + let url; + let isDone = false; + server.listen(0, function attack() { + if (isDone) { + return; + } + url ??= new URL(`http://127.0.0.1:${server.address().port}`); + const controller = new AbortController(); + fetch(url, { signal: controller.signal }) + .then(res => res.arrayBuffer()) + .catch(() => {}) + .then(attack); + }); + + let prev = Infinity; + let count = 0; + const interval = setInterval(() => { + isDone = true; + gc(); + const next = process.memoryUsage().heapUsed; + if (next <= prev) { + expect(true).toBe(true); + clearInterval(interval); + } else if (count++ > 20) { + clearInterval(interval); + expect.unreachable(); + } else { + prev = next; + } + }, 1e3); +}); diff --git a/test/js/web/fetch/fetch-url-after-redirect.test.ts b/test/js/web/fetch/fetch-url-after-redirect.test.ts new file mode 100644 index 0000000000..46ebcdc67c --- /dev/null +++ b/test/js/web/fetch/fetch-url-after-redirect.test.ts @@ -0,0 +1,55 @@ +import { once } from "node:events"; +import { createServer } from "node:http"; +import { promisify } from "node:util"; +import { test, expect } from "bun:test"; + +test("after redirecting the url of the response is set to the target url", async () => { + // redirect-1 -> redirect-2 -> target + await using server = createServer((req, res) => { + switch (res.req.url) { + case "/redirect-1": + res.writeHead(302, undefined, { Location: "/redirect-2" }); + res.end(); + break; + case "/redirect-2": + res.writeHead(302, undefined, { Location: "/redirect-3" }); + res.end(); + break; + case "/redirect-3": + res.writeHead(302, undefined, { Location: "/target" }); + res.end(); + break; + case "/target": + res.writeHead(200, "dummy", { "Content-Type": "text/plain" }); + res.end(); + break; + } + }); + + const listenAsync = promisify(server.listen.bind(server)); + await listenAsync(0); + const { port } = server.address(); + const response = await fetch(`http://127.0.0.1:${port}/redirect-1`); + + expect(response.url).toBe(`http://127.0.0.1:${port}/target`); +}); + +test("location header with non-ASCII character redirects to a properly encoded url", async () => { + // redirect -> %EC%95%88%EB%85%95 (안녕), not %C3%AC%C2%95%C2%88%C3%AB%C2%85%C2%95 + await using server = createServer((req, res) => { + if (res.req.url.endsWith("/redirect")) { + res.writeHead(302, undefined, { Location: `/${Buffer.from("안녕").toString("binary")}` }); + res.end(); + } else { + res.writeHead(200, "dummy", { "Content-Type": "text/plain" }); + res.end(); + } + }); + + const listenAsync = promisify(server.listen.bind(server)); + await listenAsync(0); + const { port } = server.address(); + const response = await fetch(`http://127.0.0.1:${port}/redirect`); + + expect(response.url).toBe(`http://127.0.0.1:${port}/${encodeURIComponent("안녕")}`); +}); diff --git a/test/js/web/fetch/headers-case.test.ts b/test/js/web/fetch/headers-case.test.ts new file mode 100644 index 0000000000..63446c6b39 --- /dev/null +++ b/test/js/web/fetch/headers-case.test.ts @@ -0,0 +1,26 @@ +"use strict"; + +import { createServer } from "node:http"; +import { once } from "node:events"; +import { test, expect } from "bun:test"; + +test.todo("Headers retain keys case-sensitive", async () => { + await using server = createServer((req, res) => { + expect(req.rawHeaders.includes("Content-Type")).toBe(true); + + res.end(); + }).listen(0); + + await once(server, "listening"); + + const url = `http://localhost:${server.address().port}`; + for (const headers of [ + new Headers([["Content-Type", "text/plain"]]), + { "Content-Type": "text/plain" }, + [["Content-Type", "text/plain"]], + ]) { + await fetch(url, { headers }); + } + // see https://github.com/nodejs/undici/pull/3183 + await fetch(new Request(url, { headers: [["Content-Type", "text/plain"]] }), { method: "GET" }); +}); diff --git a/test/js/web/fetch/headers.undici.test.ts b/test/js/web/fetch/headers.undici.test.ts new file mode 100644 index 0000000000..915d045dcc --- /dev/null +++ b/test/js/web/fetch/headers.undici.test.ts @@ -0,0 +1,685 @@ +import { expect, test, describe } from "bun:test"; +import { createServer } from "node:http"; +import { once } from "node:events"; + +describe("Headers initialization", () => { + test("allows undefined", () => { + expect(() => new Headers()).not.toThrow(); + }); + + describe("with array of header entries", () => { + test("fails on invalid array-based init", () => { + expect(() => new Headers([["undici", "fetch"], ["fetch"]])).toThrow(TypeError); + expect(() => new Headers(["undici", "fetch", "fetch"])).toThrow(TypeError); + expect(() => new Headers([0, 1, 2])).toThrow(TypeError); + }); + + test("allows even length init", () => { + const init = [ + ["undici", "fetch"], + ["fetch", "undici"], + ]; + expect(() => new Headers(init)).not.toThrow(); + }); + + test("fails for event flattened init", () => { + const init = ["undici", "fetch", "fetch", "undici"]; + expect(() => new Headers(init)).toThrow(TypeError); + }); + }); + + test("with object of header entries", () => { + const init = { + undici: "fetch", + fetch: "undici", + }; + expect(() => new Headers(init)).not.toThrow(); + }); + + test("fails silently if a boxed primitive object is passed", () => { + /* eslint-disable no-new-wrappers */ + expect(() => new Headers(new Number())).not.toThrow(); + expect(() => new Headers(new Boolean())).not.toThrow(); + expect(() => new Headers(new String())).not.toThrow(); + /* eslint-enable no-new-wrappers */ + }); + + test("fails if primitive is passed", () => { + const expectedTypeError = TypeError; + expect(() => new Headers(1)).toThrow(expectedTypeError); + expect(() => new Headers("1")).toThrow(expectedTypeError); + }); + + test("allows some weird stuff (because of webidl)", () => { + expect(() => { + new Headers(function () {}); // eslint-disable-line no-new + }).not.toThrow(); + + expect(() => { + new Headers(Function); // eslint-disable-line no-new + }).not.toThrow(); + }); + + test("allows a myriad of header values to be passed", () => { + // Headers constructor uses Headers.append + + expect(() => { + new Headers([ + ["a", ["b", "c"]], + ["d", ["e", "f"]], + ]); + }).not.toThrow(); + expect(() => new Headers([["key", null]])).not.toThrow(); // allow null values + expect(() => new Headers([["key"]])).toThrow(); + expect(() => new Headers([["key", "value", "value2"]])).toThrow(); + }); + + test("accepts headers as objects with array values", () => { + const headers = new Headers({ + c: "5", + b: ["3", "4"], + a: ["1", "2"], + }); + + expect([...headers.entries()]).toEqual([ + ["a", "1,2"], + ["b", "3,4"], + ["c", "5"], + ]); + }); +}); + +describe("Headers append", () => { + test("adds valid header entry to instance", () => { + const headers = new Headers(); + + const name = "undici"; + const value = "fetch"; + expect(() => headers.append(name, value)).not.toThrow(); + expect(headers.get(name)).toBe(value); + }); + + test("adds valid header to existing entry", () => { + const headers = new Headers(); + + const name = "undici"; + const value1 = "fetch1"; + const value2 = "fetch2"; + const value3 = "fetch3"; + headers.append(name, value1); + expect(headers.get(name)).toBe(value1); + expect(() => headers.append(name, value2)).not.toThrow(); + expect(() => headers.append(name, value3)).not.toThrow(); + expect(headers.get(name)).toEqual([value1, value2, value3].join(", ")); + }); + + test("throws on invalid entry", () => { + const headers = new Headers(); + + expect(() => headers.append()).toThrow(); + expect(() => headers.append("undici")).toThrow(); + expect(() => headers.append("invalid @ header ? name", "valid value")).toThrow(); + }); +}); + +describe("Headers delete", () => { + test("deletes valid header entry from instance", () => { + const headers = new Headers(); + + const name = "undici"; + const value = "fetch"; + headers.append(name, value); + expect(headers.get(name)).toBe(value); + expect(() => headers.delete(name)).not.toThrow(); + expect(headers.get(name)).toBeNull(); + }); + + test("does not mutate internal list when no match is found", () => { + const headers = new Headers(); + const name = "undici"; + const value = "fetch"; + headers.append(name, value); + expect(headers.get(name)).toBe(value); + expect(() => headers.delete("not-undici")).not.toThrow(); + expect(headers.get(name)).toBe(value); + }); + + test("throws on invalid entry", () => { + const headers = new Headers(); + + expect(() => headers.delete()).toThrow(); + expect(() => headers.delete("invalid @ header ? name")).toThrow(); + }); + + // https://github.com/nodejs/undici/issues/2429 + test("`Headers#delete` returns undefined", () => { + const headers = new Headers({ test: "test" }); + + expect(headers.delete("test")).toBeUndefined(); + expect(headers.delete("test2")).toBeUndefined(); + }); +}); + +describe("Headers get", () => { + test("returns null if not found in instance", () => { + const headers = new Headers(); + headers.append("undici", "fetch"); + + expect(headers.get("not-undici")).toBeNull(); + }); + + test("returns header values from valid header name", () => { + const headers = new Headers(); + + const name = "undici"; + const value1 = "fetch1"; + const value2 = "fetch2"; + headers.append(name, value1); + expect(headers.get(name)).toBe(value1); + headers.append(name, value2); + expect(headers.get(name)).toEqual([value1, value2].join(", ")); + }); + + test("throws on invalid entry", () => { + const headers = new Headers(); + + expect(() => headers.get()).toThrow(); + expect(() => headers.get("invalid @ header ? name")).toThrow(); + }); +}); + +describe("Headers has", () => { + test("returns boolean existence for a header name", () => { + const headers = new Headers(); + + const name = "undici"; + headers.append("not-undici", "fetch"); + expect(headers.has(name)).toBe(false); + headers.append(name, "fetch"); + expect(headers.has(name)).toBe(true); + }); + + test("throws on invalid entry", () => { + const headers = new Headers(); + + expect(() => headers.has()).toThrow(); + expect(() => headers.has("invalid @ header ? name")).toThrow(); + }); +}); + +describe("Headers set", async () => { + test("sets valid header entry to instance", () => { + const headers = new Headers(); + + const name = "undici"; + const value = "fetch"; + headers.append("not-undici", "fetch"); + expect(() => headers.set(name, value)).not.toThrow(); + expect(headers.get(name)).toBe(value); + }); + + test("overwrites existing entry", () => { + const headers = new Headers(); + + const name = "undici"; + const value1 = "fetch1"; + const value2 = "fetch2"; + expect(() => headers.set(name, value1)).not.toThrow(); + expect(headers.get(name)).toBe(value1); + expect(() => headers.set(name, value2)).not.toThrow(); + expect(headers.get(name)).toBe(value2); + }); + + test("allows setting a myriad of values", () => { + const headers = new Headers(); + + expect(() => headers.set("a", ["b", "c"])).not.toThrow(); + expect(() => headers.set("b", null)).not.toThrow(); + expect(() => headers.set("c")).toThrow(); + expect(() => headers.set("c", "d", "e")).not.toThrow(); + }); + + test("throws on invalid entry", () => { + const headers = new Headers(); + + expect(() => headers.set()).toThrow(); + expect(() => headers.set("undici")).toThrow(); + expect(() => headers.set("invalid @ header ? name", "valid value")).toThrow(); + }); + + // https://github.com/nodejs/undici/issues/2431 + test("`Headers#set` returns undefined", () => { + const headers = new Headers(); + + expect(headers.set("a", "b")).toBeUndefined(); + + expect(headers.set("c", "d") instanceof Map).toBe(false); + }); +}); + +describe("Headers forEach", async () => { + const headers = new Headers([ + ["a", "b"], + ["c", "d"], + ]); + + test("standard", () => { + expect(typeof headers.forEach).toBe("function"); + + headers.forEach((value, key, headerInstance) => { + expect(value === "b" || value === "d").toBeTrue(); + expect(key === "a" || key === "c").toBeTrue(); + expect(headers).toBe(headerInstance); + }); + }); + + test("with thisArg", () => { + const thisArg = { a: Math.random() }; + headers.forEach(function () { + expect(this).toBe(thisArg); + }, thisArg); + }); +}); + +describe("Headers as Iterable", () => { + test("should freeze values while iterating", () => { + const init = [ + ["foo", "123"], + ["bar", "456"], + ]; + const expected = [ + ["foo", "123"], + ["x-x-bar", "456"], + ]; + const headers = new Headers(init); + for (const [key, val] of headers) { + headers.delete(key); + headers.set(`x-${key}`, val); + } + expect([...headers]).toEqual(expected); + }); + + test("returns combined and sorted entries using .forEach()", () => { + const init = [ + ["a", "1"], + ["b", "2"], + ["c", "3"], + ["abc", "4"], + ["b", "5"], + ]; + const expected = [ + ["a", "1"], + ["abc", "4"], + ["b", "2, 5"], + ["c", "3"], + ]; + const headers = new Headers(init); + const that = {}; + let i = 0; + headers.forEach(function (value, key, _headers) { + expect(expected[i++]).toEqual([key, value]); + expect(this).toBe(that); + }, that); + }); + + test("returns combined and sorted entries using .entries()", () => { + const init = [ + ["a", "1"], + ["b", "2"], + ["c", "3"], + ["abc", "4"], + ["b", "5"], + ]; + const expected = [ + ["a", "1"], + ["abc", "4"], + ["b", "2, 5"], + ["c", "3"], + ]; + const headers = new Headers(init); + let i = 0; + for (const header of headers.entries()) { + expect(header).toEqual(expected[i++]); + } + }); + + test("returns combined and sorted keys using .keys()", () => { + const init = [ + ["a", "1"], + ["b", "2"], + ["c", "3"], + ["abc", "4"], + ["b", "5"], + ]; + const expected = ["a", "abc", "b", "c"]; + const headers = new Headers(init); + let i = 0; + for (const key of headers.keys()) { + expect(key).toEqual(expected[i++]); + } + }); + + test("returns combined and sorted values using .values()", () => { + const init = [ + ["a", "1"], + ["b", "2"], + ["c", "3"], + ["abc", "4"], + ["b", "5"], + ]; + const expected = ["1", "4", "2, 5", "3"]; + const headers = new Headers(init); + let i = 0; + for (const value of headers.values()) { + expect(value).toEqual(expected[i++]); + } + }); + + test("returns combined and sorted entries using for...of loop", () => { + const init = [ + ["a", "1"], + ["b", "2"], + ["c", "3"], + ["abc", "4"], + ["b", "5"], + ["d", ["6", "7"]], + ]; + const expected = [ + ["a", "1"], + ["abc", "4"], + ["b", "2, 5"], + ["c", "3"], + ["d", "6,7"], + ]; + let i = 0; + for (const header of new Headers(init)) { + expect(header).toEqual(expected[i++]); + } + }); + + test("validate append ordering", () => { + const headers = new Headers([ + ["b", "2"], + ["c", "3"], + ["e", "5"], + ]); + headers.append("d", "4"); + headers.append("a", "1"); + headers.append("f", "6"); + headers.append("c", "7"); + headers.append("abc", "8"); + + const expected = [ + ...new Map([ + ["a", "1"], + ["abc", "8"], + ["b", "2"], + ["c", "3, 7"], + ["d", "4"], + ["e", "5"], + ["f", "6"], + ]), + ]; + + expect([...headers]).toEqual(expected); + }); + + test("always use the same prototype Iterator", () => { + const HeadersIteratorNext = Function.call.bind(new Headers()[Symbol.iterator]().next); + + const init = [ + ["a", "1"], + ["b", "2"], + ]; + + const headers = new Headers(init); + const iterator = headers[Symbol.iterator](); + expect(HeadersIteratorNext(iterator)).toEqual({ value: init[0], done: false }); + expect(HeadersIteratorNext(iterator)).toEqual({ value: init[1], done: false }); + expect(HeadersIteratorNext(iterator)).toEqual({ value: undefined, done: true }); + }); +}); + +test("arg validation", () => { + const headers = new Headers(); + + // constructor + expect(() => { + // eslint-disable-next-line + new Headers(0); + }).toThrow(TypeError); + + // get [Symbol.toStringTag] + expect(() => { + Object.prototype.toString.call(Headers.prototype); + }).not.toThrow(); + + // toString + expect(() => { + Headers.prototype.toString.call(null); + }).not.toThrow(); + + // append + expect(() => { + Headers.prototype.append.call(null); + }).toThrow(TypeError); + expect(() => { + headers.append(); + }).toThrow(TypeError); + + // delete + expect(() => { + Headers.prototype.delete.call(null); + }).toThrow(TypeError); + expect(() => { + headers.delete(); + }).toThrow(TypeError); + + // get + expect(() => { + Headers.prototype.get.call(null); + }).toThrow(TypeError); + expect(() => { + headers.get(); + }).toThrow(TypeError); + + // has + expect(() => { + Headers.prototype.has.call(null); + }).toThrow(TypeError); + expect(() => { + headers.has(); + }).toThrow(TypeError); + + // set + expect(() => { + Headers.prototype.set.call(null); + }).toThrow(TypeError); + expect(() => { + headers.set(); + }).toThrow(TypeError); + + // forEach + expect(() => { + Headers.prototype.forEach.call(null); + }).toThrow(TypeError); + expect(() => { + headers.forEach(); + }).toThrow(TypeError); + expect(() => { + headers.forEach(1); + }).toThrow(TypeError); + + // inspect + expect(() => { + Headers.prototype[Symbol.for("nodejs.util.inspect.custom")].call(null); + }).toThrow(TypeError); +}); + +describe("function signature verification", async () => { + test("function length", () => { + expect(Headers.prototype.append.length, 2); + expect(Headers.prototype.constructor.length, 0); + expect(Headers.prototype.delete.length, 1); + expect(Headers.prototype.entries.length, 0); + expect(Headers.prototype.forEach.length, 1); + expect(Headers.prototype.get.length, 1); + expect(Headers.prototype.has.length, 1); + expect(Headers.prototype.keys.length, 0); + expect(Headers.prototype.set.length, 2); + expect(Headers.prototype.values.length, 0); + expect(Headers.prototype[Symbol.iterator].length, 0); + expect(Headers.prototype.toString.length, 0); + }); + + test("function equality", () => { + expect(Headers.prototype.entries, Headers.prototype[Symbol.iterator]); + expect(Headers.prototype.toString, Object.prototype.toString); + }); + + test("toString and Symbol.toStringTag", () => { + expect(Object.prototype.toString.call(Headers.prototype)).toBe("[object Headers]"); + expect(Headers.prototype[Symbol.toStringTag]).toBe("Headers"); + expect(Headers.prototype.toString.call(null)).toBe("[object Null]"); + }); +}); + +test("various init paths of Headers", () => { + const h1 = new Headers(); + const h2 = new Headers({}); + const h3 = new Headers(undefined); + expect([...h1.entries()].length).toBe(0); + expect([...h2.entries()].length).toBe(0); + expect([...h3.entries()].length).toBe(0); +}); + +test("invalid headers", () => { + expect(() => new Headers({ "abcdefghijklmnopqrstuvwxyz0123456789!#$%&'*+-.^_`|~": "test" })).not.toThrow(); + + const chars = '"(),/:;<=>?@[\\]{}'.split(""); + + for (const char of chars) { + expect(() => new Headers({ [char]: "test" })).toThrow(TypeError); + } + + for (const byte of ["\r", "\n", "\t", " ", String.fromCharCode(128), ""]) { + expect(() => { + new Headers().set(byte, "test"); + }).toThrow(TypeError); + } + + for (const byte of ["\0", "\r", "\n"]) { + expect(() => { + new Headers().set("a", `a${byte}b`); + }).toThrow(TypeError); + } + + expect(() => { + new Headers().set("a", "\r"); + }).not.toThrow(TypeError); + + expect(() => { + new Headers().set("a", "\n"); + }).not.toThrow(TypeError); + expect(() => { + new Headers().set("a", Symbol("symbol")); + }).toThrow(TypeError); +}); + +test("headers that might cause a ReDoS", () => { + expect(() => { + // This test will time out if the ReDoS attack is successful. + const headers = new Headers(); + const attack = "a" + "\t".repeat(500_000) + "\ta"; + headers.append("fhqwhgads", attack); + }).not.toThrow(TypeError); +}); + +describe("Headers.prototype.getSetCookie", () => { + test("Mutating the returned list does not affect the set-cookie list", () => { + const h = new Headers([ + ["set-cookie", "a=b"], + ["set-cookie", "c=d"], + ]); + + const old = h.getSetCookie(); + h.getSetCookie().push("oh=no"); + const now = h.getSetCookie(); + + expect(old).toEqual(now); + }); + + // https://github.com/nodejs/undici/issues/1935 + test("When Headers are cloned, so are the cookies (single entry)", async () => { + await using server = createServer((req, res) => { + res.setHeader("Set-Cookie", "test=onetwo"); + res.end("Hello World!"); + }).listen(0); + + await once(server, "listening"); + + const res = await fetch(`http://localhost:${server.address().port}`); + const entries = Object.fromEntries(res.headers.entries()); + + expect(res.headers.getSetCookie()).toEqual(["test=onetwo"]); + expect("set-cookie" in entries).toBeTrue(); + }); + + test("When Headers are cloned, so are the cookies (multiple entries)", async () => { + await using server = createServer((req, res) => { + res.setHeader("Set-Cookie", ["test=onetwo", "test=onetwothree"]); + res.end("Hello World!"); + }).listen(0); + + await once(server, "listening"); + + const res = await fetch(`http://localhost:${server.address().port}`); + const entries = Object.fromEntries(res.headers.entries()); + + expect(res.headers.getSetCookie()).toEqual(["test=onetwo", "test=onetwothree"]); + expect("set-cookie" in entries).toBeTrue(); + }); + + test("When Headers are cloned, so are the cookies (Headers constructor)", () => { + const headers = new Headers([ + ["set-cookie", "a"], + ["set-cookie", "b"], + ]); + + expect([...headers]).toEqual([...new Headers(headers)]); + }); +}); + +test("When the value is updated, update the cache", () => { + const expected = [ + ["a", "a"], + ["b", "b"], + ["c", "c"], + ]; + const headers = new Headers(expected); + expect([...headers]).toEqual(expected); + headers.append("d", "d"); + expect([...headers]).toEqual([...expected, ["d", "d"]]); +}); + +test("Symbol.iterator is only accessed once", () => { + let called = 0; + const dict = new Proxy( + {}, + { + get() { + called++; + + return function* () {}; + }, + }, + ); + + new Headers(dict); // eslint-disable-line no-new + expect(called).toBe(1); +}); + +test("Invalid Symbol.iterators", () => { + expect(() => new Headers({ [Symbol.iterator]: null })).toThrow(TypeError); + expect(() => new Headers({ [Symbol.iterator]: undefined })).toThrow(TypeError); +});