Support file: URLs in fetch (#3858)

* Support file: URLs in `fetch`

* Update url.zig

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>
This commit is contained in:
Jarred Sumner
2023-07-28 15:44:05 -07:00
committed by GitHub
parent e7c80b90b8
commit 7a1ebec26f
6 changed files with 299 additions and 201 deletions

View File

@@ -315,6 +315,11 @@ extern "C" void BunString__toWTFString(BunString* bunString)
}
}
extern "C" BunString URL__getFileURLString(BunString* filePath)
{
return Bun::toStringRef(WTF::URL::fileURLWithFileSystemPath(Bun::toWTFString(*filePath)).stringWithoutFragmentIdentifier());
}
extern "C" WTF::URL* URL__fromJS(EncodedJSValue encodedValue, JSC::JSGlobalObject* globalObject)
{
auto throwScope = DECLARE_THROW_SCOPE(globalObject->vm());

View File

@@ -4751,8 +4751,8 @@ pub const JSValue = enum(JSValueReprInt) {
/// It knows not to free it
/// This mimicks the implementation in JavaScriptCore's C++
pub inline fn ensureStillAlive(this: JSValue) void {
if (this.isEmpty() or this.isNumber() or this.isBoolean() or this.isUndefinedOrNull()) return;
std.mem.doNotOptimizeAway(@as(C_API.JSObjectRef, @ptrCast(this.asVoid())));
if (!this.isCell()) return;
std.mem.doNotOptimizeAway(this.asEncoded().asPtr);
}
pub inline fn asNullableVoid(this: JSValue) ?*anyopaque {
@@ -5391,12 +5391,19 @@ pub const URL = opaque {
extern fn URL__pathname(*URL) String;
extern fn URL__getHrefFromJS(JSValue, *JSC.JSGlobalObject) String;
extern fn URL__getHref(*String) String;
extern fn URL__getFileURLString(*String) String;
pub fn hrefFromString(str: bun.String) String {
JSC.markBinding(@src());
var input = str;
return URL__getHref(&input);
}
pub fn fileURLFromString(str: bun.String) String {
JSC.markBinding(@src());
var input = str;
return URL__getFileURLString(&input);
}
/// This percent-encodes the URL, punycode-encodes the hostname, and returns the result
/// If it fails, the tag is marked Dead
pub fn hrefFromJS(value: JSValue, globalObject: *JSC.JSGlobalObject) String {

View File

@@ -1034,6 +1034,7 @@ pub const Fetch = struct {
var hostname: ?[]u8 = null;
var url_proxy_buffer: []const u8 = undefined;
var is_file_url = false;
// TODO: move this into a DRYer implementation
// The status quo is very repetitive and very bug prone
@@ -1061,127 +1062,131 @@ pub const Fetch = struct {
err,
);
};
is_file_url = url.isFile();
url_proxy_buffer = url.href;
if (args.nextEat()) |options| {
if (options.isObject() or options.jsType() == .DOMWrapper) {
if (options.fastGet(ctx.ptr(), .method)) |method_| {
var slice_ = method_.toSlice(ctx.ptr(), getAllocator(ctx));
defer slice_.deinit();
method = Method.which(slice_.slice()) orelse .GET;
} else {
method = request.method;
}
if (options.fastGet(ctx.ptr(), .body)) |body__| {
if (Body.Value.fromJS(ctx.ptr(), body__)) |body_const| {
var body_value = body_const;
// TODO: buffer ReadableStream?
// we have to explicitly check for InternalBlob
body = body_value.useAsAnyBlob();
if (!is_file_url) {
if (args.nextEat()) |options| {
if (options.isObject() or options.jsType() == .DOMWrapper) {
if (options.fastGet(ctx.ptr(), .method)) |method_| {
var slice_ = method_.toSlice(ctx.ptr(), getAllocator(ctx));
defer slice_.deinit();
method = Method.which(slice_.slice()) orelse .GET;
} else {
// clean hostname if any
if (hostname) |host| {
bun.default_allocator.free(host);
}
// an error was thrown
return JSC.JSValue.jsUndefined();
method = request.method;
}
} else {
body = request.body.value.useAsAnyBlob();
}
if (options.fastGet(ctx.ptr(), .headers)) |headers_| {
if (headers_.as(FetchHeaders)) |headers__| {
if (headers__.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(headers__, bun.default_allocator, .{ .body = &body }) catch unreachable;
// TODO: make this one pass
} else if (FetchHeaders.createFromJS(ctx.ptr(), headers_)) |headers__| {
if (headers__.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(headers__, bun.default_allocator, .{ .body = &body }) catch unreachable;
headers__.deref();
} else if (request.headers) |head| {
if (head.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(head, bun.default_allocator, .{ .body = &body }) catch unreachable;
}
} else if (request.headers) |head| {
headers = Headers.from(head, bun.default_allocator, .{ .body = &body }) catch unreachable;
}
if (options.get(ctx, "timeout")) |timeout_value| {
if (timeout_value.isBoolean()) {
disable_timeout = !timeout_value.asBoolean();
} else if (timeout_value.isNumber()) {
disable_timeout = timeout_value.to(i32) == 0;
}
}
if (options.getOptionalEnum(ctx, "redirect", FetchRedirect) catch {
return .zero;
}) |redirect_value| {
redirect_type = redirect_value;
}
if (options.get(ctx, "keepalive")) |keepalive_value| {
if (keepalive_value.isBoolean()) {
disable_keepalive = !keepalive_value.asBoolean();
} else if (keepalive_value.isNumber()) {
disable_keepalive = keepalive_value.to(i32) == 0;
}
}
if (options.get(globalThis, "verbose")) |verb| {
verbose = verb.toBoolean();
}
if (options.get(globalThis, "signal")) |signal_arg| {
if (signal_arg.as(JSC.WebCore.AbortSignal)) |signal_| {
_ = signal_.ref();
signal = signal_;
}
}
if (options.get(globalThis, "proxy")) |proxy_arg| {
if (proxy_arg.isString() and proxy_arg.getLength(ctx) > 0) {
var href = JSC.URL.hrefFromJS(proxy_arg, globalThis);
if (href.tag == .Dead) {
const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, "fetch() proxy URL is invalid", .{}, ctx);
if (options.fastGet(ctx.ptr(), .body)) |body__| {
if (Body.Value.fromJS(ctx.ptr(), body__)) |body_const| {
var body_value = body_const;
// TODO: buffer ReadableStream?
// we have to explicitly check for InternalBlob
body = body_value.useAsAnyBlob();
} else {
// clean hostname if any
if (hostname) |host| {
bun.default_allocator.free(host);
}
bun.default_allocator.free(url_proxy_buffer);
return JSPromise.rejectedPromiseValue(globalThis, err);
// an error was thrown
return JSC.JSValue.jsUndefined();
}
} else {
body = request.body.value.useAsAnyBlob();
}
if (options.fastGet(ctx.ptr(), .headers)) |headers_| {
if (headers_.as(FetchHeaders)) |headers__| {
if (headers__.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(headers__, bun.default_allocator, .{ .body = &body }) catch unreachable;
// TODO: make this one pass
} else if (FetchHeaders.createFromJS(ctx.ptr(), headers_)) |headers__| {
if (headers__.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(headers__, bun.default_allocator, .{ .body = &body }) catch unreachable;
headers__.deref();
} else if (request.headers) |head| {
if (head.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(head, bun.default_allocator, .{ .body = &body }) catch unreachable;
}
} else if (request.headers) |head| {
headers = Headers.from(head, bun.default_allocator, .{ .body = &body }) catch unreachable;
}
if (options.get(ctx, "timeout")) |timeout_value| {
if (timeout_value.isBoolean()) {
disable_timeout = !timeout_value.asBoolean();
} else if (timeout_value.isNumber()) {
disable_timeout = timeout_value.to(i32) == 0;
}
}
if (options.getOptionalEnum(ctx, "redirect", FetchRedirect) catch {
return .zero;
}) |redirect_value| {
redirect_type = redirect_value;
}
if (options.get(ctx, "keepalive")) |keepalive_value| {
if (keepalive_value.isBoolean()) {
disable_keepalive = !keepalive_value.asBoolean();
} else if (keepalive_value.isNumber()) {
disable_keepalive = keepalive_value.to(i32) == 0;
}
}
if (options.get(globalThis, "verbose")) |verb| {
verbose = verb.toBoolean();
}
if (options.get(globalThis, "signal")) |signal_arg| {
if (signal_arg.as(JSC.WebCore.AbortSignal)) |signal_| {
_ = signal_.ref();
signal = signal_;
}
}
if (options.get(globalThis, "proxy")) |proxy_arg| {
if (proxy_arg.isString() and proxy_arg.getLength(ctx) > 0) {
var href = JSC.URL.hrefFromJS(proxy_arg, globalThis);
if (href.tag == .Dead) {
const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, "fetch() proxy URL is invalid", .{}, ctx);
// clean hostname if any
if (hostname) |host| {
bun.default_allocator.free(host);
}
bun.default_allocator.free(url_proxy_buffer);
return JSPromise.rejectedPromiseValue(globalThis, err);
}
defer href.deref();
var buffer = std.fmt.allocPrint(bun.default_allocator, "{s}{}", .{ url_proxy_buffer, href }) catch {
globalThis.throwOutOfMemory();
return .zero;
};
url = ZigURL.parse(buffer[0..url.href.len]);
is_file_url = url.isFile();
proxy = ZigURL.parse(buffer[url.href.len..]);
bun.default_allocator.free(url_proxy_buffer);
url_proxy_buffer = buffer;
}
defer href.deref();
var buffer = std.fmt.allocPrint(bun.default_allocator, "{s}{}", .{ url_proxy_buffer, href }) catch {
globalThis.throwOutOfMemory();
return .zero;
};
url = ZigURL.parse(buffer[0..url.href.len]);
proxy = ZigURL.parse(buffer[url.href.len..]);
bun.default_allocator.free(url_proxy_buffer);
url_proxy_buffer = buffer;
}
}
}
} else {
method = request.method;
body = request.body.value.useAsAnyBlob();
if (request.headers) |head| {
if (head.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
} else {
method = request.method;
body = request.body.value.useAsAnyBlob();
if (request.headers) |head| {
if (head.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(head, bun.default_allocator, .{ .body = &body }) catch unreachable;
}
if (request.signal) |signal_| {
_ = signal_.ref();
signal = signal_;
}
headers = Headers.from(head, bun.default_allocator, .{ .body = &body }) catch unreachable;
}
if (request.signal) |signal_| {
_ = signal_.ref();
signal = signal_;
}
}
} else if (bun.String.tryFromJS(first_arg, globalThis)) |str| {
@@ -1203,105 +1208,108 @@ pub const Fetch = struct {
return JSPromise.rejectedPromiseValue(globalThis, err);
};
url_proxy_buffer = url.href;
is_file_url = url.isFile();
if (args.nextEat()) |options| {
if (options.isObject() or options.jsType() == .DOMWrapper) {
if (options.fastGet(ctx.ptr(), .method)) |method_| {
var slice_ = method_.toSlice(ctx.ptr(), getAllocator(ctx));
defer slice_.deinit();
method = Method.which(slice_.slice()) orelse .GET;
}
if (options.fastGet(ctx.ptr(), .body)) |body__| {
if (Body.Value.fromJS(ctx.ptr(), body__)) |body_const| {
var body_value = body_const;
// TODO: buffer ReadableStream?
// we have to explicitly check for InternalBlob
body = body_value.useAsAnyBlob();
} else {
// clean hostname if any
if (hostname) |host| {
bun.default_allocator.free(host);
}
// an error was thrown
return JSC.JSValue.jsUndefined();
if (!is_file_url) {
if (args.nextEat()) |options| {
if (options.isObject() or options.jsType() == .DOMWrapper) {
if (options.fastGet(ctx.ptr(), .method)) |method_| {
var slice_ = method_.toSlice(ctx.ptr(), getAllocator(ctx));
defer slice_.deinit();
method = Method.which(slice_.slice()) orelse .GET;
}
}
if (options.fastGet(ctx.ptr(), .headers)) |headers_| {
if (headers_.as(FetchHeaders)) |headers__| {
if (headers__.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(headers__, bun.default_allocator, .{ .body = &body }) catch unreachable;
// TODO: make this one pass
} else if (FetchHeaders.createFromJS(ctx.ptr(), headers_)) |headers__| {
defer headers__.deref();
if (headers__.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(headers__, bun.default_allocator, .{ .body = &body }) catch unreachable;
} else {
// Converting the headers failed; return null and
// let the set exception get thrown
return .zero;
}
}
if (options.get(ctx, "timeout")) |timeout_value| {
if (timeout_value.isBoolean()) {
disable_timeout = !timeout_value.asBoolean();
} else if (timeout_value.isNumber()) {
disable_timeout = timeout_value.to(i32) == 0;
}
}
if (options.getOptionalEnum(ctx, "redirect", FetchRedirect) catch {
return .zero;
}) |redirect_value| {
redirect_type = redirect_value;
}
if (options.get(ctx, "keepalive")) |keepalive_value| {
if (keepalive_value.isBoolean()) {
disable_keepalive = !keepalive_value.asBoolean();
} else if (keepalive_value.isNumber()) {
disable_keepalive = keepalive_value.to(i32) == 0;
}
}
if (options.get(globalThis, "verbose")) |verb| {
verbose = verb.toBoolean();
}
if (options.get(globalThis, "signal")) |signal_arg| {
if (signal_arg.as(JSC.WebCore.AbortSignal)) |signal_| {
_ = signal_.ref();
signal = signal_;
}
}
if (options.getTruthy(globalThis, "proxy")) |proxy_arg| {
if (proxy_arg.isString() and proxy_arg.getLength(globalThis) > 0) {
var href = JSC.URL.hrefFromJS(proxy_arg, globalThis);
if (href.tag == .Dead) {
const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, "fetch() proxy URL is invalid", .{}, ctx);
if (options.fastGet(ctx.ptr(), .body)) |body__| {
if (Body.Value.fromJS(ctx.ptr(), body__)) |body_const| {
var body_value = body_const;
// TODO: buffer ReadableStream?
// we have to explicitly check for InternalBlob
body = body_value.useAsAnyBlob();
} else {
// clean hostname if any
if (hostname) |host| {
bun.default_allocator.free(host);
}
bun.default_allocator.free(url_proxy_buffer);
return JSPromise.rejectedPromiseValue(globalThis, err);
// an error was thrown
return JSC.JSValue.jsUndefined();
}
defer href.deref();
var buffer = std.fmt.allocPrint(bun.default_allocator, "{s}{}", .{ url_proxy_buffer, href }) catch {
globalThis.throwOutOfMemory();
}
if (options.fastGet(ctx.ptr(), .headers)) |headers_| {
if (headers_.as(FetchHeaders)) |headers__| {
if (headers__.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(headers__, bun.default_allocator, .{ .body = &body }) catch unreachable;
// TODO: make this one pass
} else if (FetchHeaders.createFromJS(ctx.ptr(), headers_)) |headers__| {
defer headers__.deref();
if (headers__.fastGet(JSC.FetchHeaders.HTTPHeaderName.Host)) |_hostname| {
hostname = _hostname.toOwnedSliceZ(bun.default_allocator) catch unreachable;
}
headers = Headers.from(headers__, bun.default_allocator, .{ .body = &body }) catch unreachable;
} else {
// Converting the headers failed; return null and
// let the set exception get thrown
return .zero;
};
url = ZigURL.parse(buffer[0..url.href.len]);
proxy = ZigURL.parse(buffer[url.href.len..]);
bun.default_allocator.free(url_proxy_buffer);
url_proxy_buffer = buffer;
}
}
if (options.get(ctx, "timeout")) |timeout_value| {
if (timeout_value.isBoolean()) {
disable_timeout = !timeout_value.asBoolean();
} else if (timeout_value.isNumber()) {
disable_timeout = timeout_value.to(i32) == 0;
}
}
if (options.getOptionalEnum(ctx, "redirect", FetchRedirect) catch {
return .zero;
}) |redirect_value| {
redirect_type = redirect_value;
}
if (options.get(ctx, "keepalive")) |keepalive_value| {
if (keepalive_value.isBoolean()) {
disable_keepalive = !keepalive_value.asBoolean();
} else if (keepalive_value.isNumber()) {
disable_keepalive = keepalive_value.to(i32) == 0;
}
}
if (options.get(globalThis, "verbose")) |verb| {
verbose = verb.toBoolean();
}
if (options.get(globalThis, "signal")) |signal_arg| {
if (signal_arg.as(JSC.WebCore.AbortSignal)) |signal_| {
_ = signal_.ref();
signal = signal_;
}
}
if (options.getTruthy(globalThis, "proxy")) |proxy_arg| {
if (proxy_arg.isString() and proxy_arg.getLength(globalThis) > 0) {
var href = JSC.URL.hrefFromJS(proxy_arg, globalThis);
if (href.tag == .Dead) {
const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, "fetch() proxy URL is invalid", .{}, ctx);
// clean hostname if any
if (hostname) |host| {
bun.default_allocator.free(host);
}
bun.default_allocator.free(url_proxy_buffer);
return JSPromise.rejectedPromiseValue(globalThis, err);
}
defer href.deref();
var buffer = std.fmt.allocPrint(bun.default_allocator, "{s}{}", .{ url_proxy_buffer, href }) catch {
globalThis.throwOutOfMemory();
return .zero;
};
url = ZigURL.parse(buffer[0..url.href.len]);
proxy = ZigURL.parse(buffer[url.href.len..]);
bun.default_allocator.free(url_proxy_buffer);
url_proxy_buffer = buffer;
}
}
}
}
@@ -1318,14 +1326,73 @@ pub const Fetch = struct {
return JSPromise.rejectedPromiseValue(globalThis, err);
}
// This is not 100% correct.
// We don't pass along headers, we ignore method, we ignore status code...
// But it's better than status quo.
if (is_file_url) {
defer bun.default_allocator.free(url_proxy_buffer);
var path_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
const PercentEncoding = @import("../../url.zig").PercentEncoding;
var path_buf2: [bun.MAX_PATH_BYTES]u8 = undefined;
var stream = std.io.fixedBufferStream(&path_buf2);
const url_path_decoded = path_buf2[0 .. PercentEncoding.decode(
@TypeOf(&stream.writer()),
&stream.writer(),
url.path,
) catch {
globalThis.throwOutOfMemory();
return .zero;
}];
const temp_file_path = bun.path.joinAbsStringBuf(
globalThis.bunVM().bundler.fs.top_level_dir,
&path_buf,
&[_]string{
globalThis.bunVM().main,
"../",
url_path_decoded,
},
.auto,
);
var file_url_string = JSC.URL.fileURLFromString(bun.String.fromUTF8(temp_file_path));
defer file_url_string.deref();
const bun_file = Blob.findOrCreateFileFromPath(
.{
.path = .{
.string = bun.PathString.init(
temp_file_path,
),
},
},
globalThis,
);
var response = bun.default_allocator.create(Response) catch @panic("out of memory");
response.* = Response{
.body = Body{
.init = Body.Init{
.status_code = 200,
},
.value = .{ .Blob = bun_file },
},
.allocator = bun.default_allocator,
.url = file_url_string.toOwnedSlice(bun.default_allocator) catch @panic("out of memory"),
};
return JSPromise.resolvedPromiseValue(globalThis, response.toJS(globalThis));
}
if (url.protocol.len > 0) {
if (!(url.isHTTP() or url.isHTTPS())) {
defer bun.default_allocator.free(url_proxy_buffer);
const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, "protocol must be http: or https:", .{}, ctx);
return JSPromise.rejectedPromiseValue(globalThis, err);
}
}
if (!method.hasRequestBody() and body.size() > 0) {
defer bun.default_allocator.free(url_proxy_buffer);
const err = JSC.toTypeError(.ERR_INVALID_ARG_VALUE, fetch_error_unexpected_body, .{}, ctx);
return JSPromise.rejectedPromiseValue(globalThis, err);
}

View File

@@ -36,6 +36,10 @@ pub const URL = struct {
username: string = "",
port_was_automatically_set: bool = false,
pub fn isFile(this: *const URL) bool {
return strings.eqlComptime(this.protocol, "file");
}
pub fn fromJS(js_value: JSC.JSValue, globalObject: *JSC.JSGlobalObject, allocator: std.mem.Allocator) !URL {
var href = JSC.URL.hrefFromJS(globalObject, js_value);
if (href.tag == .Dead) {
@@ -63,6 +67,10 @@ pub const URL = struct {
return this.hostname.len == 0 or strings.eqlComptime(this.hostname, "localhost") or strings.eqlComptime(this.hostname, "0.0.0.0");
}
pub inline fn isUnix(this: *const URL) bool {
return strings.hasPrefixComptime(this.protocol, "unix");
}
pub fn displayProtocol(this: *const URL) string {
if (this.protocol.len > 0) {
return this.protocol;

View File

@@ -1203,3 +1203,13 @@ it("new Request(https://example.com, otherRequest) uses url from left instead of
expect(req2.url).toBe("http://localhost/def");
expect(req2.headers.get("foo")).toBe("bar");
});
it("fetch() file:// works", async () => {
expect(await (await fetch(import.meta.url)).text()).toEqual(await Bun.file(import.meta.path).text());
expect(await (await fetch(new URL("fetch.test.ts", import.meta.url))).text()).toEqual(
await Bun.file(Bun.fileURLToPath(new URL("fetch.test.ts", import.meta.url))).text(),
);
expect(await (await fetch(new URL("file with space in the name.txt", import.meta.url))).text()).toEqual(
await Bun.file(Bun.fileURLToPath(new URL("file with space in the name.txt", import.meta.url))).text(),
);
});

View File

@@ -0,0 +1 @@
hello!