From 9ab6365a136c6be63ecb5d55426e16b968aced9e Mon Sep 17 00:00:00 2001 From: Prithvish Baidya Date: Tue, 6 Jan 2026 04:34:20 +0530 Subject: [PATCH] Add support for Requester Pays in S3 operations (#25514) - Introduced `requestPayer` option in S3-related functions and structures to handle Requester Pays buckets. - Updated S3 client methods to accept and propagate the `requestPayer` flag. - Enhanced documentation for the `requestPayer` option in the S3 type definitions. - Adjusted existing S3 operations to utilize the `requestPayer` parameter where applicable, ensuring compatibility with AWS S3's Requester Pays feature. - Ensured that the new functionality is integrated into multipart uploads and simple requests. ### What does this PR do? This change allows users to specify whether they are willing to pay for data transfer costs when accessing objects in Requester Pays buckets, improving flexibility and compliance with AWS S3's billing model. This closes #25499 ### How did you verify your code works? I have added a new test file to verify this functionality, and all my tests pass. I also tested this against an actual S3 bucket which can only be accessed if requester pays. I can confirm that it's accessible with `requestPayer` is `true`, and the default of `false` does not allow access. An example bucket is here: s3://hl-mainnet-evm-blocks/0/0/1.rmp.lz4 (my usecase is indexing [hyperliquid block data](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/hyperevm/raw-hyperevm-block-data) which is stored in s3, and I want to use bun to index faster) --------- Co-authored-by: Alistair Smith Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Ciro Spaciari --- packages/bun-types/s3.d.ts | 24 ++ src/bun.js/api/server/RequestContext.zig | 2 +- src/bun.js/rare_data.zig | 1 + src/bun.js/webcore/Blob.zig | 16 +- src/bun.js/webcore/ReadableStream.zig | 2 +- src/bun.js/webcore/S3Client.zig | 20 +- src/bun.js/webcore/S3File.zig | 34 +- src/bun.js/webcore/blob/Store.zig | 5 +- src/bun.js/webcore/fetch.zig | 3 +- src/s3/client.zig | 18 + src/s3/credentials.zig | 442 +++++++++-------------- src/s3/multipart.zig | 7 + src/s3/simple_request.zig | 2 + test/js/bun/s3/s3-requester-pays.test.ts | 251 +++++++++++++ 14 files changed, 528 insertions(+), 299 deletions(-) create mode 100644 test/js/bun/s3/s3-requester-pays.test.ts diff --git a/packages/bun-types/s3.d.ts b/packages/bun-types/s3.d.ts index 1a8c4a088f..8f376282ac 100644 --- a/packages/bun-types/s3.d.ts +++ b/packages/bun-types/s3.d.ts @@ -321,6 +321,30 @@ declare module "bun" { | "SNOW" | "STANDARD_IA"; + /** + * When set to `true`, confirms that the requester knows they will be charged + * for the request and data transfer costs. Required for accessing objects + * in Requester Pays buckets. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html + * + * @example + * // Accessing a file in a Requester Pays bucket + * const file = s3.file("data.csv", { + * bucket: "requester-pays-bucket", + * requestPayer: true + * }); + * const content = await file.text(); + * + * @example + * // Uploading to a Requester Pays bucket + * await s3.write("output.json", data, { + * bucket: "requester-pays-bucket", + * requestPayer: true + * }); + */ + requestPayer?: boolean; + /** * @deprecated The size of the internal buffer in bytes. Defaults to 5 MiB. use `partSize` and `queueSize` instead. */ diff --git a/src/bun.js/api/server/RequestContext.zig b/src/bun.js/api/server/RequestContext.zig index 282c90a3ad..4d74dcdeeb 100644 --- a/src/bun.js/api/server/RequestContext.zig +++ b/src/bun.js/api/server/RequestContext.zig @@ -1479,7 +1479,7 @@ pub fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, const path = blob.store.?.data.s3.path(); const env = globalThis.bunVM().transpiler.env; - S3.stat(credentials, path, @ptrCast(&onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null) catch {}; // TODO: properly propagate exception upwards + S3.stat(credentials, path, @ptrCast(&onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null, blob.store.?.data.s3.request_payer) catch {}; // TODO: properly propagate exception upwards return; } this.renderMetadata(); diff --git a/src/bun.js/rare_data.zig b/src/bun.js/rare_data.zig index 0315e60da3..734a2c59d9 100644 --- a/src/bun.js/rare_data.zig +++ b/src/bun.js/rare_data.zig @@ -487,6 +487,7 @@ pub fn s3DefaultClient(rare: *RareData, globalThis: *jsc.JSGlobalObject) jsc.JSV null, null, null, + false, globalThis, ) catch |err| switch (err) { error.OutOfMemory => bun.outOfMemory(), diff --git a/src/bun.js/webcore/Blob.zig b/src/bun.js/webcore/Blob.zig index 62ec61b635..4b340fb34d 100644 --- a/src/bun.js/webcore/Blob.zig +++ b/src/bun.js/webcore/Blob.zig @@ -972,6 +972,7 @@ fn writeFileWithEmptySourceToDestination(ctx: *jsc.JSGlobalObject, destination_b aws_options.acl, proxy_url, aws_options.storage_class, + aws_options.request_payer, Wrapper.resolve, Wrapper.new(.{ .promise = promise, @@ -1119,6 +1120,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, proxy_url, + aws_options.request_payer, null, undefined, ); @@ -1160,6 +1162,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl aws_options.acl, proxy_url, aws_options.storage_class, + aws_options.request_payer, Wrapper.resolve, Wrapper.new(.{ .store = source_store, @@ -1188,6 +1191,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, proxy_url, + aws_options.request_payer, null, undefined, ); @@ -1393,6 +1397,7 @@ pub fn writeFileInternal(globalThis: *jsc.JSGlobalObject, path_or_blob_: *PathOr destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, proxy_url, + aws_options.request_payer, null, undefined, ); @@ -1454,6 +1459,7 @@ pub fn writeFileInternal(globalThis: *jsc.JSGlobalObject, path_or_blob_: *PathOr destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, proxy_url, + aws_options.request_payer, null, undefined, ); @@ -2227,16 +2233,17 @@ const S3BlobDownloadTask = struct { const path = this.blob.store.?.data.s3.path(); this.poll_ref.ref(globalThis.bunVM()); + const s3_store = &this.blob.store.?.data.s3; if (blob.offset > 0) { const len: ?usize = if (blob.size != Blob.max_size) @intCast(blob.size) else null; const offset: usize = @intCast(blob.offset); - try S3.downloadSlice(credentials, path, offset, len, @ptrCast(&S3BlobDownloadTask.onS3DownloadResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null); + try S3.downloadSlice(credentials, path, offset, len, @ptrCast(&S3BlobDownloadTask.onS3DownloadResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null, s3_store.request_payer); } else if (blob.size == Blob.max_size) { - try S3.download(credentials, path, @ptrCast(&S3BlobDownloadTask.onS3DownloadResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null); + try S3.download(credentials, path, @ptrCast(&S3BlobDownloadTask.onS3DownloadResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null, s3_store.request_payer); } else { const len: usize = @intCast(blob.size); const offset: usize = @intCast(blob.offset); - try S3.downloadSlice(credentials, path, offset, len, @ptrCast(&S3BlobDownloadTask.onS3DownloadResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null); + try S3.downloadSlice(credentials, path, offset, len, @ptrCast(&S3BlobDownloadTask.onS3DownloadResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null, s3_store.request_payer); } return promise; } @@ -2410,6 +2417,7 @@ pub fn pipeReadableStreamToBlob(this: *Blob, globalThis: *jsc.JSGlobalObject, re this.contentTypeOrMimeType(), aws_options.content_disposition, proxy_url, + aws_options.request_payer, null, undefined, ); @@ -2654,6 +2662,7 @@ pub fn getWriter( if (content_disposition_str) |cd| cd.slice() else null, proxy_url, credentialsWithOptions.storage_class, + credentialsWithOptions.request_payer, ); } } @@ -2666,6 +2675,7 @@ pub fn getWriter( null, proxy_url, null, + s3.request_payer, ); } diff --git a/src/bun.js/webcore/ReadableStream.zig b/src/bun.js/webcore/ReadableStream.zig index b79bdb75da..4d8bfed8da 100644 --- a/src/bun.js/webcore/ReadableStream.zig +++ b/src/bun.js/webcore/ReadableStream.zig @@ -335,7 +335,7 @@ pub fn fromBlobCopyRef(globalThis: *JSGlobalObject, blob: *const Blob, recommend const proxy = globalThis.bunVM().transpiler.env.getHttpProxy(true, null); const proxy_url = if (proxy) |p| p.href else null; - return bun.S3.readableStream(credentials, path, blob.offset, if (blob.size != Blob.max_size) blob.size else null, proxy_url, globalThis); + return bun.S3.readableStream(credentials, path, blob.offset, if (blob.size != Blob.max_size) blob.size else null, proxy_url, s3.request_payer, globalThis); }, } } diff --git a/src/bun.js/webcore/S3Client.zig b/src/bun.js/webcore/S3Client.zig index 69cb2a8f6d..941b8afec6 100644 --- a/src/bun.js/webcore/S3Client.zig +++ b/src/bun.js/webcore/S3Client.zig @@ -88,18 +88,20 @@ pub const S3Client = struct { options: bun.S3.MultiPartUploadOptions = .{}, acl: ?bun.S3.ACL = null, storage_class: ?bun.S3.StorageClass = null, + request_payer: bool = false, pub fn constructor(globalThis: *jsc.JSGlobalObject, callframe: *jsc.CallFrame) bun.JSError!*@This() { const arguments = callframe.arguments_old(1).slice(); var args = jsc.CallFrame.ArgumentsSlice.init(globalThis.bunVM(), arguments); defer args.deinit(); - var aws_options = try S3Credentials.getCredentialsWithOptions(globalThis.bunVM().transpiler.env.getS3Credentials(), .{}, args.nextEat(), null, null, globalThis); + var aws_options = try S3Credentials.getCredentialsWithOptions(globalThis.bunVM().transpiler.env.getS3Credentials(), .{}, args.nextEat(), null, null, false, globalThis); defer aws_options.deinit(); return S3Client.new(.{ .credentials = aws_options.credentials.dupe(), .options = aws_options.options, .acl = aws_options.acl, .storage_class = aws_options.storage_class, + .request_payer = aws_options.request_payer, }); } @@ -135,7 +137,7 @@ pub const S3Client = struct { }; errdefer path.deinit(); const options = args.nextEat(); - var blob = Blob.new(try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class)); + var blob = Blob.new(try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class, ptr.request_payer)); return blob.toJS(globalThis); } @@ -152,7 +154,7 @@ pub const S3Client = struct { errdefer path.deinit(); const options = args.nextEat(); - var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class); + var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class, ptr.request_payer); defer blob.detach(); return S3File.getPresignUrlFrom(&blob, globalThis, options); } @@ -169,7 +171,7 @@ pub const S3Client = struct { }; errdefer path.deinit(); const options = args.nextEat(); - var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class); + var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class, ptr.request_payer); defer blob.detach(); return S3File.S3BlobStatTask.exists(globalThis, &blob); } @@ -186,7 +188,7 @@ pub const S3Client = struct { }; errdefer path.deinit(); const options = args.nextEat(); - var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class); + var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class, ptr.request_payer); defer blob.detach(); return S3File.S3BlobStatTask.size(globalThis, &blob); } @@ -203,7 +205,7 @@ pub const S3Client = struct { }; errdefer path.deinit(); const options = args.nextEat(); - var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class); + var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class, ptr.request_payer); defer blob.detach(); return S3File.S3BlobStatTask.stat(globalThis, &blob); } @@ -221,7 +223,7 @@ pub const S3Client = struct { }; const options = args.nextEat(); - var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class); + var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class, ptr.request_payer); defer blob.detach(); var blob_internal: PathOrBlob = .{ .blob = blob }; return Blob.writeFileInternal(globalThis, &blob_internal, data, .{ @@ -236,7 +238,7 @@ pub const S3Client = struct { const object_keys = args[0]; const options = args[1]; - var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, .{ .string = bun.PathString.empty }, options, ptr.credentials, ptr.options, null, null); + var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, .{ .string = bun.PathString.empty }, options, ptr.credentials, ptr.options, null, null, ptr.request_payer); defer blob.detach(); return blob.store.?.data.s3.listObjects(blob.store.?, globalThis, object_keys, options); @@ -251,7 +253,7 @@ pub const S3Client = struct { }; errdefer path.deinit(); const options = args.nextEat(); - var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class); + var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl, ptr.storage_class, ptr.request_payer); defer blob.detach(); return blob.store.?.data.s3.unlink(blob.store.?, globalThis, options); } diff --git a/src/bun.js/webcore/S3File.zig b/src/bun.js/webcore/S3File.zig index 9e424ed65a..7994be5cf9 100644 --- a/src/bun.js/webcore/S3File.zig +++ b/src/bun.js/webcore/S3File.zig @@ -251,8 +251,9 @@ pub fn constructS3FileWithS3CredentialsAndOptions( default_options: bun.S3.MultiPartUploadOptions, default_acl: ?bun.S3.ACL, default_storage_class: ?bun.S3.StorageClass, + default_request_payer: bool, ) bun.JSError!Blob { - var aws_options = try S3.S3Credentials.getCredentialsWithOptions(default_credentials.*, default_options, options, default_acl, default_storage_class, globalObject); + var aws_options = try S3.S3Credentials.getCredentialsWithOptions(default_credentials.*, default_options, options, default_acl, default_storage_class, default_request_payer, globalObject); defer aws_options.deinit(); const store = brk: { @@ -266,6 +267,7 @@ pub fn constructS3FileWithS3CredentialsAndOptions( store.data.s3.options = aws_options.options; store.data.s3.acl = aws_options.acl; store.data.s3.storage_class = aws_options.storage_class; + store.data.s3.request_payer = aws_options.request_payer; var blob = Blob.initWithStore(store, globalObject); if (options) |opts| { @@ -302,13 +304,14 @@ pub fn constructS3FileWithS3Credentials( options: ?jsc.JSValue, existing_credentials: S3.S3Credentials, ) bun.JSError!Blob { - var aws_options = try S3.S3Credentials.getCredentialsWithOptions(existing_credentials, .{}, options, null, null, globalObject); + var aws_options = try S3.S3Credentials.getCredentialsWithOptions(existing_credentials, .{}, options, null, null, false, globalObject); defer aws_options.deinit(); const store = bun.handleOom(Blob.Store.initS3(path, null, aws_options.credentials, bun.default_allocator)); errdefer store.deinit(); store.data.s3.options = aws_options.options; store.data.s3.acl = aws_options.acl; store.data.s3.storage_class = aws_options.storage_class; + store.data.s3.request_payer = aws_options.request_payer; var blob = Blob.initWithStore(store, globalObject); if (options) |opts| { @@ -413,11 +416,12 @@ pub const S3BlobStatTask = struct { }); this.store.ref(); const promise = this.promise.value(); - const credentials = blob.store.?.data.s3.getCredentials(); - const path = blob.store.?.data.s3.path(); + const s3_store = &blob.store.?.data.s3; + const credentials = s3_store.getCredentials(); + const path = s3_store.path(); const env = globalThis.bunVM().transpiler.env; - try S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3ExistsResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null); + try S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3ExistsResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null, s3_store.request_payer); return promise; } pub fn stat(globalThis: *jsc.JSGlobalObject, blob: *Blob) bun.JSTerminated!JSValue { @@ -428,11 +432,12 @@ pub const S3BlobStatTask = struct { }); this.store.ref(); const promise = this.promise.value(); - const credentials = blob.store.?.data.s3.getCredentials(); - const path = blob.store.?.data.s3.path(); + const s3_store = &blob.store.?.data.s3; + const credentials = s3_store.getCredentials(); + const path = s3_store.path(); const env = globalThis.bunVM().transpiler.env; - try S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3StatResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null); + try S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3StatResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null, s3_store.request_payer); return promise; } pub fn size(globalThis: *jsc.JSGlobalObject, blob: *Blob) bun.JSTerminated!JSValue { @@ -443,11 +448,12 @@ pub const S3BlobStatTask = struct { }); this.store.ref(); const promise = this.promise.value(); - const credentials = blob.store.?.data.s3.getCredentials(); - const path = blob.store.?.data.s3.path(); + const s3_store = &blob.store.?.data.s3; + const credentials = s3_store.getCredentials(); + const path = s3_store.path(); const env = globalThis.bunVM().transpiler.env; - try S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null); + try S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null, s3_store.request_payer); return promise; } @@ -466,13 +472,14 @@ pub fn getPresignUrlFrom(this: *Blob, globalThis: *jsc.JSGlobalObject, extra_opt var method: bun.http.Method = .GET; var expires: usize = 86400; // 1 day default + const s3 = &this.store.?.data.s3; var credentialsWithOptions: S3.S3CredentialsWithOptions = .{ - .credentials = this.store.?.data.s3.getCredentials().*, + .credentials = s3.getCredentials().*, + .request_payer = s3.request_payer, }; defer { credentialsWithOptions.deinit(); } - const s3 = &this.store.?.data.s3; if (extra_options) |options| { if (options.isObject()) { @@ -495,6 +502,7 @@ pub fn getPresignUrlFrom(this: *Blob, globalThis: *jsc.JSGlobalObject, extra_opt .method = method, .acl = credentialsWithOptions.acl, .storage_class = credentialsWithOptions.storage_class, + .request_payer = credentialsWithOptions.request_payer, }, false, .{ .expires = expires }) catch |sign_err| { return S3.throwSignError(sign_err, globalThis); }; diff --git a/src/bun.js/webcore/blob/Store.zig b/src/bun.js/webcore/blob/Store.zig index a812e2019e..0d03c339da 100644 --- a/src/bun.js/webcore/blob/Store.zig +++ b/src/bun.js/webcore/blob/Store.zig @@ -295,6 +295,7 @@ pub const S3 = struct { options: bun.S3.MultiPartUploadOptions = .{}, acl: ?bun.S3.ACL = null, storage_class: ?bun.S3.StorageClass = null, + request_payer: bool = false, pub fn isSeekable(_: *const @This()) ?bool { return true; @@ -306,7 +307,7 @@ pub const S3 = struct { } pub fn getCredentialsWithOptions(this: *const @This(), options: ?JSValue, globalObject: *JSGlobalObject) bun.JSError!bun.S3.S3CredentialsWithOptions { - return S3Credentials.getCredentialsWithOptions(this.getCredentials().*, this.options, options, this.acl, this.storage_class, globalObject); + return S3Credentials.getCredentialsWithOptions(this.getCredentials().*, this.options, options, this.acl, this.storage_class, this.request_payer, globalObject); } pub fn path(this: *@This()) []const u8 { @@ -365,7 +366,7 @@ pub const S3 = struct { .promise = promise, .store = store, // store is needed in case of not found error .global = globalThis, - }), proxy); + }), proxy, aws_options.request_payer); return value; } diff --git a/src/bun.js/webcore/fetch.zig b/src/bun.js/webcore/fetch.zig index 1d0a744b45..e2dba55b9e 100644 --- a/src/bun.js/webcore/fetch.zig +++ b/src/bun.js/webcore/fetch.zig @@ -1215,7 +1215,7 @@ pub fn Bun__fetch_( if (try options.getTruthyComptime(globalThis, "s3")) |s3_options| { if (s3_options.isObject()) { s3_options.ensureStillAlive(); - credentialsWithOptions = try s3.S3Credentials.getCredentialsWithOptions(credentialsWithOptions.credentials, .{}, s3_options, null, null, globalThis); + credentialsWithOptions = try s3.S3Credentials.getCredentialsWithOptions(credentialsWithOptions.credentials, .{}, s3_options, null, null, false, globalThis); } } } @@ -1303,6 +1303,7 @@ pub fn Bun__fetch_( if (headers) |h| (h.getContentType()) else null, if (headers) |h| h.getContentDisposition() else null, proxy_url, + credentialsWithOptions.request_payer, @ptrCast(&Wrapper.resolve), s3_stream, ); diff --git a/src/s3/client.zig b/src/s3/client.zig index 6adf666a4a..f3ac2e4d1a 100644 --- a/src/s3/client.zig +++ b/src/s3/client.zig @@ -26,12 +26,14 @@ pub fn stat( callback: *const fn (S3StatResult, *anyopaque) bun.JSTerminated!void, callback_context: *anyopaque, proxy_url: ?[]const u8, + request_payer: bool, ) bun.JSTerminated!void { try S3SimpleRequest.executeSimpleS3Request(this, .{ .path = path, .method = .HEAD, .proxy_url = proxy_url, .body = "", + .request_payer = request_payer, }, .{ .stat = callback }, callback_context); } @@ -41,12 +43,14 @@ pub fn download( callback: *const fn (S3DownloadResult, *anyopaque) bun.JSTerminated!void, callback_context: *anyopaque, proxy_url: ?[]const u8, + request_payer: bool, ) bun.JSTerminated!void { try S3SimpleRequest.executeSimpleS3Request(this, .{ .path = path, .method = .GET, .proxy_url = proxy_url, .body = "", + .request_payer = request_payer, }, .{ .download = callback }, callback_context); } @@ -58,6 +62,7 @@ pub fn downloadSlice( callback: *const fn (S3DownloadResult, *anyopaque) bun.JSTerminated!void, callback_context: *anyopaque, proxy_url: ?[]const u8, + request_payer: bool, ) bun.JSTerminated!void { const range = brk: { if (size) |size_| { @@ -77,6 +82,7 @@ pub fn downloadSlice( .proxy_url = proxy_url, .body = "", .range = range, + .request_payer = request_payer, }, .{ .download = callback }, callback_context); } @@ -86,12 +92,14 @@ pub fn delete( callback: *const fn (S3DeleteResult, *anyopaque) bun.JSTerminated!void, callback_context: *anyopaque, proxy_url: ?[]const u8, + request_payer: bool, ) bun.JSTerminated!void { try S3SimpleRequest.executeSimpleS3Request(this, .{ .path = path, .method = .DELETE, .proxy_url = proxy_url, .body = "", + .request_payer = request_payer, }, .{ .delete = callback }, callback_context); } @@ -233,6 +241,7 @@ pub fn upload( acl: ?ACL, proxy_url: ?[]const u8, storage_class: ?StorageClass, + request_payer: bool, callback: *const fn (S3UploadResult, *anyopaque) bun.JSTerminated!void, callback_context: *anyopaque, ) bun.JSTerminated!void { @@ -245,6 +254,7 @@ pub fn upload( .content_disposition = content_disposition, .acl = acl, .storage_class = storage_class, + .request_payer = request_payer, }, .{ .upload = callback }, callback_context); } /// returns a writable stream that writes to the s3 path @@ -257,6 +267,7 @@ pub fn writableStream( content_disposition: ?[]const u8, proxy: ?[]const u8, storage_class: ?StorageClass, + request_payer: bool, ) bun.JSError!jsc.JSValue { const Wrapper = struct { pub fn callback(result: S3UploadResult, sink: *jsc.WebCore.NetworkSink) bun.JSTerminated!void { @@ -300,6 +311,7 @@ pub fn writableStream( .content_type = if (content_type) |ct| bun.handleOom(bun.default_allocator.dupe(u8, ct)) else null, .content_disposition = if (content_disposition) |cd| bun.handleOom(bun.default_allocator.dupe(u8, cd)) else null, .storage_class = storage_class, + .request_payer = request_payer, .callback = @ptrCast(&Wrapper.callback), .callback_context = undefined, @@ -440,6 +452,7 @@ pub fn uploadStream( content_type: ?[]const u8, content_disposition: ?[]const u8, proxy: ?[]const u8, + request_payer: bool, callback: ?*const fn (S3UploadResult, *anyopaque) void, callback_context: *anyopaque, ) bun.JSError!jsc.JSValue { @@ -483,6 +496,7 @@ pub fn uploadStream( .options = options, .acl = acl, .storage_class = storage_class, + .request_payer = request_payer, .vm = jsc.VirtualMachine.get(), }); @@ -513,6 +527,7 @@ pub fn downloadStream( offset: usize, size: ?usize, proxy_url: ?[]const u8, + request_payer: bool, callback: *const fn (chunk: bun.MutableString, has_more: bool, err: ?Error.S3Error, *anyopaque) void, callback_context: *anyopaque, ) void { @@ -533,6 +548,7 @@ pub fn downloadStream( var result = this.signRequest(.{ .path = path, .method = .GET, + .request_payer = request_payer, }, false, null) catch |sign_err| { if (range) |range_| bun.default_allocator.free(range_); const error_code_and_message = Error.getSignErrorCodeAndMessage(sign_err); @@ -606,6 +622,7 @@ pub fn readableStream( offset: usize, size: ?usize, proxy_url: ?[]const u8, + request_payer: bool, globalThis: *jsc.JSGlobalObject, ) bun.JSError!jsc.JSValue { var reader = jsc.WebCore.ByteStream.Source.new(.{ @@ -670,6 +687,7 @@ pub fn readableStream( offset, size, proxy_url, + request_payer, S3DownloadStreamWrapper.opaqueCallback, S3DownloadStreamWrapper.new(.{ .readable_stream_ref = jsc.WebCore.ReadableStream.Strong.init(.{ diff --git a/src/s3/credentials.zig b/src/s3/credentials.zig index 05668dc780..602f7f4399 100644 --- a/src/s3/credentials.zig +++ b/src/s3/credentials.zig @@ -35,7 +35,7 @@ pub const S3Credentials = struct { return hasher.final(); } - pub fn getCredentialsWithOptions(this: S3Credentials, default_options: MultiPartUploadOptions, options: ?jsc.JSValue, default_acl: ?ACL, default_storage_class: ?StorageClass, globalObject: *jsc.JSGlobalObject) bun.JSError!S3CredentialsWithOptions { + pub fn getCredentialsWithOptions(this: S3Credentials, default_options: MultiPartUploadOptions, options: ?jsc.JSValue, default_acl: ?ACL, default_storage_class: ?StorageClass, default_request_payer: bool, globalObject: *jsc.JSGlobalObject) bun.JSError!S3CredentialsWithOptions { bun.analytics.Features.s3 += 1; // get ENV config var new_credentials = S3CredentialsWithOptions{ @@ -43,6 +43,7 @@ pub const S3Credentials = struct { .options = default_options, .acl = default_acl, .storage_class = default_storage_class, + .request_payer = default_request_payer, }; errdefer { new_credentials.deinit(); @@ -227,6 +228,10 @@ pub const S3Credentials = struct { } } } + + if (try opts.getBooleanStrict(globalObject, "requestPayer")) |request_payer| { + new_credentials.request_payer = request_payer; + } } } return new_credentials; @@ -347,7 +352,9 @@ pub const S3Credentials = struct { session_token: []const u8 = "", acl: ?ACL = null, storage_class: ?StorageClass = null, - _headers: [8]picohttp.Header = .{ + request_payer: bool = false, + _headers: [9]picohttp.Header = .{ + .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, @@ -416,6 +423,7 @@ pub const S3Credentials = struct { content_disposition: ?[]const u8 = null, acl: ?ACL = null, storage_class: ?StorageClass = null, + request_payer: bool = false, }; /// This is not used for signing but for console.log output, is just nice to have pub fn guessBucket(endpoint: []const u8) ?[]const u8 { @@ -629,133 +637,16 @@ pub const S3Credentials = struct { errdefer bun.default_allocator.free(amz_date); const amz_day = amz_date[0..8]; - const signed_headers = if (signQuery) "host" else brk: { - if (content_md5 != null) { - if (storage_class != null) { - if (acl != null) { - if (content_disposition != null) { - if (session_token != null) { - break :brk "content-disposition;content-md5;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class"; - } else { - break :brk "content-disposition;content-md5;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-storage-class"; - } - } else { - if (session_token != null) { - break :brk "content-md5;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class"; - } else { - break :brk "content-md5;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-storage-class"; - } - } - } else { - if (content_disposition != null) { - if (session_token != null) { - break :brk "content-disposition;content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class"; - } else { - break :brk "content-disposition;content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class"; - } - } else { - if (session_token != null) { - break :brk "content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class"; - } else { - break :brk "content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class"; - } - } - } - } else { - if (acl != null) { - if (content_disposition != null) { - if (session_token != null) { - break :brk "content-disposition;content-md5;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-security-token"; - } else { - break :brk "content-disposition;content-md5;host;x-amz-acl;x-amz-content-sha256;x-amz-date"; - } - } else { - if (session_token != null) { - break :brk "content-md5;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-security-token"; - } else { - break :brk "content-md5;host;x-amz-acl;x-amz-content-sha256;x-amz-date"; - } - } - } else { - if (content_disposition != null) { - if (session_token != null) { - break :brk "content-disposition;content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-security-token"; - } else { - break :brk "content-disposition;content-md5;host;x-amz-content-sha256;x-amz-date"; - } - } else { - if (session_token != null) { - break :brk "content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-security-token"; - } else { - break :brk "content-md5;host;x-amz-content-sha256;x-amz-date"; - } - } - } - } - } else { - if (storage_class != null) { - if (acl != null) { - if (content_disposition != null) { - if (session_token != null) { - break :brk "content-disposition;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class"; - } else { - break :brk "content-disposition;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-storage-class"; - } - } else { - if (session_token != null) { - break :brk "host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class"; - } else { - break :brk "host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-storage-class"; - } - } - } else { - if (content_disposition != null) { - if (session_token != null) { - break :brk "content-disposition;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class"; - } else { - break :brk "content-disposition;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class"; - } - } else { - if (session_token != null) { - break :brk "host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class"; - } else { - break :brk "host;x-amz-content-sha256;x-amz-date;x-amz-storage-class"; - } - } - } - } else { - if (acl != null) { - if (content_disposition != null) { - if (session_token != null) { - break :brk "content-disposition;host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-security-token"; - } else { - break :brk "content-disposition;host;x-amz-acl;x-amz-content-sha256;x-amz-date"; - } - } else { - if (session_token != null) { - break :brk "host;x-amz-acl;x-amz-content-sha256;x-amz-date;x-amz-security-token"; - } else { - break :brk "host;x-amz-acl;x-amz-content-sha256;x-amz-date"; - } - } - } else { - if (content_disposition != null) { - if (session_token != null) { - break :brk "content-disposition;host;x-amz-content-sha256;x-amz-date;x-amz-security-token"; - } else { - break :brk "content-disposition;host;x-amz-content-sha256;x-amz-date"; - } - } else { - if (session_token != null) { - break :brk "host;x-amz-content-sha256;x-amz-date;x-amz-security-token"; - } else { - break :brk "host;x-amz-content-sha256;x-amz-date"; - } - } - } - } - } + const request_payer = signOptions.request_payer; + const header_key = SignedHeaders.Key{ + .content_disposition = content_disposition != null, + .content_md5 = content_md5 != null, + .acl = acl != null, + .request_payer = request_payer, + .session_token = session_token != null, + .storage_class = storage_class != null, }; + const signed_headers = if (signQuery) "host" else SignedHeaders.get(header_key); const service_name = "s3"; @@ -800,9 +691,9 @@ pub const S3Credentials = struct { const canonical = brk_canonical: { var stack_fallback = std.heap.stackFallback(512, bun.default_allocator); const allocator = stack_fallback.get(); - var query_parts: bun.BoundedArray([]const u8, 10) = .{}; + var query_parts: bun.BoundedArray([]const u8, 11) = .{}; - // Add parameters in alphabetical order: Content-MD5, X-Amz-Acl, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, X-Amz-Expires, X-Amz-Security-Token, X-Amz-SignedHeaders, x-amz-storage-class + // Add parameters in alphabetical order: Content-MD5, X-Amz-Acl, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, X-Amz-Expires, X-Amz-Security-Token, X-Amz-SignedHeaders, x-amz-request-payer, x-amz-storage-class if (encoded_content_md5) |encoded_content_md5_value| { try query_parts.append(try std.fmt.allocPrint(allocator, "Content-MD5={s}", .{encoded_content_md5_value})); @@ -826,6 +717,10 @@ pub const S3Credentials = struct { try query_parts.append(try std.fmt.allocPrint(allocator, "X-Amz-SignedHeaders=host", .{})); + if (request_payer) { + try query_parts.append(try std.fmt.allocPrint(allocator, "x-amz-request-payer=requester", .{})); + } + if (storage_class) |storage_class_value| { try query_parts.append(try std.fmt.allocPrint(allocator, "x-amz-storage-class={s}", .{storage_class_value})); } @@ -851,9 +746,9 @@ pub const S3Credentials = struct { // Build final URL with query parameters in alphabetical order to match canonical request var url_stack_fallback = std.heap.stackFallback(512, bun.default_allocator); const url_allocator = url_stack_fallback.get(); - var url_query_parts: bun.BoundedArray([]const u8, 10) = .{}; + var url_query_parts: bun.BoundedArray([]const u8, 12) = .{}; - // Add parameters in alphabetical order: Content-MD5, X-Amz-Acl, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, X-Amz-Expires, X-Amz-Security-Token, X-Amz-SignedHeaders, x-amz-storage-class, X-Amz-Signature + // Add parameters in alphabetical order: Content-MD5, X-Amz-Acl, X-Amz-Algorithm, X-Amz-Credential, X-Amz-Date, X-Amz-Expires, X-Amz-Security-Token, X-Amz-Signature, X-Amz-SignedHeaders, x-amz-request-payer, x-amz-storage-class if (encoded_content_md5) |encoded_content_md5_value| { try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "Content-MD5={s}", .{encoded_content_md5_value})); @@ -879,6 +774,10 @@ pub const S3Credentials = struct { try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "X-Amz-SignedHeaders=host", .{})); + if (request_payer) { + try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "x-amz-request-payer=requester", .{})); + } + if (storage_class) |storage_class_value| { try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "x-amz-storage-class={s}", .{storage_class_value})); } @@ -894,143 +793,22 @@ pub const S3Credentials = struct { break :brk try std.fmt.allocPrint(bun.default_allocator, "{s}://{s}{s}?{s}", .{ protocol, host, normalizedPath, url_query_string.items }); } else { - const canonical = brk_canonical: { - if (content_md5) |content_md5_value| { - if (storage_class) |storage_class_value| { - if (acl) |acl_value| { - if (content_disposition) |disposition| { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, content_md5_value, host, acl_value, aws_content_hash, amz_date, token, storage_class_value, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, content_md5_value, host, acl_value, aws_content_hash, amz_date, storage_class_value, signed_headers, aws_content_hash }); - } - } else { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", content_md5_value, host, acl_value, aws_content_hash, amz_date, token, storage_class_value, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", content_md5_value, host, acl_value, aws_content_hash, amz_date, storage_class_value, signed_headers, aws_content_hash }); - } - } - } else { - if (content_disposition) |disposition| { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, content_md5_value, host, aws_content_hash, amz_date, token, storage_class_value, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, content_md5_value, host, aws_content_hash, amz_date, storage_class_value, signed_headers, aws_content_hash }); - } - } else { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", content_md5_value, host, aws_content_hash, amz_date, token, storage_class_value, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", content_md5_value, host, aws_content_hash, amz_date, storage_class_value, signed_headers, aws_content_hash }); - } - } - } - } else { - if (acl) |acl_value| { - if (content_disposition) |disposition| { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, content_md5_value, host, acl_value, aws_content_hash, amz_date, token, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, content_md5_value, host, acl_value, aws_content_hash, amz_date, signed_headers, aws_content_hash }); - } - } else { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", content_md5_value, host, acl_value, aws_content_hash, amz_date, token, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", content_md5_value, host, acl_value, aws_content_hash, amz_date, signed_headers, aws_content_hash }); - } - } - } else { - if (content_disposition) |disposition| { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, content_md5_value, host, aws_content_hash, amz_date, token, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, content_md5_value, host, aws_content_hash, amz_date, signed_headers, aws_content_hash }); - } - } else { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", content_md5_value, host, aws_content_hash, amz_date, token, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-md5:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\n\n{s}\n{s}", .{ - method_name, - normalizedPath, - if (search_params) |p| p[1..] else "", - content_md5_value, - host, - aws_content_hash, - amz_date, - signed_headers, - aws_content_hash, - }); - } - } - } - } - } else { - if (storage_class) |storage_class_value| { - if (acl) |acl_value| { - if (content_disposition) |disposition| { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, host, acl_value, aws_content_hash, amz_date, token, storage_class_value, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, host, acl_value, aws_content_hash, amz_date, storage_class_value, signed_headers, aws_content_hash }); - } - } else { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", host, acl_value, aws_content_hash, amz_date, token, storage_class_value, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", host, acl_value, aws_content_hash, amz_date, storage_class_value, signed_headers, aws_content_hash }); - } - } - } else { - if (content_disposition) |disposition| { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, host, aws_content_hash, amz_date, token, storage_class_value, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, host, aws_content_hash, amz_date, storage_class_value, signed_headers, aws_content_hash }); - } - } else { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", host, aws_content_hash, amz_date, token, storage_class_value, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-storage-class:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", host, aws_content_hash, amz_date, storage_class_value, signed_headers, aws_content_hash }); - } - } - } - } else { - if (acl) |acl_value| { - if (content_disposition) |disposition| { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, host, acl_value, aws_content_hash, amz_date, token, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, host, acl_value, aws_content_hash, amz_date, signed_headers, aws_content_hash }); - } - } else { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", host, acl_value, aws_content_hash, amz_date, token, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\nx-amz-acl:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", host, acl_value, aws_content_hash, amz_date, signed_headers, aws_content_hash }); - } - } - } else { - if (content_disposition) |disposition| { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, host, aws_content_hash, amz_date, token, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\ncontent-disposition:{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", disposition, host, aws_content_hash, amz_date, signed_headers, aws_content_hash }); - } - } else { - if (session_token) |token| { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\nx-amz-security-token:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", host, aws_content_hash, amz_date, token, signed_headers, aws_content_hash }); - } else { - break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\nx-amz-content-sha256:{s}\nx-amz-date:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, if (search_params) |p| p[1..] else "", host, aws_content_hash, amz_date, signed_headers, aws_content_hash }); - } - } - } - } - } - }; + const canonical = try CanonicalRequest.format( + &tmp_buffer, + header_key, + method_name, + normalizedPath, + if (search_params) |p| p[1..] else "", + content_disposition, + content_md5, + host, + acl, + aws_content_hash, + amz_date, + session_token, + storage_class, + signed_headers, + ); var sha_digest = std.mem.zeroes(bun.sha.SHA256.Digest); bun.sha.SHA256.hash(canonical, &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine()); @@ -1067,6 +845,7 @@ pub const S3Credentials = struct { .authorization = authorization, .acl = signOptions.acl, .storage_class = signOptions.storage_class, + .request_payer = request_payer, .url = try std.fmt.allocPrint(bun.default_allocator, "{s}://{s}{s}{s}", .{ protocol, host, normalizedPath, if (search_params) |s| s else "" }), ._headers = [_]picohttp.Header{ .{ .name = "x-amz-content-sha256", .value = aws_content_hash }, @@ -1077,6 +856,7 @@ pub const S3Credentials = struct { .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, + .{ .name = "", .value = "" }, }, ._headers_len = 4, }; @@ -1111,6 +891,11 @@ pub const S3Credentials = struct { result._headers_len += 1; } + if (request_payer) { + result._headers[result._headers_len] = .{ .name = "x-amz-request-payer", .value = "requester" }; + result._headers_len += 1; + } + return result; } }; @@ -1121,6 +906,8 @@ pub const S3CredentialsWithOptions = struct { acl: ?ACL = null, storage_class: ?StorageClass = null, content_disposition: ?[]const u8 = null, + /// indicates if requester pays for the request (for requester pays buckets) + request_payer: bool = false, /// indicates if the credentials have changed changed_credentials: bool = false, /// indicates if the virtual hosted style is used @@ -1144,6 +931,123 @@ pub const S3CredentialsWithOptions = struct { } }; +/// Comptime-generated lookup table for signed headers strings. +/// Headers must be in alphabetical order per AWS Signature V4 spec. +const SignedHeaders = struct { + const Key = packed struct(u6) { + content_disposition: bool, + content_md5: bool, + acl: bool, + request_payer: bool, + session_token: bool, + storage_class: bool, + }; + + fn generate(comptime key: Key) []const u8 { + return (if (key.content_disposition) "content-disposition;" else "") ++ + (if (key.content_md5) "content-md5;" else "") ++ + "host;" ++ + (if (key.acl) "x-amz-acl;" else "") ++ + "x-amz-content-sha256;x-amz-date" ++ + (if (key.request_payer) ";x-amz-request-payer" else "") ++ + (if (key.session_token) ";x-amz-security-token" else "") ++ + (if (key.storage_class) ";x-amz-storage-class" else ""); + } + + const table = init: { + var t: [64][]const u8 = undefined; + for (0..64) |i| { + t[i] = generate(@bitCast(@as(u6, @intCast(i)))); + } + break :init t; + }; + + pub fn get(key: Key) []const u8 { + return table[@as(u6, @bitCast(key))]; + } +}; + +/// Comptime-generated format strings for canonical request. +/// Uses the same key as SignedHeaders to select the right format. +const CanonicalRequest = struct { + fn fmtString(comptime key: SignedHeaders.Key) []const u8 { + return "{s}\n{s}\n{s}\n" ++ // method, path, query + (if (key.content_disposition) "content-disposition:{s}\n" else "") ++ + (if (key.content_md5) "content-md5:{s}\n" else "") ++ + "host:{s}\n" ++ + (if (key.acl) "x-amz-acl:{s}\n" else "") ++ + "x-amz-content-sha256:{s}\nx-amz-date:{s}\n" ++ + (if (key.request_payer) "x-amz-request-payer:requester\n" else "") ++ + (if (key.session_token) "x-amz-security-token:{s}\n" else "") ++ + (if (key.storage_class) "x-amz-storage-class:{s}\n" else "") ++ + "\n{s}\n{s}"; // signed_headers, hash + } + + inline fn formatForKey( + comptime key: SignedHeaders.Key, + buf: []u8, + method: []const u8, + path: []const u8, + query: []const u8, + content_disposition: ?[]const u8, + content_md5: ?[]const u8, + host: []const u8, + acl: ?[]const u8, + hash: []const u8, + date: []const u8, + session_token: ?[]const u8, + storage_class: ?[]const u8, + signed_headers: []const u8, + ) error{NoSpaceLeft}![]u8 { + return std.fmt.bufPrint(buf, fmtString(key), .{ method, path, query } ++ + (if (key.content_disposition) .{content_disposition.?} else .{}) ++ + (if (key.content_md5) .{content_md5.?} else .{}) ++ + .{host} ++ + (if (key.acl) .{acl.?} else .{}) ++ + .{ hash, date } ++ + (if (key.session_token) .{session_token.?} else .{}) ++ + (if (key.storage_class) .{storage_class.?} else .{}) ++ + .{ signed_headers, hash }); + } + + pub fn format( + buf: []u8, + key: SignedHeaders.Key, + method: []const u8, + path: []const u8, + query: []const u8, + content_disposition: ?[]const u8, + content_md5: ?[]const u8, + host: []const u8, + acl: ?[]const u8, + hash: []const u8, + date: []const u8, + session_token: ?[]const u8, + storage_class: ?[]const u8, + signed_headers: []const u8, + ) error{NoSpaceLeft}![]u8 { + // Dispatch to the right comptime-specialized function based on runtime key + return switch (@as(u6, @bitCast(key))) { + inline 0...63 => |idx| formatForKey( + @bitCast(idx), + buf, + method, + path, + query, + content_disposition, + content_md5, + host, + acl, + hash, + date, + session_token, + storage_class, + signed_headers, + ), + }; + } +}; + const std = @import("std"); const ACL = @import("./acl.zig").ACL; const MultiPartUploadOptions = @import("./multipart_options.zig").MultiPartUploadOptions; diff --git a/src/s3/multipart.zig b/src/s3/multipart.zig index 17bcfe70a5..13fe6dad67 100644 --- a/src/s3/multipart.zig +++ b/src/s3/multipart.zig @@ -105,6 +105,7 @@ pub const MultiPartUpload = struct { options: MultiPartUploadOptions = .{}, acl: ?ACL = null, storage_class: ?Storageclass = null, + request_payer: bool = false, credentials: *S3Credentials, poll_ref: bun.Async.KeepAlive = bun.Async.KeepAlive.init(), vm: *jsc.VirtualMachine, @@ -239,6 +240,7 @@ pub const MultiPartUpload = struct { .proxy_url = this.ctx.proxyUrl(), .body = this.data, .search_params = search_params, + .request_payer = this.ctx.request_payer, }, .{ .part = @ptrCast(&onPartResponse) }, this); } pub fn start(this: *@This()) bun.JSTerminated!void { @@ -310,6 +312,7 @@ pub const MultiPartUpload = struct { .content_disposition = this.content_disposition, .acl = this.acl, .storage_class = this.storage_class, + .request_payer = this.request_payer, }, .{ .upload = @ptrCast(&singleSendUploadResponse) }, this); return; @@ -565,6 +568,7 @@ pub const MultiPartUpload = struct { .proxy_url = this.proxyUrl(), .body = this.multipart_upload_list.slice(), .search_params = searchParams, + .request_payer = this.request_payer, }, .{ .commit = @ptrCast(&onCommitMultiPartRequest) }, this); } fn rollbackMultiPartRequest(this: *@This()) bun.JSTerminated!void { @@ -580,6 +584,7 @@ pub const MultiPartUpload = struct { .proxy_url = this.proxyUrl(), .body = "", .search_params = search_params, + .request_payer = this.request_payer, }, .{ .upload = @ptrCast(&onRollbackMultiPartRequest) }, this); } fn enqueuePart(this: *@This(), chunk: []const u8, allocated_size: usize, needs_clone: bool) bun.JSTerminated!bool { @@ -599,6 +604,7 @@ pub const MultiPartUpload = struct { .content_disposition = this.content_disposition, .acl = this.acl, .storage_class = this.storage_class, + .request_payer = this.request_payer, }, .{ .download = @ptrCast(&startMultiPartRequestResult) }, this); } else if (this.state == .multipart_completed) { try part.start(); @@ -676,6 +682,7 @@ pub const MultiPartUpload = struct { .content_disposition = this.content_disposition, .acl = this.acl, .storage_class = this.storage_class, + .request_payer = this.request_payer, }, .{ .upload = @ptrCast(&singleSendUploadResponse) }, this) catch {}; // TODO: properly propagate exception upwards } else { // we need to split diff --git a/src/s3/simple_request.zig b/src/s3/simple_request.zig index 40c08470af..32fb032549 100644 --- a/src/s3/simple_request.zig +++ b/src/s3/simple_request.zig @@ -358,6 +358,7 @@ pub const S3SimpleRequestOptions = struct { range: ?[]const u8 = null, acl: ?ACL = null, storage_class: ?StorageClass = null, + request_payer: bool = false, }; pub fn executeSimpleS3Request( @@ -373,6 +374,7 @@ pub fn executeSimpleS3Request( .content_disposition = options.content_disposition, .acl = options.acl, .storage_class = options.storage_class, + .request_payer = options.request_payer, }, false, null) catch |sign_err| { if (options.range) |range_| bun.default_allocator.free(range_); const error_code_and_message = getSignErrorCodeAndMessage(sign_err); diff --git a/test/js/bun/s3/s3-requester-pays.test.ts b/test/js/bun/s3/s3-requester-pays.test.ts new file mode 100644 index 0000000000..09b834918c --- /dev/null +++ b/test/js/bun/s3/s3-requester-pays.test.ts @@ -0,0 +1,251 @@ +import { S3Client, type S3Options } from "bun"; +import { describe, expect, it } from "bun:test"; + +describe("s3 - Requester Pays", () => { + const s3Options: S3Options = { + accessKeyId: "test", + secretAccessKey: "test", + region: "eu-west-3", + bucket: "my_bucket", + }; + + it("should include x-amz-request-payer header when requestPayer is true", async () => { + let reqHeaders: Headers | undefined = undefined; + using server = Bun.serve({ + port: 0, + async fetch(req) { + reqHeaders = req.headers; + return new Response("", { + headers: { + "Content-Type": "text/plain", + }, + status: 200, + }); + }, + }); + + await S3Client.file("test_file", { + ...s3Options, + endpoint: server.url.href, + requestPayer: true, + }).write("Test content"); + + expect(reqHeaders!.get("authorization")).toInclude("x-amz-request-payer"); + expect(reqHeaders!.get("x-amz-request-payer")).toBe("requester"); + }); + + it("should NOT include x-amz-request-payer header when requestPayer is false", async () => { + let reqHeaders: Headers | undefined = undefined; + using server = Bun.serve({ + port: 0, + async fetch(req) { + reqHeaders = req.headers; + return new Response("", { + headers: { + "Content-Type": "text/plain", + }, + status: 200, + }); + }, + }); + + await S3Client.file("test_file", { + ...s3Options, + endpoint: server.url.href, + requestPayer: false, + }).write("Test content"); + + expect(reqHeaders!.get("authorization")).not.toInclude("x-amz-request-payer"); + expect(reqHeaders!.get("x-amz-request-payer")).toBeNull(); + }); + + it("should NOT include x-amz-request-payer header by default", async () => { + let reqHeaders: Headers | undefined = undefined; + using server = Bun.serve({ + port: 0, + async fetch(req) { + reqHeaders = req.headers; + return new Response("", { + headers: { + "Content-Type": "text/plain", + }, + status: 200, + }); + }, + }); + + await S3Client.file("test_file", { + ...s3Options, + endpoint: server.url.href, + }).write("Test content"); + + expect(reqHeaders!.get("authorization")).not.toInclude("x-amz-request-payer"); + expect(reqHeaders!.get("x-amz-request-payer")).toBeNull(); + }); + + it("should work with S3Client instance", async () => { + let reqHeaders: Headers | undefined = undefined; + using server = Bun.serve({ + port: 0, + async fetch(req) { + reqHeaders = req.headers; + return new Response("", { + headers: { + "Content-Type": "text/plain", + }, + status: 200, + }); + }, + }); + + const client = new S3Client({ + ...s3Options, + endpoint: server.url.href, + requestPayer: true, + }); + + await client.file("test_file").write("Test content"); + + expect(reqHeaders!.get("authorization")).toInclude("x-amz-request-payer"); + expect(reqHeaders!.get("x-amz-request-payer")).toBe("requester"); + }); + + it("should work with file-level options overriding client options", async () => { + let reqHeaders: Headers | undefined = undefined; + using server = Bun.serve({ + port: 0, + async fetch(req) { + reqHeaders = req.headers; + return new Response("", { + headers: { + "Content-Type": "text/plain", + }, + status: 200, + }); + }, + }); + + // Client has requestPayer: false, but file overrides with true + const client = new S3Client({ + ...s3Options, + endpoint: server.url.href, + requestPayer: false, + }); + + await client.file("test_file", { requestPayer: true }).write("Test content"); + + expect(reqHeaders!.get("authorization")).toInclude("x-amz-request-payer"); + expect(reqHeaders!.get("x-amz-request-payer")).toBe("requester"); + }); + + it("should include x-amz-request-payer in read operations", async () => { + let reqHeaders: Headers | undefined = undefined; + const body = "Test content from requester pays bucket"; + using server = Bun.serve({ + port: 0, + async fetch(req) { + reqHeaders = req.headers; + return new Response(body, { + headers: { + "Content-Type": "text/plain", + "Content-Length": String(body.length), + }, + status: 200, + }); + }, + }); + + const file = S3Client.file("test_file", { + ...s3Options, + endpoint: server.url.href, + requestPayer: true, + }); + + await file.text(); + + expect(reqHeaders!.get("authorization")).toInclude("x-amz-request-payer"); + expect(reqHeaders!.get("x-amz-request-payer")).toBe("requester"); + }); + + it("should include x-amz-request-payer in HEAD requests (exists/size/stat)", async () => { + let reqHeaders: Headers | undefined = undefined; + let reqMethod: string | undefined = undefined; + using server = Bun.serve({ + port: 0, + async fetch(req) { + reqHeaders = req.headers; + reqMethod = req.method; + return new Response("", { + headers: { + "Content-Type": "text/plain", + "Content-Length": "100", + }, + status: 200, + }); + }, + }); + + const file = S3Client.file("test_file", { + ...s3Options, + endpoint: server.url.href, + requestPayer: true, + }); + + await file.exists(); + + expect(reqMethod).toBe("HEAD"); + expect(reqHeaders!.get("authorization")).toInclude("x-amz-request-payer"); + expect(reqHeaders!.get("x-amz-request-payer")).toBe("requester"); + }); + + it("should include x-amz-request-payer in DELETE requests", async () => { + let reqHeaders: Headers | undefined = undefined; + let reqMethod: string | undefined = undefined; + using server = Bun.serve({ + port: 0, + async fetch(req) { + reqHeaders = req.headers; + reqMethod = req.method; + return new Response("", { + status: 204, + }); + }, + }); + + const file = S3Client.file("test_file", { + ...s3Options, + endpoint: server.url.href, + requestPayer: true, + }); + + await file.delete(); + + expect(reqMethod).toBe("DELETE"); + expect(reqHeaders!.get("authorization")).toInclude("x-amz-request-payer"); + expect(reqHeaders!.get("x-amz-request-payer")).toBe("requester"); + }); + + it("should include x-amz-request-payer in presigned URLs", async () => { + const file = S3Client.file("test_file", { + ...s3Options, + requestPayer: true, + }); + + const presignedUrl = file.presign({ expiresIn: 3600 }); + const url = new URL(presignedUrl); + + expect(url.searchParams.get("x-amz-request-payer")).toBe("requester"); + }); + + it("should NOT include x-amz-request-payer in presigned URLs when requestPayer is false", async () => { + const file = S3Client.file("test_file", { + ...s3Options, + requestPayer: false, + }); + + const presignedUrl = file.presign({ expiresIn: 3600 }); + const url = new URL(presignedUrl); + + expect(url.searchParams.get("x-amz-request-payer")).toBeNull(); + }); +});