From 5d3f37d7ae74a96b836e86c111bc9e63a1e0dfb5 Mon Sep 17 00:00:00 2001 From: Ciro Spaciari Date: Thu, 15 Jan 2026 18:09:33 -0800 Subject: [PATCH] feat(s3): add Content-Encoding header support for S3 uploads (#26149) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Add support for setting the `Content-Encoding` header in S3 `.write()` and `.writer()` calls, following the same pattern as `Content-Disposition`. This allows users to specify the encoding of uploaded content: ```typescript // With .write() await s3file.write("compressed data", { contentEncoding: "gzip" }); // With .writer() const writer = s3file.writer({ contentEncoding: "gzip" }); writer.write("compressed data"); await writer.end(); // With bucket.write() await bucket.write("key", data, { contentEncoding: "br" }); ``` ## Implementation - Extended `SignedHeaders.Key` from 6 bits to 7 bits (64→128 combinations) to accommodate the new header - Added `content_encoding` to `S3CredentialsWithOptions`, `SignOptions`, and `SignResult` structs - Updated `CanonicalRequest` format strings to include `content-encoding` in AWS SigV4 signing - Added `getContentEncoding()` method to `Headers` for fetch-based S3 uploads - Expanded `_headers` array from 9 to 10 elements - Pass `content_encoding` through all S3 upload paths (upload, uploadStream, writableStream) ## Test plan - Added tests for "should be able to set content-encoding" - Added tests for "should be able to set content-encoding in writer" - Tests verify the Content-Encoding header is properly set on uploaded objects via presigned URL fetch - All 4 new tests pass with `bun bd test` and fail with `USE_SYSTEM_BUN=1` (confirming the feature is new) ## Changelog > Describe your changes in 1-2 sentences. These will be featured on [bun.sh/blog](https://bun.sh/blog) and Bun's release notes. Added `contentEncoding` option to S3 `.write()` and `.writer()` methods, allowing users to set the `Content-Encoding` header when uploading objects. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.5 --- src/bun.js/webcore/Blob.zig | 20 ++++++++++- src/bun.js/webcore/fetch.zig | 3 +- src/http/Headers.zig | 3 ++ src/s3/client.zig | 8 ++++- src/s3/credentials.zig | 66 +++++++++++++++++++++++++++++++----- src/s3/multipart.zig | 9 +++++ src/s3/simple_request.zig | 4 ++- test/js/bun/s3/s3.test.ts | 40 ++++++++++++++++++++++ 8 files changed, 141 insertions(+), 12 deletions(-) diff --git a/src/bun.js/webcore/Blob.zig b/src/bun.js/webcore/Blob.zig index 8e06b05c47..e150209e87 100644 --- a/src/bun.js/webcore/Blob.zig +++ b/src/bun.js/webcore/Blob.zig @@ -968,6 +968,7 @@ fn writeFileWithEmptySourceToDestination(ctx: *jsc.JSGlobalObject, destination_b "", destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, + aws_options.content_encoding, aws_options.acl, proxy_url, aws_options.storage_class, @@ -1120,6 +1121,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl aws_options.storage_class, destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, + aws_options.content_encoding, proxy_url, aws_options.request_payer, null, @@ -1160,6 +1162,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl bytes.slice(), destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, + aws_options.content_encoding, aws_options.acl, proxy_url, aws_options.storage_class, @@ -1191,6 +1194,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl aws_options.storage_class, destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, + aws_options.content_encoding, proxy_url, aws_options.request_payer, null, @@ -1398,6 +1402,7 @@ pub fn writeFileInternal(globalThis: *jsc.JSGlobalObject, path_or_blob_: *PathOr aws_options.storage_class, destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, + aws_options.content_encoding, proxy_url, aws_options.request_payer, null, @@ -1460,6 +1465,7 @@ pub fn writeFileInternal(globalThis: *jsc.JSGlobalObject, path_or_blob_: *PathOr aws_options.storage_class, destination_blob.contentTypeOrMimeType(), aws_options.content_disposition, + aws_options.content_encoding, proxy_url, aws_options.request_payer, null, @@ -2438,6 +2444,7 @@ pub fn pipeReadableStreamToBlob(this: *Blob, globalThis: *jsc.JSGlobalObject, re aws_options.storage_class, this.contentTypeOrMimeType(), aws_options.content_disposition, + aws_options.content_encoding, proxy_url, aws_options.request_payer, null, @@ -2674,7 +2681,16 @@ pub fn getWriter( } content_disposition_str = try content_disposition.toSlice(globalThis, bun.default_allocator); } - const credentialsWithOptions = try s3.getCredentialsWithOptions(options, globalThis); + var content_encoding_str: ?ZigString.Slice = null; + defer if (content_encoding_str) |ce| ce.deinit(); + if (try options.getTruthy(globalThis, "contentEncoding")) |content_encoding| { + if (!content_encoding.isString()) { + return globalThis.throwInvalidArgumentType("write", "options.contentEncoding", "string"); + } + content_encoding_str = try content_encoding.toSlice(globalThis, bun.default_allocator); + } + var credentialsWithOptions = try s3.getCredentialsWithOptions(options, globalThis); + defer credentialsWithOptions.deinit(); return try S3.writableStream( credentialsWithOptions.credentials.dupe(), path, @@ -2682,6 +2698,7 @@ pub fn getWriter( credentialsWithOptions.options, this.contentTypeOrMimeType(), if (content_disposition_str) |cd| cd.slice() else null, + if (content_encoding_str) |ce| ce.slice() else null, proxy_url, credentialsWithOptions.storage_class, credentialsWithOptions.request_payer, @@ -2695,6 +2712,7 @@ pub fn getWriter( .{}, this.contentTypeOrMimeType(), null, + null, proxy_url, null, s3.request_payer, diff --git a/src/bun.js/webcore/fetch.zig b/src/bun.js/webcore/fetch.zig index a0c2bdbccf..e7fe2a3f2b 100644 --- a/src/bun.js/webcore/fetch.zig +++ b/src/bun.js/webcore/fetch.zig @@ -1319,6 +1319,7 @@ fn fetchImpl( credentialsWithOptions.storage_class, if (headers) |h| (h.getContentType()) else null, if (headers) |h| h.getContentDisposition() else null, + if (headers) |h| h.getContentEncoding() else null, proxy_url, credentialsWithOptions.request_payer, @ptrCast(&Wrapper.resolve), @@ -1360,7 +1361,7 @@ fn fetchImpl( } const content_type = if (headers) |h| (h.getContentType()) else null; - var header_buffer: [10]picohttp.Header = undefined; + var header_buffer: [s3.S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined; if (range) |range_| { const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ }); diff --git a/src/http/Headers.zig b/src/http/Headers.zig index f9a41ad546..3fa7884c90 100644 --- a/src/http/Headers.zig +++ b/src/http/Headers.zig @@ -79,6 +79,9 @@ pub fn deinit(this: *Headers) void { pub fn getContentDisposition(this: *const Headers) ?[]const u8 { return this.get("content-disposition"); } +pub fn getContentEncoding(this: *const Headers) ?[]const u8 { + return this.get("content-encoding"); +} pub fn getContentType(this: *const Headers) ?[]const u8 { return this.get("content-type"); } diff --git a/src/s3/client.zig b/src/s3/client.zig index f3ac2e4d1a..4cdf4e63a6 100644 --- a/src/s3/client.zig +++ b/src/s3/client.zig @@ -238,6 +238,7 @@ pub fn upload( content: []const u8, content_type: ?[]const u8, content_disposition: ?[]const u8, + content_encoding: ?[]const u8, acl: ?ACL, proxy_url: ?[]const u8, storage_class: ?StorageClass, @@ -252,6 +253,7 @@ pub fn upload( .body = content, .content_type = content_type, .content_disposition = content_disposition, + .content_encoding = content_encoding, .acl = acl, .storage_class = storage_class, .request_payer = request_payer, @@ -265,6 +267,7 @@ pub fn writableStream( options: MultiPartUploadOptions, content_type: ?[]const u8, content_disposition: ?[]const u8, + content_encoding: ?[]const u8, proxy: ?[]const u8, storage_class: ?StorageClass, request_payer: bool, @@ -310,6 +313,7 @@ pub fn writableStream( .proxy = if (proxy_url.len > 0) bun.handleOom(bun.default_allocator.dupe(u8, proxy_url)) else "", .content_type = if (content_type) |ct| bun.handleOom(bun.default_allocator.dupe(u8, ct)) else null, .content_disposition = if (content_disposition) |cd| bun.handleOom(bun.default_allocator.dupe(u8, cd)) else null, + .content_encoding = if (content_encoding) |ce| bun.handleOom(bun.default_allocator.dupe(u8, ce)) else null, .storage_class = storage_class, .request_payer = request_payer, @@ -451,6 +455,7 @@ pub fn uploadStream( storage_class: ?StorageClass, content_type: ?[]const u8, content_disposition: ?[]const u8, + content_encoding: ?[]const u8, proxy: ?[]const u8, request_payer: bool, callback: ?*const fn (S3UploadResult, *anyopaque) void, @@ -489,6 +494,7 @@ pub fn uploadStream( .proxy = if (proxy_url.len > 0) bun.handleOom(bun.default_allocator.dupe(u8, proxy_url)) else "", .content_type = if (content_type) |ct| bun.handleOom(bun.default_allocator.dupe(u8, ct)) else null, .content_disposition = if (content_disposition) |cd| bun.handleOom(bun.default_allocator.dupe(u8, cd)) else null, + .content_encoding = if (content_encoding) |ce| bun.handleOom(bun.default_allocator.dupe(u8, ce)) else null, .callback = @ptrCast(&S3UploadStreamWrapper.resolve), .callback_context = undefined, .globalThis = globalThis, @@ -559,7 +565,7 @@ pub fn downloadStream( return; }; - var header_buffer: [10]picohttp.Header = undefined; + var header_buffer: [S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined; const headers = brk: { if (range) |range_| { const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ }); diff --git a/src/s3/credentials.zig b/src/s3/credentials.zig index 1ec0614874..10588c1334 100644 --- a/src/s3/credentials.zig +++ b/src/s3/credentials.zig @@ -244,6 +244,21 @@ pub const S3Credentials = struct { } } + if (try opts.getTruthyComptime(globalObject, "contentEncoding")) |js_value| { + if (!js_value.isEmptyOrUndefinedOrNull()) { + if (js_value.isString()) { + const str = try bun.String.fromJS(js_value, globalObject); + defer str.deref(); + if (str.tag != .Empty and str.tag != .Dead) { + new_credentials._contentEncodingSlice = str.toUTF8(bun.default_allocator); + new_credentials.content_encoding = new_credentials._contentEncodingSlice.?.slice(); + } + } else { + return globalObject.throwInvalidArgumentTypeValue("contentEncoding", "string", js_value); + } + } + } + if (try opts.getBooleanStrict(globalObject, "requestPayer")) |request_payer| { new_credentials.request_payer = request_payer; } @@ -363,12 +378,15 @@ pub const S3Credentials = struct { url: []const u8, content_disposition: []const u8 = "", + content_encoding: []const u8 = "", content_md5: []const u8 = "", session_token: []const u8 = "", acl: ?ACL = null, storage_class: ?StorageClass = null, request_payer: bool = false, - _headers: [9]picohttp.Header = .{ + _headers: [MAX_HEADERS]picohttp.Header = .{ + .{ .name = "", .value = "" }, + .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, @@ -381,6 +399,8 @@ pub const S3Credentials = struct { }, _headers_len: u8 = 0, + pub const MAX_HEADERS = 11; + pub fn headers(this: *const @This()) []const picohttp.Header { return this._headers[0..this._headers_len]; } @@ -408,6 +428,10 @@ pub const S3Credentials = struct { bun.freeSensitive(bun.default_allocator, this.content_disposition); } + if (this.content_encoding.len > 0) { + bun.freeSensitive(bun.default_allocator, this.content_encoding); + } + if (this.host.len > 0) { bun.freeSensitive(bun.default_allocator, this.host); } @@ -437,6 +461,7 @@ pub const S3Credentials = struct { search_params: ?[]const u8 = null, content_disposition: ?[]const u8 = null, content_type: ?[]const u8 = null, + content_encoding: ?[]const u8 = null, acl: ?ACL = null, storage_class: ?StorageClass = null, request_payer: bool = false, @@ -551,6 +576,10 @@ pub const S3Credentials = struct { if (content_type != null and content_type.?.len == 0) { content_type = null; } + var content_encoding = signOptions.content_encoding; + if (content_encoding != null and content_encoding.?.len == 0) { + content_encoding = null; + } const session_token: ?[]const u8 = if (this.sessionToken.len == 0) null else this.sessionToken; const acl: ?[]const u8 = if (signOptions.acl) |acl_value| acl_value.toString() else null; @@ -660,6 +689,7 @@ pub const S3Credentials = struct { const request_payer = signOptions.request_payer; const header_key = SignedHeaders.Key{ .content_disposition = content_disposition != null, + .content_encoding = content_encoding != null, .content_md5 = content_md5 != null, .acl = acl != null, .request_payer = request_payer, @@ -849,6 +879,7 @@ pub const S3Credentials = struct { normalizedPath, if (search_params) |p| p[1..] else "", content_disposition, + content_encoding, content_md5, host, acl, @@ -906,6 +937,8 @@ pub const S3Credentials = struct { .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, .{ .name = "", .value = "" }, + .{ .name = "", .value = "" }, + .{ .name = "", .value = "" }, }, ._headers_len = 4, }; @@ -933,6 +966,13 @@ pub const S3Credentials = struct { result._headers_len += 1; } + if (content_encoding) |ce| { + const content_encoding_value = bun.handleOom(bun.default_allocator.dupe(u8, ce)); + result.content_encoding = content_encoding_value; + result._headers[result._headers_len] = .{ .name = "content-encoding", .value = content_encoding_value }; + result._headers_len += 1; + } + if (content_md5) |c_md5| { const content_md5_value = bun.handleOom(bun.default_allocator.dupe(u8, c_md5)); result.content_md5 = content_md5_value; @@ -956,6 +996,7 @@ pub const S3CredentialsWithOptions = struct { storage_class: ?StorageClass = null, content_disposition: ?[]const u8 = null, content_type: ?[]const u8 = null, + content_encoding: ?[]const u8 = null, /// indicates if requester pays for the request (for requester pays buckets) request_payer: bool = false, /// indicates if the credentials have changed @@ -970,6 +1011,7 @@ pub const S3CredentialsWithOptions = struct { _sessionTokenSlice: ?jsc.ZigString.Slice = null, _contentDispositionSlice: ?jsc.ZigString.Slice = null, _contentTypeSlice: ?jsc.ZigString.Slice = null, + _contentEncodingSlice: ?jsc.ZigString.Slice = null, pub fn deinit(this: *@This()) void { if (this._accessKeyIdSlice) |slice| slice.deinit(); @@ -980,14 +1022,16 @@ pub const S3CredentialsWithOptions = struct { if (this._sessionTokenSlice) |slice| slice.deinit(); if (this._contentDispositionSlice) |slice| slice.deinit(); if (this._contentTypeSlice) |slice| slice.deinit(); + if (this._contentEncodingSlice) |slice| slice.deinit(); } }; /// Comptime-generated lookup table for signed headers strings. /// Headers must be in alphabetical order per AWS Signature V4 spec. const SignedHeaders = struct { - const Key = packed struct(u6) { + const Key = packed struct(u7) { content_disposition: bool, + content_encoding: bool, content_md5: bool, acl: bool, request_payer: bool, @@ -997,6 +1041,7 @@ const SignedHeaders = struct { fn generate(comptime key: Key) []const u8 { return (if (key.content_disposition) "content-disposition;" else "") ++ + (if (key.content_encoding) "content-encoding;" else "") ++ (if (key.content_md5) "content-md5;" else "") ++ "host;" ++ (if (key.acl) "x-amz-acl;" else "") ++ @@ -1007,15 +1052,15 @@ const SignedHeaders = struct { } const table = init: { - var t: [64][]const u8 = undefined; - for (0..64) |i| { - t[i] = generate(@bitCast(@as(u6, @intCast(i)))); + var t: [128][]const u8 = undefined; + for (0..128) |i| { + t[i] = generate(@bitCast(@as(u7, @intCast(i)))); } break :init t; }; pub fn get(key: Key) []const u8 { - return table[@as(u6, @bitCast(key))]; + return table[@as(u7, @bitCast(key))]; } }; @@ -1025,6 +1070,7 @@ const CanonicalRequest = struct { fn fmtString(comptime key: SignedHeaders.Key) []const u8 { return "{s}\n{s}\n{s}\n" ++ // method, path, query (if (key.content_disposition) "content-disposition:{s}\n" else "") ++ + (if (key.content_encoding) "content-encoding:{s}\n" else "") ++ (if (key.content_md5) "content-md5:{s}\n" else "") ++ "host:{s}\n" ++ (if (key.acl) "x-amz-acl:{s}\n" else "") ++ @@ -1042,6 +1088,7 @@ const CanonicalRequest = struct { path: []const u8, query: []const u8, content_disposition: ?[]const u8, + content_encoding: ?[]const u8, content_md5: ?[]const u8, host: []const u8, acl: ?[]const u8, @@ -1053,6 +1100,7 @@ const CanonicalRequest = struct { ) error{NoSpaceLeft}![]u8 { return std.fmt.bufPrint(buf, fmtString(key), .{ method, path, query } ++ (if (key.content_disposition) .{content_disposition.?} else .{}) ++ + (if (key.content_encoding) .{content_encoding.?} else .{}) ++ (if (key.content_md5) .{content_md5.?} else .{}) ++ .{host} ++ (if (key.acl) .{acl.?} else .{}) ++ @@ -1069,6 +1117,7 @@ const CanonicalRequest = struct { path: []const u8, query: []const u8, content_disposition: ?[]const u8, + content_encoding: ?[]const u8, content_md5: ?[]const u8, host: []const u8, acl: ?[]const u8, @@ -1079,14 +1128,15 @@ const CanonicalRequest = struct { signed_headers: []const u8, ) error{NoSpaceLeft}![]u8 { // Dispatch to the right comptime-specialized function based on runtime key - return switch (@as(u6, @bitCast(key))) { - inline 0...63 => |idx| formatForKey( + return switch (@as(u7, @bitCast(key))) { + inline 0...127 => |idx| formatForKey( @bitCast(idx), buf, method, path, query, content_disposition, + content_encoding, content_md5, host, acl, diff --git a/src/s3/multipart.zig b/src/s3/multipart.zig index 13fe6dad67..0f1816a19d 100644 --- a/src/s3/multipart.zig +++ b/src/s3/multipart.zig @@ -117,6 +117,7 @@ pub const MultiPartUpload = struct { proxy: []const u8, content_type: ?[]const u8 = null, content_disposition: ?[]const u8 = null, + content_encoding: ?[]const u8 = null, upload_id: []const u8 = "", uploadid_buffer: bun.MutableString = .{ .allocator = bun.default_allocator, .list = .{} }, @@ -284,6 +285,11 @@ pub const MultiPartUpload = struct { bun.default_allocator.free(cd); } } + if (this.content_encoding) |ce| { + if (ce.len > 0) { + bun.default_allocator.free(ce); + } + } this.credentials.deref(); this.uploadid_buffer.deinit(); for (this.multipart_etags.items) |tag| { @@ -310,6 +316,7 @@ pub const MultiPartUpload = struct { .body = this.buffered.slice(), .content_type = this.content_type, .content_disposition = this.content_disposition, + .content_encoding = this.content_encoding, .acl = this.acl, .storage_class = this.storage_class, .request_payer = this.request_payer, @@ -602,6 +609,7 @@ pub const MultiPartUpload = struct { .search_params = "?uploads=", .content_type = this.content_type, .content_disposition = this.content_disposition, + .content_encoding = this.content_encoding, .acl = this.acl, .storage_class = this.storage_class, .request_payer = this.request_payer, @@ -680,6 +688,7 @@ pub const MultiPartUpload = struct { .body = this.buffered.slice(), .content_type = this.content_type, .content_disposition = this.content_disposition, + .content_encoding = this.content_encoding, .acl = this.acl, .storage_class = this.storage_class, .request_payer = this.request_payer, diff --git a/src/s3/simple_request.zig b/src/s3/simple_request.zig index 32fb032549..ab3f3b552f 100644 --- a/src/s3/simple_request.zig +++ b/src/s3/simple_request.zig @@ -351,6 +351,7 @@ pub const S3SimpleRequestOptions = struct { search_params: ?[]const u8 = null, content_type: ?[]const u8 = null, content_disposition: ?[]const u8 = null, + content_encoding: ?[]const u8 = null, // http request options body: []const u8, @@ -372,6 +373,7 @@ pub fn executeSimpleS3Request( .method = options.method, .search_params = options.search_params, .content_disposition = options.content_disposition, + .content_encoding = options.content_encoding, .acl = options.acl, .storage_class = options.storage_class, .request_payer = options.request_payer, @@ -383,7 +385,7 @@ pub fn executeSimpleS3Request( }; const headers = brk: { - var header_buffer: [10]picohttp.Header = undefined; + var header_buffer: [S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined; if (options.range) |range_| { const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ }); break :brk bun.handleOom(bun.http.Headers.fromPicoHttpHeaders(_headers, bun.default_allocator)); diff --git a/test/js/bun/s3/s3.test.ts b/test/js/bun/s3/s3.test.ts index a947d12004..53c56a5d0a 100644 --- a/test/js/bun/s3/s3.test.ts +++ b/test/js/bun/s3/s3.test.ts @@ -445,6 +445,46 @@ for (let credentials of allCredentials) { } }); + it("should be able to set content-encoding", async () => { + await using tmpfile = await tmp(); + { + const s3file = bucket.file(tmpfile.name, options!); + await s3file.write("Hello Bun!", { contentEncoding: "gzip" }); + // Use decompress: false since content isn't actually gzip-compressed + const response = await fetch(s3file.presign(), { decompress: false }); + expect(response.headers.get("content-encoding")).toBe("gzip"); + } + { + const s3file = bucket.file(tmpfile.name, options!); + await s3file.write("Hello Bun!", { contentEncoding: "br" }); + // Use decompress: false since content isn't actually br-compressed + const response = await fetch(s3file.presign(), { decompress: false }); + expect(response.headers.get("content-encoding")).toBe("br"); + } + { + await bucket.write(tmpfile.name, "Hello Bun!", { + ...options, + contentEncoding: "identity", + }); + const response = await fetch(bucket.file(tmpfile.name, options!).presign(), { decompress: false }); + expect(response.headers.get("content-encoding")).toBe("identity"); + } + }); + it("should be able to set content-encoding in writer", async () => { + await using tmpfile = await tmp(); + { + const s3file = bucket.file(tmpfile.name, options!); + const writer = s3file.writer({ + contentEncoding: "gzip", + }); + writer.write("Hello Bun!!"); + await writer.end(); + // Use decompress: false since content isn't actually gzip-compressed + const response = await fetch(s3file.presign(), { decompress: false }); + expect(response.headers.get("content-encoding")).toBe("gzip"); + } + }); + it("should be able to upload large files using bucket.write + readable Request", async () => { await using tmpfile = await tmp(); {