feat(s3): add custom metadata support (x-amz-meta-*)

Add support for S3 custom metadata headers that allow users to attach
key-value pairs to S3 objects.

**Write path**: Accept `metadata: Record<string, string>` in S3Options
- Convert `{ sku: "12345" }` to `x-amz-meta-sku: 12345` headers
- Works with `write()`, `writer()`, and presigned PUT URLs
- Keys are automatically normalized to lowercase per AWS requirements

**Read path**: Return metadata in S3Stats via `stat()`
- Extract x-amz-meta-* headers from response
- Strip prefix and return as `metadata: Record<string, string>`

**Presigned URLs**: Include metadata as signed headers in PUT URLs
- Client uploading to presigned URL must include matching headers

Added `metadata` option to S3Options for attaching custom metadata (x-amz-meta-* headers) to S3 objects during upload. Added `metadata` property to S3Stats returned by `stat()` to retrieve stored metadata. Keys are automatically lowercased per AWS requirements. Maximum ~2KB total metadata supported.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Ciro Spaciari MacBook
2026-01-15 18:06:04 -08:00
parent 8da29af1ae
commit 2fe0b053e9
12 changed files with 774 additions and 58 deletions

View File

@@ -345,6 +345,32 @@ declare module "bun" {
*/
requestPayer?: boolean;
/**
* User-defined metadata to store with the S3 object as `x-amz-meta-*` headers.
* Keys are automatically lowercased per AWS requirements.
* Maximum total metadata size is ~2KB.
*
* @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
*
* @example
* // Setting custom metadata on upload
* await s3.write("document.pdf", data, {
* metadata: {
* sku: "12345",
* category: "reports"
* }
* });
*
* @example
* // Metadata with presigned PUT URL
* const url = file.presign({
* method: "PUT",
* metadata: { "custom-key": "value" }
* });
* // Client must include matching header: x-amz-meta-custom-key: value
*/
metadata?: Record<string, string>;
/**
* @deprecated The size of the internal buffer in bytes. Defaults to 5 MiB. use `partSize` and `queueSize` instead.
*/
@@ -400,6 +426,16 @@ declare module "bun" {
lastModified: Date;
etag: string;
type: string;
/**
* User-defined metadata retrieved from `x-amz-meta-*` headers.
* Keys have the `x-amz-meta-` prefix stripped and are lowercased.
*
* @example
* // Retrieving metadata
* const stat = await file.stat();
* console.log(stat.metadata); // { sku: "12345", category: "reports" }
*/
metadata: Record<string, string>;
}
/**

View File

@@ -25,6 +25,10 @@ export default [
getter: "getContentType",
cache: true,
},
metadata: {
getter: "getMetadata",
cache: true,
},
},
}),
];

View File

@@ -973,6 +973,7 @@ fn writeFileWithEmptySourceToDestination(ctx: *jsc.JSGlobalObject, destination_b
proxy_url,
aws_options.storage_class,
aws_options.request_payer,
aws_options.metadata,
Wrapper.resolve,
Wrapper.new(.{
.promise = promise,
@@ -1124,6 +1125,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl
aws_options.content_encoding,
proxy_url,
aws_options.request_payer,
aws_options.metadata,
null,
undefined,
);
@@ -1167,6 +1169,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl
proxy_url,
aws_options.storage_class,
aws_options.request_payer,
aws_options.metadata,
Wrapper.resolve,
Wrapper.new(.{
.store = source_store,
@@ -1197,6 +1200,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl
aws_options.content_encoding,
proxy_url,
aws_options.request_payer,
aws_options.metadata,
null,
undefined,
);
@@ -1405,6 +1409,7 @@ pub fn writeFileInternal(globalThis: *jsc.JSGlobalObject, path_or_blob_: *PathOr
aws_options.content_encoding,
proxy_url,
aws_options.request_payer,
aws_options.metadata,
null,
undefined,
);
@@ -1468,6 +1473,7 @@ pub fn writeFileInternal(globalThis: *jsc.JSGlobalObject, path_or_blob_: *PathOr
aws_options.content_encoding,
proxy_url,
aws_options.request_payer,
aws_options.metadata,
null,
undefined,
);
@@ -2447,6 +2453,7 @@ pub fn pipeReadableStreamToBlob(this: *Blob, globalThis: *jsc.JSGlobalObject, re
aws_options.content_encoding,
proxy_url,
aws_options.request_payer,
aws_options.metadata,
null,
undefined,
);
@@ -2702,20 +2709,24 @@ pub fn getWriter(
proxy_url,
credentialsWithOptions.storage_class,
credentialsWithOptions.request_payer,
credentialsWithOptions.metadata,
);
}
}
// Dupe metadata since multipart upload takes ownership
const metadata_dupe = if (s3.metadata) |meta| meta.dupe(bun.default_allocator) else null;
return try S3.writableStream(
s3.getCredentials(),
path,
globalThis,
.{},
s3.options,
this.contentTypeOrMimeType(),
null,
null,
proxy_url,
null,
s3.storage_class,
s3.request_payer,
metadata_dupe,
);
}

View File

@@ -268,6 +268,9 @@ pub fn constructS3FileWithS3CredentialsAndOptions(
store.data.s3.acl = aws_options.acl;
store.data.s3.storage_class = aws_options.storage_class;
store.data.s3.request_payer = aws_options.request_payer;
// Transfer metadata ownership to the store
store.data.s3.metadata = aws_options.metadata;
aws_options.metadata = null; // Prevent deinit from freeing
var blob = Blob.initWithStore(store, globalObject);
if (options) |opts| {
@@ -312,6 +315,9 @@ pub fn constructS3FileWithS3Credentials(
store.data.s3.acl = aws_options.acl;
store.data.s3.storage_class = aws_options.storage_class;
store.data.s3.request_payer = aws_options.request_payer;
// Transfer metadata ownership to the store
store.data.s3.metadata = aws_options.metadata;
aws_options.metadata = null; // Prevent deinit from freeing
var blob = Blob.initWithStore(store, globalObject);
if (options) |opts| {
@@ -399,6 +405,7 @@ pub const S3BlobStatTask = struct {
stat_result.etag,
stat_result.contentType,
stat_result.lastModified,
stat_result.headers,
globalThis,
)).toJS(globalThis));
},
@@ -505,6 +512,7 @@ pub fn getPresignUrlFrom(this: *Blob, globalThis: *jsc.JSGlobalObject, extra_opt
.request_payer = credentialsWithOptions.request_payer,
.content_disposition = credentialsWithOptions.content_disposition,
.content_type = credentialsWithOptions.content_type,
.metadata = credentialsWithOptions.metadata,
}, false, .{ .expires = expires }) catch |sign_err| {
return S3.throwSignError(sign_err, globalThis);
};

View File

@@ -11,30 +11,66 @@ pub const S3Stat = struct {
etag: bun.String,
contentType: bun.String,
lastModified: f64,
/// Cached JS object containing x-amz-meta-* headers as key-value pairs.
/// Keys have the "x-amz-meta-" prefix stripped.
metadata: jsc.JSValue,
pub fn constructor(globalThis: *jsc.JSGlobalObject, _: *jsc.CallFrame) bun.JSError!*@This() {
return globalThis.throwInvalidArguments("S3Stat is not constructable", .{});
}
/// Initialize S3Stat from stat result data.
/// `headers` should be the raw response headers - this function will extract x-amz-meta-* headers.
pub fn init(
size: u64,
etag: []const u8,
contentType: []const u8,
lastModified: []const u8,
headers: []const picohttp.Header,
globalThis: *jsc.JSGlobalObject,
) bun.JSError!*@This() {
var date_str = bun.String.init(lastModified);
defer date_str.deref();
const last_modified = try date_str.parseDate(globalThis);
// Build metadata JS object from x-amz-meta-* headers
const metadata_obj = try buildMetadataObject(headers, globalThis);
jsc.JSValue.protect(metadata_obj);
return S3Stat.new(.{
.size = size,
.etag = bun.String.cloneUTF8(etag),
.contentType = bun.String.cloneUTF8(contentType),
.lastModified = last_modified,
.metadata = metadata_obj,
});
}
/// Extract x-amz-meta-* headers and build a JS object with stripped key names.
fn buildMetadataObject(headers: []const picohttp.Header, globalThis: *jsc.JSGlobalObject) bun.JSError!jsc.JSValue {
const prefix = "x-amz-meta-";
const prefix_len = prefix.len;
// Create empty JS object
const obj = jsc.JSValue.createEmptyObject(globalThis, 0);
for (headers) |header| {
// Case-insensitive check for x-amz-meta- prefix
if (header.name.len > prefix_len and
strings.eqlCaseInsensitiveASCII(header.name[0..prefix_len], prefix, true))
{
// Strip the prefix to get the user's key name
const key = header.name[prefix_len..];
const value_js = try bun.String.createUTF8ForJS(globalThis, header.value);
// put() accepts []const u8 directly and wraps it in ZigString
obj.put(globalThis, key, value_js);
}
}
return obj;
}
pub fn getSize(this: *@This(), _: *jsc.JSGlobalObject) jsc.JSValue {
return jsc.JSValue.jsNumber(this.size);
}
@@ -51,12 +87,19 @@ pub const S3Stat = struct {
return jsc.JSValue.fromDateNumber(globalObject, this.lastModified);
}
pub fn getMetadata(this: *@This(), _: *jsc.JSGlobalObject) jsc.JSValue {
return this.metadata;
}
pub fn finalize(this: *@This()) void {
this.etag.deref();
this.contentType.deref();
jsc.JSValue.unprotect(this.metadata);
bun.destroy(this);
}
};
const bun = @import("bun");
const jsc = bun.jsc;
const picohttp = bun.picohttp;
const strings = bun.strings;

View File

@@ -296,6 +296,8 @@ pub const S3 = struct {
acl: ?bun.S3.ACL = null,
storage_class: ?bun.S3.StorageClass = null,
request_payer: bool = false,
/// User-defined metadata (x-amz-meta-* headers)
metadata: ?bun.S3.MetadataMap = null,
pub fn isSeekable(_: *const @This()) ?bool {
return true;
@@ -464,6 +466,10 @@ pub const S3 = struct {
credentials.deref();
this.credentials = null;
}
if (this.metadata) |*meta| {
meta.deinit(bun.default_allocator);
this.metadata = null;
}
}
const S3Credentials = bun.S3.S3Credentials;

View File

@@ -1322,6 +1322,7 @@ fn fetchImpl(
if (headers) |h| h.getContentEncoding() else null,
proxy_url,
credentialsWithOptions.request_payer,
credentialsWithOptions.metadata,
@ptrCast(&Wrapper.resolve),
s3_stream,
);
@@ -1361,7 +1362,7 @@ fn fetchImpl(
}
const content_type = if (headers) |h| (h.getContentType()) else null;
var header_buffer: [s3.S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined;
var header_buffer: [s3.MAX_HEADERS + 1]picohttp.Header = undefined;
if (range) |range_| {
const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ });

View File

@@ -10,6 +10,8 @@ pub const getJSSignError = Error.getJSSignError;
pub const S3Credentials = Credentials.S3Credentials;
pub const S3CredentialsWithOptions = Credentials.S3CredentialsWithOptions;
pub const MetadataMap = Credentials.MetadataMap;
pub const MAX_HEADERS = Credentials.MAX_HEADERS;
pub const S3HttpSimpleTask = S3SimpleRequest.S3HttpSimpleTask;
pub const S3UploadResult = S3SimpleRequest.S3UploadResult;
@@ -243,6 +245,7 @@ pub fn upload(
proxy_url: ?[]const u8,
storage_class: ?StorageClass,
request_payer: bool,
metadata: ?Credentials.MetadataMap,
callback: *const fn (S3UploadResult, *anyopaque) bun.JSTerminated!void,
callback_context: *anyopaque,
) bun.JSTerminated!void {
@@ -257,6 +260,7 @@ pub fn upload(
.acl = acl,
.storage_class = storage_class,
.request_payer = request_payer,
.metadata = metadata,
}, .{ .upload = callback }, callback_context);
}
/// returns a writable stream that writes to the s3 path
@@ -271,6 +275,7 @@ pub fn writableStream(
proxy: ?[]const u8,
storage_class: ?StorageClass,
request_payer: bool,
metadata: ?Credentials.MetadataMap,
) bun.JSError!jsc.JSValue {
const Wrapper = struct {
pub fn callback(result: S3UploadResult, sink: *jsc.WebCore.NetworkSink) bun.JSTerminated!void {
@@ -316,6 +321,7 @@ pub fn writableStream(
.content_encoding = if (content_encoding) |ce| bun.handleOom(bun.default_allocator.dupe(u8, ce)) else null,
.storage_class = storage_class,
.request_payer = request_payer,
.metadata = metadata,
.callback = @ptrCast(&Wrapper.callback),
.callback_context = undefined,
@@ -458,6 +464,7 @@ pub fn uploadStream(
content_encoding: ?[]const u8,
proxy: ?[]const u8,
request_payer: bool,
metadata: ?Credentials.MetadataMap,
callback: ?*const fn (S3UploadResult, *anyopaque) void,
callback_context: *anyopaque,
) bun.JSError!jsc.JSValue {
@@ -503,6 +510,7 @@ pub fn uploadStream(
.acl = acl,
.storage_class = storage_class,
.request_payer = request_payer,
.metadata = metadata,
.vm = jsc.VirtualMachine.get(),
});
@@ -565,7 +573,7 @@ pub fn downloadStream(
return;
};
var header_buffer: [S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined;
var header_buffer: [Credentials.MAX_HEADERS + 1]picohttp.Header = undefined;
const headers = brk: {
if (range) |range_| {
const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ });

View File

@@ -1,3 +1,67 @@
/// Maximum number of user-defined metadata headers.
/// AWS S3 limits total metadata to ~2KB (https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html).
/// With typical key-value pairs of ~50-100 bytes, 32 headers provides ample capacity.
const MAX_METADATA_HEADERS: usize = 32;
/// Number of base S3 headers (x-amz-content-sha256, x-amz-date, Host, Authorization, etc.)
const BASE_HEADERS: usize = 9;
/// Total maximum headers: base headers + metadata headers
pub const MAX_HEADERS: usize = BASE_HEADERS + MAX_METADATA_HEADERS;
/// Buffer size for signed headers string (e.g., "content-disposition;host;x-amz-meta-foo;...")
/// Enough for all standard headers plus MAX_METADATA_HEADERS metadata keys of ~50 chars each
const SIGNED_HEADERS_BUF_SIZE: usize = 2048;
/// Buffer size for canonical request building with metadata
/// Needs space for method, path, query, all headers with values, signed_headers, and hash
const CANONICAL_REQUEST_BUF_SIZE: usize = 8192;
/// Holds user-defined metadata key-value pairs for S3 objects (x-amz-meta-* headers).
/// Keys are normalized to lowercase per AWS S3 requirements.
/// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
pub const MetadataMap = struct {
keys: []const []const u8,
values: []const []const u8,
pub fn count(this: @This()) usize {
return this.keys.len;
}
/// Deep-copy the metadata map, duplicating all keys and values.
pub fn dupe(this: @This(), allocator: std.mem.Allocator) ?MetadataMap {
const n = this.keys.len;
if (n == 0) return null;
const new_keys = allocator.alloc([]const u8, n) catch return null;
const new_values = allocator.alloc([]const u8, n) catch {
allocator.free(new_keys);
return null;
};
for (0..n) |i| {
new_keys[i] = allocator.dupe(u8, this.keys[i]) catch return null;
new_values[i] = allocator.dupe(u8, this.values[i]) catch return null;
}
return .{
.keys = new_keys,
.values = new_values,
};
}
pub fn deinit(this: *@This(), allocator: std.mem.Allocator) void {
for (this.keys) |key| {
allocator.free(key);
}
for (this.values) |value| {
allocator.free(value);
}
allocator.free(this.keys);
allocator.free(this.values);
}
};
pub const S3Credentials = struct {
const RefCount = bun.ptr.RefCount(@This(), "ref_count", deinit, .{});
pub const ref = RefCount.ref;
@@ -262,6 +326,64 @@ pub const S3Credentials = struct {
if (try opts.getBooleanStrict(globalObject, "requestPayer")) |request_payer| {
new_credentials.request_payer = request_payer;
}
// Parse metadata object
if (try opts.getTruthyComptime(globalObject, "metadata")) |metadata_value| {
if (!metadata_value.isEmptyOrUndefinedOrNull()) {
if (metadata_value.isObject()) {
const metadata_obj = try metadata_value.toObject(globalObject);
// Count properties first
var property_count: usize = 0;
var iter = try jsc.JSPropertyIterator(.{ .skip_empty_name = true, .include_value = true }).init(globalObject, metadata_obj);
defer iter.deinit();
while (try iter.next()) |_| {
property_count += 1;
}
if (property_count > 0) {
// Allocate arrays for keys and values
const keys = bun.default_allocator.alloc([]const u8, property_count) catch bun.outOfMemory();
const values = bun.default_allocator.alloc([]const u8, property_count) catch bun.outOfMemory();
// Second pass to extract keys and values
var iter2 = try jsc.JSPropertyIterator(.{ .skip_empty_name = true, .include_value = true }).init(globalObject, metadata_obj);
defer iter2.deinit();
var i: usize = 0;
while (try iter2.next()) |key| {
// Convert key to lowercase (AWS requires lowercase metadata keys)
const key_owned = try key.toOwnedSlice(bun.default_allocator);
const key_lower = bun.default_allocator.alloc(u8, key_owned.len) catch bun.outOfMemory();
for (key_owned, 0..) |c, j| {
key_lower[j] = if (c >= 'A' and c <= 'Z') c + 32 else c;
}
bun.default_allocator.free(key_owned);
keys[i] = key_lower;
// Get value - metadata values must be strings
const val = iter2.value;
if (val.isString()) {
const val_str = try bun.String.fromJS(val, globalObject);
defer val_str.deref();
const val_slice = val_str.toUTF8(bun.default_allocator);
values[i] = bun.default_allocator.dupe(u8, val_slice.slice()) catch bun.outOfMemory();
val_slice.deinit();
} else {
// Non-string value, use empty string or skip
values[i] = "";
}
i += 1;
}
new_credentials.metadata = .{
.keys = keys,
.values = values,
};
}
} else {
return globalObject.throwInvalidArgumentTypeValue("metadata", "object", metadata_value);
}
}
}
}
}
return new_credentials;
@@ -384,23 +506,12 @@ pub const S3Credentials = struct {
acl: ?ACL = null,
storage_class: ?StorageClass = null,
request_payer: bool = false,
_headers: [MAX_HEADERS]picohttp.Header = .{
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
},
/// Stores allocated metadata header names ("x-amz-meta-*") for cleanup
_metadata_header_names: ?[][]u8 = null,
/// Header storage: BASE_HEADERS (9) + MAX_METADATA_HEADERS (32) = MAX_HEADERS (41)
_headers: [MAX_HEADERS]picohttp.Header = [_]picohttp.Header{.{ .name = "", .value = "" }} ** MAX_HEADERS,
_headers_len: u8 = 0,
pub const MAX_HEADERS = 11;
pub fn headers(this: *const @This()) []const picohttp.Header {
return this._headers[0..this._headers_len];
}
@@ -447,6 +558,14 @@ pub const S3Credentials = struct {
if (this.content_md5.len > 0) {
bun.default_allocator.free(this.content_md5);
}
// Free allocated metadata header names
if (this._metadata_header_names) |names| {
for (names) |name| {
bun.default_allocator.free(name);
}
bun.default_allocator.free(names);
}
}
};
@@ -465,6 +584,8 @@ pub const S3Credentials = struct {
acl: ?ACL = null,
storage_class: ?StorageClass = null,
request_payer: bool = false,
/// User-defined metadata (x-amz-meta-* headers)
metadata: ?MetadataMap = null,
};
/// This is not used for signing but for console.log output, is just nice to have
pub fn guessBucket(endpoint: []const u8) ?[]const u8 {
@@ -687,6 +808,7 @@ pub const S3Credentials = struct {
const amz_day = amz_date[0..8];
const request_payer = signOptions.request_payer;
const metadata = signOptions.metadata;
const header_key = SignedHeaders.Key{
.content_disposition = content_disposition != null,
.content_encoding = content_encoding != null,
@@ -696,7 +818,97 @@ pub const S3Credentials = struct {
.session_token = session_token != null,
.storage_class = storage_class != null,
};
const signed_headers = if (signQuery) "host" else SignedHeaders.get(header_key);
// Build signed_headers string - for metadata we need to do this dynamically
// because metadata keys vary at runtime. Metadata header names (x-amz-meta-*)
// come after x-amz-date and before x-amz-request-payer alphabetically.
// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
var signed_headers_buf: [SIGNED_HEADERS_BUF_SIZE]u8 = undefined;
const signed_headers: []const u8 = brk_signed: {
if (signQuery) {
// For presigned PUT URLs with metadata, include metadata headers
if (metadata) |meta| {
const meta_count = meta.count();
if (meta_count > 0) {
var fba = std.heap.FixedBufferAllocator.init(&signed_headers_buf);
var list = std.array_list.Managed(u8).init(fba.allocator());
list.appendSlice("host") catch {};
// Sort metadata keys alphabetically
var indices: [MAX_METADATA_HEADERS]usize = undefined;
for (0..meta_count) |i| {
indices[i] = i;
}
std.mem.sort(usize, indices[0..meta_count], meta.keys, struct {
fn lessThan(keys: []const []const u8, a: usize, b: usize) bool {
return std.mem.lessThan(u8, keys[a], keys[b]);
}
}.lessThan);
// Add metadata headers (alphabetically after "host")
for (indices[0..meta_count]) |idx| {
list.appendSlice(";x-amz-meta-") catch {};
list.appendSlice(meta.keys[idx]) catch {};
}
if (list.items.len > 0) {
break :brk_signed list.items;
}
}
}
break :brk_signed "host";
}
if (metadata) |meta| {
// Build signed headers dynamically with metadata
// AWS Sig V4 requires headers in alphabetical order
var fba = std.heap.FixedBufferAllocator.init(&signed_headers_buf);
var list = std.array_list.Managed(u8).init(fba.allocator());
// Headers in alphabetical order (same as SignedHeaders.generate):
// content-disposition, content-encoding, content-md5, host, x-amz-acl, x-amz-content-sha256, x-amz-date,
// [x-amz-meta-*], x-amz-request-payer, x-amz-security-token, x-amz-storage-class
if (content_disposition != null) list.appendSlice("content-disposition;") catch {};
if (content_encoding != null) list.appendSlice("content-encoding;") catch {};
if (content_md5 != null) list.appendSlice("content-md5;") catch {};
list.appendSlice("host;") catch {};
if (acl != null) list.appendSlice("x-amz-acl;") catch {};
list.appendSlice("x-amz-content-sha256;x-amz-date") catch {};
// Add metadata headers (sorted by key)
// Keys are already lowercase from parsing (AWS requirement)
const meta_count = meta.count();
if (meta_count > 0) {
// Create sorted indices for metadata keys
var indices: [MAX_METADATA_HEADERS]usize = undefined;
for (0..meta_count) |i| {
indices[i] = i;
}
// Sort indices by key for AWS Sig V4 canonical header ordering
std.mem.sort(usize, indices[0..meta_count], meta.keys, struct {
fn lessThan(keys: []const []const u8, a: usize, b: usize) bool {
return std.mem.lessThan(u8, keys[a], keys[b]);
}
}.lessThan);
// Append sorted metadata header names (x-amz-meta-{key})
for (indices[0..meta_count]) |idx| {
list.appendSlice(";x-amz-meta-") catch {};
list.appendSlice(meta.keys[idx]) catch {};
}
}
if (request_payer) list.appendSlice(";x-amz-request-payer") catch {};
if (session_token != null) list.appendSlice(";x-amz-security-token") catch {};
if (storage_class != null) list.appendSlice(";x-amz-storage-class") catch {};
// If buffer overflowed, fall back to non-metadata path
if (list.items.len == 0) {
// Fallback: metadata too large, use base signed_headers without metadata
break :brk_signed SignedHeaders.get(header_key);
}
break :brk_signed list.items;
}
break :brk_signed SignedHeaders.get(header_key);
};
const service_name = "s3";
@@ -778,7 +990,10 @@ pub const S3Credentials = struct {
try query_parts.append(try std.fmt.allocPrint(allocator, "X-Amz-Security-Token={s}", .{token}));
}
try query_parts.append(try std.fmt.allocPrint(allocator, "X-Amz-SignedHeaders=host", .{}));
// URL-encode the signed headers (semicolons become %3B)
var signed_headers_encoded_buf: [SIGNED_HEADERS_BUF_SIZE]u8 = undefined;
const encoded_signed_headers = encodeURIComponent(signed_headers, &signed_headers_encoded_buf, true) catch signed_headers;
try query_parts.append(try std.fmt.allocPrint(allocator, "X-Amz-SignedHeaders={s}", .{encoded_signed_headers}));
if (encoded_content_disposition) |cd| {
try query_parts.append(try std.fmt.allocPrint(allocator, "response-content-disposition={s}", .{cd}));
@@ -805,7 +1020,45 @@ pub const S3Credentials = struct {
allocator.free(part);
}
break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\n\nhost\n{s}", .{ method_name, normalizedPath, query_string.items, host, aws_content_hash });
// Canonical request: METHOD\nPATH\nQUERY\nCANONICAL_HEADERS\n\nSIGNED_HEADERS\nPAYLOAD_HASH
// For presigned URLs with metadata, include both host and metadata headers
if (metadata) |meta| {
const meta_count = meta.count();
if (meta_count > 0) {
// Build canonical headers with metadata (sorted alphabetically)
var canonical_headers_buf: [4096]u8 = undefined;
var headers_fba = std.heap.FixedBufferAllocator.init(&canonical_headers_buf);
var headers_list = std.array_list.Managed(u8).init(headers_fba.allocator());
// Add host header first (comes before x-amz-meta-* alphabetically)
headers_list.appendSlice("host:") catch {};
headers_list.appendSlice(host) catch {};
headers_list.append('\n') catch {};
// Sort metadata indices for alphabetical ordering
var meta_indices: [MAX_METADATA_HEADERS]usize = undefined;
for (0..meta_count) |i| {
meta_indices[i] = i;
}
std.mem.sort(usize, meta_indices[0..meta_count], meta.keys, struct {
fn lessThan(keys: []const []const u8, a: usize, b: usize) bool {
return std.mem.lessThan(u8, keys[a], keys[b]);
}
}.lessThan);
// Add metadata headers (x-amz-meta-{key}:{value})
for (meta_indices[0..meta_count]) |idx| {
headers_list.appendSlice("x-amz-meta-") catch {};
headers_list.appendSlice(meta.keys[idx]) catch {};
headers_list.append(':') catch {};
headers_list.appendSlice(meta.values[idx]) catch {};
headers_list.append('\n') catch {};
}
break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\n{s}\n{s}\n{s}", .{ method_name, normalizedPath, query_string.items, headers_list.items, signed_headers, aws_content_hash });
}
}
break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, query_string.items, host, signed_headers, aws_content_hash });
};
var sha_digest = std.mem.zeroes(bun.sha.SHA256.Digest);
bun.sha.SHA256.hash(canonical, &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine());
@@ -843,7 +1096,10 @@ pub const S3Credentials = struct {
try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "X-Amz-Signature={s}", .{std.fmt.bytesToHex(signature[0..DIGESTED_HMAC_256_LEN], .lower)}));
try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "X-Amz-SignedHeaders=host", .{}));
// URL-encode the signed headers for the final URL
var url_signed_headers_encoded_buf: [SIGNED_HEADERS_BUF_SIZE]u8 = undefined;
const url_encoded_signed_headers = encodeURIComponent(signed_headers, &url_signed_headers_encoded_buf, true) catch signed_headers;
try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "X-Amz-SignedHeaders={s}", .{url_encoded_signed_headers}));
if (encoded_content_disposition) |cd| {
try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "response-content-disposition={s}", .{cd}));
@@ -872,25 +1128,187 @@ pub const S3Credentials = struct {
break :brk try std.fmt.allocPrint(bun.default_allocator, "{s}://{s}{s}?{s}", .{ protocol, host, normalizedPath, url_query_string.items });
} else {
const canonical = try CanonicalRequest.format(
&tmp_buffer,
header_key,
method_name,
normalizedPath,
if (search_params) |p| p[1..] else "",
content_disposition,
content_encoding,
content_md5,
host,
acl,
aws_content_hash,
amz_date,
session_token,
storage_class,
signed_headers,
);
// Build canonical request - if metadata exists, build manually
var sha_digest = std.mem.zeroes(bun.sha.SHA256.Digest);
bun.sha.SHA256.hash(canonical, &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine());
if (metadata) |meta| {
// Build canonical request manually with metadata headers
// Format per AWS Sig V4: METHOD\nPATH\nQUERY\nHEADERS\n\nSIGNED_HEADERS\nHASH
// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
var canonical_buf: [CANONICAL_REQUEST_BUF_SIZE]u8 = undefined;
var fba = std.heap.FixedBufferAllocator.init(&canonical_buf);
var writer = fba.allocator().alloc(u8, CANONICAL_REQUEST_BUF_SIZE) catch unreachable;
var pos: usize = 0;
// METHOD\n
@memcpy(writer[pos..][0..method_name.len], method_name);
pos += method_name.len;
writer[pos] = '\n';
pos += 1;
// PATH\n
@memcpy(writer[pos..][0..normalizedPath.len], normalizedPath);
pos += normalizedPath.len;
writer[pos] = '\n';
pos += 1;
// QUERY\n
const query_str = if (search_params) |p| p[1..] else "";
@memcpy(writer[pos..][0..query_str.len], query_str);
pos += query_str.len;
writer[pos] = '\n';
pos += 1;
// HEADERS (sorted alphabetically)
if (content_disposition) |cd| {
const hdr = "content-disposition:";
@memcpy(writer[pos..][0..hdr.len], hdr);
pos += hdr.len;
@memcpy(writer[pos..][0..cd.len], cd);
pos += cd.len;
writer[pos] = '\n';
pos += 1;
}
if (content_encoding) |ce| {
const hdr = "content-encoding:";
@memcpy(writer[pos..][0..hdr.len], hdr);
pos += hdr.len;
@memcpy(writer[pos..][0..ce.len], ce);
pos += ce.len;
writer[pos] = '\n';
pos += 1;
}
if (content_md5) |md5| {
const hdr = "content-md5:";
@memcpy(writer[pos..][0..hdr.len], hdr);
pos += hdr.len;
@memcpy(writer[pos..][0..md5.len], md5);
pos += md5.len;
writer[pos] = '\n';
pos += 1;
}
// host
const host_hdr = "host:";
@memcpy(writer[pos..][0..host_hdr.len], host_hdr);
pos += host_hdr.len;
@memcpy(writer[pos..][0..host.len], host);
pos += host.len;
writer[pos] = '\n';
pos += 1;
if (acl) |acl_val| {
const hdr = "x-amz-acl:";
@memcpy(writer[pos..][0..hdr.len], hdr);
pos += hdr.len;
@memcpy(writer[pos..][0..acl_val.len], acl_val);
pos += acl_val.len;
writer[pos] = '\n';
pos += 1;
}
// x-amz-content-sha256
const hash_hdr = "x-amz-content-sha256:";
@memcpy(writer[pos..][0..hash_hdr.len], hash_hdr);
pos += hash_hdr.len;
@memcpy(writer[pos..][0..aws_content_hash.len], aws_content_hash);
pos += aws_content_hash.len;
writer[pos] = '\n';
pos += 1;
// x-amz-date
const date_hdr = "x-amz-date:";
@memcpy(writer[pos..][0..date_hdr.len], date_hdr);
pos += date_hdr.len;
@memcpy(writer[pos..][0..amz_date.len], amz_date);
pos += amz_date.len;
writer[pos] = '\n';
pos += 1;
// x-amz-meta-* headers (sorted by key)
const meta_count = meta.count();
if (meta_count > 0) {
var indices: [MAX_METADATA_HEADERS]usize = undefined;
for (0..meta_count) |i| {
indices[i] = i;
}
std.mem.sort(usize, indices[0..meta_count], meta.keys, struct {
fn lessThan(keys: []const []const u8, a: usize, b: usize) bool {
return std.mem.lessThan(u8, keys[a], keys[b]);
}
}.lessThan);
for (indices[0..meta_count]) |idx| {
const prefix = "x-amz-meta-";
@memcpy(writer[pos..][0..prefix.len], prefix);
pos += prefix.len;
@memcpy(writer[pos..][0..meta.keys[idx].len], meta.keys[idx]);
pos += meta.keys[idx].len;
writer[pos] = ':';
pos += 1;
@memcpy(writer[pos..][0..meta.values[idx].len], meta.values[idx]);
pos += meta.values[idx].len;
writer[pos] = '\n';
pos += 1;
}
}
if (request_payer) {
const hdr = "x-amz-request-payer:requester\n";
@memcpy(writer[pos..][0..hdr.len], hdr);
pos += hdr.len;
}
if (session_token) |token| {
const hdr = "x-amz-security-token:";
@memcpy(writer[pos..][0..hdr.len], hdr);
pos += hdr.len;
@memcpy(writer[pos..][0..token.len], token);
pos += token.len;
writer[pos] = '\n';
pos += 1;
}
if (storage_class) |sc| {
const hdr = "x-amz-storage-class:";
@memcpy(writer[pos..][0..hdr.len], hdr);
pos += hdr.len;
@memcpy(writer[pos..][0..sc.len], sc);
pos += sc.len;
writer[pos] = '\n';
pos += 1;
}
// Empty line separator
writer[pos] = '\n';
pos += 1;
// SIGNED_HEADERS
@memcpy(writer[pos..][0..signed_headers.len], signed_headers);
pos += signed_headers.len;
writer[pos] = '\n';
pos += 1;
// HASH
@memcpy(writer[pos..][0..aws_content_hash.len], aws_content_hash);
pos += aws_content_hash.len;
bun.sha.SHA256.hash(writer[0..pos], &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine());
} else {
const canonical = try CanonicalRequest.format(
&tmp_buffer,
header_key,
method_name,
normalizedPath,
if (search_params) |p| p[1..] else "",
content_disposition,
content_encoding,
content_md5,
host,
acl,
aws_content_hash,
amz_date,
session_token,
storage_class,
signed_headers,
);
bun.sha.SHA256.hash(canonical, &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine());
}
const signValue = try std.fmt.bufPrint(&tmp_buffer, "AWS4-HMAC-SHA256\n{s}\n{s}/{s}/{s}/aws4_request\n{s}", .{ amz_date, amz_day, region, service_name, std.fmt.bytesToHex(sha_digest[0..bun.sha.SHA256.digest], .lower) });
@@ -927,21 +1345,13 @@ pub const S3Credentials = struct {
.storage_class = signOptions.storage_class,
.request_payer = request_payer,
.url = try std.fmt.allocPrint(bun.default_allocator, "{s}://{s}{s}{s}", .{ protocol, host, normalizedPath, if (search_params) |s| s else "" }),
._headers = [_]picohttp.Header{
.{ .name = "x-amz-content-sha256", .value = aws_content_hash },
.{ .name = "x-amz-date", .value = amz_date },
.{ .name = "Host", .value = host },
.{ .name = "Authorization", .value = authorization[0..] },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
.{ .name = "", .value = "" },
},
._headers_len = 4,
};
// Set up the base headers
result._headers[0] = .{ .name = "x-amz-content-sha256", .value = aws_content_hash };
result._headers[1] = .{ .name = "x-amz-date", .value = amz_date };
result._headers[2] = .{ .name = "Host", .value = host };
result._headers[3] = .{ .name = "Authorization", .value = authorization[0..] };
if (acl) |acl_value| {
result._headers[result._headers_len] = .{ .name = "x-amz-acl", .value = acl_value };
@@ -985,6 +1395,34 @@ pub const S3Credentials = struct {
result._headers_len += 1;
}
// Add metadata headers (x-amz-meta-*)
// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
if (metadata) |meta| {
const meta_count = meta.count();
if (meta_count > 0) {
// Allocate array to hold the "x-amz-meta-{key}" header names
// These need to be allocated because picohttp.Header stores slices
const header_names = bun.handleOom(bun.default_allocator.alloc([]u8, meta_count));
result._metadata_header_names = header_names;
for (0..meta_count) |i| {
const key = meta.keys[i];
// Build "x-amz-meta-{key}" header name
const prefix = "x-amz-meta-";
const header_name = bun.handleOom(bun.default_allocator.alloc(u8, prefix.len + key.len));
@memcpy(header_name[0..prefix.len], prefix);
@memcpy(header_name[prefix.len..], key);
header_names[i] = header_name;
result._headers[result._headers_len] = .{
.name = header_name,
.value = meta.values[i],
};
result._headers_len += 1;
}
}
}
return result;
}
};
@@ -1003,6 +1441,8 @@ pub const S3CredentialsWithOptions = struct {
changed_credentials: bool = false,
/// indicates if the virtual hosted style is used
virtual_hosted_style: bool = false,
/// User-defined metadata (x-amz-meta-* headers)
metadata: ?MetadataMap = null,
_accessKeyIdSlice: ?jsc.ZigString.Slice = null,
_secretAccessKeySlice: ?jsc.ZigString.Slice = null,
_regionSlice: ?jsc.ZigString.Slice = null,
@@ -1023,6 +1463,7 @@ pub const S3CredentialsWithOptions = struct {
if (this._contentDispositionSlice) |slice| slice.deinit();
if (this._contentTypeSlice) |slice| slice.deinit();
if (this._contentEncodingSlice) |slice| slice.deinit();
if (this.metadata) |*meta| meta.deinit(bun.default_allocator);
}
};

View File

@@ -106,6 +106,7 @@ pub const MultiPartUpload = struct {
acl: ?ACL = null,
storage_class: ?Storageclass = null,
request_payer: bool = false,
metadata: ?s3creds.MetadataMap = null,
credentials: *S3Credentials,
poll_ref: bun.Async.KeepAlive = bun.Async.KeepAlive.init(),
vm: *jsc.VirtualMachine,
@@ -290,6 +291,9 @@ pub const MultiPartUpload = struct {
bun.default_allocator.free(ce);
}
}
if (this.metadata) |*meta| {
meta.deinit(bun.default_allocator);
}
this.credentials.deref();
this.uploadid_buffer.deinit();
for (this.multipart_etags.items) |tag| {
@@ -320,6 +324,7 @@ pub const MultiPartUpload = struct {
.acl = this.acl,
.storage_class = this.storage_class,
.request_payer = this.request_payer,
.metadata = this.metadata,
}, .{ .upload = @ptrCast(&singleSendUploadResponse) }, this);
return;
@@ -613,6 +618,7 @@ pub const MultiPartUpload = struct {
.acl = this.acl,
.storage_class = this.storage_class,
.request_payer = this.request_payer,
.metadata = this.metadata,
}, .{ .download = @ptrCast(&startMultiPartRequestResult) }, this);
} else if (this.state == .multipart_completed) {
try part.start();
@@ -692,6 +698,7 @@ pub const MultiPartUpload = struct {
.acl = this.acl,
.storage_class = this.storage_class,
.request_payer = this.request_payer,
.metadata = this.metadata,
}, .{ .upload = @ptrCast(&singleSendUploadResponse) }, this) catch {}; // TODO: properly propagate exception upwards
} else {
// we need to split
@@ -787,7 +794,8 @@ pub const MultiPartUpload = struct {
const std = @import("std");
const ACL = @import("./acl.zig").ACL;
const MultiPartUploadOptions = @import("./multipart_options.zig").MultiPartUploadOptions;
const S3Credentials = @import("./credentials.zig").S3Credentials;
const s3creds = @import("./credentials.zig");
const S3Credentials = s3creds.S3Credentials;
const S3Error = @import("./error.zig").S3Error;
const Storageclass = @import("./storage_class.zig").StorageClass;

View File

@@ -7,6 +7,9 @@ pub const S3StatResult = union(enum) {
lastModified: []const u8 = "",
/// format: text/plain, contentType is not owned and need to be copied if used after this callback
contentType: []const u8 = "",
/// Raw headers from response - caller should filter for x-amz-meta-* headers.
/// Headers are not owned and need to be copied if used after this callback.
headers: []const picohttp.Header = &.{},
},
not_found: S3Error,
@@ -231,6 +234,7 @@ pub const S3HttpSimpleTask = struct {
.lastModified = response.headers.get("last-modified") orelse "",
.contentType = response.headers.get("content-type") orelse "",
.size = if (response.headers.get("content-length")) |content_len| (std.fmt.parseInt(usize, content_len, 10) catch 0) else 0,
.headers = response.headers.list,
},
}, this.callback_context);
},
@@ -360,6 +364,7 @@ pub const S3SimpleRequestOptions = struct {
acl: ?ACL = null,
storage_class: ?StorageClass = null,
request_payer: bool = false,
metadata: ?s3creds.MetadataMap = null,
};
pub fn executeSimpleS3Request(
@@ -377,6 +382,7 @@ pub fn executeSimpleS3Request(
.acl = options.acl,
.storage_class = options.storage_class,
.request_payer = options.request_payer,
.metadata = options.metadata,
}, false, null) catch |sign_err| {
if (options.range) |range_| bun.default_allocator.free(range_);
const error_code_and_message = getSignErrorCodeAndMessage(sign_err);
@@ -385,7 +391,8 @@ pub fn executeSimpleS3Request(
};
const headers = brk: {
var header_buffer: [S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined;
// MAX_HEADERS (41) + 1 for the additional header we mix in
var header_buffer: [s3creds.MAX_HEADERS + 1]picohttp.Header = undefined;
if (options.range) |range_| {
const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ });
break :brk bun.handleOom(bun.http.Headers.fromPicoHttpHeaders(_headers, bun.default_allocator));
@@ -444,8 +451,9 @@ const std = @import("std");
const ACL = @import("./acl.zig").ACL;
const StorageClass = @import("./storage_class.zig").StorageClass;
const S3Credentials = @import("./credentials.zig").S3Credentials;
const SignResult = @import("./credentials.zig").S3Credentials.SignResult;
const s3creds = @import("./credentials.zig");
const S3Credentials = s3creds.S3Credentials;
const SignResult = S3Credentials.SignResult;
const S3Error = @import("./error.zig").S3Error;
const getSignErrorCodeAndMessage = @import("./error.zig").getSignErrorCodeAndMessage;

View File

@@ -1674,3 +1674,145 @@ describe.skipIf(!minioCredentials)("Archive with S3", () => {
await s3File.delete();
});
});
// Metadata tests - test with MinIO only since it properly supports x-amz-meta-* headers
describe.skipIf(!minioCredentials?.endpoint)("S3 Metadata (x-amz-meta-*)", () => {
const credentials: S3Options = {
accessKeyId: minioCredentials!.accessKeyId,
secretAccessKey: minioCredentials!.secretAccessKey,
endpoint: minioCredentials!.endpoint,
bucket: minioCredentials!.bucket,
};
it("should upload and retrieve metadata via S3File.write() and stat()", async () => {
const client = new Bun.S3Client(credentials);
const key = randomUUIDv7() + ".txt";
const file = client.file(key);
// Write with metadata
await file.write("test content", {
metadata: { sku: "12345", category: "test" },
});
// Retrieve and verify metadata
const stat = await file.stat();
expect(stat.metadata).toBeDefined();
expect(stat.metadata.sku).toBe("12345");
expect(stat.metadata.category).toBe("test");
// Cleanup
await file.unlink();
});
it("should upload metadata via S3Client.write() static method", async () => {
const key = randomUUIDv7() + ".txt";
// Upload with metadata via S3Client.write()
await S3Client.write(key, "test content", {
...credentials,
metadata: { "custom-key": "custom-value", author: "bun" },
});
// Verify metadata via S3 stat
const client = new Bun.S3Client(credentials);
const stat = await client.stat(key);
expect(stat.metadata["custom-key"]).toBe("custom-value");
expect(stat.metadata.author).toBe("bun");
// Cleanup
await client.delete(key);
});
it("should normalize metadata keys to lowercase", async () => {
const client = new Bun.S3Client(credentials);
const key = randomUUIDv7() + ".txt";
const file = client.file(key);
// Write with mixed-case metadata keys
await file.write("test content", {
metadata: { SKU: "12345", MyCustomKey: "value" },
});
// Keys should be normalized to lowercase
const stat = await file.stat();
expect(stat.metadata.sku).toBe("12345");
expect(stat.metadata.mycustomkey).toBe("value");
// Uppercase keys should not exist
expect(stat.metadata.SKU).toBeUndefined();
expect(stat.metadata.MyCustomKey).toBeUndefined();
// Cleanup
await file.unlink();
});
it("should return empty metadata object when no metadata is set", async () => {
const client = new Bun.S3Client(credentials);
const key = randomUUIDv7() + ".txt";
const file = client.file(key);
// Write without metadata
await file.write("test content");
// stat should return empty metadata object
const stat = await file.stat();
expect(stat.metadata).toBeDefined();
expect(typeof stat.metadata).toBe("object");
expect(Object.keys(stat.metadata).length).toBe(0);
// Cleanup
await file.unlink();
});
it("presigned PUT URL should include metadata in signed headers", async () => {
const client = new Bun.S3Client(credentials);
const key = randomUUIDv7() + ".txt";
const file = client.file(key);
// Generate presigned PUT URL with metadata
const url = file.presign({
method: "PUT",
expiresIn: 3600,
metadata: { "signed-key": "signed-value" },
});
// The presigned URL should contain metadata headers in the signature
expect(url).toContain("x-amz-meta-signed-key");
// Upload using presigned URL with matching metadata header
const uploadRes = await fetch(url, {
method: "PUT",
headers: { "x-amz-meta-signed-key": "signed-value" },
body: "presigned content",
});
expect(uploadRes.status).toBe(200);
// Verify metadata was stored
const stat = await file.stat();
expect(stat.metadata["signed-key"]).toBe("signed-value");
// Cleanup
await file.unlink();
});
it("should support metadata with writer() streaming upload", async () => {
const client = new Bun.S3Client(credentials);
const key = randomUUIDv7() + ".txt";
const file = client.file(key, {
metadata: { "stream-key": "stream-value" },
});
// Write using streaming writer
const writer = file.writer();
writer.write("chunk 1 ");
writer.write("chunk 2 ");
writer.write("chunk 3");
await writer.end();
// Verify metadata
const stat = await file.stat();
expect(stat.metadata["stream-key"]).toBe("stream-value");
// Cleanup
await file.unlink();
});
});