mirror of
https://github.com/oven-sh/bun
synced 2026-02-16 05:42:43 +00:00
feat(s3): add custom metadata support (x-amz-meta-*)
Add support for S3 custom metadata headers that allow users to attach
key-value pairs to S3 objects.
**Write path**: Accept `metadata: Record<string, string>` in S3Options
- Convert `{ sku: "12345" }` to `x-amz-meta-sku: 12345` headers
- Works with `write()`, `writer()`, and presigned PUT URLs
- Keys are automatically normalized to lowercase per AWS requirements
**Read path**: Return metadata in S3Stats via `stat()`
- Extract x-amz-meta-* headers from response
- Strip prefix and return as `metadata: Record<string, string>`
**Presigned URLs**: Include metadata as signed headers in PUT URLs
- Client uploading to presigned URL must include matching headers
Added `metadata` option to S3Options for attaching custom metadata (x-amz-meta-* headers) to S3 objects during upload. Added `metadata` property to S3Stats returned by `stat()` to retrieve stored metadata. Keys are automatically lowercased per AWS requirements. Maximum ~2KB total metadata supported.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
36
packages/bun-types/s3.d.ts
vendored
36
packages/bun-types/s3.d.ts
vendored
@@ -345,6 +345,32 @@ declare module "bun" {
|
||||
*/
|
||||
requestPayer?: boolean;
|
||||
|
||||
/**
|
||||
* User-defined metadata to store with the S3 object as `x-amz-meta-*` headers.
|
||||
* Keys are automatically lowercased per AWS requirements.
|
||||
* Maximum total metadata size is ~2KB.
|
||||
*
|
||||
* @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
|
||||
*
|
||||
* @example
|
||||
* // Setting custom metadata on upload
|
||||
* await s3.write("document.pdf", data, {
|
||||
* metadata: {
|
||||
* sku: "12345",
|
||||
* category: "reports"
|
||||
* }
|
||||
* });
|
||||
*
|
||||
* @example
|
||||
* // Metadata with presigned PUT URL
|
||||
* const url = file.presign({
|
||||
* method: "PUT",
|
||||
* metadata: { "custom-key": "value" }
|
||||
* });
|
||||
* // Client must include matching header: x-amz-meta-custom-key: value
|
||||
*/
|
||||
metadata?: Record<string, string>;
|
||||
|
||||
/**
|
||||
* @deprecated The size of the internal buffer in bytes. Defaults to 5 MiB. use `partSize` and `queueSize` instead.
|
||||
*/
|
||||
@@ -400,6 +426,16 @@ declare module "bun" {
|
||||
lastModified: Date;
|
||||
etag: string;
|
||||
type: string;
|
||||
/**
|
||||
* User-defined metadata retrieved from `x-amz-meta-*` headers.
|
||||
* Keys have the `x-amz-meta-` prefix stripped and are lowercased.
|
||||
*
|
||||
* @example
|
||||
* // Retrieving metadata
|
||||
* const stat = await file.stat();
|
||||
* console.log(stat.metadata); // { sku: "12345", category: "reports" }
|
||||
*/
|
||||
metadata: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -25,6 +25,10 @@ export default [
|
||||
getter: "getContentType",
|
||||
cache: true,
|
||||
},
|
||||
metadata: {
|
||||
getter: "getMetadata",
|
||||
cache: true,
|
||||
},
|
||||
},
|
||||
}),
|
||||
];
|
||||
|
||||
@@ -973,6 +973,7 @@ fn writeFileWithEmptySourceToDestination(ctx: *jsc.JSGlobalObject, destination_b
|
||||
proxy_url,
|
||||
aws_options.storage_class,
|
||||
aws_options.request_payer,
|
||||
aws_options.metadata,
|
||||
Wrapper.resolve,
|
||||
Wrapper.new(.{
|
||||
.promise = promise,
|
||||
@@ -1124,6 +1125,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl
|
||||
aws_options.content_encoding,
|
||||
proxy_url,
|
||||
aws_options.request_payer,
|
||||
aws_options.metadata,
|
||||
null,
|
||||
undefined,
|
||||
);
|
||||
@@ -1167,6 +1169,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl
|
||||
proxy_url,
|
||||
aws_options.storage_class,
|
||||
aws_options.request_payer,
|
||||
aws_options.metadata,
|
||||
Wrapper.resolve,
|
||||
Wrapper.new(.{
|
||||
.store = source_store,
|
||||
@@ -1197,6 +1200,7 @@ pub fn writeFileWithSourceDestination(ctx: *jsc.JSGlobalObject, source_blob: *Bl
|
||||
aws_options.content_encoding,
|
||||
proxy_url,
|
||||
aws_options.request_payer,
|
||||
aws_options.metadata,
|
||||
null,
|
||||
undefined,
|
||||
);
|
||||
@@ -1405,6 +1409,7 @@ pub fn writeFileInternal(globalThis: *jsc.JSGlobalObject, path_or_blob_: *PathOr
|
||||
aws_options.content_encoding,
|
||||
proxy_url,
|
||||
aws_options.request_payer,
|
||||
aws_options.metadata,
|
||||
null,
|
||||
undefined,
|
||||
);
|
||||
@@ -1468,6 +1473,7 @@ pub fn writeFileInternal(globalThis: *jsc.JSGlobalObject, path_or_blob_: *PathOr
|
||||
aws_options.content_encoding,
|
||||
proxy_url,
|
||||
aws_options.request_payer,
|
||||
aws_options.metadata,
|
||||
null,
|
||||
undefined,
|
||||
);
|
||||
@@ -2447,6 +2453,7 @@ pub fn pipeReadableStreamToBlob(this: *Blob, globalThis: *jsc.JSGlobalObject, re
|
||||
aws_options.content_encoding,
|
||||
proxy_url,
|
||||
aws_options.request_payer,
|
||||
aws_options.metadata,
|
||||
null,
|
||||
undefined,
|
||||
);
|
||||
@@ -2702,20 +2709,24 @@ pub fn getWriter(
|
||||
proxy_url,
|
||||
credentialsWithOptions.storage_class,
|
||||
credentialsWithOptions.request_payer,
|
||||
credentialsWithOptions.metadata,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Dupe metadata since multipart upload takes ownership
|
||||
const metadata_dupe = if (s3.metadata) |meta| meta.dupe(bun.default_allocator) else null;
|
||||
return try S3.writableStream(
|
||||
s3.getCredentials(),
|
||||
path,
|
||||
globalThis,
|
||||
.{},
|
||||
s3.options,
|
||||
this.contentTypeOrMimeType(),
|
||||
null,
|
||||
null,
|
||||
proxy_url,
|
||||
null,
|
||||
s3.storage_class,
|
||||
s3.request_payer,
|
||||
metadata_dupe,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -268,6 +268,9 @@ pub fn constructS3FileWithS3CredentialsAndOptions(
|
||||
store.data.s3.acl = aws_options.acl;
|
||||
store.data.s3.storage_class = aws_options.storage_class;
|
||||
store.data.s3.request_payer = aws_options.request_payer;
|
||||
// Transfer metadata ownership to the store
|
||||
store.data.s3.metadata = aws_options.metadata;
|
||||
aws_options.metadata = null; // Prevent deinit from freeing
|
||||
|
||||
var blob = Blob.initWithStore(store, globalObject);
|
||||
if (options) |opts| {
|
||||
@@ -312,6 +315,9 @@ pub fn constructS3FileWithS3Credentials(
|
||||
store.data.s3.acl = aws_options.acl;
|
||||
store.data.s3.storage_class = aws_options.storage_class;
|
||||
store.data.s3.request_payer = aws_options.request_payer;
|
||||
// Transfer metadata ownership to the store
|
||||
store.data.s3.metadata = aws_options.metadata;
|
||||
aws_options.metadata = null; // Prevent deinit from freeing
|
||||
|
||||
var blob = Blob.initWithStore(store, globalObject);
|
||||
if (options) |opts| {
|
||||
@@ -399,6 +405,7 @@ pub const S3BlobStatTask = struct {
|
||||
stat_result.etag,
|
||||
stat_result.contentType,
|
||||
stat_result.lastModified,
|
||||
stat_result.headers,
|
||||
globalThis,
|
||||
)).toJS(globalThis));
|
||||
},
|
||||
@@ -505,6 +512,7 @@ pub fn getPresignUrlFrom(this: *Blob, globalThis: *jsc.JSGlobalObject, extra_opt
|
||||
.request_payer = credentialsWithOptions.request_payer,
|
||||
.content_disposition = credentialsWithOptions.content_disposition,
|
||||
.content_type = credentialsWithOptions.content_type,
|
||||
.metadata = credentialsWithOptions.metadata,
|
||||
}, false, .{ .expires = expires }) catch |sign_err| {
|
||||
return S3.throwSignError(sign_err, globalThis);
|
||||
};
|
||||
|
||||
@@ -11,30 +11,66 @@ pub const S3Stat = struct {
|
||||
etag: bun.String,
|
||||
contentType: bun.String,
|
||||
lastModified: f64,
|
||||
/// Cached JS object containing x-amz-meta-* headers as key-value pairs.
|
||||
/// Keys have the "x-amz-meta-" prefix stripped.
|
||||
metadata: jsc.JSValue,
|
||||
|
||||
pub fn constructor(globalThis: *jsc.JSGlobalObject, _: *jsc.CallFrame) bun.JSError!*@This() {
|
||||
return globalThis.throwInvalidArguments("S3Stat is not constructable", .{});
|
||||
}
|
||||
|
||||
/// Initialize S3Stat from stat result data.
|
||||
/// `headers` should be the raw response headers - this function will extract x-amz-meta-* headers.
|
||||
pub fn init(
|
||||
size: u64,
|
||||
etag: []const u8,
|
||||
contentType: []const u8,
|
||||
lastModified: []const u8,
|
||||
headers: []const picohttp.Header,
|
||||
globalThis: *jsc.JSGlobalObject,
|
||||
) bun.JSError!*@This() {
|
||||
var date_str = bun.String.init(lastModified);
|
||||
defer date_str.deref();
|
||||
const last_modified = try date_str.parseDate(globalThis);
|
||||
|
||||
// Build metadata JS object from x-amz-meta-* headers
|
||||
const metadata_obj = try buildMetadataObject(headers, globalThis);
|
||||
jsc.JSValue.protect(metadata_obj);
|
||||
|
||||
return S3Stat.new(.{
|
||||
.size = size,
|
||||
.etag = bun.String.cloneUTF8(etag),
|
||||
.contentType = bun.String.cloneUTF8(contentType),
|
||||
.lastModified = last_modified,
|
||||
.metadata = metadata_obj,
|
||||
});
|
||||
}
|
||||
|
||||
/// Extract x-amz-meta-* headers and build a JS object with stripped key names.
|
||||
fn buildMetadataObject(headers: []const picohttp.Header, globalThis: *jsc.JSGlobalObject) bun.JSError!jsc.JSValue {
|
||||
const prefix = "x-amz-meta-";
|
||||
const prefix_len = prefix.len;
|
||||
|
||||
// Create empty JS object
|
||||
const obj = jsc.JSValue.createEmptyObject(globalThis, 0);
|
||||
|
||||
for (headers) |header| {
|
||||
// Case-insensitive check for x-amz-meta- prefix
|
||||
if (header.name.len > prefix_len and
|
||||
strings.eqlCaseInsensitiveASCII(header.name[0..prefix_len], prefix, true))
|
||||
{
|
||||
// Strip the prefix to get the user's key name
|
||||
const key = header.name[prefix_len..];
|
||||
const value_js = try bun.String.createUTF8ForJS(globalThis, header.value);
|
||||
|
||||
// put() accepts []const u8 directly and wraps it in ZigString
|
||||
obj.put(globalThis, key, value_js);
|
||||
}
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
pub fn getSize(this: *@This(), _: *jsc.JSGlobalObject) jsc.JSValue {
|
||||
return jsc.JSValue.jsNumber(this.size);
|
||||
}
|
||||
@@ -51,12 +87,19 @@ pub const S3Stat = struct {
|
||||
return jsc.JSValue.fromDateNumber(globalObject, this.lastModified);
|
||||
}
|
||||
|
||||
pub fn getMetadata(this: *@This(), _: *jsc.JSGlobalObject) jsc.JSValue {
|
||||
return this.metadata;
|
||||
}
|
||||
|
||||
pub fn finalize(this: *@This()) void {
|
||||
this.etag.deref();
|
||||
this.contentType.deref();
|
||||
jsc.JSValue.unprotect(this.metadata);
|
||||
bun.destroy(this);
|
||||
}
|
||||
};
|
||||
|
||||
const bun = @import("bun");
|
||||
const jsc = bun.jsc;
|
||||
const picohttp = bun.picohttp;
|
||||
const strings = bun.strings;
|
||||
|
||||
@@ -296,6 +296,8 @@ pub const S3 = struct {
|
||||
acl: ?bun.S3.ACL = null,
|
||||
storage_class: ?bun.S3.StorageClass = null,
|
||||
request_payer: bool = false,
|
||||
/// User-defined metadata (x-amz-meta-* headers)
|
||||
metadata: ?bun.S3.MetadataMap = null,
|
||||
|
||||
pub fn isSeekable(_: *const @This()) ?bool {
|
||||
return true;
|
||||
@@ -464,6 +466,10 @@ pub const S3 = struct {
|
||||
credentials.deref();
|
||||
this.credentials = null;
|
||||
}
|
||||
if (this.metadata) |*meta| {
|
||||
meta.deinit(bun.default_allocator);
|
||||
this.metadata = null;
|
||||
}
|
||||
}
|
||||
|
||||
const S3Credentials = bun.S3.S3Credentials;
|
||||
|
||||
@@ -1322,6 +1322,7 @@ fn fetchImpl(
|
||||
if (headers) |h| h.getContentEncoding() else null,
|
||||
proxy_url,
|
||||
credentialsWithOptions.request_payer,
|
||||
credentialsWithOptions.metadata,
|
||||
@ptrCast(&Wrapper.resolve),
|
||||
s3_stream,
|
||||
);
|
||||
@@ -1361,7 +1362,7 @@ fn fetchImpl(
|
||||
}
|
||||
|
||||
const content_type = if (headers) |h| (h.getContentType()) else null;
|
||||
var header_buffer: [s3.S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined;
|
||||
var header_buffer: [s3.MAX_HEADERS + 1]picohttp.Header = undefined;
|
||||
|
||||
if (range) |range_| {
|
||||
const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ });
|
||||
|
||||
@@ -10,6 +10,8 @@ pub const getJSSignError = Error.getJSSignError;
|
||||
|
||||
pub const S3Credentials = Credentials.S3Credentials;
|
||||
pub const S3CredentialsWithOptions = Credentials.S3CredentialsWithOptions;
|
||||
pub const MetadataMap = Credentials.MetadataMap;
|
||||
pub const MAX_HEADERS = Credentials.MAX_HEADERS;
|
||||
|
||||
pub const S3HttpSimpleTask = S3SimpleRequest.S3HttpSimpleTask;
|
||||
pub const S3UploadResult = S3SimpleRequest.S3UploadResult;
|
||||
@@ -243,6 +245,7 @@ pub fn upload(
|
||||
proxy_url: ?[]const u8,
|
||||
storage_class: ?StorageClass,
|
||||
request_payer: bool,
|
||||
metadata: ?Credentials.MetadataMap,
|
||||
callback: *const fn (S3UploadResult, *anyopaque) bun.JSTerminated!void,
|
||||
callback_context: *anyopaque,
|
||||
) bun.JSTerminated!void {
|
||||
@@ -257,6 +260,7 @@ pub fn upload(
|
||||
.acl = acl,
|
||||
.storage_class = storage_class,
|
||||
.request_payer = request_payer,
|
||||
.metadata = metadata,
|
||||
}, .{ .upload = callback }, callback_context);
|
||||
}
|
||||
/// returns a writable stream that writes to the s3 path
|
||||
@@ -271,6 +275,7 @@ pub fn writableStream(
|
||||
proxy: ?[]const u8,
|
||||
storage_class: ?StorageClass,
|
||||
request_payer: bool,
|
||||
metadata: ?Credentials.MetadataMap,
|
||||
) bun.JSError!jsc.JSValue {
|
||||
const Wrapper = struct {
|
||||
pub fn callback(result: S3UploadResult, sink: *jsc.WebCore.NetworkSink) bun.JSTerminated!void {
|
||||
@@ -316,6 +321,7 @@ pub fn writableStream(
|
||||
.content_encoding = if (content_encoding) |ce| bun.handleOom(bun.default_allocator.dupe(u8, ce)) else null,
|
||||
.storage_class = storage_class,
|
||||
.request_payer = request_payer,
|
||||
.metadata = metadata,
|
||||
|
||||
.callback = @ptrCast(&Wrapper.callback),
|
||||
.callback_context = undefined,
|
||||
@@ -458,6 +464,7 @@ pub fn uploadStream(
|
||||
content_encoding: ?[]const u8,
|
||||
proxy: ?[]const u8,
|
||||
request_payer: bool,
|
||||
metadata: ?Credentials.MetadataMap,
|
||||
callback: ?*const fn (S3UploadResult, *anyopaque) void,
|
||||
callback_context: *anyopaque,
|
||||
) bun.JSError!jsc.JSValue {
|
||||
@@ -503,6 +510,7 @@ pub fn uploadStream(
|
||||
.acl = acl,
|
||||
.storage_class = storage_class,
|
||||
.request_payer = request_payer,
|
||||
.metadata = metadata,
|
||||
.vm = jsc.VirtualMachine.get(),
|
||||
});
|
||||
|
||||
@@ -565,7 +573,7 @@ pub fn downloadStream(
|
||||
return;
|
||||
};
|
||||
|
||||
var header_buffer: [S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined;
|
||||
var header_buffer: [Credentials.MAX_HEADERS + 1]picohttp.Header = undefined;
|
||||
const headers = brk: {
|
||||
if (range) |range_| {
|
||||
const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ });
|
||||
|
||||
@@ -1,3 +1,67 @@
|
||||
/// Maximum number of user-defined metadata headers.
|
||||
/// AWS S3 limits total metadata to ~2KB (https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html).
|
||||
/// With typical key-value pairs of ~50-100 bytes, 32 headers provides ample capacity.
|
||||
const MAX_METADATA_HEADERS: usize = 32;
|
||||
|
||||
/// Number of base S3 headers (x-amz-content-sha256, x-amz-date, Host, Authorization, etc.)
|
||||
const BASE_HEADERS: usize = 9;
|
||||
|
||||
/// Total maximum headers: base headers + metadata headers
|
||||
pub const MAX_HEADERS: usize = BASE_HEADERS + MAX_METADATA_HEADERS;
|
||||
|
||||
/// Buffer size for signed headers string (e.g., "content-disposition;host;x-amz-meta-foo;...")
|
||||
/// Enough for all standard headers plus MAX_METADATA_HEADERS metadata keys of ~50 chars each
|
||||
const SIGNED_HEADERS_BUF_SIZE: usize = 2048;
|
||||
|
||||
/// Buffer size for canonical request building with metadata
|
||||
/// Needs space for method, path, query, all headers with values, signed_headers, and hash
|
||||
const CANONICAL_REQUEST_BUF_SIZE: usize = 8192;
|
||||
|
||||
/// Holds user-defined metadata key-value pairs for S3 objects (x-amz-meta-* headers).
|
||||
/// Keys are normalized to lowercase per AWS S3 requirements.
|
||||
/// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
|
||||
pub const MetadataMap = struct {
|
||||
keys: []const []const u8,
|
||||
values: []const []const u8,
|
||||
|
||||
pub fn count(this: @This()) usize {
|
||||
return this.keys.len;
|
||||
}
|
||||
|
||||
/// Deep-copy the metadata map, duplicating all keys and values.
|
||||
pub fn dupe(this: @This(), allocator: std.mem.Allocator) ?MetadataMap {
|
||||
const n = this.keys.len;
|
||||
if (n == 0) return null;
|
||||
|
||||
const new_keys = allocator.alloc([]const u8, n) catch return null;
|
||||
const new_values = allocator.alloc([]const u8, n) catch {
|
||||
allocator.free(new_keys);
|
||||
return null;
|
||||
};
|
||||
|
||||
for (0..n) |i| {
|
||||
new_keys[i] = allocator.dupe(u8, this.keys[i]) catch return null;
|
||||
new_values[i] = allocator.dupe(u8, this.values[i]) catch return null;
|
||||
}
|
||||
|
||||
return .{
|
||||
.keys = new_keys,
|
||||
.values = new_values,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(this: *@This(), allocator: std.mem.Allocator) void {
|
||||
for (this.keys) |key| {
|
||||
allocator.free(key);
|
||||
}
|
||||
for (this.values) |value| {
|
||||
allocator.free(value);
|
||||
}
|
||||
allocator.free(this.keys);
|
||||
allocator.free(this.values);
|
||||
}
|
||||
};
|
||||
|
||||
pub const S3Credentials = struct {
|
||||
const RefCount = bun.ptr.RefCount(@This(), "ref_count", deinit, .{});
|
||||
pub const ref = RefCount.ref;
|
||||
@@ -262,6 +326,64 @@ pub const S3Credentials = struct {
|
||||
if (try opts.getBooleanStrict(globalObject, "requestPayer")) |request_payer| {
|
||||
new_credentials.request_payer = request_payer;
|
||||
}
|
||||
|
||||
// Parse metadata object
|
||||
if (try opts.getTruthyComptime(globalObject, "metadata")) |metadata_value| {
|
||||
if (!metadata_value.isEmptyOrUndefinedOrNull()) {
|
||||
if (metadata_value.isObject()) {
|
||||
const metadata_obj = try metadata_value.toObject(globalObject);
|
||||
// Count properties first
|
||||
var property_count: usize = 0;
|
||||
var iter = try jsc.JSPropertyIterator(.{ .skip_empty_name = true, .include_value = true }).init(globalObject, metadata_obj);
|
||||
defer iter.deinit();
|
||||
while (try iter.next()) |_| {
|
||||
property_count += 1;
|
||||
}
|
||||
|
||||
if (property_count > 0) {
|
||||
// Allocate arrays for keys and values
|
||||
const keys = bun.default_allocator.alloc([]const u8, property_count) catch bun.outOfMemory();
|
||||
const values = bun.default_allocator.alloc([]const u8, property_count) catch bun.outOfMemory();
|
||||
|
||||
// Second pass to extract keys and values
|
||||
var iter2 = try jsc.JSPropertyIterator(.{ .skip_empty_name = true, .include_value = true }).init(globalObject, metadata_obj);
|
||||
defer iter2.deinit();
|
||||
var i: usize = 0;
|
||||
while (try iter2.next()) |key| {
|
||||
// Convert key to lowercase (AWS requires lowercase metadata keys)
|
||||
const key_owned = try key.toOwnedSlice(bun.default_allocator);
|
||||
const key_lower = bun.default_allocator.alloc(u8, key_owned.len) catch bun.outOfMemory();
|
||||
for (key_owned, 0..) |c, j| {
|
||||
key_lower[j] = if (c >= 'A' and c <= 'Z') c + 32 else c;
|
||||
}
|
||||
bun.default_allocator.free(key_owned);
|
||||
keys[i] = key_lower;
|
||||
|
||||
// Get value - metadata values must be strings
|
||||
const val = iter2.value;
|
||||
if (val.isString()) {
|
||||
const val_str = try bun.String.fromJS(val, globalObject);
|
||||
defer val_str.deref();
|
||||
const val_slice = val_str.toUTF8(bun.default_allocator);
|
||||
values[i] = bun.default_allocator.dupe(u8, val_slice.slice()) catch bun.outOfMemory();
|
||||
val_slice.deinit();
|
||||
} else {
|
||||
// Non-string value, use empty string or skip
|
||||
values[i] = "";
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
new_credentials.metadata = .{
|
||||
.keys = keys,
|
||||
.values = values,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
return globalObject.throwInvalidArgumentTypeValue("metadata", "object", metadata_value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return new_credentials;
|
||||
@@ -384,23 +506,12 @@ pub const S3Credentials = struct {
|
||||
acl: ?ACL = null,
|
||||
storage_class: ?StorageClass = null,
|
||||
request_payer: bool = false,
|
||||
_headers: [MAX_HEADERS]picohttp.Header = .{
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
},
|
||||
/// Stores allocated metadata header names ("x-amz-meta-*") for cleanup
|
||||
_metadata_header_names: ?[][]u8 = null,
|
||||
/// Header storage: BASE_HEADERS (9) + MAX_METADATA_HEADERS (32) = MAX_HEADERS (41)
|
||||
_headers: [MAX_HEADERS]picohttp.Header = [_]picohttp.Header{.{ .name = "", .value = "" }} ** MAX_HEADERS,
|
||||
_headers_len: u8 = 0,
|
||||
|
||||
pub const MAX_HEADERS = 11;
|
||||
|
||||
pub fn headers(this: *const @This()) []const picohttp.Header {
|
||||
return this._headers[0..this._headers_len];
|
||||
}
|
||||
@@ -447,6 +558,14 @@ pub const S3Credentials = struct {
|
||||
if (this.content_md5.len > 0) {
|
||||
bun.default_allocator.free(this.content_md5);
|
||||
}
|
||||
|
||||
// Free allocated metadata header names
|
||||
if (this._metadata_header_names) |names| {
|
||||
for (names) |name| {
|
||||
bun.default_allocator.free(name);
|
||||
}
|
||||
bun.default_allocator.free(names);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -465,6 +584,8 @@ pub const S3Credentials = struct {
|
||||
acl: ?ACL = null,
|
||||
storage_class: ?StorageClass = null,
|
||||
request_payer: bool = false,
|
||||
/// User-defined metadata (x-amz-meta-* headers)
|
||||
metadata: ?MetadataMap = null,
|
||||
};
|
||||
/// This is not used for signing but for console.log output, is just nice to have
|
||||
pub fn guessBucket(endpoint: []const u8) ?[]const u8 {
|
||||
@@ -687,6 +808,7 @@ pub const S3Credentials = struct {
|
||||
|
||||
const amz_day = amz_date[0..8];
|
||||
const request_payer = signOptions.request_payer;
|
||||
const metadata = signOptions.metadata;
|
||||
const header_key = SignedHeaders.Key{
|
||||
.content_disposition = content_disposition != null,
|
||||
.content_encoding = content_encoding != null,
|
||||
@@ -696,7 +818,97 @@ pub const S3Credentials = struct {
|
||||
.session_token = session_token != null,
|
||||
.storage_class = storage_class != null,
|
||||
};
|
||||
const signed_headers = if (signQuery) "host" else SignedHeaders.get(header_key);
|
||||
|
||||
// Build signed_headers string - for metadata we need to do this dynamically
|
||||
// because metadata keys vary at runtime. Metadata header names (x-amz-meta-*)
|
||||
// come after x-amz-date and before x-amz-request-payer alphabetically.
|
||||
// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
|
||||
var signed_headers_buf: [SIGNED_HEADERS_BUF_SIZE]u8 = undefined;
|
||||
const signed_headers: []const u8 = brk_signed: {
|
||||
if (signQuery) {
|
||||
// For presigned PUT URLs with metadata, include metadata headers
|
||||
if (metadata) |meta| {
|
||||
const meta_count = meta.count();
|
||||
if (meta_count > 0) {
|
||||
var fba = std.heap.FixedBufferAllocator.init(&signed_headers_buf);
|
||||
var list = std.array_list.Managed(u8).init(fba.allocator());
|
||||
list.appendSlice("host") catch {};
|
||||
|
||||
// Sort metadata keys alphabetically
|
||||
var indices: [MAX_METADATA_HEADERS]usize = undefined;
|
||||
for (0..meta_count) |i| {
|
||||
indices[i] = i;
|
||||
}
|
||||
std.mem.sort(usize, indices[0..meta_count], meta.keys, struct {
|
||||
fn lessThan(keys: []const []const u8, a: usize, b: usize) bool {
|
||||
return std.mem.lessThan(u8, keys[a], keys[b]);
|
||||
}
|
||||
}.lessThan);
|
||||
|
||||
// Add metadata headers (alphabetically after "host")
|
||||
for (indices[0..meta_count]) |idx| {
|
||||
list.appendSlice(";x-amz-meta-") catch {};
|
||||
list.appendSlice(meta.keys[idx]) catch {};
|
||||
}
|
||||
if (list.items.len > 0) {
|
||||
break :brk_signed list.items;
|
||||
}
|
||||
}
|
||||
}
|
||||
break :brk_signed "host";
|
||||
}
|
||||
if (metadata) |meta| {
|
||||
// Build signed headers dynamically with metadata
|
||||
// AWS Sig V4 requires headers in alphabetical order
|
||||
var fba = std.heap.FixedBufferAllocator.init(&signed_headers_buf);
|
||||
var list = std.array_list.Managed(u8).init(fba.allocator());
|
||||
|
||||
// Headers in alphabetical order (same as SignedHeaders.generate):
|
||||
// content-disposition, content-encoding, content-md5, host, x-amz-acl, x-amz-content-sha256, x-amz-date,
|
||||
// [x-amz-meta-*], x-amz-request-payer, x-amz-security-token, x-amz-storage-class
|
||||
if (content_disposition != null) list.appendSlice("content-disposition;") catch {};
|
||||
if (content_encoding != null) list.appendSlice("content-encoding;") catch {};
|
||||
if (content_md5 != null) list.appendSlice("content-md5;") catch {};
|
||||
list.appendSlice("host;") catch {};
|
||||
if (acl != null) list.appendSlice("x-amz-acl;") catch {};
|
||||
list.appendSlice("x-amz-content-sha256;x-amz-date") catch {};
|
||||
|
||||
// Add metadata headers (sorted by key)
|
||||
// Keys are already lowercase from parsing (AWS requirement)
|
||||
const meta_count = meta.count();
|
||||
if (meta_count > 0) {
|
||||
// Create sorted indices for metadata keys
|
||||
var indices: [MAX_METADATA_HEADERS]usize = undefined;
|
||||
for (0..meta_count) |i| {
|
||||
indices[i] = i;
|
||||
}
|
||||
// Sort indices by key for AWS Sig V4 canonical header ordering
|
||||
std.mem.sort(usize, indices[0..meta_count], meta.keys, struct {
|
||||
fn lessThan(keys: []const []const u8, a: usize, b: usize) bool {
|
||||
return std.mem.lessThan(u8, keys[a], keys[b]);
|
||||
}
|
||||
}.lessThan);
|
||||
|
||||
// Append sorted metadata header names (x-amz-meta-{key})
|
||||
for (indices[0..meta_count]) |idx| {
|
||||
list.appendSlice(";x-amz-meta-") catch {};
|
||||
list.appendSlice(meta.keys[idx]) catch {};
|
||||
}
|
||||
}
|
||||
|
||||
if (request_payer) list.appendSlice(";x-amz-request-payer") catch {};
|
||||
if (session_token != null) list.appendSlice(";x-amz-security-token") catch {};
|
||||
if (storage_class != null) list.appendSlice(";x-amz-storage-class") catch {};
|
||||
|
||||
// If buffer overflowed, fall back to non-metadata path
|
||||
if (list.items.len == 0) {
|
||||
// Fallback: metadata too large, use base signed_headers without metadata
|
||||
break :brk_signed SignedHeaders.get(header_key);
|
||||
}
|
||||
break :brk_signed list.items;
|
||||
}
|
||||
break :brk_signed SignedHeaders.get(header_key);
|
||||
};
|
||||
|
||||
const service_name = "s3";
|
||||
|
||||
@@ -778,7 +990,10 @@ pub const S3Credentials = struct {
|
||||
try query_parts.append(try std.fmt.allocPrint(allocator, "X-Amz-Security-Token={s}", .{token}));
|
||||
}
|
||||
|
||||
try query_parts.append(try std.fmt.allocPrint(allocator, "X-Amz-SignedHeaders=host", .{}));
|
||||
// URL-encode the signed headers (semicolons become %3B)
|
||||
var signed_headers_encoded_buf: [SIGNED_HEADERS_BUF_SIZE]u8 = undefined;
|
||||
const encoded_signed_headers = encodeURIComponent(signed_headers, &signed_headers_encoded_buf, true) catch signed_headers;
|
||||
try query_parts.append(try std.fmt.allocPrint(allocator, "X-Amz-SignedHeaders={s}", .{encoded_signed_headers}));
|
||||
|
||||
if (encoded_content_disposition) |cd| {
|
||||
try query_parts.append(try std.fmt.allocPrint(allocator, "response-content-disposition={s}", .{cd}));
|
||||
@@ -805,7 +1020,45 @@ pub const S3Credentials = struct {
|
||||
allocator.free(part);
|
||||
}
|
||||
|
||||
break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\n\nhost\n{s}", .{ method_name, normalizedPath, query_string.items, host, aws_content_hash });
|
||||
// Canonical request: METHOD\nPATH\nQUERY\nCANONICAL_HEADERS\n\nSIGNED_HEADERS\nPAYLOAD_HASH
|
||||
// For presigned URLs with metadata, include both host and metadata headers
|
||||
if (metadata) |meta| {
|
||||
const meta_count = meta.count();
|
||||
if (meta_count > 0) {
|
||||
// Build canonical headers with metadata (sorted alphabetically)
|
||||
var canonical_headers_buf: [4096]u8 = undefined;
|
||||
var headers_fba = std.heap.FixedBufferAllocator.init(&canonical_headers_buf);
|
||||
var headers_list = std.array_list.Managed(u8).init(headers_fba.allocator());
|
||||
|
||||
// Add host header first (comes before x-amz-meta-* alphabetically)
|
||||
headers_list.appendSlice("host:") catch {};
|
||||
headers_list.appendSlice(host) catch {};
|
||||
headers_list.append('\n') catch {};
|
||||
|
||||
// Sort metadata indices for alphabetical ordering
|
||||
var meta_indices: [MAX_METADATA_HEADERS]usize = undefined;
|
||||
for (0..meta_count) |i| {
|
||||
meta_indices[i] = i;
|
||||
}
|
||||
std.mem.sort(usize, meta_indices[0..meta_count], meta.keys, struct {
|
||||
fn lessThan(keys: []const []const u8, a: usize, b: usize) bool {
|
||||
return std.mem.lessThan(u8, keys[a], keys[b]);
|
||||
}
|
||||
}.lessThan);
|
||||
|
||||
// Add metadata headers (x-amz-meta-{key}:{value})
|
||||
for (meta_indices[0..meta_count]) |idx| {
|
||||
headers_list.appendSlice("x-amz-meta-") catch {};
|
||||
headers_list.appendSlice(meta.keys[idx]) catch {};
|
||||
headers_list.append(':') catch {};
|
||||
headers_list.appendSlice(meta.values[idx]) catch {};
|
||||
headers_list.append('\n') catch {};
|
||||
}
|
||||
|
||||
break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\n{s}\n{s}\n{s}", .{ method_name, normalizedPath, query_string.items, headers_list.items, signed_headers, aws_content_hash });
|
||||
}
|
||||
}
|
||||
break :brk_canonical try std.fmt.bufPrint(&tmp_buffer, "{s}\n{s}\n{s}\nhost:{s}\n\n{s}\n{s}", .{ method_name, normalizedPath, query_string.items, host, signed_headers, aws_content_hash });
|
||||
};
|
||||
var sha_digest = std.mem.zeroes(bun.sha.SHA256.Digest);
|
||||
bun.sha.SHA256.hash(canonical, &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine());
|
||||
@@ -843,7 +1096,10 @@ pub const S3Credentials = struct {
|
||||
|
||||
try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "X-Amz-Signature={s}", .{std.fmt.bytesToHex(signature[0..DIGESTED_HMAC_256_LEN], .lower)}));
|
||||
|
||||
try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "X-Amz-SignedHeaders=host", .{}));
|
||||
// URL-encode the signed headers for the final URL
|
||||
var url_signed_headers_encoded_buf: [SIGNED_HEADERS_BUF_SIZE]u8 = undefined;
|
||||
const url_encoded_signed_headers = encodeURIComponent(signed_headers, &url_signed_headers_encoded_buf, true) catch signed_headers;
|
||||
try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "X-Amz-SignedHeaders={s}", .{url_encoded_signed_headers}));
|
||||
|
||||
if (encoded_content_disposition) |cd| {
|
||||
try url_query_parts.append(try std.fmt.allocPrint(url_allocator, "response-content-disposition={s}", .{cd}));
|
||||
@@ -872,25 +1128,187 @@ pub const S3Credentials = struct {
|
||||
|
||||
break :brk try std.fmt.allocPrint(bun.default_allocator, "{s}://{s}{s}?{s}", .{ protocol, host, normalizedPath, url_query_string.items });
|
||||
} else {
|
||||
const canonical = try CanonicalRequest.format(
|
||||
&tmp_buffer,
|
||||
header_key,
|
||||
method_name,
|
||||
normalizedPath,
|
||||
if (search_params) |p| p[1..] else "",
|
||||
content_disposition,
|
||||
content_encoding,
|
||||
content_md5,
|
||||
host,
|
||||
acl,
|
||||
aws_content_hash,
|
||||
amz_date,
|
||||
session_token,
|
||||
storage_class,
|
||||
signed_headers,
|
||||
);
|
||||
// Build canonical request - if metadata exists, build manually
|
||||
var sha_digest = std.mem.zeroes(bun.sha.SHA256.Digest);
|
||||
bun.sha.SHA256.hash(canonical, &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine());
|
||||
if (metadata) |meta| {
|
||||
// Build canonical request manually with metadata headers
|
||||
// Format per AWS Sig V4: METHOD\nPATH\nQUERY\nHEADERS\n\nSIGNED_HEADERS\nHASH
|
||||
// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
|
||||
var canonical_buf: [CANONICAL_REQUEST_BUF_SIZE]u8 = undefined;
|
||||
var fba = std.heap.FixedBufferAllocator.init(&canonical_buf);
|
||||
var writer = fba.allocator().alloc(u8, CANONICAL_REQUEST_BUF_SIZE) catch unreachable;
|
||||
var pos: usize = 0;
|
||||
|
||||
// METHOD\n
|
||||
@memcpy(writer[pos..][0..method_name.len], method_name);
|
||||
pos += method_name.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
|
||||
// PATH\n
|
||||
@memcpy(writer[pos..][0..normalizedPath.len], normalizedPath);
|
||||
pos += normalizedPath.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
|
||||
// QUERY\n
|
||||
const query_str = if (search_params) |p| p[1..] else "";
|
||||
@memcpy(writer[pos..][0..query_str.len], query_str);
|
||||
pos += query_str.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
|
||||
// HEADERS (sorted alphabetically)
|
||||
if (content_disposition) |cd| {
|
||||
const hdr = "content-disposition:";
|
||||
@memcpy(writer[pos..][0..hdr.len], hdr);
|
||||
pos += hdr.len;
|
||||
@memcpy(writer[pos..][0..cd.len], cd);
|
||||
pos += cd.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
}
|
||||
if (content_encoding) |ce| {
|
||||
const hdr = "content-encoding:";
|
||||
@memcpy(writer[pos..][0..hdr.len], hdr);
|
||||
pos += hdr.len;
|
||||
@memcpy(writer[pos..][0..ce.len], ce);
|
||||
pos += ce.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
}
|
||||
if (content_md5) |md5| {
|
||||
const hdr = "content-md5:";
|
||||
@memcpy(writer[pos..][0..hdr.len], hdr);
|
||||
pos += hdr.len;
|
||||
@memcpy(writer[pos..][0..md5.len], md5);
|
||||
pos += md5.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
}
|
||||
// host
|
||||
const host_hdr = "host:";
|
||||
@memcpy(writer[pos..][0..host_hdr.len], host_hdr);
|
||||
pos += host_hdr.len;
|
||||
@memcpy(writer[pos..][0..host.len], host);
|
||||
pos += host.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
|
||||
if (acl) |acl_val| {
|
||||
const hdr = "x-amz-acl:";
|
||||
@memcpy(writer[pos..][0..hdr.len], hdr);
|
||||
pos += hdr.len;
|
||||
@memcpy(writer[pos..][0..acl_val.len], acl_val);
|
||||
pos += acl_val.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
}
|
||||
// x-amz-content-sha256
|
||||
const hash_hdr = "x-amz-content-sha256:";
|
||||
@memcpy(writer[pos..][0..hash_hdr.len], hash_hdr);
|
||||
pos += hash_hdr.len;
|
||||
@memcpy(writer[pos..][0..aws_content_hash.len], aws_content_hash);
|
||||
pos += aws_content_hash.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
|
||||
// x-amz-date
|
||||
const date_hdr = "x-amz-date:";
|
||||
@memcpy(writer[pos..][0..date_hdr.len], date_hdr);
|
||||
pos += date_hdr.len;
|
||||
@memcpy(writer[pos..][0..amz_date.len], amz_date);
|
||||
pos += amz_date.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
|
||||
// x-amz-meta-* headers (sorted by key)
|
||||
const meta_count = meta.count();
|
||||
if (meta_count > 0) {
|
||||
var indices: [MAX_METADATA_HEADERS]usize = undefined;
|
||||
for (0..meta_count) |i| {
|
||||
indices[i] = i;
|
||||
}
|
||||
std.mem.sort(usize, indices[0..meta_count], meta.keys, struct {
|
||||
fn lessThan(keys: []const []const u8, a: usize, b: usize) bool {
|
||||
return std.mem.lessThan(u8, keys[a], keys[b]);
|
||||
}
|
||||
}.lessThan);
|
||||
|
||||
for (indices[0..meta_count]) |idx| {
|
||||
const prefix = "x-amz-meta-";
|
||||
@memcpy(writer[pos..][0..prefix.len], prefix);
|
||||
pos += prefix.len;
|
||||
@memcpy(writer[pos..][0..meta.keys[idx].len], meta.keys[idx]);
|
||||
pos += meta.keys[idx].len;
|
||||
writer[pos] = ':';
|
||||
pos += 1;
|
||||
@memcpy(writer[pos..][0..meta.values[idx].len], meta.values[idx]);
|
||||
pos += meta.values[idx].len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (request_payer) {
|
||||
const hdr = "x-amz-request-payer:requester\n";
|
||||
@memcpy(writer[pos..][0..hdr.len], hdr);
|
||||
pos += hdr.len;
|
||||
}
|
||||
if (session_token) |token| {
|
||||
const hdr = "x-amz-security-token:";
|
||||
@memcpy(writer[pos..][0..hdr.len], hdr);
|
||||
pos += hdr.len;
|
||||
@memcpy(writer[pos..][0..token.len], token);
|
||||
pos += token.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
}
|
||||
if (storage_class) |sc| {
|
||||
const hdr = "x-amz-storage-class:";
|
||||
@memcpy(writer[pos..][0..hdr.len], hdr);
|
||||
pos += hdr.len;
|
||||
@memcpy(writer[pos..][0..sc.len], sc);
|
||||
pos += sc.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
}
|
||||
|
||||
// Empty line separator
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
|
||||
// SIGNED_HEADERS
|
||||
@memcpy(writer[pos..][0..signed_headers.len], signed_headers);
|
||||
pos += signed_headers.len;
|
||||
writer[pos] = '\n';
|
||||
pos += 1;
|
||||
|
||||
// HASH
|
||||
@memcpy(writer[pos..][0..aws_content_hash.len], aws_content_hash);
|
||||
pos += aws_content_hash.len;
|
||||
|
||||
bun.sha.SHA256.hash(writer[0..pos], &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine());
|
||||
} else {
|
||||
const canonical = try CanonicalRequest.format(
|
||||
&tmp_buffer,
|
||||
header_key,
|
||||
method_name,
|
||||
normalizedPath,
|
||||
if (search_params) |p| p[1..] else "",
|
||||
content_disposition,
|
||||
content_encoding,
|
||||
content_md5,
|
||||
host,
|
||||
acl,
|
||||
aws_content_hash,
|
||||
amz_date,
|
||||
session_token,
|
||||
storage_class,
|
||||
signed_headers,
|
||||
);
|
||||
bun.sha.SHA256.hash(canonical, &sha_digest, jsc.VirtualMachine.get().rareData().boringEngine());
|
||||
}
|
||||
|
||||
const signValue = try std.fmt.bufPrint(&tmp_buffer, "AWS4-HMAC-SHA256\n{s}\n{s}/{s}/{s}/aws4_request\n{s}", .{ amz_date, amz_day, region, service_name, std.fmt.bytesToHex(sha_digest[0..bun.sha.SHA256.digest], .lower) });
|
||||
|
||||
@@ -927,21 +1345,13 @@ pub const S3Credentials = struct {
|
||||
.storage_class = signOptions.storage_class,
|
||||
.request_payer = request_payer,
|
||||
.url = try std.fmt.allocPrint(bun.default_allocator, "{s}://{s}{s}{s}", .{ protocol, host, normalizedPath, if (search_params) |s| s else "" }),
|
||||
._headers = [_]picohttp.Header{
|
||||
.{ .name = "x-amz-content-sha256", .value = aws_content_hash },
|
||||
.{ .name = "x-amz-date", .value = amz_date },
|
||||
.{ .name = "Host", .value = host },
|
||||
.{ .name = "Authorization", .value = authorization[0..] },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
.{ .name = "", .value = "" },
|
||||
},
|
||||
._headers_len = 4,
|
||||
};
|
||||
// Set up the base headers
|
||||
result._headers[0] = .{ .name = "x-amz-content-sha256", .value = aws_content_hash };
|
||||
result._headers[1] = .{ .name = "x-amz-date", .value = amz_date };
|
||||
result._headers[2] = .{ .name = "Host", .value = host };
|
||||
result._headers[3] = .{ .name = "Authorization", .value = authorization[0..] };
|
||||
|
||||
if (acl) |acl_value| {
|
||||
result._headers[result._headers_len] = .{ .name = "x-amz-acl", .value = acl_value };
|
||||
@@ -985,6 +1395,34 @@ pub const S3Credentials = struct {
|
||||
result._headers_len += 1;
|
||||
}
|
||||
|
||||
// Add metadata headers (x-amz-meta-*)
|
||||
// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
|
||||
if (metadata) |meta| {
|
||||
const meta_count = meta.count();
|
||||
if (meta_count > 0) {
|
||||
// Allocate array to hold the "x-amz-meta-{key}" header names
|
||||
// These need to be allocated because picohttp.Header stores slices
|
||||
const header_names = bun.handleOom(bun.default_allocator.alloc([]u8, meta_count));
|
||||
result._metadata_header_names = header_names;
|
||||
|
||||
for (0..meta_count) |i| {
|
||||
const key = meta.keys[i];
|
||||
// Build "x-amz-meta-{key}" header name
|
||||
const prefix = "x-amz-meta-";
|
||||
const header_name = bun.handleOom(bun.default_allocator.alloc(u8, prefix.len + key.len));
|
||||
@memcpy(header_name[0..prefix.len], prefix);
|
||||
@memcpy(header_name[prefix.len..], key);
|
||||
header_names[i] = header_name;
|
||||
|
||||
result._headers[result._headers_len] = .{
|
||||
.name = header_name,
|
||||
.value = meta.values[i],
|
||||
};
|
||||
result._headers_len += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
};
|
||||
@@ -1003,6 +1441,8 @@ pub const S3CredentialsWithOptions = struct {
|
||||
changed_credentials: bool = false,
|
||||
/// indicates if the virtual hosted style is used
|
||||
virtual_hosted_style: bool = false,
|
||||
/// User-defined metadata (x-amz-meta-* headers)
|
||||
metadata: ?MetadataMap = null,
|
||||
_accessKeyIdSlice: ?jsc.ZigString.Slice = null,
|
||||
_secretAccessKeySlice: ?jsc.ZigString.Slice = null,
|
||||
_regionSlice: ?jsc.ZigString.Slice = null,
|
||||
@@ -1023,6 +1463,7 @@ pub const S3CredentialsWithOptions = struct {
|
||||
if (this._contentDispositionSlice) |slice| slice.deinit();
|
||||
if (this._contentTypeSlice) |slice| slice.deinit();
|
||||
if (this._contentEncodingSlice) |slice| slice.deinit();
|
||||
if (this.metadata) |*meta| meta.deinit(bun.default_allocator);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -106,6 +106,7 @@ pub const MultiPartUpload = struct {
|
||||
acl: ?ACL = null,
|
||||
storage_class: ?Storageclass = null,
|
||||
request_payer: bool = false,
|
||||
metadata: ?s3creds.MetadataMap = null,
|
||||
credentials: *S3Credentials,
|
||||
poll_ref: bun.Async.KeepAlive = bun.Async.KeepAlive.init(),
|
||||
vm: *jsc.VirtualMachine,
|
||||
@@ -290,6 +291,9 @@ pub const MultiPartUpload = struct {
|
||||
bun.default_allocator.free(ce);
|
||||
}
|
||||
}
|
||||
if (this.metadata) |*meta| {
|
||||
meta.deinit(bun.default_allocator);
|
||||
}
|
||||
this.credentials.deref();
|
||||
this.uploadid_buffer.deinit();
|
||||
for (this.multipart_etags.items) |tag| {
|
||||
@@ -320,6 +324,7 @@ pub const MultiPartUpload = struct {
|
||||
.acl = this.acl,
|
||||
.storage_class = this.storage_class,
|
||||
.request_payer = this.request_payer,
|
||||
.metadata = this.metadata,
|
||||
}, .{ .upload = @ptrCast(&singleSendUploadResponse) }, this);
|
||||
|
||||
return;
|
||||
@@ -613,6 +618,7 @@ pub const MultiPartUpload = struct {
|
||||
.acl = this.acl,
|
||||
.storage_class = this.storage_class,
|
||||
.request_payer = this.request_payer,
|
||||
.metadata = this.metadata,
|
||||
}, .{ .download = @ptrCast(&startMultiPartRequestResult) }, this);
|
||||
} else if (this.state == .multipart_completed) {
|
||||
try part.start();
|
||||
@@ -692,6 +698,7 @@ pub const MultiPartUpload = struct {
|
||||
.acl = this.acl,
|
||||
.storage_class = this.storage_class,
|
||||
.request_payer = this.request_payer,
|
||||
.metadata = this.metadata,
|
||||
}, .{ .upload = @ptrCast(&singleSendUploadResponse) }, this) catch {}; // TODO: properly propagate exception upwards
|
||||
} else {
|
||||
// we need to split
|
||||
@@ -787,7 +794,8 @@ pub const MultiPartUpload = struct {
|
||||
const std = @import("std");
|
||||
const ACL = @import("./acl.zig").ACL;
|
||||
const MultiPartUploadOptions = @import("./multipart_options.zig").MultiPartUploadOptions;
|
||||
const S3Credentials = @import("./credentials.zig").S3Credentials;
|
||||
const s3creds = @import("./credentials.zig");
|
||||
const S3Credentials = s3creds.S3Credentials;
|
||||
const S3Error = @import("./error.zig").S3Error;
|
||||
const Storageclass = @import("./storage_class.zig").StorageClass;
|
||||
|
||||
|
||||
@@ -7,6 +7,9 @@ pub const S3StatResult = union(enum) {
|
||||
lastModified: []const u8 = "",
|
||||
/// format: text/plain, contentType is not owned and need to be copied if used after this callback
|
||||
contentType: []const u8 = "",
|
||||
/// Raw headers from response - caller should filter for x-amz-meta-* headers.
|
||||
/// Headers are not owned and need to be copied if used after this callback.
|
||||
headers: []const picohttp.Header = &.{},
|
||||
},
|
||||
not_found: S3Error,
|
||||
|
||||
@@ -231,6 +234,7 @@ pub const S3HttpSimpleTask = struct {
|
||||
.lastModified = response.headers.get("last-modified") orelse "",
|
||||
.contentType = response.headers.get("content-type") orelse "",
|
||||
.size = if (response.headers.get("content-length")) |content_len| (std.fmt.parseInt(usize, content_len, 10) catch 0) else 0,
|
||||
.headers = response.headers.list,
|
||||
},
|
||||
}, this.callback_context);
|
||||
},
|
||||
@@ -360,6 +364,7 @@ pub const S3SimpleRequestOptions = struct {
|
||||
acl: ?ACL = null,
|
||||
storage_class: ?StorageClass = null,
|
||||
request_payer: bool = false,
|
||||
metadata: ?s3creds.MetadataMap = null,
|
||||
};
|
||||
|
||||
pub fn executeSimpleS3Request(
|
||||
@@ -377,6 +382,7 @@ pub fn executeSimpleS3Request(
|
||||
.acl = options.acl,
|
||||
.storage_class = options.storage_class,
|
||||
.request_payer = options.request_payer,
|
||||
.metadata = options.metadata,
|
||||
}, false, null) catch |sign_err| {
|
||||
if (options.range) |range_| bun.default_allocator.free(range_);
|
||||
const error_code_and_message = getSignErrorCodeAndMessage(sign_err);
|
||||
@@ -385,7 +391,8 @@ pub fn executeSimpleS3Request(
|
||||
};
|
||||
|
||||
const headers = brk: {
|
||||
var header_buffer: [S3Credentials.SignResult.MAX_HEADERS + 1]picohttp.Header = undefined;
|
||||
// MAX_HEADERS (41) + 1 for the additional header we mix in
|
||||
var header_buffer: [s3creds.MAX_HEADERS + 1]picohttp.Header = undefined;
|
||||
if (options.range) |range_| {
|
||||
const _headers = result.mixWithHeader(&header_buffer, .{ .name = "range", .value = range_ });
|
||||
break :brk bun.handleOom(bun.http.Headers.fromPicoHttpHeaders(_headers, bun.default_allocator));
|
||||
@@ -444,8 +451,9 @@ const std = @import("std");
|
||||
const ACL = @import("./acl.zig").ACL;
|
||||
const StorageClass = @import("./storage_class.zig").StorageClass;
|
||||
|
||||
const S3Credentials = @import("./credentials.zig").S3Credentials;
|
||||
const SignResult = @import("./credentials.zig").S3Credentials.SignResult;
|
||||
const s3creds = @import("./credentials.zig");
|
||||
const S3Credentials = s3creds.S3Credentials;
|
||||
const SignResult = S3Credentials.SignResult;
|
||||
|
||||
const S3Error = @import("./error.zig").S3Error;
|
||||
const getSignErrorCodeAndMessage = @import("./error.zig").getSignErrorCodeAndMessage;
|
||||
|
||||
@@ -1674,3 +1674,145 @@ describe.skipIf(!minioCredentials)("Archive with S3", () => {
|
||||
await s3File.delete();
|
||||
});
|
||||
});
|
||||
|
||||
// Metadata tests - test with MinIO only since it properly supports x-amz-meta-* headers
|
||||
describe.skipIf(!minioCredentials?.endpoint)("S3 Metadata (x-amz-meta-*)", () => {
|
||||
const credentials: S3Options = {
|
||||
accessKeyId: minioCredentials!.accessKeyId,
|
||||
secretAccessKey: minioCredentials!.secretAccessKey,
|
||||
endpoint: minioCredentials!.endpoint,
|
||||
bucket: minioCredentials!.bucket,
|
||||
};
|
||||
|
||||
it("should upload and retrieve metadata via S3File.write() and stat()", async () => {
|
||||
const client = new Bun.S3Client(credentials);
|
||||
const key = randomUUIDv7() + ".txt";
|
||||
const file = client.file(key);
|
||||
|
||||
// Write with metadata
|
||||
await file.write("test content", {
|
||||
metadata: { sku: "12345", category: "test" },
|
||||
});
|
||||
|
||||
// Retrieve and verify metadata
|
||||
const stat = await file.stat();
|
||||
expect(stat.metadata).toBeDefined();
|
||||
expect(stat.metadata.sku).toBe("12345");
|
||||
expect(stat.metadata.category).toBe("test");
|
||||
|
||||
// Cleanup
|
||||
await file.unlink();
|
||||
});
|
||||
|
||||
it("should upload metadata via S3Client.write() static method", async () => {
|
||||
const key = randomUUIDv7() + ".txt";
|
||||
|
||||
// Upload with metadata via S3Client.write()
|
||||
await S3Client.write(key, "test content", {
|
||||
...credentials,
|
||||
metadata: { "custom-key": "custom-value", author: "bun" },
|
||||
});
|
||||
|
||||
// Verify metadata via S3 stat
|
||||
const client = new Bun.S3Client(credentials);
|
||||
const stat = await client.stat(key);
|
||||
expect(stat.metadata["custom-key"]).toBe("custom-value");
|
||||
expect(stat.metadata.author).toBe("bun");
|
||||
|
||||
// Cleanup
|
||||
await client.delete(key);
|
||||
});
|
||||
|
||||
it("should normalize metadata keys to lowercase", async () => {
|
||||
const client = new Bun.S3Client(credentials);
|
||||
const key = randomUUIDv7() + ".txt";
|
||||
const file = client.file(key);
|
||||
|
||||
// Write with mixed-case metadata keys
|
||||
await file.write("test content", {
|
||||
metadata: { SKU: "12345", MyCustomKey: "value" },
|
||||
});
|
||||
|
||||
// Keys should be normalized to lowercase
|
||||
const stat = await file.stat();
|
||||
expect(stat.metadata.sku).toBe("12345");
|
||||
expect(stat.metadata.mycustomkey).toBe("value");
|
||||
// Uppercase keys should not exist
|
||||
expect(stat.metadata.SKU).toBeUndefined();
|
||||
expect(stat.metadata.MyCustomKey).toBeUndefined();
|
||||
|
||||
// Cleanup
|
||||
await file.unlink();
|
||||
});
|
||||
|
||||
it("should return empty metadata object when no metadata is set", async () => {
|
||||
const client = new Bun.S3Client(credentials);
|
||||
const key = randomUUIDv7() + ".txt";
|
||||
const file = client.file(key);
|
||||
|
||||
// Write without metadata
|
||||
await file.write("test content");
|
||||
|
||||
// stat should return empty metadata object
|
||||
const stat = await file.stat();
|
||||
expect(stat.metadata).toBeDefined();
|
||||
expect(typeof stat.metadata).toBe("object");
|
||||
expect(Object.keys(stat.metadata).length).toBe(0);
|
||||
|
||||
// Cleanup
|
||||
await file.unlink();
|
||||
});
|
||||
|
||||
it("presigned PUT URL should include metadata in signed headers", async () => {
|
||||
const client = new Bun.S3Client(credentials);
|
||||
const key = randomUUIDv7() + ".txt";
|
||||
const file = client.file(key);
|
||||
|
||||
// Generate presigned PUT URL with metadata
|
||||
const url = file.presign({
|
||||
method: "PUT",
|
||||
expiresIn: 3600,
|
||||
metadata: { "signed-key": "signed-value" },
|
||||
});
|
||||
|
||||
// The presigned URL should contain metadata headers in the signature
|
||||
expect(url).toContain("x-amz-meta-signed-key");
|
||||
|
||||
// Upload using presigned URL with matching metadata header
|
||||
const uploadRes = await fetch(url, {
|
||||
method: "PUT",
|
||||
headers: { "x-amz-meta-signed-key": "signed-value" },
|
||||
body: "presigned content",
|
||||
});
|
||||
expect(uploadRes.status).toBe(200);
|
||||
|
||||
// Verify metadata was stored
|
||||
const stat = await file.stat();
|
||||
expect(stat.metadata["signed-key"]).toBe("signed-value");
|
||||
|
||||
// Cleanup
|
||||
await file.unlink();
|
||||
});
|
||||
|
||||
it("should support metadata with writer() streaming upload", async () => {
|
||||
const client = new Bun.S3Client(credentials);
|
||||
const key = randomUUIDv7() + ".txt";
|
||||
const file = client.file(key, {
|
||||
metadata: { "stream-key": "stream-value" },
|
||||
});
|
||||
|
||||
// Write using streaming writer
|
||||
const writer = file.writer();
|
||||
writer.write("chunk 1 ");
|
||||
writer.write("chunk 2 ");
|
||||
writer.write("chunk 3");
|
||||
await writer.end();
|
||||
|
||||
// Verify metadata
|
||||
const stat = await file.stat();
|
||||
expect(stat.metadata["stream-key"]).toBe("stream-value");
|
||||
|
||||
// Cleanup
|
||||
await file.unlink();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user