fix(buffer): handle string allocation failures in encoding operations (#25214)

## Summary
- Add proper bounds checking for encoding operations that produce larger
output than input
- Handle allocation failures gracefully by returning appropriate errors
- Add defensive checks in string initialization functions

## Test plan
- Added test case for encoding operations with large buffers
- Verified existing buffer tests still pass

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Bot <claude-bot@bun.sh>
Co-authored-by: Claude <noreply@anthropic.com>

src/bun.js/bindings/JSBuffer.cpp

@@ -1741,6 +1741,16 @@ JSC::EncodedJSValue jsBufferToStringFromBytes(JSGlobalObject* lexicalGlobalObjec
         return Bun::ERR::STRING_TOO_LONG(scope, lexicalGlobalObject);
     }
 
+    // Check encoding-specific output size limits
+    // For hex, output is 2x input size
+    if (encoding == BufferEncodingType::hex && bytes.size() > WTF::String::MaxLength / 2) {
+        return Bun::ERR::STRING_TOO_LONG(scope, lexicalGlobalObject);
+    }
+    // For base64, output is ceil(input * 4 / 3)
+    if ((encoding == BufferEncodingType::base64 || encoding == BufferEncodingType::base64url) && bytes.size() > (WTF::String::MaxLength / 4) * 3) {
+        return Bun::ERR::STRING_TOO_LONG(scope, lexicalGlobalObject);
+    }
+
     switch (encoding) {
     case WebCore::BufferEncodingType::buffer: {
         auto* buffer = createUninitializedBuffer(lexicalGlobalObject, bytes.size());

src/bun.js/webcore/encoding.zig

@@ -197,11 +197,17 @@ pub fn toBunStringComptime(input: []const u8, comptime encoding: Encoding) bun.S
     switch (comptime encoding) {
         .ascii => {
             const str, const chars = bun.String.createUninitialized(.latin1, input.len);
+            if (str.tag == .Dead) {
+                return str;
+            }
             strings.copyLatin1IntoASCII(chars, input);
             return str;
         },
         .latin1 => {
             const str, const chars = bun.String.createUninitialized(.latin1, input.len);
+            if (str.tag == .Dead) {
+                return str;
+            }
             @memcpy(chars, input);
             return str;
         },
@@ -220,6 +226,9 @@ pub fn toBunStringComptime(input: []const u8, comptime encoding: Encoding) bun.S
             if (input.len / 2 == 0) return bun.String.empty;
 
             const str, const chars = bun.String.createUninitialized(.utf16, input.len / 2);
+            if (str.tag == .Dead) {
+                return str;
+            }
             var output_bytes = std.mem.sliceAsBytes(chars);
             output_bytes[output_bytes.len - 1] = 0;
 
@@ -229,6 +238,9 @@ pub fn toBunStringComptime(input: []const u8, comptime encoding: Encoding) bun.S
 
         .hex => {
             const str, const chars = bun.String.createUninitialized(.latin1, input.len * 2);
+            if (str.tag == .Dead) {
+                return str;
+            }
 
             const wrote = strings.encodeBytesToHex(chars, input);
             bun.assert(wrote == chars.len);

src/string.zig

@@ -131,6 +131,9 @@ pub const String = extern struct {
     fn createUninitializedLatin1(len: usize) struct { String, []u8 } {
         bun.assert(len > 0);
         const string = bun.cpp.BunString__fromLatin1Unitialized(len);
+        if (string.tag == .Dead) {
+            return .{ string, &.{} };
+        }
         _ = validateRefCount(string);
         const wtf = string.value.WTFStringImpl;
         return .{
@@ -142,6 +145,9 @@ pub const String = extern struct {
     fn createUninitializedUTF16(len: usize) struct { String, []u16 } {
         bun.assert(len > 0);
         const string = bun.cpp.BunString__fromUTF16Unitialized(len);
+        if (string.tag == .Dead) {
+            return .{ string, &.{} };
+        }
         _ = validateRefCount(string);
         const wtf = string.value.WTFStringImpl;
         return .{

test/js/node/buffer.test.js

@@ -2865,6 +2865,16 @@ for (let withOverridenBufferWrite of [false, true]) {
         expect(buf.hexSlice(3, 4)).toStrictEqual("33");
       });
 
+      // Regression test: large buffers that would produce strings exceeding max string length
+      it("Buffer.hexSlice() throws for large buffers", () => {
+        const { MAX_STRING_LENGTH } = require("buffer").constants;
+        // Hex output is 2x input size, so buffer size > MAX_STRING_LENGTH/2 will overflow
+        const largeBuffer = Buffer.allocUnsafe(Math.floor(MAX_STRING_LENGTH / 2) + 1);
+        expect(() => largeBuffer.hexSlice()).toThrow(
+          `Cannot create a string longer than ${MAX_STRING_LENGTH} characters`,
+        );
+      });
+
       it("Buffer.ucs2Slice()", () => {
         const buf = Buffer.from("あいうえお", "ucs2");