Add detailed comment explaining OpenSSL v3 default digest behavior

This commit adds comprehensive documentation in the code explaining:
- How OpenSSL v3 provides SHA256 as the default digest for RSA keys
- The exact source code locations in OpenSSL where this happens
- Why BoringSSL behaves differently (no automatic default)
- Why this fix is necessary for Node.js compatibility

The comment includes specific file paths and line numbers from OpenSSL v3
source code for future reference and maintenance.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/bun.js/bindings/node/crypto/CryptoSignJob.cpp

@@ -349,6 +349,50 @@ std::optional<SignJobCtx> SignJobCtx::fromJS(JSGlobalObject* globalObject, Throw
             ERR::CRYPTO_INVALID_DIGEST(scope, globalObject, algorithmView);
             return {};
         }
+    } else {
+        // OpenSSL v3 Default Digest Behavior for RSA Keys
+        // ================================================
+        // When Node.js calls crypto.sign() or crypto.verify() with a null/undefined algorithm,
+        // it passes NULL to OpenSSL's EVP_DigestSignInit/EVP_DigestVerifyInit functions.
+        // 
+        // OpenSSL v3 then automatically provides a default digest for RSA keys through the
+        // following mechanism:
+        //
+        // 1. In crypto/evp/m_sigver.c:215-220 (do_sigver_init function):
+        //    When mdname is NULL and type is NULL, OpenSSL calls:
+        //    evp_keymgmt_util_get_deflt_digest_name(tmp_keymgmt, provkey, locmdname, sizeof(locmdname))
+        //
+        // 2. In crypto/evp/keymgmt_lib.c:533-571 (evp_keymgmt_util_get_deflt_digest_name function):
+        //    This queries the key management provider for OSSL_PKEY_PARAM_DEFAULT_DIGEST
+        //
+        // 3. In providers/implementations/keymgmt/rsa_kmgmt.c:
+        //    - Line 54: #define RSA_DEFAULT_MD "SHA256"
+        //    - Lines 351-355: For RSA keys (non-PSS), it returns RSA_DEFAULT_MD ("SHA256")
+        //      if ((p = OSSL_PARAM_locate(params, OSSL_PKEY_PARAM_DEFAULT_DIGEST)) != NULL
+        //          && (rsa_type != RSA_FLAG_TYPE_RSASSAPSS
+        //              || ossl_rsa_pss_params_30_is_unrestricted(pss_params))) {
+        //          if (!OSSL_PARAM_set_utf8_string(p, RSA_DEFAULT_MD))
+        //              return 0;
+        //      }
+        //
+        // BoringSSL Difference:
+        // =====================
+        // BoringSSL (used by Bun) does not have this automatic default mechanism.
+        // When NULL is passed as the digest to EVP_DigestVerifyInit for RSA keys,
+        // BoringSSL returns error 0x06000077 (NO_DEFAULT_DIGEST).
+        //
+        // This Fix:
+        // =========
+        // To achieve Node.js/OpenSSL compatibility, we explicitly set SHA256 as the
+        // default digest for RSA keys when no algorithm is specified, matching the
+        // OpenSSL behavior documented above.
+        //
+        // For Ed25519/Ed448 keys (one-shot variants), we intentionally leave digest
+        // as null since these algorithms perform their own hashing internally and
+        // don't require a separate digest algorithm.
+        if (keyObject.asymmetricKey().isRsaVariant()) {
+            digest = Digest::FromName("SHA256"_s);
+        }
     }
 
     if (mode == Mode::Verify) {

test/regression/issue/11029-crypto-verify-null-algorithm.test.ts

@@ -0,0 +1,135 @@
+import { test, expect } from "bun:test";
+import crypto from "crypto";
+
+// Regression test for issue #11029
+// crypto.verify() should support null/undefined algorithm parameter
+test("crypto.verify with null algorithm should work for RSA keys", () => {
+  // Generate RSA key pair
+  const { publicKey, privateKey } = crypto.generateKeyPairSync("rsa", {
+    modulusLength: 2048,
+    publicKeyEncoding: {
+      type: "spki",
+      format: "pem",
+    },
+    privateKeyEncoding: {
+      type: "pkcs8",
+      format: "pem",
+    },
+  });
+
+  const data = Buffer.from("test data");
+
+  // Sign with null algorithm (should use default SHA256 for RSA)
+  const signature = crypto.sign(null, data, privateKey);
+  expect(signature).toBeInstanceOf(Buffer);
+
+  // Verify with null algorithm should succeed
+  const isVerified = crypto.verify(null, data, publicKey, signature);
+  expect(isVerified).toBe(true);
+
+  // Verify with wrong data should fail
+  const wrongData = Buffer.from("wrong data");
+  const isVerifiedWrong = crypto.verify(null, wrongData, publicKey, signature);
+  expect(isVerifiedWrong).toBe(false);
+});
+
+test("crypto.verify with undefined algorithm should work for RSA keys", () => {
+  const { publicKey, privateKey } = crypto.generateKeyPairSync("rsa", {
+    modulusLength: 2048,
+    publicKeyEncoding: {
+      type: "spki",
+      format: "pem",
+    },
+    privateKeyEncoding: {
+      type: "pkcs8",
+      format: "pem",
+    },
+  });
+
+  const data = Buffer.from("test data");
+  const signature = crypto.sign(undefined, data, privateKey);
+
+  // Verify with undefined algorithm
+  const isVerified = crypto.verify(undefined, data, publicKey, signature);
+  expect(isVerified).toBe(true);
+});
+
+test("crypto.verify with null algorithm should work for Ed25519 keys", () => {
+  // Generate Ed25519 key pair (one-shot variant that doesn't need digest)
+  const { publicKey, privateKey } = crypto.generateKeyPairSync("ed25519", {
+    publicKeyEncoding: {
+      type: "spki",
+      format: "pem",
+    },
+    privateKeyEncoding: {
+      type: "pkcs8",
+      format: "pem",
+    },
+  });
+
+  const data = Buffer.from("test data");
+
+  // Ed25519 should work with null algorithm (no digest needed)
+  const signature = crypto.sign(null, data, privateKey);
+  expect(signature).toBeInstanceOf(Buffer);
+
+  const isVerified = crypto.verify(null, data, publicKey, signature);
+  expect(isVerified).toBe(true);
+});
+
+test("crypto.verify cross-verification between null and explicit SHA256", () => {
+  const { publicKey, privateKey } = crypto.generateKeyPairSync("rsa", {
+    modulusLength: 2048,
+    publicKeyEncoding: {
+      type: "spki",
+      format: "pem",
+    },
+    privateKeyEncoding: {
+      type: "pkcs8",
+      format: "pem",
+    },
+  });
+
+  const data = Buffer.from("test data");
+
+  // Sign with SHA256
+  const signatureSHA256 = crypto.sign("SHA256", data, privateKey);
+
+  // Should be able to verify with null (defaults to SHA256 for RSA)
+  const isVerifiedWithNull = crypto.verify(null, data, publicKey, signatureSHA256);
+  expect(isVerifiedWithNull).toBe(true);
+
+  // Sign with null
+  const signatureNull = crypto.sign(null, data, privateKey);
+
+  // Should be able to verify with explicit SHA256
+  const isVerifiedWithSHA256 = crypto.verify("SHA256", data, publicKey, signatureNull);
+  expect(isVerifiedWithSHA256).toBe(true);
+});
+
+test("crypto.createVerify should also work with RSA keys", () => {
+  const { publicKey, privateKey } = crypto.generateKeyPairSync("rsa", {
+    modulusLength: 2048,
+    publicKeyEncoding: {
+      type: "spki",
+      format: "pem",
+    },
+    privateKeyEncoding: {
+      type: "pkcs8",
+      format: "pem",
+    },
+  });
+
+  const data = Buffer.from("test data");
+
+  // Create signature using createSign
+  const signer = crypto.createSign("SHA256");
+  signer.update(data);
+  const signature = signer.sign(privateKey);
+
+  // Verify using createVerify
+  const verifier = crypto.createVerify("SHA256");
+  verifier.update(data);
+  const isVerified = verifier.verify(publicKey, signature);
+  expect(isVerified).toBe(true);
+});