mirrors/bun.sh
1
0
Fork 0
mirror of https://github.com/oven-sh/bun synced 2026-02-10 10:58:56 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
4dd2b5989facdfff17b35456c8426c0cbf73f00a
bun.sh/test/js/web/encoding/text-decoder.test.js
190n e23491391b bun run prettier (#19807) 
Co-authored-by: 190n <7763597+190n@users.noreply.github.com>
2025-05-20 20:01:38 -07:00

582 lines
30 KiB
JavaScript
Raw Blame History

import { describe, expect, it } from "bun:test";
import { gc as gcTrace, withoutAggressiveGC } from "harness";
const getByteLength = str => {
// returns the byte length of an utf8 string
var s = str.length;
for (var i = str.length - 1; i >= 0; i--) {
var code = str.charCodeAt(i);
if (code > 0x7f && code <= 0x7ff) s++;
else if (code > 0x7ff && code <= 0xffff) s += 2;
if (code >= 0xdc00 && code <= 0xdfff) i--; //trail surrogate
}
return s;
};
describe("TextDecoder", () => {
it("should not crash on empty text", () => {
const decoder = new TextDecoder();
gcTrace(true);
const fixtures = [new Uint8Array(), new Uint8Array([]), new Buffer(0), new ArrayBuffer(0), new Uint16Array(0)];
for (let input of fixtures) {
expect(decoder.decode(input)).toBe("");
}
// Cause a de-opt
try {
decoder.decode([NaN, Symbol("s")]);
} catch (e) {}
// DOMJIT test
for (let i = 0; i < 90000; i++) {
decoder.decode(fixtures[0]);
}
gcTrace(true);
});
it("should decode ascii text", () => {
const decoder = new TextDecoder("latin1");
gcTrace(true);
expect(decoder.encoding).toBe("windows-1252");
gcTrace(true);
expect(decoder.decode(new Uint8Array([0x41, 0x42, 0x43]))).toBe("ABC");
gcTrace(true);
// hit the SIMD code path
const result = [
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33,
];
gcTrace(true);
expect(decoder.decode(Uint8Array.from(result))).toBe(String.fromCharCode(...result));
gcTrace(true);
});
it("should decode unicode text", () => {
const decoder = new TextDecoder();
gcTrace(true);
const inputBytes = [226, 157, 164, 239, 184, 143, 32, 82, 101, 100, 32, 72, 101, 97, 114, 116];
for (var repeat = 1; repeat < 100; repeat++) {
var text = `❤️ Red Heart`.repeat(repeat);
var bytes = Array.from({ length: repeat }, () => inputBytes).flat();
var decoded = decoder.decode(Uint8Array.from(bytes));
expect(decoder.encoding).toBe("utf-8");
expect(decoded).toBe(text);
gcTrace(true);
}
});
describe("typedArrays", () => {
var text = `ABC DEF GHI JKL MNO PQR STU VWX YZ ABC DEF GHI JKL MNO PQR STU V`;
var bytes = new TextEncoder().encode(text);
var decoder = new TextDecoder();
for (let TypedArray of [
Uint8Array,
Uint16Array,
Uint32Array,
Int8Array,
Int16Array,
Int32Array,
Float16Array,
Float32Array,
Float64Array,
DataView,
BigInt64Array,
BigUint64Array,
]) {
it(`should decode ${TypedArray.name}`, () => {
const decoded = decoder.decode(new TypedArray(bytes.buffer));
expect(decoded).toBe(text);
});
}
it("DOMJIT call", () => {
const array = new Uint8Array(bytes.buffer);
withoutAggressiveGC(() => {
for (let i = 0; i < 100_000; i++) {
const decoded = decoder.decode(array);
expect(decoded).toBe(text);
}
});
});
});
it("should decode unicode text with multiple consecutive emoji", () => {
const decoder = new TextDecoder();
const encoder = new TextEncoder();
gcTrace(true);
var text = `❤️❤️❤️❤️❤️❤️ Red Heart`;
text += ` ✨ Sparkles 🔥 Fire 😀 😃 😄 😁 😆 😅 😂 🤣 🥲 ☺️ 😊 😇 🙂 🙃 😉 😌 😍 🥰 😘 😗 😙 😚 😋 😛 😝 😜 🤪 🤨 🧐 🤓 😎 🥸 🤩 🥳 😏 😒 😞 😔 😟 😕 🙁 ☹️ 😣 😖 😫 😩 🥺 😢 😭 😤 😠 😡 🤬 🤯 😳 🥵 🥶 😱 😨 😰`;
gcTrace(true);
const result = decoder.decode(encoder.encode(text));
expect(result).toBe(text);
expect(result).toBeUTF16String();
gcTrace(true);
const bytes = new Uint8Array(getByteLength(text) * 8);
gcTrace(true);
const amount = encoder.encodeInto(text, bytes);
gcTrace(true);
expect(decoder.decode(bytes.subarray(0, amount.written))).toBe(text);
gcTrace(true);
});
it("coerces the fatal flag to boolean : 1 -> true", () => {
const decoder = new TextDecoder("utf-8", { fatal: 1 });
expect(decoder.fatal).toBe(true);
});
it("coerces the fatal flag to boolean : 0 -> false", () => {
const decoder = new TextDecoder("utf-8", { fatal: 0 });
expect(decoder.fatal).toBe(false);
});
it("coerces the fatal flag to boolean : string -> true", () => {
const decoder = new TextDecoder("utf-8", { fatal: "string" });
expect(decoder.fatal).toBe(true);
});
it("coerces the fatal flag to boolean : null -> false", () => {
const decoder = new TextDecoder("utf-8", { fatal: null });
expect(decoder.fatal).toBe(false);
});
it("coerces the fatal flag to boolean : empty-string -> false", () => {
const decoder = new TextDecoder("utf-8", { fatal: "" });
expect(decoder.fatal).toBe(false);
});
it("should respect fatal when encountering invalid data", () => {
const decoder = new TextDecoder("utf-8", { fatal: true });
expect(() => {
decoder.decode(new Uint8Array([0xc0])); // Invalid UTF8
}).toThrow(Error);
let err;
try {
decoder.decode(new Uint8Array([0xc0, 0x80])); // Invalid UTF8
} catch (e) {
err = e;
}
expect(err.code).toBe("ERR_ENCODING_INVALID_ENCODED_DATA");
});
it("should not trim invalid byte sequences when fatal is false", () => {
const buf = Buffer.from([77, 97, 110, 32, 208, 129, 240, 164, 173]);
const received = new TextDecoder("utf-8").decode(buf);
const expected = "Man Ё\ufffd";
expect(received).toBe(expected);
});
it("should trim when stream is true", () => {
const buf = Buffer.from([77, 97, 110, 32, 208, 129, 240, 164, 173]);
const received = new TextDecoder("utf-8").decode(buf, { stream: true });
const expected = "Man Ё";
expect(received).toBe(expected);
});
it("constructor should set values", () => {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false });
expect(decoder.fatal).toBe(true);
expect(decoder.ignoreBOM).toBe(false);
});
it("should throw on invalid input", () => {
expect(() => {
const decoder = new TextDecoder("utf-8", { fatal: 10, ignoreBOM: {} });
}).toThrow();
});
it("should support undifined", () => {
const decoder = new TextDecoder(undefined);
expect(decoder.encoding).toBe("utf-8");
});
it("should support undefined options", () => {
expect(() => {
const decoder = new TextDecoder("utf-8", undefined);
}).not.toThrow();
});
});
describe("TextDecoder ignoreBOM", () => {
it.each([
{
encoding: "utf-8",
bytes: [0xef, 0xbb, 0xbf, 0x61, 0x62, 0x63],
},
{
encoding: "utf-16le",
bytes: [0xff, 0xfe, 0x61, 0x00, 0x62, 0x00, 0x63, 0x00],
},
])("should ignoreBOM for: %o", ({ encoding, bytes }) => {
const BOM = "\uFEFF";
const array = new Uint8Array(bytes);
const decoder_ignore_bom = new TextDecoder(encoding, { ignoreBOM: true });
expect(decoder_ignore_bom.decode(array)).toStrictEqual(`${BOM}abc`);
const decoder_not_ignore_bom = new TextDecoder(encoding, { ignoreBOM: false });
expect(decoder_not_ignore_bom.decode(array)).toStrictEqual("abc");
const decoder_not_ignore_bom_default = new TextDecoder(encoding);
expect(decoder_not_ignore_bom_default.decode(array)).toStrictEqual(`abc`);
});
});
it("truncated sequences", () => {
const assert_equals = (a, b) => expect(a).toBe(b);
// Truncated sequences
assert_equals(new TextDecoder().decode(new Uint8Array([0xf0])), "\uFFFD");
assert_equals(new TextDecoder().decode(new Uint8Array([0xf0, 0x9f])), "\uFFFD");
assert_equals(new TextDecoder().decode(new Uint8Array([0xf0, 0x9f, 0x92])), "\uFFFD");
// Errors near end-of-queue
assert_equals(new TextDecoder().decode(new Uint8Array([0xf0, 0x9f, 0x41])), "\uFFFDA");
assert_equals(new TextDecoder().decode(new Uint8Array([0xf0, 0x41, 0x42])), "\uFFFDAB");
assert_equals(new TextDecoder().decode(new Uint8Array([0xf0, 0x41, 0xf0])), "\uFFFDA\uFFFD");
assert_equals(new TextDecoder().decode(new Uint8Array([0xf0, 0x8f, 0x92])), "\uFFFD\uFFFD\uFFFD");
});
it.each([
[0xc0, 0x80], // 192
[0xc1, 0x80], // 193
])(`should handle %d`, (...input) => {
const decoder = new TextDecoder();
const output = decoder.decode(Uint8Array.from(input));
expect(output).toBe("\uFFFD\uFFFD");
});
// https://github.com/nodejs/node/blob/492032f34c1bf264eae01dc5cdfc77c8032b8552/test/fixtures/wpt/encoding/textdecoder-fatal-streaming.any.js#L4
it("Fatal flag, non-streaming cases", () => {
[
{ encoding: "utf-8", sequence: [0xc0] },
{ encoding: "utf-16le", sequence: [0x00] },
{ encoding: "utf-16be", sequence: [0x00] },
].forEach(function (testCase) {
expect(
() => {
var decoder = new TextDecoder(testCase.encoding, { fatal: true });
decoder.decode(new Uint8Array(testCase.sequence));
},
"Unterminated " + testCase.encoding + " sequence should throw if fatal flag is set",
).toThrow();
expect(
new TextDecoder(testCase.encoding).decode(new Uint8Array([testCase.sequence])),
"Unterminated UTF-8 sequence should emit replacement character if fatal flag is unset",
).toBe("\uFFFD");
});
});
describe("stream", () => {
{
// https://github.com/nodejs/node/blob/492032f34c1bf264eae01dc5cdfc77c8032b8552/test/fixtures/wpt/encoding/textdecoder-arguments.any.js#L3
it("TextDecoder decode() with explicit undefined", () => {
const decoder = new TextDecoder();
// Just passing nothing.
expect(decoder.decode(undefined), "Undefined as first arg should decode to empty string").toBe("");
// Flushing an incomplete sequence.
decoder.decode(new Uint8Array([0xc9]), { stream: true });
expect(decoder.decode(undefined), "Undefined as first arg should flush the stream").toBe("\uFFFD");
});
it("TextDecoder decode() with undefined and undefined", () => {
const decoder = new TextDecoder();
// Just passing nothing.
expect(decoder.decode(undefined, undefined), "Undefined as first arg should decode to empty string").toBe("");
// Flushing an incomplete sequence.
decoder.decode(new Uint8Array([0xc9]), { stream: true });
expect(decoder.decode(undefined, undefined), "Undefined as first arg should flush the stream").toBe("\uFFFD");
});
it("TextDecoder decode() with undefined and options", () => {
const decoder = new TextDecoder();
// Just passing nothing.
expect(decoder.decode(undefined, {}), "Undefined as first arg should decode to empty string").toBe("");
// Flushing an incomplete sequence.
decoder.decode(new Uint8Array([0xc9]), { stream: true });
expect(decoder.decode(undefined, {}), "Undefined as first arg should flush the stream").toBe("\uFFFD");
});
}
{
// https://github.com/nodejs/node/blob/492032f34c1bf264eae01dc5cdfc77c8032b8552/test/fixtures/wpt/encoding/textdecoder-eof.any.js#L14
it("TextDecoder end-of-queue handling using stream: true", () => {
const decoder = new TextDecoder();
decoder.decode(new Uint8Array([0xf0]), { stream: true });
expect(decoder.decode()).toBe("\uFFFD");
decoder.decode(new Uint8Array([0xf0]), { stream: true });
decoder.decode(new Uint8Array([0x9f]), { stream: true });
expect(decoder.decode()).toBe("\uFFFD");
decoder.decode(new Uint8Array([0xf0, 0x9f]), { stream: true });
expect(decoder.decode(new Uint8Array([0x92]))).toBe("\uFFFD");
expect(decoder.decode(new Uint8Array([0xf0, 0x9f]), { stream: true })).toBe("");
expect(decoder.decode(new Uint8Array([0x41]), { stream: true })).toBe("\uFFFDA");
expect(decoder.decode()).toBe("");
expect(decoder.decode(new Uint8Array([0xf0, 0x41, 0x42]), { stream: true })).toBe("\uFFFDAB");
expect(decoder.decode()).toBe("");
expect(decoder.decode(new Uint8Array([0xf0, 0x41, 0xf0]), { stream: true })).toBe("\uFFFDA");
expect(decoder.decode()).toBe("\uFFFD");
expect(decoder.decode(new Uint8Array([0xf0]), { stream: true })).toBe("");
expect(decoder.decode(new Uint8Array([0x8f]), { stream: true })).toBe("\uFFFD\uFFFD");
expect(decoder.decode(new Uint8Array([0x92]), { stream: true })).toBe("\uFFFD");
expect(decoder.decode()).toBe("");
});
}
{
// https://github.com/WebKit/WebKit/blob/443e796d1538654c34f2690e39600c70c8052b63/LayoutTests/imported/w3c/web-platform-tests/encoding/textdecoder-fatal-streaming.any.js#L22
it("Fatal flag, streaming cases", () => {
var decoder = new TextDecoder("utf-16le", { fatal: true });
var odd = new Uint8Array([0x00]);
var even = new Uint8Array([0x00, 0x00]);
expect(decoder.decode(odd, { stream: true })).toBe("");
expect(decoder.decode(odd, { stream: true })).toBe("\u0000");
expect(() => {
decoder.decode(even, { stream: true });
decoder.decode(odd);
}).toThrow(TypeError);
expect(() => {
decoder.decode(odd, { stream: true });
decoder.decode(even);
}).toThrow(TypeError);
expect(decoder.decode(even, { stream: true })).toBe("\u0000");
expect(() => {
decoder.decode(odd);
}).toThrow(TypeError);
// expect(decoder.decode(odd)).toBe("\u0000");
});
}
{
// https://github.com/nodejs/node/blob/926503b66910d9ec895c33c7fd94361fd78dea72/test/fixtures/wpt/encoding/textdecoder-streaming.any.js#L6
// META: title=Encoding API: Streaming decode
// META: global=window,worker
// META: script=resources/encodings.js
// META: script=/common/sab.js
var string = "\x00123ABCabc\x80\xFF\u0100\u1000\uFFFD\uD800\uDC00\uDBFF\uDFFF";
var octets = {
"utf-8": [
0x00, 0x31, 0x32, 0x33, 0x41, 0x42, 0x43, 0x61, 0x62, 0x63, 0xc2, 0x80, 0xc3, 0xbf, 0xc4, 0x80, 0xe1, 0x80,
0x80, 0xef, 0xbf, 0xbd, 0xf0, 0x90, 0x80, 0x80, 0xf4, 0x8f, 0xbf, 0xbf,
],
"utf-16le": [
0x00, 0x00, 0x31, 0x00, 0x32, 0x00, 0x33, 0x00, 0x41, 0x00, 0x42, 0x00, 0x43, 0x00, 0x61, 0x00, 0x62, 0x00,
0x63, 0x00, 0x80, 0x00, 0xff, 0x00, 0x00, 0x01, 0x00, 0x10, 0xfd, 0xff, 0x00, 0xd8, 0x00, 0xdc, 0xff, 0xdb,
0xff, 0xdf,
],
"utf-16be": [
0x00, 0x00, 0x00, 0x31, 0x00, 0x32, 0x00, 0x33, 0x00, 0x41, 0x00, 0x42, 0x00, 0x43, 0x00, 0x61, 0x00, 0x62,
0x00, 0x63, 0x00, 0x80, 0x00, 0xff, 0x01, 0x00, 0x10, 0x00, 0xff, 0xfd, 0xd8, 0x00, 0xdc, 0x00, 0xdb, 0xff,
0xdf, 0xff,
],
};
[ArrayBuffer, SharedArrayBuffer].forEach(arrayBufferOrSharedArrayBuffer => {
Object.keys(octets).forEach(function (encoding) {
for (var len = 1; len <= 5; ++len) {
it(
"Streaming decode: " + encoding + ", " + len + " byte window (" + arrayBufferOrSharedArrayBuffer.name + ")",
() => {
var encoded = octets[encoding];
var out = "";
var decoder = new TextDecoder(encoding);
for (var i = 0; i < encoded.length; i += len) {
var sub = [];
for (var j = i; j < encoded.length && j < i + len; ++j) {
sub.push(encoded[j]);
}
var uintArray = new Uint8Array(new arrayBufferOrSharedArrayBuffer(sub.length));
uintArray.set(sub);
out += decoder.decode(uintArray, { stream: true });
}
out += decoder.decode();
expect(out).toEqual(string);
},
);
}
});
it(`Streaming decode: UTF-8 chunk tests (${arrayBufferOrSharedArrayBuffer.name})`, () => {
function bytes(byteArray) {
const view = new Uint8Array(new arrayBufferOrSharedArrayBuffer(byteArray.length));
view.set(byteArray);
return view;
}
const decoder = new TextDecoder();
expect(decoder.decode(bytes([0xc1]), { stream: true })).toEqual("\uFFFD");
expect(decoder.decode()).toEqual("");
expect(decoder.decode(bytes([0xf5]), { stream: true })).toEqual("\uFFFD");
expect(decoder.decode()).toEqual("");
expect(decoder.decode(bytes([0xe0, 0x41]), { stream: true })).toEqual("\uFFFDA");
expect(decoder.decode(bytes([0x42]))).toEqual("B");
expect(decoder.decode(bytes([0xe0, 0x80]), { stream: true })).toEqual("\uFFFD\uFFFD");
expect(decoder.decode(bytes([0x80]))).toEqual("\uFFFD");
expect(decoder.decode(bytes([0xed, 0xa0]), { stream: true })).toEqual("\uFFFD\uFFFD");
expect(decoder.decode(bytes([0x80]))).toEqual("\uFFFD");
expect(decoder.decode(bytes([0xf0, 0x41]), { stream: true })).toEqual("\uFFFDA");
expect(decoder.decode(bytes([0x42]), { stream: true })).toEqual("B");
expect(decoder.decode(bytes([0x43]))).toEqual("C");
expect(decoder.decode(bytes([0xf0, 0x80]), { stream: true })).toEqual("\uFFFD\uFFFD");
expect(decoder.decode(bytes([0x80]), { stream: true })).toEqual("\uFFFD");
expect(decoder.decode(bytes([0x80]))).toEqual("\uFFFD");
expect(decoder.decode(bytes([0xf4, 0xa0]), { stream: true })).toEqual("\uFFFD\uFFFD");
expect(decoder.decode(bytes([0x80]), { stream: true })).toEqual("\uFFFD");
expect(decoder.decode(bytes([0x80]))).toEqual("\uFFFD");
expect(decoder.decode(bytes([0xf0, 0x90, 0x41]), { stream: true })).toEqual("\uFFFDA");
expect(decoder.decode(bytes([0x42]))).toEqual("B");
// 4-byte UTF-8 sequences always correspond to non-BMP characters. Here
// we make sure that, although the first 3 bytes are enough to emit the
// lead surrogate, it only gets emitted when the fourth byte is read.
expect(decoder.decode(bytes([0xf0, 0x9f, 0x92]), { stream: true })).toEqual("");
expect(decoder.decode(bytes([0xa9]))).toEqual("\u{1F4A9}");
});
});
}
});
it("should not crash with a getter that throws", () => {
expect(() =>
new TextDecoder().decode(new Uint8Array(32), {
get stream() {
throw new Error("stream get error");
},
}),
).toThrowErrorMatchingInlineSnapshot(`"stream get error"`);
});
Reference in New Issue View Git Blame Copy Permalink