mirror of
https://github.com/oven-sh/bun
synced 2026-02-15 21:32:05 +00:00
fix test failure in aarch64 and add test for simd code path
This commit is contained in:
Jarred Sumner
@@ -956,31 +956,52 @@ pub inline fn copyU16IntoU8(output_: []u8, comptime InputType: type, input_: Inp
|
||||
if (comptime Environment.allow_assert) {
|
||||
std.debug.assert(input_.len <= output_.len);
|
||||
}
|
||||
if (comptime !JSC.is_bindgen and Environment.isAarch64) {
|
||||
// faster on aarch64
|
||||
// but it only uses SSE2 when it could use AVX2
|
||||
// so it's better to let llvm auto-vectorize it
|
||||
JSC.WTF.copyLCharsFromUCharSource(output_.ptr, InputType, input_);
|
||||
} else {
|
||||
var output = output_;
|
||||
var input = input_;
|
||||
if (comptime Environment.allow_assert) {
|
||||
std.debug.assert(input.len <= output.len);
|
||||
|
||||
var output = output_;
|
||||
var input = input_;
|
||||
if (comptime Environment.allow_assert) {
|
||||
std.debug.assert(input.len <= output.len);
|
||||
}
|
||||
|
||||
// https://zig.godbolt.org/z/9rTn1orcY
|
||||
|
||||
const group = @as(usize, 16);
|
||||
// end at the last group of 16 bytes
|
||||
var input_ptr = input.ptr;
|
||||
var output_ptr = output.ptr;
|
||||
|
||||
if (comptime Environment.enableSIMD) {
|
||||
const last_vector_ptr = input.ptr + (@min(input.len, output.len) & ~(group - 1));
|
||||
while (last_vector_ptr != input_ptr) {
|
||||
const input_vec1: @Vector(group, u16) = input_ptr[0..group].*;
|
||||
output_ptr[0] = @truncate(u8, input_vec1[0]);
|
||||
output_ptr[1] = @truncate(u8, input_vec1[1]);
|
||||
output_ptr[2] = @truncate(u8, input_vec1[2]);
|
||||
output_ptr[3] = @truncate(u8, input_vec1[3]);
|
||||
output_ptr[4] = @truncate(u8, input_vec1[4]);
|
||||
output_ptr[5] = @truncate(u8, input_vec1[5]);
|
||||
output_ptr[6] = @truncate(u8, input_vec1[6]);
|
||||
output_ptr[7] = @truncate(u8, input_vec1[7]);
|
||||
output_ptr[8] = @truncate(u8, input_vec1[8]);
|
||||
output_ptr[9] = @truncate(u8, input_vec1[9]);
|
||||
output_ptr[10] = @truncate(u8, input_vec1[10]);
|
||||
output_ptr[11] = @truncate(u8, input_vec1[11]);
|
||||
output_ptr[12] = @truncate(u8, input_vec1[12]);
|
||||
output_ptr[13] = @truncate(u8, input_vec1[13]);
|
||||
output_ptr[14] = @truncate(u8, input_vec1[14]);
|
||||
output_ptr[15] = @truncate(u8, input_vec1[15]);
|
||||
|
||||
output_ptr += group;
|
||||
input_ptr += group;
|
||||
}
|
||||
}
|
||||
|
||||
// https://zig.godbolt.org/z/Y1qa9PTo1
|
||||
// https://github.com/ziglang/zig/issues/11830
|
||||
// this auto-vectorizes on x64 and aarch64
|
||||
var input_ptr = input.ptr;
|
||||
var output_ptr = output.ptr;
|
||||
const last_input_ptr = input_ptr + @min(input.len, output.len);
|
||||
|
||||
const last_input_ptr = input_ptr + @min(input.len, output.len);
|
||||
|
||||
while (last_input_ptr != input_ptr) {
|
||||
output_ptr[0] = @truncate(u8, input_ptr[0]);
|
||||
output_ptr += 1;
|
||||
input_ptr += 1;
|
||||
}
|
||||
while (last_input_ptr != input_ptr) {
|
||||
output_ptr[0] = @truncate(u8, input_ptr[0]);
|
||||
output_ptr += 1;
|
||||
input_ptr += 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -428,18 +428,49 @@ it("read", () => {
|
||||
reset();
|
||||
});
|
||||
|
||||
it("write", () => {
|
||||
// this is for checking the simd code path
|
||||
it("write long utf16 string works", () => {
|
||||
const long = "😀😃😄😁😆😅😂🤣☺️😊😊😇".repeat(1000);
|
||||
const buf = Buffer.alloc(long.length * 2);
|
||||
buf.write(long, 0, "utf16le");
|
||||
expect(buf.toString("utf16le")).toBe(long);
|
||||
for (let offset = 0; offset < long.length; offset += 48) {
|
||||
expect(buf.toString("utf16le", offset, offset + 4)).toBe("😀");
|
||||
expect(buf.toString("utf16le", offset, offset + 8)).toBe("😀😃");
|
||||
expect(buf.toString("utf16le", offset, offset + 12)).toBe("😀😃😄");
|
||||
expect(buf.toString("utf16le", offset, offset + 16)).toBe("😀😃😄😁");
|
||||
expect(buf.toString("utf16le", offset, offset + 20)).toBe("😀😃😄😁😆");
|
||||
expect(buf.toString("utf16le", offset, offset + 24)).toBe("😀😃😄😁😆😅");
|
||||
expect(buf.toString("utf16le", offset, offset + 28)).toBe("😀😃😄😁😆😅😂");
|
||||
expect(buf.toString("utf16le", offset, offset + 32)).toBe(
|
||||
"😀😃😄😁😆😅😂🤣",
|
||||
);
|
||||
expect(buf.toString("utf16le", offset, offset + 36)).toBe(
|
||||
"😀😃😄😁😆😅😂🤣☺️",
|
||||
);
|
||||
expect(buf.toString("utf16le", offset, offset + 40)).toBe(
|
||||
"😀😃😄😁😆😅😂🤣☺️😊",
|
||||
);
|
||||
expect(buf.toString("utf16le", offset, offset + 44)).toBe(
|
||||
"😀😃😄😁😆😅😂🤣☺️😊😊",
|
||||
);
|
||||
expect(buf.toString("utf16le", offset, offset + 48)).toBe(
|
||||
"😀😃😄😁😆😅😂🤣☺️😊😊😇",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("write", () => {
|
||||
const resultMap = new Map([
|
||||
['utf8', Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
['ucs2', Buffer.from([102, 0, 111, 0, 111, 0, 0, 0, 0])],
|
||||
['ascii', Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
['latin1', Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
['binary', Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
['utf16le', Buffer.from([102, 0, 111, 0, 111, 0, 0, 0, 0])],
|
||||
['base64', Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
['base64url', Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
['hex', Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
["utf8", Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
["ucs2", Buffer.from([102, 0, 111, 0, 111, 0, 0, 0, 0])],
|
||||
["ascii", Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
["latin1", Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
["binary", Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
["utf16le", Buffer.from([102, 0, 111, 0, 111, 0, 0, 0, 0])],
|
||||
["base64", Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
["base64url", Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
["hex", Buffer.from([102, 111, 111, 0, 0, 0, 0, 0, 0])],
|
||||
]);
|
||||
|
||||
let buf = Buffer.alloc(9);
|
||||
@@ -447,46 +478,51 @@ it("write", () => {
|
||||
new Uint8Array(buf.buffer).fill(0);
|
||||
}
|
||||
|
||||
|
||||
// utf8, ucs2, ascii, latin1, utf16le
|
||||
const encodings = ['utf8', 'utf-8', 'ucs2', 'ucs-2', 'ascii', 'latin1',
|
||||
'binary', 'utf16le', 'utf-16le'];
|
||||
const encodings = [
|
||||
"utf8",
|
||||
"utf-8",
|
||||
"ucs2",
|
||||
"ucs-2",
|
||||
"ascii",
|
||||
"latin1",
|
||||
"binary",
|
||||
"utf16le",
|
||||
"utf-16le",
|
||||
];
|
||||
|
||||
encodings
|
||||
.reduce((es, e) => es.concat(e, e.toUpperCase()), [])
|
||||
.forEach((encoding) => {
|
||||
reset();
|
||||
|
||||
const len = Buffer.byteLength('foo', encoding);
|
||||
expect(buf.write('foo', 0, len, encoding)).toBe(len);
|
||||
const len = Buffer.byteLength("foo", encoding);
|
||||
expect(buf.write("foo", 0, len, encoding)).toBe(len);
|
||||
|
||||
if (encoding.includes('-'))
|
||||
encoding = encoding.replace('-', '');
|
||||
if (encoding.includes("-")) encoding = encoding.replace("-", "");
|
||||
|
||||
expect(buf).toStrictEqual(resultMap.get(encoding.toLowerCase()));
|
||||
});
|
||||
|
||||
// base64
|
||||
['base64', 'BASE64', 'base64url', 'BASE64URL'].forEach((encoding) => {
|
||||
reset()
|
||||
["base64", "BASE64", "base64url", "BASE64URL"].forEach((encoding) => {
|
||||
reset();
|
||||
|
||||
const len = Buffer.byteLength('Zm9v', encoding);
|
||||
const len = Buffer.byteLength("Zm9v", encoding);
|
||||
|
||||
expect(buf.write('Zm9v', 0, len, encoding)).toBe(len);
|
||||
expect(buf.write("Zm9v", 0, len, encoding)).toBe(len);
|
||||
expect(buf).toStrictEqual(resultMap.get(encoding.toLowerCase()));
|
||||
});
|
||||
|
||||
|
||||
// hex
|
||||
['hex', 'HEX'].forEach((encoding) => {
|
||||
["hex", "HEX"].forEach((encoding) => {
|
||||
reset();
|
||||
const len = Buffer.byteLength('666f6f', encoding);
|
||||
const len = Buffer.byteLength("666f6f", encoding);
|
||||
|
||||
expect(buf.write('666f6f', 0, len, encoding)).toBe(len);
|
||||
expect(buf.write("666f6f", 0, len, encoding)).toBe(len);
|
||||
expect(buf).toStrictEqual(resultMap.get(encoding.toLowerCase()));
|
||||
});
|
||||
|
||||
|
||||
// UCS-2 overflow CVE-2018-12115
|
||||
for (let i = 1; i < 4; i++) {
|
||||
// Allocate two Buffers sequentially off the pool. Run more than once in case
|
||||
@@ -494,32 +530,31 @@ it("write", () => {
|
||||
const x = Buffer.allocUnsafe(4).fill(0);
|
||||
const y = Buffer.allocUnsafe(4).fill(1);
|
||||
// Should not write anything, pos 3 doesn't have enough room for a 16-bit char
|
||||
expect(x.write('ыыыыыы', 3, 'ucs2')).toBe(0);
|
||||
expect(x.write("ыыыыыы", 3, "ucs2")).toBe(0);
|
||||
// CVE-2018-12115 experienced via buffer overrun to next block in the pool
|
||||
expect(Buffer.compare(y, Buffer.alloc(4, 1))).toBe(0);
|
||||
}
|
||||
|
||||
// // Should not write any data when there is no space for 16-bit chars
|
||||
const z = Buffer.alloc(4, 0);
|
||||
expect(z.write('\u0001', 3, 'ucs2')).toBe(0);
|
||||
expect(z.write("\u0001", 3, "ucs2")).toBe(0);
|
||||
expect(Buffer.compare(z, Buffer.alloc(4, 0))).toBe(0);
|
||||
// Make sure longer strings are written up to the buffer end.
|
||||
expect(z.write('abcd', 2)).toBe(2);
|
||||
expect(z.write("abcd", 2)).toBe(2);
|
||||
expect([...z]).toStrictEqual([0, 0, 0x61, 0x62]);
|
||||
|
||||
//Large overrun could corrupt the process with utf8
|
||||
expect(Buffer.alloc(4).write('a'.repeat(100), 3, 'utf8')).toBe(1);
|
||||
expect(Buffer.alloc(4).write("a".repeat(100), 3, "utf8")).toBe(1);
|
||||
|
||||
// Large overrun could corrupt the process
|
||||
expect(Buffer.alloc(4).write('ыыыыыы'.repeat(100), 3, 'utf16le')).toBe(0);
|
||||
expect(Buffer.alloc(4).write("ыыыыыы".repeat(100), 3, "utf16le")).toBe(0);
|
||||
|
||||
{
|
||||
// .write() does not affect the byte after the written-to slice of the Buffer.
|
||||
// Refs: https://github.com/nodejs/node/issues/26422
|
||||
const buf = Buffer.alloc(8);
|
||||
expect(buf.write('ыы', 1, 'utf16le')).toBe(4);
|
||||
expect(buf.write("ыы", 1, "utf16le")).toBe(4);
|
||||
expect([...buf]).toStrictEqual([0, 0x4b, 0x04, 0x4b, 0x04, 0, 0, 0]);
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
@@ -743,7 +778,7 @@ it("Buffer.toString(base64)", () => {
|
||||
|
||||
it("Buffer can be mocked", () => {
|
||||
function MockBuffer() {
|
||||
const noop = function () { };
|
||||
const noop = function () {};
|
||||
const res = Buffer.alloc(0);
|
||||
for (const op in Buffer.prototype) {
|
||||
if (typeof res[op] === "function") {
|
||||
@@ -830,7 +865,7 @@ it("Buffer.from (Node.js test/test-buffer-from.js)", () => {
|
||||
new MyBadPrimitive(),
|
||||
Symbol(),
|
||||
5n,
|
||||
(one, two, three) => { },
|
||||
(one, two, three) => {},
|
||||
undefined,
|
||||
null,
|
||||
].forEach((input) => {
|
||||
@@ -872,7 +907,7 @@ it("new Buffer() (Node.js test/test-buffer-new.js)", () => {
|
||||
// Now test protecting users from doing stupid things
|
||||
|
||||
expect(function () {
|
||||
function AB() { }
|
||||
function AB() {}
|
||||
Object.setPrototypeOf(AB, ArrayBuffer);
|
||||
Object.setPrototypeOf(AB.prototype, ArrayBuffer.prototype);
|
||||
Buffer.from(new AB());
|
||||
|
||||
Reference in New Issue
Block a user