fix macro string escaping (#3967)

* handle macro escaping

* remove printer

* use `js_lexer.decodeEscapeSequences`

src/js_ast.zig

@@ -25,6 +25,7 @@ const JSONParser = bun.JSON;
 const is_bindgen = std.meta.globalOption("bindgen", bool) orelse false;
 const ComptimeStringMap = bun.ComptimeStringMap;
 const JSPrinter = @import("./js_printer.zig");
+const js_lexer = @import("./js_lexer.zig");
 const ThreadlocalArena = @import("./mimalloc_arena.zig").Arena;
 
 /// This is the index to the automatically-generated part containing code that
@@ -2359,7 +2360,6 @@ pub const E = struct {
         }
 
         pub fn toJS(s: *String, allocator: std.mem.Allocator, globalObject: *JSC.JSGlobalObject) JSC.JSValue {
-            s.resolveRopeIfNeeded(allocator);
             if (!s.isPresent()) {
                 var emp = bun.String.empty;
                 return emp.toJS(globalObject);
@@ -2372,8 +2372,15 @@ pub const E = struct {
             }
 
             {
-                var out = bun.String.create(s.slice(allocator));
+                s.resolveRopeIfNeeded(allocator);
+
+                const decoded = js_lexer.decodeUTF8(s.slice(allocator), allocator) catch unreachable;
+                defer allocator.free(decoded);
+
+                var out = bun.String.createUninitializedUTF16(decoded.len);
                 defer out.deref();
+                @memcpy(@constCast(out.utf16()), decoded);
+
                 return out.toJS(globalObject);
             }
         }
@@ -9951,12 +9958,8 @@ pub const Macro = struct {
                         },
                         .String => {
                             var bun_str = value.toBunString(this.global);
-                            if (bun_str.is8Bit()) {
-                                if (strings.isAllASCII(bun_str.latin1())) {
-                                    return Expr.init(E.String, E.String.init(this.allocator.dupe(u8, bun_str.latin1()) catch unreachable), this.caller.loc);
-                                }
-                            }
 
+                            // encode into utf16 so the printer escapes the string correctly
                             var utf16_bytes = this.allocator.alloc(u16, bun_str.length()) catch unreachable;
                             var out_slice = utf16_bytes[0 .. (bun_str.encodeInto(std.mem.sliceAsBytes(utf16_bytes), .utf16le) catch 0) / 2];
                             return Expr.init(E.String, E.String.init(out_slice), this.caller.loc);

src/js_lexer.zig

@@ -75,6 +75,19 @@ pub const JSONOptions = struct {
     was_originally_macro: bool = false,
 };
 
+pub fn decodeUTF8(bytes: string, allocator: std.mem.Allocator) ![]const u16 {
+    var log = logger.Log.init(allocator);
+    defer log.deinit();
+    var source = logger.Source.initEmptyFile("");
+    var lexer = try NewLexer(.{}).init(&log, source, allocator);
+    defer lexer.deinit();
+
+    var buf = std.ArrayList(u16).init(allocator);
+    try lexer.decodeEscapeSequences(0, bytes, @TypeOf(buf), &buf);
+
+    return buf.items;
+}
+
 pub fn NewLexer(
     comptime json_options: JSONOptions,
 ) type {

test/transpiler/macro-test.test.ts

@@ -1,4 +1,4 @@
-import { identity } from "./macro.ts" assert { type: "macro" };
+import { identity, escape, addStringsUTF16, addStrings } from "./macro.ts" assert { type: "macro" };
 
 test("latin1 string", () => {
   expect(identity("©")).toBe("©");
@@ -8,6 +8,67 @@ test("ascii string", () => {
   expect(identity("abc")).toBe("abc");
 });
 
+test("escaping", () => {
+  expect(identity("\\")).toBe("\\");
+  expect(identity("\f")).toBe("\f");
+  expect(identity("\n")).toBe("\n");
+  expect(identity("\r")).toBe("\r");
+  expect(identity("\t")).toBe("\t");
+  expect(identity("\v")).toBe("\v");
+  expect(identity("\0")).toBe("\0");
+  expect(identity("'")).toBe("'");
+  expect(identity('"')).toBe('"');
+  expect(identity("`")).toBe("`");
+  // prettier-ignore
+  expect(identity("\'")).toBe("\'");
+  // prettier-ignore
+  expect(identity('\"')).toBe('\"');
+  // prettier-ignore
+  expect(identity("\`")).toBe("\`");
+  expect(identity("$")).toBe("$");
+  expect(identity("\x00")).toBe("\x00");
+  expect(identity("\x0B")).toBe("\x0B");
+  expect(identity("\x0C")).toBe("\x0C");
+
+  expect(identity("\\")).toBe("\\");
+
+  expect(escape()).toBe("\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C");
+
+  expect(addStrings("abc")).toBe("abc\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\n")).toBe("\n\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\r")).toBe("\r\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\t")).toBe("\t\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("©")).toBe("©\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\x00")).toBe("\x00\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\x0B")).toBe("\x0B\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\x0C")).toBe("\x0C\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\\")).toBe("\\\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\f")).toBe("\f\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\v")).toBe("\v\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("\0")).toBe("\0\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("'")).toBe("'\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings('"')).toBe('"\\\f\n\r\t\v\0\'"`$\x00\x0B\x0C©');
+  expect(addStrings("`")).toBe("`\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+  expect(addStrings("😊")).toBe("😊\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C©");
+
+  expect(addStringsUTF16("abc")).toBe("abc\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\n")).toBe("\n\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\r")).toBe("\r\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\t")).toBe("\t\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("©")).toBe("©\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\x00")).toBe("\x00\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\x0B")).toBe("\x0B\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\x0C")).toBe("\x0C\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\\")).toBe("\\\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\f")).toBe("\f\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\v")).toBe("\v\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("\0")).toBe("\0\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("'")).toBe("'\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16('"')).toBe('"\\\f\n\r\t\v\0\'"`$\x00\x0B\x0C😊');
+  expect(addStringsUTF16("`")).toBe("`\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+  expect(addStringsUTF16("😊")).toBe("😊\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C😊");
+});
+
 test("utf16 string", () => {
   expect(identity("😊 Smiling Face with Smiling Eyes Emoji")).toBe("😊 Smiling Face with Smiling Eyes Emoji");
 });

test/transpiler/macro.ts

@@ -1,3 +1,15 @@
 export function identity(arg: any) {
   return arg;
 }
+
+export function escape() {
+  return "\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C";
+}
+
+export function addStrings(arg: string) {
+  return arg + "\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C" + "©";
+}
+
+export function addStringsUTF16(arg: string) {
+  return arg + "\\\f\n\r\t\v\0'\"`$\x00\x0B\x0C" + "😊";
+}