diff --git a/src/shell/interpreter.zig b/src/shell/interpreter.zig index 774c5c98f1..1e894fd1ed 100644 --- a/src/shell/interpreter.zig +++ b/src/shell/interpreter.zig @@ -1065,6 +1065,20 @@ pub const Interpreter = struct { return shell.ParseError.Lex; } + if (comptime bun.Environment.allow_assert) { + const print = bun.Output.scoped(.ShellTokens, true); + var test_tokens = std.ArrayList(shell.Test.TestToken).initCapacity(arena.allocator(), lex_result.tokens.len) catch @panic("OOPS"); + defer test_tokens.deinit(); + for (lex_result.tokens) |tok| { + const test_tok = shell.Test.TestToken.from_real(tok, lex_result.strpool); + test_tokens.append(test_tok) catch @panic("OOPS"); + } + + const str = std.json.stringifyAlloc(bun.default_allocator, test_tokens.items[0..], .{}) catch @panic("OOPS"); + defer bun.default_allocator.free(str); + print("Tokens: {s}", .{str}); + } + out_parser.* = try bun.shell.Parser.new(arena.allocator(), lex_result, jsobjs); const script_ast = try out_parser.*.?.parse(); diff --git a/src/shell/shell.zig b/src/shell/shell.zig index 8665f26275..edd603f03a 100644 --- a/src/shell/shell.zig +++ b/src/shell/shell.zig @@ -2629,6 +2629,16 @@ pub fn NewLexer(comptime encoding: StringEncoding) type { } continue; } + // Treat newline preceded by backslash as whitespace + else if (char == '\n') { + if (comptime bun.Environment.allow_assert) { + std.debug.assert(input.escaped); + } + if (self.chars.state != .Double) { + try self.break_word_impl(true, true, false); + } + continue; + } try self.appendCharToStrPool(char); } @@ -2979,9 +2989,11 @@ pub fn NewLexer(comptime encoding: StringEncoding) type { }, .normal => try self.tokens.append(.OpenParen), } + const prev_quote_state = self.chars.state; var sublexer = self.make_sublexer(kind); try sublexer.lex(); self.continue_from_sublexer(&sublexer); + self.chars.state = prev_quote_state; } fn appendStringToStrPool(self: *@This(), bunstr: bun.String) !void { @@ -3440,6 +3452,7 @@ pub fn ShellCharIter(comptime encoding: StringEncoding) type { else => return .{ .char = char, .escaped = false }, } }, + // We checked `self.state == .Single` above so this is impossible .Single => unreachable, } diff --git a/test/js/bun/shell/bunshell.test.ts b/test/js/bun/shell/bunshell.test.ts index 403cbfb408..b69fa71006 100644 --- a/test/js/bun/shell/bunshell.test.ts +++ b/test/js/bun/shell/bunshell.test.ts @@ -351,6 +351,176 @@ describe("bunshell", () => { expect(stdout.toString()).toEqual(`noice\n`); }); + // Ported from GNU bash "quote.tests" + // https://github.com/bminor/bash/blob/f3b6bd19457e260b65d11f2712ec3da56cef463f/tests/quote.tests#L1 + // Some backtick tests are skipped, because of insane behavior: + // For some reason, even though $(...) and `...` are suppoed to be equivalent, + // doing: + // echo "`echo 'foo\ + // bar'`" + // + // gives: + // foobar + // + // While doing the same, but with $(...): + // echo "$(echo 'foo\ + // bar')" + // + // gives: + // foo\ + // bar + // + // I'm not sure why, this isn't documented behavior, so I'm choosing to ignore it. + describe("gnu_quote", () => { + // An unfortunate consequence of our use of String.raw and tagged template + // functions for the shell make it so that we have to use { raw: string } to do + // backtick command substitution + const BACKTICK = { raw: "`" }; + + // Single Quote + TestBuilder.command` +echo 'foo +bar' +echo 'foo +bar' +echo 'foo\ +bar' +` + .stdout("foo\nbar\nfoo\nbar\nfoo\\\nbar\n") + .runAsTest("Single Quote"); + + TestBuilder.command` +echo "foo +bar" +echo "foo +bar" +echo "foo\ +bar" +` + .stdout("foo\nbar\nfoo\nbar\nfoobar\n") + .runAsTest("Double Quote"); + + TestBuilder.command` +echo ${BACKTICK}echo 'foo +bar'${BACKTICK} +echo ${BACKTICK}echo 'foo +bar'${BACKTICK} +echo ${BACKTICK}echo 'foo\ +bar'${BACKTICK} +` + .stdout( + `foo bar +foo bar +foobar\n`, + ) + .todo("insane backtick behavior") + .runAsTest("Backslash Single Quote"); + + TestBuilder.command` +echo "${BACKTICK}echo 'foo +bar'${BACKTICK}" +echo "${BACKTICK}echo 'foo +bar'${BACKTICK}" +echo "${BACKTICK}echo 'foo\ +bar'${BACKTICK}" +` + .stdout( + `foo +bar +foo +bar +foobar\n`, + ) + .todo("insane backtick behavior") + .runAsTest("Double Quote Backslash Single Quote"); + + TestBuilder.command` +echo $(echo 'foo +bar') +echo $(echo 'foo +bar') +echo $(echo 'foo\ +bar') +` + .stdout( + `foo bar +foo bar +foo\\ bar\n`, + ) + .runAsTest("Dollar Paren Single Quote"); + + TestBuilder.command` +echo "$(echo 'foo +bar')" +echo "$(echo 'foo +bar')" +echo "$(echo 'foo\ +bar')" +` + .stdout( + `foo +bar +foo +bar +foo\\ +bar\n`, + ) + .runAsTest("Dollar Paren Double Quote"); + + TestBuilder.command` +echo "$(echo 'foo +bar')" +echo "$(echo 'foo +bar')" +echo "$(echo 'foo\ +bar')" +` + .stdout( + `foo +bar +foo +bar +foo\\ +bar\n`, + ) + .runAsTest("Double Quote Dollar Paren Single Quote"); + }); + + describe("escaped_newline", () => { + const printArgs = /* ts */ `console.log(JSON.stringify(process.argv))`; + + TestBuilder.command/* sh */ `${BUN} run ./code.ts hi hello \ + on a newline! + ` + .ensureTempDir() + .file("code.ts", printArgs) + .stdout(out => expect(JSON.parse(out).slice(2)).toEqual(["hi", "hello", "on", "a", "newline!"])) + .runAsTest("single"); + + TestBuilder.command/* sh */ `${BUN} run ./code.ts hi hello \ + on a newline! \ + and \ + a few \ + others! + ` + .ensureTempDir() + .file("code.ts", printArgs) + .stdout(out => + expect(JSON.parse(out).slice(2)).toEqual(["hi", "hello", "on", "a", "newline!", "and", "a", "few", "others!"]), + ) + .runAsTest("many"); + + TestBuilder.command/* sh */ `${BUN} run ./code.ts hi hello \ + on a newline! \ + ooga" +booga" + ` + .ensureTempDir() + .file("code.ts", printArgs) + .stdout(out => expect(JSON.parse(out).slice(2)).toEqual(["hi", "hello", "on", "a", "newline!", "ooga\nbooga"])) + .runAsTest("quotes"); + }); + describe("glob expansion", () => { // Issue #8403: https://github.com/oven-sh/bun/issues/8403 TestBuilder.command`ls *.sdfljsfsdf`