authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 13:16:37+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
log2183c4bb444d80921783fc5a26d217c0e4a68d31
tree1e5261202f7e2d3b6c1ab55791edf5bec39cb08d
parenta5d1fb1e49891c70fd384e1cf38e9d2f4eac6ee9
signature Commit is signed but in an unrecognized format.

std-c tokenizer string concatenation

1 files changed, 17 insertions(+), 7 deletions(-)

lib/std/c/tokenizer.zig+17-7
...@@ -272,6 +272,7 @@ pub const Tokenizer = struct {...@@ -272,6 +272,7 @@ pub const Tokenizer = struct {
272 U,272 U,
273 L,273 L,
274 StringLiteral,274 StringLiteral,
275 AfterStringLiteral,
275 CharLiteralStart,276 CharLiteralStart,
276 CharLiteral,277 CharLiteral,
277 EscapeSequence,278 EscapeSequence,
...@@ -565,8 +566,7 @@ pub const Tokenizer = struct {...@@ -565,8 +566,7 @@ pub const Tokenizer = struct {
565 state = .EscapeSequence;566 state = .EscapeSequence;
566 },567 },
567 '"' => {568 '"' => {
568 self.index += 1;569 state = .AfterStringLiteral;
569 break;
570 },570 },
571 '\n', '\r' => {571 '\n', '\r' => {
572 result.id = .Invalid;572 result.id = .Invalid;
...@@ -574,6 +574,15 @@ pub const Tokenizer = struct {...@@ -574,6 +574,15 @@ pub const Tokenizer = struct {
574 },574 },
575 else => {},575 else => {},
576 },576 },
577 .AfterStringLiteral => switch (c) {
578 '"' => {
579 state = .StringLiteral;
580 },
581 '\n'...'\r', ' ' => {},
582 else => {
583 break;
584 },
585 },
577 .CharLiteralStart => switch (c) {586 .CharLiteralStart => switch (c) {
578 '\\' => {587 '\\' => {
579 string = false;588 string = false;
...@@ -1109,6 +1118,7 @@ pub const Tokenizer = struct {...@@ -1109,6 +1118,7 @@ pub const Tokenizer = struct {
1109 }1118 }
1110 } else if (self.index == self.source.buffer.len) {1119 } else if (self.index == self.source.buffer.len) {
1111 switch (state) {1120 switch (state) {
1121 .AfterStringLiteral,
1112 .Start => {},1122 .Start => {},
1113 .u, .u8, .U, .L, .Identifier => {1123 .u, .u8, .U, .L, .Identifier => {
1114 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;1124 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
...@@ -1351,11 +1361,11 @@ test "line continuation" {...@@ -1351,11 +1361,11 @@ test "line continuation" {
13511361
1352test "string prefix" {1362test "string prefix" {
1353 expectTokens(1363 expectTokens(
1354 \\"foo"1364 \\"foo" "bar"
1355 \\u"foo"1365 \\u"foo" "bar"
1356 \\u8"foo"1366 \\u8"foo" "bar"
1357 \\U"foo"1367 \\U"foo" "bar"
1358 \\L"foo"1368 \\L"foo" "bar"
1359 \\'foo'1369 \\'foo'
1360 \\u'foo'1370 \\u'foo'
1361 \\U'foo'1371 \\U'foo'