authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-06-12 13:17:01+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:57+02:00
log5c2510ba58d2c5b0b3082244ab9b9320662d1d65
treeb42d85d96a54523eb78cc8299331612b93ec2411
parent0b24b58fb24b462c85ebba53ee838e87887e4d25
signaturelock-open Commit is signed but in an unrecognized format.

grammar: match tokenizer newline handling

The tokenizer currently handles termination of comments/multiline string literal lines differently from the grammar. The tokenizer forbids e.g. tab characters at the end of comments before a newline and requires \r to be directly followed by \n. The tokenizer behavior here is intended, see c2b8afcac9e427102370dc.

2 files changed, 35 insertions(+), 41 deletions(-)

doc/langref/grammar.peg+7-4
......@@ -430,10 +430,13 @@ string_char
430430 <- '\\"'
431431 / !["] non_control_utf8
432432
433container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+
434doc_comment <- ('///' non_control_utf8* [ \n]* skip)+
435line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8*
436line_string <- '\\\\' non_control_utf8* [ \n]*
433container_doc_comment <- ('//!' non_control_utf8* newline skip)+
434doc_comment <- ('///' non_control_utf8* newline skip)+
435line_comment
436 <- '//' ![!/] non_control_utf8* newline
437 / '////' non_control_utf8* newline
438line_string <- '\\\\' non_control_utf8* newline
439newline <- "\n" / "\r\n" / eof
437440skip <- ([ \n\t\r] / line_comment)*
438441
439442CHAR_LITERAL <- ['] char_char* ['] skip
lib/std/zig/parser_generated_oracle.zig+28-37
......@@ -2194,18 +2194,7 @@ const Parser = struct {
21942194 } and blk_4: {
21952195 while (p.parsenon_control_utf8()) {}
21962196 break :blk_4 true;
2197 } and blk_4: {
2198 while ((p.i < p.source.len and switch (p.source[p.i]) {
2199 ' '...' ',
2200 '\n'...'\n',
2201 => blk_5: {
2202 p.i += 1;
2203 break :blk_5 true;
2204 },
2205 else => false,
2206 })) {}
2207 break :blk_4 true;
2208 } and p.parseskip()) break :blk_3 true;
2197 } and p.parsenewline() and p.parseskip()) break :blk_3 true;
22092198 p.i = pos_3;
22102199 break :blk_3 false;
22112200 }) {
......@@ -2233,18 +2222,7 @@ const Parser = struct {
22332222 } and blk_4: {
22342223 while (p.parsenon_control_utf8()) {}
22352224 break :blk_4 true;
2236 } and blk_4: {
2237 while ((p.i < p.source.len and switch (p.source[p.i]) {
2238 ' '...' ',
2239 '\n'...'\n',
2240 => blk_5: {
2241 p.i += 1;
2242 break :blk_5 true;
2243 },
2244 else => false,
2245 })) {}
2246 break :blk_4 true;
2247 } and p.parseskip()) break :blk_3 true;
2225 } and p.parsenewline() and p.parseskip()) break :blk_3 true;
22482226 p.i = pos_3;
22492227 break :blk_3 false;
22502228 }) {
......@@ -2281,7 +2259,7 @@ const Parser = struct {
22812259 } and blk_1: {
22822260 while (p.parsenon_control_utf8()) {}
22832261 break :blk_1 true;
2284 }) break :blk_0 true;
2262 } and p.parsenewline()) break :blk_0 true;
22852263 p.i = pos_0;
22862264 if (blk_1: {
22872265 if (std.mem.startsWith(u8, p.source[p.i..], "////")) {
......@@ -2292,7 +2270,7 @@ const Parser = struct {
22922270 } and blk_1: {
22932271 while (p.parsenon_control_utf8()) {}
22942272 break :blk_1 true;
2295 }) break :blk_0 true;
2273 } and p.parsenewline()) break :blk_0 true;
22962274 p.i = pos_0;
22972275 break :blk_0 false;
22982276 };
......@@ -2309,19 +2287,32 @@ const Parser = struct {
23092287 } and blk_1: {
23102288 while (p.parsenon_control_utf8()) {}
23112289 break :blk_1 true;
2312 } and blk_1: {
2313 while ((p.i < p.source.len and switch (p.source[p.i]) {
2314 ' '...' ',
2315 '\n'...'\n',
2316 => blk_2: {
2317 p.i += 1;
2318 break :blk_2 true;
2319 },
2320 else => false,
2321 })) {}
2322 break :blk_1 true;
2290 } and p.parsenewline()) break :blk_0 true;
2291 p.i = pos_0;
2292 break :blk_0 false;
2293 };
2294 }
2295 pub fn parsenewline(p: *Parser) bool {
2296 return blk_0: {
2297 const pos_0 = p.i;
2298 if (blk_1: {
2299 if (std.mem.startsWith(u8, p.source[p.i..], "\n")) {
2300 p.i += 1;
2301 break :blk_1 true;
2302 }
2303 break :blk_1 false;
23232304 }) break :blk_0 true;
23242305 p.i = pos_0;
2306 if (blk_1: {
2307 if (std.mem.startsWith(u8, p.source[p.i..], "\r\n")) {
2308 p.i += 2;
2309 break :blk_1 true;
2310 }
2311 break :blk_1 false;
2312 }) break :blk_0 true;
2313 p.i = pos_0;
2314 if (p.parseeof()) break :blk_0 true;
2315 p.i = pos_0;
23252316 break :blk_0 false;
23262317 };
23272318 }