| author |  Isaac Freund <mail@isaacfreund.com> 2026-07-03 15:54:40+02:00 |
| committer | Isaac Freund <mail@isaacfreund.com> 2026-07-08 11:49:00+02:00 |
| log | 080ad9a4522f7d8eb756d47ffd0c7bfa61851446 |
| tree | 3f3462ab55231f3ec9082016ea3222531b46b569 |
| parent | 62720ad9f9e65dd8c62579c90142cb043d60b01c |
| signature |  Commit is signed but in an unrecognized format. |
grammar: handle byte order mark
The tokenizer allows the unicode byte order mark (U+FEFF) to be present
at the start of the file, but the grammar currently does not.
Fixing the grammar requires some care due to how same-line doc comments
are forbidden in the grammar.3 files changed, 29 insertions(+), 4 deletions(-)
doc- langref
doc/langref/grammar.peg+5-2
| ... | @@ -450,11 +450,14 @@ ExprPrefix <- ASTERISK | ... | @@ -450,11 +450,14 @@ ExprPrefix <- ASTERISK |
| 450 | 450 | ||
| 451 | # *** Tokens *** | 451 | # *** Tokens *** |
| 452 | 452 | ||
| 453 | # https://en.wikipedia.org/wiki/Byte_order_mark | ||
| 454 | byte_order_mark <- '\357\273\277' | ||
| 455 | |||
| 453 | # Unfortunately, there is not a standard way to match the start of the file in PEG. | 456 | # Unfortunately, there is not a standard way to match the start of the file in PEG. |
| 454 | # This rule definition is compatible with the original peg(1) tool but is not | 457 | # This rule definition is compatible with the original peg(1) tool but is not |
| 455 | # portable. It is however trivial to implement an equivalent rule in hand written | 458 | # portable. It is however trivial to implement an equivalent rule in hand written |
| 456 | # parsers and other PEG tooling should have similar mechanisms. | 459 | # parsers and other PEG tooling should have similar mechanisms. |
| 457 | sof <- &{ (yy->__pos == 0) } | 460 | sof <- &{ (yy->__pos == 0) } byte_order_mark? |
| 458 | eof <- !. | 461 | eof <- !. |
| 459 | 462 | ||
| 460 | ox80_oxBF <- [\200-\277] | 463 | ox80_oxBF <- [\200-\277] |
| ... | @@ -536,7 +539,7 @@ line_string <- '\\\\' non_control_utf8* newline | ... | @@ -536,7 +539,7 @@ line_string <- '\\\\' non_control_utf8* newline |
| 536 | # the newline terminating a multiline string literal or doc comment visible | 539 | # the newline terminating a multiline string literal or doc comment visible |
| 537 | # to the pre_op_white non-terminal. | 540 | # to the pre_op_white non-terminal. |
| 538 | newline <- &("\n" / "\r\n" / eof) | 541 | newline <- &("\n" / "\r\n" / eof) |
| 539 | skip <- ([ \n\t\r] / line_comment)* | 542 | skip <- sof? ([ \n\t\r] / line_comment)* |
| 540 | skip_require_newline <- [ \t\r]* ([\n] / line_comment) skip | 543 | skip_require_newline <- [ \t\r]* ([\n] / line_comment) skip |
| 541 | pre_op_white <- ([ \n\t\r] / line_comment)+ | 544 | pre_op_white <- ([ \n\t\r] / line_comment)+ |
| 542 | post_op_white <- [ \n\t\r] skip | 545 | post_op_white <- [ \n\t\r] skip |
lib/std/zig/parser_fuzz.zig+8
| ... | @@ -119,6 +119,14 @@ test "catch capture whitespace" { | ... | @@ -119,6 +119,14 @@ test "catch capture whitespace" { |
| 119 | try checkAgainstOracle("test{0 catch |h|0;}"); | 119 | try checkAgainstOracle("test{0 catch |h|0;}"); |
| 120 | } | 120 | } |
| 121 | 121 | ||
| 122 | // Found using AFL++ | ||
| 123 | test "byte order mark" { | ||
| 124 | // https://en.wikipedia.org/wiki/Byte_order_mark | ||
| 125 | try checkAgainstOracle("\xef\xbb\xbf"); | ||
| 126 | try checkAgainstOracle("\xef\xbb\xbf///\n0"); | ||
| 127 | } | ||
| 128 | |||
| 129 | >>>>>>> bc46a69fba (grammar: handle byte order mark) | ||
| 122 | fn checkAgainstOracle(source: [:0]const u8) !void { | 130 | fn checkAgainstOracle(source: [:0]const u8) !void { |
| 123 | var fba_buf: [1 << 18]u8 = undefined; | 131 | var fba_buf: [1 << 18]u8 = undefined; |
| 124 | var fba: std.heap.FixedBufferAllocator = .init(&fba_buf); | 132 | var fba: std.heap.FixedBufferAllocator = .init(&fba_buf); |
lib/std/zig/parser_generated_oracle.zig+16-2
| ... | @@ -2327,10 +2327,24 @@ const Parser = struct { | ... | @@ -2327,10 +2327,24 @@ const Parser = struct { |
| 2327 | break :blk_0 false; | 2327 | break :blk_0 false; |
| 2328 | }; | 2328 | }; |
| 2329 | } | 2329 | } |
| 2330 | pub fn parsebyte_order_mark(p: *Parser) Error!bool { | ||
| 2331 | return blk_0: { | ||
| 2332 | const pos_0 = p.i; | ||
| 2333 | if (blk_1: { | ||
| 2334 | if (std.mem.startsWith(u8, p.source[p.i..], "\xef\xbb\xbf")) { | ||
| 2335 | p.i += 3; | ||
| 2336 | break :blk_1 true; | ||
| 2337 | } | ||
| 2338 | break :blk_1 false; | ||
| 2339 | }) break :blk_0 true; | ||
| 2340 | p.i = pos_0; | ||
| 2341 | break :blk_0 false; | ||
| 2342 | }; | ||
| 2343 | } | ||
| 2330 | pub fn parsesof(p: *Parser) Error!bool { | 2344 | pub fn parsesof(p: *Parser) Error!bool { |
| 2331 | return blk_0: { | 2345 | return blk_0: { |
| 2332 | const pos_0 = p.i; | 2346 | const pos_0 = p.i; |
| 2333 | if ((p.i == 0)) break :blk_0 true; | 2347 | if ((p.i == 0) and (try p.parsebyte_order_mark() or true)) break :blk_0 true; |
| 2334 | p.i = pos_0; | 2348 | p.i = pos_0; |
| 2335 | break :blk_0 false; | 2349 | break :blk_0 false; |
| 2336 | }; | 2350 | }; |
| ... | @@ -2854,7 +2868,7 @@ const Parser = struct { | ... | @@ -2854,7 +2868,7 @@ const Parser = struct { |
| 2854 | pub fn parseskip(p: *Parser) Error!bool { | 2868 | pub fn parseskip(p: *Parser) Error!bool { |
| 2855 | return blk_0: { | 2869 | return blk_0: { |
| 2856 | const pos_0 = p.i; | 2870 | const pos_0 = p.i; |
| 2857 | if (blk_1: { | 2871 | if ((try p.parsesof() or true) and blk_1: { |
| 2858 | var i_1: usize = 0; | 2872 | var i_1: usize = 0; |
| 2859 | while (blk_3: { | 2873 | while (blk_3: { |
| 2860 | const pos_3 = p.i; | 2874 | const pos_3 = p.i; |