From 068d4837807570550e44db99d92b25544a50cad8 Mon Sep 17 00:00:00 2001 From: Isaac Freund Date: Tue, 16 Jun 2026 10:14:37 +0200 Subject: [PATCH] grammar: fix doc comment at start of file My attempt to handle this case in 6f151109b5fdf88eb8b has proven insufficient, the test case added by this commit was failing. Solve this with a "start of file" non-terminal in the PEG grammar, which unfortunately isn't portable but much better reflects the actual implementation and is much simpler. --- doc/langref/grammar.peg | 16 ++++-- lib/std/zig/parser_fuzz.zig | 1 + lib/std/zig/parser_generated_oracle.zig | 73 ++++++------------------- tools/gen_parser_oracle.zig | 14 ++++- 4 files changed, 41 insertions(+), 63 deletions(-) diff --git a/doc/langref/grammar.peg b/doc/langref/grammar.peg index b061d3eb5084d630ddd34c18cc49fa033f1ff995..e30b8f6029c4195e2b0faab06f4179b6302674f6 100644 --- a/doc/langref/grammar.peg +++ b/doc/langref/grammar.peg @@ -1,4 +1,4 @@ -Root <- initial_doc_comment? ContainerMembers skip eof +Root <- ContainerMembers skip eof # *** Top level *** ContainerMembers <- container_doc_comment? ContainerDeclaration* (ContainerField COMMA)* (ContainerField / ContainerDeclaration*) @@ -366,6 +366,12 @@ ParamDeclList <- (ParamDecl COMMA)* (ParamDecl / DOT3 COMMA?)? ExprList <- (Expr COMMA)* Expr? # *** Tokens *** + +# Unfortunately, there is not a standard way to match the start of the file in PEG. +# This rule definition is compatible with the original peg(1) tool but is not +# portable. It is however trivial to implement an equivalent rule in hand written +# parsers and other PEG tooling should have similar mechanisms. +sof <- &{ (yy->__pos == 0) } eof <- !. ox80_oxBF <- [\200-\277] @@ -430,12 +436,10 @@ string_char / !["] non_control_utf8 -# We forbid same-line doc comments to disambiguate the mapping to e.g. struct -# fields for documentation generation tooling. However, this requires a special -# case in the grammar to handle a doc comment right at the beginning of a file. -initial_doc_comment <- (skip '///' non_control_utf8* newline)+ skip !'//!' !KEYWORD_test !KEYWORD_comptime !eof container_doc_comment <- (skip '//!' non_control_utf8* newline)+ -doc_comment <- skip_require_newline (skip '///' non_control_utf8* newline)+ +# We forbid same-line doc comments to disambiguate the mapping to e.g. struct +# fields for documentation generation tooling. +doc_comment <- (sof / skip_require_newline) (skip '///' non_control_utf8* newline)+ line_comment <- '//' ![!/] non_control_utf8* newline / '////' non_control_utf8* newline diff --git a/lib/std/zig/parser_fuzz.zig b/lib/std/zig/parser_fuzz.zig index 8adb5f225c004414a464f1cd75eba163bbf527b8..550b03510683da942848c51358491465c0ecd0ab 100644 --- a/lib/std/zig/parser_fuzz.zig +++ b/lib/std/zig/parser_fuzz.zig @@ -50,6 +50,7 @@ test "newline required before doc comment not at start of file" { try checkAgainstOracle("///"); // error try checkAgainstOracle("///\n//!"); try checkAgainstOracle("///\ntest {}"); + try checkAgainstOracle("///\ncomptime 0"); } // Found using AFL++ diff --git a/lib/std/zig/parser_generated_oracle.zig b/lib/std/zig/parser_generated_oracle.zig index 0e3d078372a87e7569f3335dd45fb8ae5a4dd22b..943c86cf288cfe7eb4d1c3eb17db1a5b1e29a2ba 100644 --- a/lib/std/zig/parser_generated_oracle.zig +++ b/lib/std/zig/parser_generated_oracle.zig @@ -16,7 +16,7 @@ const Parser = struct { pub fn parseRoot(p: *Parser) bool { return blk_0: { const pos_0 = p.i; - if ((p.parseinitial_doc_comment() or true) and p.parseContainerMembers() and p.parseskip() and p.parseeof()) break :blk_0 true; + if (p.parseContainerMembers() and p.parseskip() and p.parseeof()) break :blk_0 true; p.i = pos_0; break :blk_0 false; }; @@ -1764,6 +1764,14 @@ const Parser = struct { break :blk_0 false; }; } + pub fn parsesof(p: *Parser) bool { + return blk_0: { + const pos_0 = p.i; + if ((p.i == 0)) break :blk_0 true; + p.i = pos_0; + break :blk_0 false; + }; + } pub fn parseeof(p: *Parser) bool { return blk_0: { const pos_0 = p.i; @@ -2083,60 +2091,6 @@ const Parser = struct { break :blk_0 false; }; } - pub fn parseinitial_doc_comment(p: *Parser) bool { - return blk_0: { - const pos_0 = p.i; - if (blk_1: { - var match_1 = false; - while (blk_3: { - const pos_3 = p.i; - if (p.parseskip() and blk_4: { - if (std.mem.startsWith(u8, p.source[p.i..], "///")) { - p.i += 3; - break :blk_4 true; - } - break :blk_4 false; - } and blk_4: { - while (p.parsenon_control_utf8()) {} - break :blk_4 true; - } and p.parsenewline()) break :blk_3 true; - p.i = pos_3; - break :blk_3 false; - }) { - match_1 = true; - } - break :blk_1 match_1; - } and p.parseskip() and blk_1: { - const pos_1 = p.i; - const match_1 = blk_2: { - if (std.mem.startsWith(u8, p.source[p.i..], "//!")) { - p.i += 3; - break :blk_2 true; - } - break :blk_2 false; - }; - p.i = pos_1; - break :blk_1 !match_1; - } and blk_1: { - const pos_1 = p.i; - const match_1 = p.parseKEYWORD_test(); - p.i = pos_1; - break :blk_1 !match_1; - } and blk_1: { - const pos_1 = p.i; - const match_1 = p.parseKEYWORD_comptime(); - p.i = pos_1; - break :blk_1 !match_1; - } and blk_1: { - const pos_1 = p.i; - const match_1 = p.parseeof(); - p.i = pos_1; - break :blk_1 !match_1; - }) break :blk_0 true; - p.i = pos_0; - break :blk_0 false; - }; - } pub fn parsecontainer_doc_comment(p: *Parser) bool { return blk_0: { const pos_0 = p.i; @@ -2168,7 +2122,14 @@ const Parser = struct { pub fn parsedoc_comment(p: *Parser) bool { return blk_0: { const pos_0 = p.i; - if (p.parseskip_require_newline() and blk_1: { + if (blk_2: { + const pos_2 = p.i; + if (p.parsesof()) break :blk_2 true; + p.i = pos_2; + if (p.parseskip_require_newline()) break :blk_2 true; + p.i = pos_2; + break :blk_2 false; + } and blk_1: { var match_1 = false; while (blk_3: { const pos_3 = p.i; diff --git a/tools/gen_parser_oracle.zig b/tools/gen_parser_oracle.zig index ae536622c55a042a8ad581190bbfa9e4d34ecade..bc9422f371993eb1934b701622aa7bfb4d136b6b 100644 --- a/tools/gen_parser_oracle.zig +++ b/tools/gen_parser_oracle.zig @@ -242,6 +242,7 @@ const Generator = struct { \\}}) , .{ suffix, suffix }); }, + .sof => try g.w.writeAll("(p.i == 0)"), else => unreachable, } } @@ -337,6 +338,8 @@ const Parser = struct { start: u8, end: u8, }, + /// Start of file + sof, const Index = enum(u32) { _, @@ -422,8 +425,17 @@ const Parser = struct { // Prefix <- AND Action // / ( AND / NOT )? Suffix fn parsePrefix(p: *Parser) !?Node.Index { - // We don't implement Action if (p.eatAnd()) { + // We only support a single hardcoded "start of file" Action + if (p.eat('{')) { + // Action <- '{' < [^}]* > '}' Spacing + if (std.mem.startsWith(u8, p.source[p.i..], " (yy->__pos == 0) }")) { + while (!p.eat('}')) p.i += 1; + _ = p.eatSpacing(); + return try p.addNode(.sof); + } + return null; + } const suffix = try p.parseSuffix() orelse return null; return try p.addNode(.{ .@"&" = suffix }); } -- 2.54.0