authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-06-07 21:07:09+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:57+02:00
log20a2485061b9c480d1df82f8fef2bec13a23e74a
tree8074f2f6af9c34cba26e61c31169616bcfa21d2d
parent82297207263575358702e84788731e2b2c4d8871
signaturelock-open Commit is signed but in an unrecognized format.

tools: add oracle generator for parser fuzzing

This script parses a PEG grammar definition and generates a simple recursive descent parser that recognizes that grammar. The intent is to use this tool to generate an oracle against which std.zig.Ast.parse() can be fuzz tested. Unfortunately, generating a "smith" that can produce all possible valid source code according to the PEG grammar definition does not seem feasible due to PEG semantics. In particular, modeling the ordered choice operator ('/') with a smith is problematic and seems to have worst-case exponential runtime. Consider that `A / B` is equivalent to `A / !A B`, it is not sufficient for a smith to choose an option from the ordered choice at random and generate an input matched by that sub-expression. The current handwritten AstSmith.zig implementation does not match the exact semantics of the PEG grammar. It has some hacks (see the `not_*` variables) to work around specific instances of the general problem with ordered choice described above but that is not an acceptable long term solution in my opinion. Generating the oracle for the fuzz test automatically from the PEG makes the oracle correct by construction and gives us a single source of truth: the PEG definition in the language spec.

2 files changed, 711 insertions(+), 0 deletions(-)

test/standalone/build.zig+1
...@@ -36,6 +36,7 @@ pub fn build(b: *std.Build) void {...@@ -36,6 +36,7 @@ pub fn build(b: *std.Build) void {
36 "../../tools/fetch_them_macos_headers.zig",36 "../../tools/fetch_them_macos_headers.zig",
37 "../../tools/gen_macos_headers_c.zig",37 "../../tools/gen_macos_headers_c.zig",
38 "../../tools/gen_outline_atomics.zig",38 "../../tools/gen_outline_atomics.zig",
39 "../../tools/gen_parser_oracle.zig",
39 "../../tools/gen_spirv_spec.zig",40 "../../tools/gen_spirv_spec.zig",
40 "../../tools/gen_stubs.zig",41 "../../tools/gen_stubs.zig",
41 "../../tools/generate_c_size_and_align_checks.zig",42 "../../tools/generate_c_size_and_align_checks.zig",
tools/gen_parser_oracle.zig created+710
...@@ -0,0 +1,710 @@
1//! Example usage:
2//! zig run ./tools/gen_parser_oracle.zig -- ./doc/langref/grammar.peg > ./lib/std/zig/parser_generated_oracle.zig
3
4// This program implements a subset of the PEG grammar definition
5// in the peg(1) man page.
6//
7// It generates a recursive descent parser that returns true if a given input is
8// matched by the grammar. This generated parser is used as an oracle for fuzz testing.
9
10const std = @import("std");
11const assert = std.debug.assert;
12const Io = std.Io;
13const mem = std.mem;
14const Allocator = mem.Allocator;
15const log = std.log;
16
17pub fn main(init: std.process.Init) !void {
18 const gpa = init.gpa;
19 const arena = init.arena.allocator();
20 const io = init.io;
21 const args = try init.minimal.args.toSlice(arena);
22
23 const grammar_path = args[1];
24
25 const grammar = try Io.Dir.cwd().readFileAlloc(io, grammar_path, gpa, .unlimited);
26 defer gpa.free(grammar);
27
28 var parser: Parser = .init(gpa, grammar);
29 defer parser.deinit();
30
31 const root = try parser.parseGrammar() orelse {
32 log.err("Invalid grammar", .{});
33 return;
34 };
35
36 var buffer: Io.Writer.Allocating = .init(gpa);
37 defer buffer.deinit();
38
39 var g: Generator = .init(&buffer.writer, &parser);
40 try g.genRoot(root);
41
42 const generated = try buffer.toOwnedSliceSentinel(0);
43 defer gpa.free(generated);
44
45 // Parse the generated Zig code and render it in the canonical format
46 var tree = try std.zig.Ast.parse(gpa, generated, .zig);
47 defer tree.deinit(gpa);
48
49 if (tree.errors.len != 0) {
50 // This should never be reached, but helps a lot when debugging this script.
51 try std.zig.printAstErrorsToStderr(gpa, io, tree, "generated", .auto);
52 return error.ParseError;
53 }
54
55 var stdout_buffer: [4096]u8 = undefined;
56 var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer);
57 const stdout = &stdout_writer.interface;
58
59 try tree.render(gpa, stdout, .{});
60 try stdout.flush();
61}
62
63const Generator = struct {
64 w: *Io.Writer,
65 p: *const Parser,
66 /// Suffix for generated identifiers, incremented for each nested scope to avoid shadowing,
67 /// Decremented at end of each generated scope to give smaller git diffs when regenerating
68 /// lib/std/zig/parser_generated_oracle.zig.
69 suffix: usize,
70
71 fn init(w: *Io.Writer, p: *const Parser) Generator {
72 return .{ .w = w, .p = p, .suffix = 0 };
73 }
74
75 const Error = Io.Writer.Error;
76 const Node = Parser.Node;
77
78 fn genRoot(g: *Generator, node: Node.Index) Error!void {
79 try g.w.writeAll(
80 \\//! This file is generated, do not edit manually! To generate, run:
81 \\//! zig run ./tools/gen_parser_oracle.zig -- ./doc/langref/grammar.peg > ./lib/std/zig/parser_generated_oracle.zig
82 \\
83 \\const std = @import("std");
84 \\
85 \\/// Returns true if the input source is in the language defined by
86 \\/// the grammar.
87 \\pub fn parse(source: []const u8) bool {
88 \\ var p: Parser = .{ .source = source, .i = 0 };
89 \\ return p.parseRoot();
90 \\}
91 \\
92 \\const Parser = struct {
93 \\ source: []const u8,
94 \\ i: usize,
95 \\
96 );
97 for (g.p.getExtra(node.get(g.p).root)) |def| {
98 try g.genDef(def);
99 }
100 try g.w.writeAll("};");
101 }
102
103 fn genDef(g: *Generator, node: Node.Index) Error!void {
104 const def = node.get(g.p).def;
105 const id = def.id.get(g.p).id;
106 assert(g.suffix == 0);
107 try g.w.print("pub fn parse{s}(p: *Parser) bool {{ return ", .{id});
108 try g.genExpr(def.expr);
109 try g.w.writeAll(";}");
110 }
111
112 fn genExpr(g: *Generator, node: Node.Index) Error!void {
113 const suffix = g.suffix;
114 g.suffix += 1;
115 defer g.suffix -= 1;
116 try g.w.print(
117 \\blk_{d}: {{
118 \\const pos_{d} = p.i;
119 , .{ suffix, suffix });
120 for (g.p.getExtra(node.get(g.p).expr)) |seq| {
121 try g.w.writeAll("if (");
122 try g.genSeq(seq);
123 try g.w.print(") break :blk_{d} true;", .{suffix});
124 try g.w.print("p.i = pos_{d};", .{suffix});
125 }
126 try g.w.print("break :blk_{d} false; }}", .{suffix});
127 }
128
129 fn genSeq(g: *Generator, node: Node.Index) Error!void {
130 const items = g.p.getExtra(node.get(g.p).seq);
131 for (items, 0..) |item, i| {
132 if (i > 0) try g.w.writeAll(" and ");
133 try g.genNode(item);
134 }
135 }
136
137 fn genNode(g: *Generator, node: Node.Index) Error!void {
138 const suffix = g.suffix;
139 g.suffix += 1;
140 defer g.suffix -= 1;
141 switch (node.get(g.p)) {
142 .id => |id| try g.w.print("p.parse{s}()", .{id}),
143 .expr => try g.genExpr(node),
144 .@"&" => @panic("'&' not supported, unused in Zig's grammar.peg"),
145 .@"!" => |child| {
146 // XXX forbid unbounded lookahead
147 try g.w.print(
148 \\blk_{d}: {{
149 \\const pos_{d} = p.i;
150 \\const match_{d} =
151 , .{ suffix, suffix, suffix });
152 try g.genNode(child);
153 try g.w.print(
154 \\;
155 \\p.i = pos_{d};
156 \\ break :blk_{d} !match_{d};
157 \\}}
158 , .{ suffix, suffix, suffix });
159 },
160 .@"?" => |child| {
161 try g.w.writeAll("(");
162 try g.genNode(child);
163 try g.w.writeAll(" or true )");
164 },
165 .@"*" => |child| {
166 try g.w.print(
167 \\blk_{d}: {{
168 \\while (
169 , .{suffix});
170 try g.genNode(child);
171 try g.w.print(
172 \\) {{}}
173 \\break :blk_{d} true; }}
174 , .{suffix});
175 },
176 .@"+" => |child| {
177 try g.w.print(
178 \\blk_{d}: {{
179 \\var match_{d} = false;
180 \\while (
181 , .{ suffix, suffix });
182 try g.genNode(child);
183 try g.w.print(
184 \\) {{ match_{d} = true; }}
185 \\break :blk_{d} match_{d}; }}
186 , .{ suffix, suffix, suffix });
187 },
188 .@"." => {
189 try g.w.print(
190 \\blk_{d}: {{
191 \\ if (p.i < p.source.len) {{
192 \\ p.i += 1;
193 \\ break :blk_{d} true;
194 \\ }}
195 \\ break :blk_{d} false;
196 \\}}
197 , .{ suffix, suffix, suffix });
198 },
199 .literal => |literal| {
200 const bytes = g.p.strings.items[literal.off..][0..literal.len];
201 try g.w.print(
202 \\blk_{d}: {{
203 \\if (std.mem.startsWith(u8, p.source[p.i..], "
204 , .{suffix});
205 try std.zig.stringEscape(bytes, g.w);
206 try g.w.print(
207 \\")) {{
208 \\p.i += {d};
209 \\ break :blk_{d} true;
210 \\}}
211 \\break :blk_{d} false;
212 \\}}
213 , .{ bytes.len, suffix, suffix });
214 },
215 .class => |ranges| {
216 try g.w.writeAll("(p.i < p.source.len and switch (p.source[p.i]) {");
217 for (g.p.getExtra(ranges)) |n| {
218 const range = n.get(g.p).range;
219 try g.w.writeAll("'");
220 try std.zig.charEscape(range.start, g.w);
221 try g.w.writeAll("'...'");
222 try std.zig.charEscape(range.end, g.w);
223 try g.w.writeAll("',");
224 }
225 try g.w.print(
226 \\=> blk_{d}: {{ p.i += 1; break :blk_{d} true; }},
227 \\else => false,
228 \\}})
229 , .{ suffix, suffix });
230 },
231 else => unreachable,
232 }
233 }
234};
235
236/// Parser implements a subset of the PEG grammar definition.
237/// We don't bother implementing the Action, BEGIN, and END rules
238/// and also omit unneeded character escape sequences.
239///
240/// The full PEG grammar found in the peg(1) man page:
241///
242/// Grammar <- Spacing Definition+ EndOfFile
243///
244/// Definition <- Identifier LEFTARROW Expression
245/// Expression <- Sequence ( SLASH Sequence )*
246/// Sequence <- Prefix*
247/// Prefix <- AND Action
248/// / ( AND / NOT )? Suffix
249/// Suffix <- Primary ( QUERY / STAR / PLUS )?
250/// Primary <- Identifier !LEFTARROW
251/// / OPEN Expression CLOSE
252/// / Literal
253/// / Class
254/// / DOT
255/// / Action
256/// / BEGIN
257/// / END
258///
259/// Identifier <- < IdentStart IdentCont* > Spacing
260/// IdentStart <- [a-zA-Z_]
261/// IdentCont <- IdentStart / [0-9]
262/// Literal <- ['] < ( !['] Char )* > ['] Spacing
263/// / ["] < ( !["] Char )* > ["] Spacing
264/// Class <- '[' < ( !']' Range )* > ']' Spacing
265/// Range <- Char '-' Char / Char
266/// Char <- '\\' [abefnrtv'"\[\]\\]
267/// / '\\' [0-3][0-7][0-7]
268/// / '\\' [0-7][0-7]?
269/// / '\\' '-'
270/// / !'\\' .
271/// LEFTARROW <- '<-' Spacing
272/// SLASH <- '/' Spacing
273/// AND <- '&' Spacing
274/// NOT <- '!' Spacing
275/// QUERY <- '?' Spacing
276/// STAR <- '*' Spacing
277/// PLUS <- '+' Spacing
278/// OPEN <- '(' Spacing
279/// CLOSE <- ')' Spacing
280/// DOT <- '.' Spacing
281/// Spacing <- ( Space / Comment )*
282/// Comment <- '#' ( !EndOfLine . )* EndOfLine
283/// Space <- ' ' / '\t' / EndOfLine
284/// EndOfLine <- '\r\n' / '\n' / '\r'
285/// EndOfFile <- !.
286/// Action <- '{' < [^}]* > '}' Spacing
287/// BEGIN <- '<' Spacing
288/// END <- '>' Spacing
289const Parser = struct {
290 gpa: Allocator,
291 /// PEG grammar source
292 source: []const u8,
293 /// Current index into source
294 i: u32,
295 nodes: std.ArrayList(Node),
296 extra: std.ArrayList(Node.Index),
297 strings: std.ArrayList(u8),
298
299 const Node = union(enum) {
300 /// Slice into extra
301 root: Slice,
302 def: struct {
303 id: Index,
304 expr: Index,
305 },
306 /// Slice into Parser.source
307 id: []const u8,
308 /// Slice into extra
309 expr: Slice,
310 /// Slice into extra
311 seq: Slice,
312 @"&": Index,
313 @"!": Index,
314 @"?": Index,
315 @"*": Index,
316 @"+": Index,
317 @".",
318 /// Slice into strings
319 literal: Slice,
320 /// Slice into extra
321 class: Slice,
322 range: struct {
323 start: u8,
324 end: u8,
325 },
326
327 const Index = enum(u32) {
328 _,
329
330 fn get(index: Index, p: *const Parser) Node {
331 return p.nodes.items[@intFromEnum(index)];
332 }
333 };
334
335 const Slice = struct {
336 off: u32,
337 len: u32,
338 };
339 };
340
341 fn init(gpa: Allocator, source: []const u8) Parser {
342 return .{
343 .gpa = gpa,
344 .source = source,
345 .i = 0,
346 .nodes = .empty,
347 .extra = .empty,
348 .strings = .empty,
349 };
350 }
351
352 fn deinit(p: *Parser) void {
353 p.nodes.deinit(p.gpa);
354 p.extra.deinit(p.gpa);
355 p.strings.deinit(p.gpa);
356 }
357
358 // Grammar <- Spacing Definition+ EndOfFile
359 // EndOfFile <- !.
360 fn parseGrammar(p: *Parser) !?Node.Index {
361 var scratch: std.ArrayList(Node.Index) = .empty;
362 defer scratch.deinit(p.gpa);
363 _ = p.eatSpacing();
364 while (try p.parseDefinition()) |def| {
365 try scratch.append(p.gpa, def);
366 }
367 if (scratch.items.len == 0) return null;
368 if (p.peek() != null) return null;
369 const defs = try p.addExtra(scratch.items);
370 return try p.addNode(.{ .root = defs });
371 }
372
373 // Definition <- Identifier LEFTARROW Expression
374 fn parseDefinition(p: *Parser) !?Node.Index {
375 const id = try p.parseIdentifier() orelse return null;
376 if (!p.eatLeftArrow()) return null;
377 const expr = try p.parseExpression() orelse return null;
378 return try p.addNode(.{ .def = .{
379 .id = id,
380 .expr = expr,
381 } });
382 }
383
384 // Expression <- Sequence ( SLASH Sequence )*
385 fn parseExpression(p: *Parser) error{OutOfMemory}!?Node.Index {
386 var scratch: std.ArrayList(Node.Index) = .empty;
387 defer scratch.deinit(p.gpa);
388 while (try p.parseSequence()) |seq| {
389 try scratch.append(p.gpa, seq);
390 if (!p.eatSlash()) break;
391 }
392 if (scratch.items.len == 0) return null;
393 const seqs = try p.addExtra(scratch.items);
394 return try p.addNode(.{ .expr = seqs });
395 }
396
397 // Sequence <- Prefix*
398 fn parseSequence(p: *Parser) !?Node.Index {
399 var scratch: std.ArrayList(Node.Index) = .empty;
400 defer scratch.deinit(p.gpa);
401 while (try p.parsePrefix()) |primary| {
402 try scratch.append(p.gpa, primary);
403 }
404 const primaries = try p.addExtra(scratch.items);
405 return try p.addNode(.{ .seq = primaries });
406 }
407
408 // Prefix <- AND Action
409 // / ( AND / NOT )? Suffix
410 fn parsePrefix(p: *Parser) !?Node.Index {
411 // We don't implement Action
412 if (p.eatAnd()) {
413 const suffix = try p.parseSuffix() orelse return null;
414 return try p.addNode(.{ .@"&" = suffix });
415 }
416 if (p.eatNot()) {
417 const suffix = try p.parseSuffix() orelse return null;
418 return try p.addNode(.{ .@"!" = suffix });
419 }
420 return try p.parseSuffix();
421 }
422
423 // Suffix <- Primary ( QUERY / STAR / PLUS )?
424 fn parseSuffix(p: *Parser) !?Node.Index {
425 const primary = try p.parsePrimary() orelse return null;
426 if (p.eatQuery()) {
427 return try p.addNode(.{ .@"?" = primary });
428 }
429 if (p.eatStar()) {
430 return try p.addNode(.{ .@"*" = primary });
431 }
432 if (p.eatPlus()) {
433 return try p.addNode(.{ .@"+" = primary });
434 }
435 return primary;
436 }
437
438 // Primary <- Identifier !LEFTARROW
439 // / OPEN Expression CLOSE
440 // / Literal
441 // / Class
442 // / DOT
443 // / Action
444 // / BEGIN
445 // / END
446 fn parsePrimary(p: *Parser) !?Node.Index {
447 const init_pos = p.savePos();
448 if (try p.parseIdentifier()) |id| {
449 const pos = p.savePos();
450 if (!p.eatLeftArrow()) {
451 p.restorePos(pos);
452 return id;
453 }
454 }
455 p.restorePos(init_pos);
456 if (p.eatOpen()) if (try p.parseExpression()) |expr| if (p.eatClose()) return expr;
457 p.restorePos(init_pos);
458 if (try p.parseLiteral()) |literal| return literal;
459 p.restorePos(init_pos);
460 if (try p.parseClass()) |class| return class;
461 p.restorePos(init_pos);
462 if (p.eatDot()) return try p.addNode(.@".");
463 // We don't implement Action, BEGIN, and END.
464 return null;
465 }
466
467 // Identifier <- < IdentStart IdentCont* > Spacing
468 // IdentStart <- [a-zA-Z_]
469 // IdentCont <- IdentStart / [0-9]
470 fn parseIdentifier(p: *Parser) !?Node.Index {
471 const start = p.i;
472 switch (p.next() orelse return null) {
473 'a'...'z', 'A'...'Z', '_' => {},
474 else => return null,
475 }
476 while (p.peek()) |cont| {
477 switch (cont) {
478 'a'...'z', 'A'...'Z', '_', '0'...'9' => p.i += 1,
479 else => break,
480 }
481 }
482 const id = p.source[start..p.i];
483 _ = p.eatSpacing();
484 return try p.addNode(.{ .id = id });
485 }
486
487 // Literal <- ['] < ( !['] Char )* > ['] Spacing
488 // / ["] < ( !["] Char )* > ["] Spacing
489 fn parseLiteral(p: *Parser) !?Node.Index {
490 const quote: u8 = if (p.eat('\'')) '\'' else if (p.eat('"')) '"' else return null;
491 const off = p.strings.items.len;
492 while (!p.eat(quote)) {
493 const byte = p.parseChar() orelse return null;
494 try p.strings.append(p.gpa, byte);
495 }
496 _ = p.eatSpacing();
497 return try p.addNode(.{ .literal = .{
498 .off = @intCast(off),
499 .len = @intCast(p.strings.items.len - off),
500 } });
501 }
502
503 // Class <- '[' < ( !']' Range )* > ']' Spacing
504 fn parseClass(p: *Parser) !?Node.Index {
505 var scratch: std.ArrayList(Node.Index) = .empty;
506 defer scratch.deinit(p.gpa);
507 if (!p.eat('[')) return null;
508 while (!p.eat(']')) {
509 const range = try p.parseRange() orelse return null;
510 try scratch.append(p.gpa, range);
511 }
512 _ = p.eatSpacing();
513 const ranges = try p.addExtra(scratch.items);
514 return try p.addNode(.{ .class = ranges });
515 }
516
517 // Range <- Char '-' Char / Char
518 fn parseRange(p: *Parser) !?Node.Index {
519 const start = p.parseChar() orelse return null;
520 const end = blk: {
521 if (p.eat('-')) {
522 break :blk p.parseChar() orelse return null;
523 }
524 break :blk start;
525 };
526 return try p.addNode(.{ .range = .{
527 .start = start,
528 .end = end,
529 } });
530 }
531
532 // Char <- '\\' [abefnrtv'"\[\]\\]
533 // / '\\' [0-3][0-7][0-7]
534 // / '\\' [0-7][0-7]?
535 // / '\\' '-'
536 // / !'\\' .
537 fn parseChar(p: *Parser) ?u8 {
538 if (p.eat('\\')) {
539 const c = p.next() orelse return null;
540 return switch (c) {
541 // Only the escape sequences actually used in the Zig grammar are implemented
542 'n' => '\n',
543 'r' => '\r',
544 't' => '\t',
545 '\'' => '\'',
546 '"' => '"',
547 '[' => '[',
548 ']' => ']',
549 '\\' => '\\',
550 '-' => '-',
551 '0'...'7' => {
552 // octal
553 if (c <= '3') {
554 const c2 = p.next() orelse return null;
555 if (c2 < '0' or c2 > '7') return null;
556 const c3 = p.next() orelse return null;
557 if (c3 < '0' or c3 > '7') return null;
558 return (c - '0') * 8 * 8 + (c2 - '0') * 8 + (c3 - '0');
559 } else {
560 if (p.peek()) |c2| {
561 if (c2 >= '0' and c2 <= '7') {
562 p.i += 1;
563 return (c - '0') * 8 + (c2 - '0');
564 }
565 }
566 return (c - '0');
567 }
568 },
569 else => null,
570 };
571 } else {
572 return p.next();
573 }
574 }
575
576 // LEFTARROW <- '<-' Spacing
577 fn eatLeftArrow(p: *Parser) bool {
578 return p.eat('<') and p.eat('-') and p.eatSpacing();
579 }
580
581 // SLASH <- '/' Spacing
582 fn eatSlash(p: *Parser) bool {
583 return p.eat('/') and p.eatSpacing();
584 }
585
586 // AND <- '&' Spacing
587 fn eatAnd(p: *Parser) bool {
588 return p.eat('&') and p.eatSpacing();
589 }
590
591 // NOT <- '!' Spacing
592 fn eatNot(p: *Parser) bool {
593 return p.eat('!') and p.eatSpacing();
594 }
595
596 // QUERY <- '?' Spacing
597 fn eatQuery(p: *Parser) bool {
598 return p.eat('?') and p.eatSpacing();
599 }
600
601 // STAR <- '*' Spacing
602 fn eatStar(p: *Parser) bool {
603 return p.eat('*') and p.eatSpacing();
604 }
605
606 // PLUS <- '+' Spacing
607 fn eatPlus(p: *Parser) bool {
608 return p.eat('+') and p.eatSpacing();
609 }
610
611 // OPEN <- '(' Spacing
612 fn eatOpen(p: *Parser) bool {
613 return p.eat('(') and p.eatSpacing();
614 }
615
616 // CLOSE <- ')' Spacing
617 fn eatClose(p: *Parser) bool {
618 return p.eat(')') and p.eatSpacing();
619 }
620
621 // DOT <- '.' Spacing
622 fn eatDot(p: *Parser) bool {
623 return p.eat('.') and p.eatSpacing();
624 }
625
626 // Spacing <- ( Space / Comment )*
627 fn eatSpacing(p: *Parser) bool {
628 while (p.eatSpace() or p.eatComment()) {}
629 return true;
630 }
631
632 // Comment <- '#' ( !EndOfLine . )* EndOfLine
633 fn eatComment(p: *Parser) bool {
634 if (!p.eat('#')) return false;
635 while (!p.eatEndOfLine()) p.i += 1;
636 return true;
637 }
638
639 // Space <- ' ' / '\t' / EndOfLine
640 fn eatSpace(p: *Parser) bool {
641 return p.eat(' ') or p.eat('\t') or p.eatEndOfLine();
642 }
643
644 // EndOfLine <- '\r\n' / '\n' / '\r'
645 fn eatEndOfLine(p: *Parser) bool {
646 return p.eat('\n');
647 }
648
649 fn peek(p: *Parser) ?u8 {
650 if (p.i < p.source.len) {
651 return p.source[p.i];
652 }
653 return null;
654 }
655
656 fn next(p: *Parser) ?u8 {
657 if (p.i < p.source.len) {
658 defer p.i += 1;
659 return p.source[p.i];
660 }
661 return null;
662 }
663
664 fn eat(p: *Parser, byte: u8) bool {
665 if (p.i < p.source.len and p.source[p.i] == byte) {
666 p.i += 1;
667 return true;
668 }
669 return false;
670 }
671
672 fn addNode(p: *Parser, node: Node) !Node.Index {
673 try p.nodes.append(p.gpa, node);
674 return @enumFromInt(p.nodes.items.len - 1);
675 }
676
677 fn addExtra(p: *Parser, nodes: []const Node.Index) !Node.Slice {
678 const off = p.extra.items.len;
679 try p.extra.appendSlice(p.gpa, nodes);
680 return .{ .off = @intCast(off), .len = @intCast(p.extra.items.len - off) };
681 }
682
683 const Pos = struct {
684 i: u32,
685 nodes_len: u32,
686 extra_len: u32,
687 strings_len: u32,
688 };
689
690 fn savePos(p: *const Parser) Pos {
691 return .{
692 .i = p.i,
693 .nodes_len = @intCast(p.nodes.items.len),
694 .extra_len = @intCast(p.extra.items.len),
695 .strings_len = @intCast(p.strings.items.len),
696 };
697 }
698
699 fn restorePos(p: *Parser, pos: Pos) void {
700 assert(p.i >= pos.i);
701 p.i = pos.i;
702 p.nodes.shrinkRetainingCapacity(pos.nodes_len);
703 p.extra.shrinkRetainingCapacity(pos.extra_len);
704 p.strings.shrinkRetainingCapacity(pos.strings_len);
705 }
706
707 fn getExtra(p: *const Parser, s: Node.Slice) []const Node.Index {
708 return p.extra.items[s.off..][0..s.len];
709 }
710};