authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-06-11 11:01:27+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:57+02:00
log6101a1372086d374dab7fa930834c5c80f87e4f9
tree918d10c2c54fec7dfe0964e2aab0dc3ad3c09e70
parent25dee862b47e235a6b1e86ea596f91ed57c83f85
signaturelock-open Commit is signed but in an unrecognized format.

grammar: match tokenizer number literal behavior


2 files changed, 112 insertions(+), 291 deletions(-)

doc/langref/grammar.peg+12-25
...@@ -134,17 +134,16 @@ PrimaryTypeExpr...@@ -134,17 +134,16 @@ PrimaryTypeExpr
134 / DOT IDENTIFIER134 / DOT IDENTIFIER
135 / DOT InitList135 / DOT InitList
136 / ErrorSetDecl136 / ErrorSetDecl
137 / FLOAT
138 / FnProto137 / FnProto
139 / GroupedExpr138 / GroupedExpr
140 / LabeledTypeExpr139 / LabeledTypeExpr
141 / IDENTIFIER !(COLON LabelableExpr)140 / IDENTIFIER !(COLON LabelableExpr)
142 / IfTypeExpr141 / IfTypeExpr
143 / INTEGER
144 / KEYWORD_comptime TypeExpr !ExprSuffix142 / KEYWORD_comptime TypeExpr !ExprSuffix
145 / KEYWORD_error DOT IDENTIFIER143 / KEYWORD_error DOT IDENTIFIER
146 / KEYWORD_anyframe144 / KEYWORD_anyframe
147 / KEYWORD_unreachable145 / KEYWORD_unreachable
146 / NUMBERLITERAL
148 / STRINGLITERAL147 / STRINGLITERAL
149148
150ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto149ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto
...@@ -369,19 +368,6 @@ ExprList <- (Expr COMMA)* Expr?...@@ -369,19 +368,6 @@ ExprList <- (Expr COMMA)* Expr?
369368
370# *** Tokens ***369# *** Tokens ***
371eof <- !.370eof <- !.
372bin <- [01]
373bin_ <- '_'? bin
374oct <- [0-7]
375oct_ <- '_'? oct
376hex <- [0-9a-fA-F]
377hex_ <- '_'? hex
378dec <- [0-9]
379dec_ <- '_'? dec
380
381bin_int <- bin bin_*
382oct_int <- oct oct_*
383dec_int <- dec dec_*
384hex_int <- hex hex_*
385371
386ox80_oxBF <- [\200-\277]372ox80_oxBF <- [\200-\277]
387oxF4 <- '\364'373oxF4 <- '\364'
...@@ -422,6 +408,9 @@ multibyte_utf8 <-...@@ -422,6 +408,9 @@ multibyte_utf8 <-
422non_control_ascii <- [\040-\176]408non_control_ascii <- [\040-\176]
423non_control_utf8 <- [\040-\377]409non_control_utf8 <- [\040-\377]
424410
411hex <- [0-9a-fA-F]
412hex_ <- '_'? hex
413
425char_escape414char_escape
426 <- "\\x" hex hex415 <- "\\x" hex hex
427 / "\\u{" hex+ "}"416 / "\\u{" hex+ "}"
...@@ -443,16 +432,14 @@ line_string <- '\\\\' non_control_utf8* [ \n]*...@@ -443,16 +432,14 @@ line_string <- '\\\\' non_control_utf8* [ \n]*
443skip <- ([ \n] / line_comment)*432skip <- ([ \n] / line_comment)*
444433
445CHAR_LITERAL <- ['] char_char ['] skip434CHAR_LITERAL <- ['] char_char ['] skip
446FLOAT435
447 <- '0x' hex_int '.' hex_int ([pP] [-+]? dec_int)? skip436digit <- [_0-9A-DF-OQ-Za-df-oq-z]
448 / dec_int '.' dec_int ([eE] [-+]? dec_int)? skip437digit_int <- digit / [eEpP]
449 / '0x' hex_int [pP] [-+]? dec_int skip438digit_float <- digit / [eEpP] [-+]?
450 / dec_int [eE] [-+]? dec_int skip439NUMBERLITERAL
451INTEGER440 <- [0-9] digit_int* '.' digit_float+ skip
452 <- '0b' bin_int skip441 / [0-9] digit_float* skip
453 / '0o' oct_int skip442
454 / '0x' hex_int skip
455 / dec_int skip
456STRINGLITERALSINGLE <- ["] string_char* ["] skip443STRINGLITERALSINGLE <- ["] string_char* ["] skip
457STRINGLITERAL444STRINGLITERAL
458 <- STRINGLITERALSINGLE445 <- STRINGLITERALSINGLE
lib/std/zig/parser_generated_oracle.zig+100-266
...@@ -921,8 +921,6 @@ const Parser = struct {...@@ -921,8 +921,6 @@ const Parser = struct {
921 p.i = pos_0;921 p.i = pos_0;
922 if (p.parseErrorSetDecl()) break :blk_0 true;922 if (p.parseErrorSetDecl()) break :blk_0 true;
923 p.i = pos_0;923 p.i = pos_0;
924 if (p.parseFLOAT()) break :blk_0 true;
925 p.i = pos_0;
926 if (p.parseFnProto()) break :blk_0 true;924 if (p.parseFnProto()) break :blk_0 true;
927 p.i = pos_0;925 p.i = pos_0;
928 if (p.parseGroupedExpr()) break :blk_0 true;926 if (p.parseGroupedExpr()) break :blk_0 true;
...@@ -943,8 +941,6 @@ const Parser = struct {...@@ -943,8 +941,6 @@ const Parser = struct {
943 p.i = pos_0;941 p.i = pos_0;
944 if (p.parseIfTypeExpr()) break :blk_0 true;942 if (p.parseIfTypeExpr()) break :blk_0 true;
945 p.i = pos_0;943 p.i = pos_0;
946 if (p.parseINTEGER()) break :blk_0 true;
947 p.i = pos_0;
948 if (p.parseKEYWORD_comptime() and p.parseTypeExpr() and blk_1: {944 if (p.parseKEYWORD_comptime() and p.parseTypeExpr() and blk_1: {
949 const pos_1 = p.i;945 const pos_1 = p.i;
950 const match_1 = p.parseExprSuffix();946 const match_1 = p.parseExprSuffix();
...@@ -958,6 +954,8 @@ const Parser = struct {...@@ -958,6 +954,8 @@ const Parser = struct {
958 p.i = pos_0;954 p.i = pos_0;
959 if (p.parseKEYWORD_unreachable()) break :blk_0 true;955 if (p.parseKEYWORD_unreachable()) break :blk_0 true;
960 p.i = pos_0;956 p.i = pos_0;
957 if (p.parseNUMBERLITERAL()) break :blk_0 true;
958 p.i = pos_0;
961 if (p.parseSTRINGLITERAL()) break :blk_0 true;959 if (p.parseSTRINGLITERAL()) break :blk_0 true;
962 p.i = pos_0;960 p.i = pos_0;
963 break :blk_0 false;961 break :blk_0 false;
...@@ -1880,169 +1878,6 @@ const Parser = struct {...@@ -1880,169 +1878,6 @@ const Parser = struct {
1880 break :blk_0 false;1878 break :blk_0 false;
1881 };1879 };
1882 }1880 }
1883 pub fn parsebin(p: *Parser) bool {
1884 return blk_0: {
1885 const pos_0 = p.i;
1886 if ((p.i < p.source.len and switch (p.source[p.i]) {
1887 '0'...'0',
1888 '1'...'1',
1889 => blk_1: {
1890 p.i += 1;
1891 break :blk_1 true;
1892 },
1893 else => false,
1894 })) break :blk_0 true;
1895 p.i = pos_0;
1896 break :blk_0 false;
1897 };
1898 }
1899 pub fn parsebin_(p: *Parser) bool {
1900 return blk_0: {
1901 const pos_0 = p.i;
1902 if ((blk_2: {
1903 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
1904 p.i += 1;
1905 break :blk_2 true;
1906 }
1907 break :blk_2 false;
1908 } or true) and p.parsebin()) break :blk_0 true;
1909 p.i = pos_0;
1910 break :blk_0 false;
1911 };
1912 }
1913 pub fn parseoct(p: *Parser) bool {
1914 return blk_0: {
1915 const pos_0 = p.i;
1916 if ((p.i < p.source.len and switch (p.source[p.i]) {
1917 '0'...'7',
1918 => blk_1: {
1919 p.i += 1;
1920 break :blk_1 true;
1921 },
1922 else => false,
1923 })) break :blk_0 true;
1924 p.i = pos_0;
1925 break :blk_0 false;
1926 };
1927 }
1928 pub fn parseoct_(p: *Parser) bool {
1929 return blk_0: {
1930 const pos_0 = p.i;
1931 if ((blk_2: {
1932 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
1933 p.i += 1;
1934 break :blk_2 true;
1935 }
1936 break :blk_2 false;
1937 } or true) and p.parseoct()) break :blk_0 true;
1938 p.i = pos_0;
1939 break :blk_0 false;
1940 };
1941 }
1942 pub fn parsehex(p: *Parser) bool {
1943 return blk_0: {
1944 const pos_0 = p.i;
1945 if ((p.i < p.source.len and switch (p.source[p.i]) {
1946 '0'...'9',
1947 'a'...'f',
1948 'A'...'F',
1949 => blk_1: {
1950 p.i += 1;
1951 break :blk_1 true;
1952 },
1953 else => false,
1954 })) break :blk_0 true;
1955 p.i = pos_0;
1956 break :blk_0 false;
1957 };
1958 }
1959 pub fn parsehex_(p: *Parser) bool {
1960 return blk_0: {
1961 const pos_0 = p.i;
1962 if ((blk_2: {
1963 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
1964 p.i += 1;
1965 break :blk_2 true;
1966 }
1967 break :blk_2 false;
1968 } or true) and p.parsehex()) break :blk_0 true;
1969 p.i = pos_0;
1970 break :blk_0 false;
1971 };
1972 }
1973 pub fn parsedec(p: *Parser) bool {
1974 return blk_0: {
1975 const pos_0 = p.i;
1976 if ((p.i < p.source.len and switch (p.source[p.i]) {
1977 '0'...'9',
1978 => blk_1: {
1979 p.i += 1;
1980 break :blk_1 true;
1981 },
1982 else => false,
1983 })) break :blk_0 true;
1984 p.i = pos_0;
1985 break :blk_0 false;
1986 };
1987 }
1988 pub fn parsedec_(p: *Parser) bool {
1989 return blk_0: {
1990 const pos_0 = p.i;
1991 if ((blk_2: {
1992 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
1993 p.i += 1;
1994 break :blk_2 true;
1995 }
1996 break :blk_2 false;
1997 } or true) and p.parsedec()) break :blk_0 true;
1998 p.i = pos_0;
1999 break :blk_0 false;
2000 };
2001 }
2002 pub fn parsebin_int(p: *Parser) bool {
2003 return blk_0: {
2004 const pos_0 = p.i;
2005 if (p.parsebin() and blk_1: {
2006 while (p.parsebin_()) {}
2007 break :blk_1 true;
2008 }) break :blk_0 true;
2009 p.i = pos_0;
2010 break :blk_0 false;
2011 };
2012 }
2013 pub fn parseoct_int(p: *Parser) bool {
2014 return blk_0: {
2015 const pos_0 = p.i;
2016 if (p.parseoct() and blk_1: {
2017 while (p.parseoct_()) {}
2018 break :blk_1 true;
2019 }) break :blk_0 true;
2020 p.i = pos_0;
2021 break :blk_0 false;
2022 };
2023 }
2024 pub fn parsedec_int(p: *Parser) bool {
2025 return blk_0: {
2026 const pos_0 = p.i;
2027 if (p.parsedec() and blk_1: {
2028 while (p.parsedec_()) {}
2029 break :blk_1 true;
2030 }) break :blk_0 true;
2031 p.i = pos_0;
2032 break :blk_0 false;
2033 };
2034 }
2035 pub fn parsehex_int(p: *Parser) bool {
2036 return blk_0: {
2037 const pos_0 = p.i;
2038 if (p.parsehex() and blk_1: {
2039 while (p.parsehex_()) {}
2040 break :blk_1 true;
2041 }) break :blk_0 true;
2042 p.i = pos_0;
2043 break :blk_0 false;
2044 };
2045 }
2046 pub fn parseox80_oxBF(p: *Parser) bool {1881 pub fn parseox80_oxBF(p: *Parser) bool {
2047 return blk_0: {1882 return blk_0: {
2048 const pos_0 = p.i;1883 const pos_0 = p.i;
...@@ -2286,6 +2121,37 @@ const Parser = struct {...@@ -2286,6 +2121,37 @@ const Parser = struct {
2286 break :blk_0 false;2121 break :blk_0 false;
2287 };2122 };
2288 }2123 }
2124 pub fn parsehex(p: *Parser) bool {
2125 return blk_0: {
2126 const pos_0 = p.i;
2127 if ((p.i < p.source.len and switch (p.source[p.i]) {
2128 '0'...'9',
2129 'a'...'f',
2130 'A'...'F',
2131 => blk_1: {
2132 p.i += 1;
2133 break :blk_1 true;
2134 },
2135 else => false,
2136 })) break :blk_0 true;
2137 p.i = pos_0;
2138 break :blk_0 false;
2139 };
2140 }
2141 pub fn parsehex_(p: *Parser) bool {
2142 return blk_0: {
2143 const pos_0 = p.i;
2144 if ((blk_2: {
2145 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
2146 p.i += 1;
2147 break :blk_2 true;
2148 }
2149 break :blk_2 false;
2150 } or true) and p.parsehex()) break :blk_0 true;
2151 p.i = pos_0;
2152 break :blk_0 false;
2153 };
2154 }
2289 pub fn parsechar_escape(p: *Parser) bool {2155 pub fn parsechar_escape(p: *Parser) bool {
2290 return blk_0: {2156 return blk_0: {
2291 const pos_0 = p.i;2157 const pos_0 = p.i;
...@@ -2587,80 +2453,36 @@ const Parser = struct {...@@ -2587,80 +2453,36 @@ const Parser = struct {
2587 break :blk_0 false;2453 break :blk_0 false;
2588 };2454 };
2589 }2455 }
2590 pub fn parseFLOAT(p: *Parser) bool {2456 pub fn parsedigit(p: *Parser) bool {
2591 return blk_0: {2457 return blk_0: {
2592 const pos_0 = p.i;2458 const pos_0 = p.i;
2593 if (blk_1: {2459 if ((p.i < p.source.len and switch (p.source[p.i]) {
2594 if (std.mem.startsWith(u8, p.source[p.i..], "0x")) {2460 '_'...'_',
2595 p.i += 2;2461 '0'...'9',
2596 break :blk_1 true;2462 'A'...'D',
2597 }2463 'F'...'O',
2598 break :blk_1 false;2464 'Q'...'Z',
2599 } and p.parsehex_int() and blk_1: {2465 'a'...'d',
2600 if (std.mem.startsWith(u8, p.source[p.i..], ".")) {2466 'f'...'o',
2467 'q'...'z',
2468 => blk_1: {
2601 p.i += 1;2469 p.i += 1;
2602 break :blk_1 true;2470 break :blk_1 true;
2603 }2471 },
2604 break :blk_1 false;2472 else => false,
2605 } and p.parsehex_int() and (blk_3: {2473 })) break :blk_0 true;
2606 const pos_3 = p.i;
2607 if ((p.i < p.source.len and switch (p.source[p.i]) {
2608 'p'...'p',
2609 'P'...'P',
2610 => blk_4: {
2611 p.i += 1;
2612 break :blk_4 true;
2613 },
2614 else => false,
2615 }) and ((p.i < p.source.len and switch (p.source[p.i]) {
2616 '-'...'-',
2617 '+'...'+',
2618 => blk_5: {
2619 p.i += 1;
2620 break :blk_5 true;
2621 },
2622 else => false,
2623 }) or true) and p.parsedec_int()) break :blk_3 true;
2624 p.i = pos_3;
2625 break :blk_3 false;
2626 } or true) and p.parseskip()) break :blk_0 true;
2627 p.i = pos_0;2474 p.i = pos_0;
2628 if (p.parsedec_int() and blk_1: {2475 break :blk_0 false;
2629 if (std.mem.startsWith(u8, p.source[p.i..], ".")) {2476 };
2630 p.i += 1;2477 }
2631 break :blk_1 true;2478 pub fn parsedigit_int(p: *Parser) bool {
2632 }2479 return blk_0: {
2633 break :blk_1 false;2480 const pos_0 = p.i;
2634 } and p.parsedec_int() and (blk_3: {2481 if (p.parsedigit()) break :blk_0 true;
2635 const pos_3 = p.i;
2636 if ((p.i < p.source.len and switch (p.source[p.i]) {
2637 'e'...'e',
2638 'E'...'E',
2639 => blk_4: {
2640 p.i += 1;
2641 break :blk_4 true;
2642 },
2643 else => false,
2644 }) and ((p.i < p.source.len and switch (p.source[p.i]) {
2645 '-'...'-',
2646 '+'...'+',
2647 => blk_5: {
2648 p.i += 1;
2649 break :blk_5 true;
2650 },
2651 else => false,
2652 }) or true) and p.parsedec_int()) break :blk_3 true;
2653 p.i = pos_3;
2654 break :blk_3 false;
2655 } or true) and p.parseskip()) break :blk_0 true;
2656 p.i = pos_0;2482 p.i = pos_0;
2657 if (blk_1: {2483 if ((p.i < p.source.len and switch (p.source[p.i]) {
2658 if (std.mem.startsWith(u8, p.source[p.i..], "0x")) {2484 'e'...'e',
2659 p.i += 2;2485 'E'...'E',
2660 break :blk_1 true;
2661 }
2662 break :blk_1 false;
2663 } and p.parsehex_int() and (p.i < p.source.len and switch (p.source[p.i]) {
2664 'p'...'p',2486 'p'...'p',
2665 'P'...'P',2487 'P'...'P',
2666 => blk_1: {2488 => blk_1: {
...@@ -2668,19 +2490,21 @@ const Parser = struct {...@@ -2668,19 +2490,21 @@ const Parser = struct {
2668 break :blk_1 true;2490 break :blk_1 true;
2669 },2491 },
2670 else => false,2492 else => false,
2671 }) and ((p.i < p.source.len and switch (p.source[p.i]) {2493 })) break :blk_0 true;
2672 '-'...'-',2494 p.i = pos_0;
2673 '+'...'+',2495 break :blk_0 false;
2674 => blk_2: {2496 };
2675 p.i += 1;2497 }
2676 break :blk_2 true;2498 pub fn parsedigit_float(p: *Parser) bool {
2677 },2499 return blk_0: {
2678 else => false,2500 const pos_0 = p.i;
2679 }) or true) and p.parsedec_int() and p.parseskip()) break :blk_0 true;2501 if (p.parsedigit()) break :blk_0 true;
2680 p.i = pos_0;2502 p.i = pos_0;
2681 if (p.parsedec_int() and (p.i < p.source.len and switch (p.source[p.i]) {2503 if ((p.i < p.source.len and switch (p.source[p.i]) {
2682 'e'...'e',2504 'e'...'e',
2683 'E'...'E',2505 'E'...'E',
2506 'p'...'p',
2507 'P'...'P',
2684 => blk_1: {2508 => blk_1: {
2685 p.i += 1;2509 p.i += 1;
2686 break :blk_1 true;2510 break :blk_1 true;
...@@ -2694,39 +2518,49 @@ const Parser = struct {...@@ -2694,39 +2518,49 @@ const Parser = struct {
2694 break :blk_2 true;2518 break :blk_2 true;
2695 },2519 },
2696 else => false,2520 else => false,
2697 }) or true) and p.parsedec_int() and p.parseskip()) break :blk_0 true;2521 }) or true)) break :blk_0 true;
2698 p.i = pos_0;2522 p.i = pos_0;
2699 break :blk_0 false;2523 break :blk_0 false;
2700 };2524 };
2701 }2525 }
2702 pub fn parseINTEGER(p: *Parser) bool {2526 pub fn parseNUMBERLITERAL(p: *Parser) bool {
2703 return blk_0: {2527 return blk_0: {
2704 const pos_0 = p.i;2528 const pos_0 = p.i;
2705 if (blk_1: {2529 if ((p.i < p.source.len and switch (p.source[p.i]) {
2706 if (std.mem.startsWith(u8, p.source[p.i..], "0b")) {2530 '0'...'9',
2707 p.i += 2;2531 => blk_1: {
2532 p.i += 1;
2708 break :blk_1 true;2533 break :blk_1 true;
2709 }2534 },
2710 break :blk_1 false;2535 else => false,
2711 } and p.parsebin_int() and p.parseskip()) break :blk_0 true;2536 }) and blk_1: {
2712 p.i = pos_0;2537 while (p.parsedigit_int()) {}
2713 if (blk_1: {2538 break :blk_1 true;
2714 if (std.mem.startsWith(u8, p.source[p.i..], "0o")) {2539 } and blk_1: {
2715 p.i += 2;2540 if (std.mem.startsWith(u8, p.source[p.i..], ".")) {
2541 p.i += 1;
2716 break :blk_1 true;2542 break :blk_1 true;
2717 }2543 }
2718 break :blk_1 false;2544 break :blk_1 false;
2719 } and p.parseoct_int() and p.parseskip()) break :blk_0 true;2545 } and blk_1: {
2720 p.i = pos_0;2546 var match_1 = false;
2721 if (blk_1: {2547 while (p.parsedigit_float()) {
2722 if (std.mem.startsWith(u8, p.source[p.i..], "0x")) {2548 match_1 = true;
2723 p.i += 2;
2724 break :blk_1 true;
2725 }2549 }
2726 break :blk_1 false;2550 break :blk_1 match_1;
2727 } and p.parsehex_int() and p.parseskip()) break :blk_0 true;2551 } and p.parseskip()) break :blk_0 true;
2728 p.i = pos_0;2552 p.i = pos_0;
2729 if (p.parsedec_int() and p.parseskip()) break :blk_0 true;2553 if ((p.i < p.source.len and switch (p.source[p.i]) {
2554 '0'...'9',
2555 => blk_1: {
2556 p.i += 1;
2557 break :blk_1 true;
2558 },
2559 else => false,
2560 }) and blk_1: {
2561 while (p.parsedigit_float()) {}
2562 break :blk_1 true;
2563 } and p.parseskip()) break :blk_0 true;
2730 p.i = pos_0;2564 p.i = pos_0;
2731 break :blk_0 false;2565 break :blk_0 false;
2732 };2566 };