authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-06-12 12:10:26+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2026-07-08 11:48:57+02:00
log0b24b58fb24b462c85ebba53ee838e87887e4d25
tree20e73d475892f8ca9f0c298d2076e6f7c28c2824
parentd11dfc9b9c9b6bb5933f10df8b618184f921c033
signaturelock-open Commit is signed but in an unrecognized format.

grammar: match tokenizer string literal behavior

The tokenizer does not yet perform UTF-8 validation. This violates the language spec, but fixing it is out of scope for this branch. Leave the intended language spec commented out in the grammar to be restored when the tokenizer is fixed.

2 files changed, 38 insertions(+), 109 deletions(-)

doc/langref/grammar.peg+21-15
...@@ -405,25 +405,30 @@ multibyte_utf8 <-...@@ -405,25 +405,30 @@ multibyte_utf8 <-
405 / oxE0 oxA0_oxBF ox80_oxBF405 / oxE0 oxA0_oxBF ox80_oxBF
406 / oxC2_oxDF ox80_oxBF406 / oxC2_oxDF ox80_oxBF
407407
408# Exclude \177 which is DEL
408non_control_ascii <- [\040-\176]409non_control_ascii <- [\040-\176]
409non_control_utf8 <- [\040-\377]410non_control_utf8 <- [\040-\176\200-\377]
411
412# XXX: the Zig tokenizer doesn't yet perform UTF-8 validation
413# When the tokenizer is fixed, switch back to these definitions
414# that forbid invalid UTF-8:
415#
416# char_char
417# <- multibyte_utf8
418# / "\\'"
419# / !['] non_control_ascii
420# string_char
421# <- multibyte_utf8
422# / '\\"'
423# / !["] non_control_ascii
410424
411hex <- [0-9a-fA-F]
412hex_ <- '_'? hex
413
414char_escape
415 <- "\\x" hex hex
416 / "\\u{" hex+ "}"
417 / "\\" [nr\\t'"]
418char_char425char_char
419 <- multibyte_utf8426 <- "\\'"
420 / char_escape427 / !['] non_control_utf8
421 / ![\\'\n] non_control_ascii
422428
423string_char429string_char
424 <- multibyte_utf8430 <- '\\"'
425 / char_escape431 / !["] non_control_utf8
426 / ![\\"\n] non_control_ascii
427432
428container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+433container_doc_comment <- ('//!' non_control_utf8* [ \n]* skip)+
429doc_comment <- ('///' non_control_utf8* [ \n]* skip)+434doc_comment <- ('///' non_control_utf8* [ \n]* skip)+
...@@ -431,7 +436,7 @@ line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8*...@@ -431,7 +436,7 @@ line_comment <- '//' ![!/] non_control_utf8* / '////' non_control_utf8*
431line_string <- '\\\\' non_control_utf8* [ \n]*436line_string <- '\\\\' non_control_utf8* [ \n]*
432skip <- ([ \n\t\r] / line_comment)*437skip <- ([ \n\t\r] / line_comment)*
433438
434CHAR_LITERAL <- ['] char_char ['] skip439CHAR_LITERAL <- ['] char_char* ['] skip
435440
436digit <- [_0-9A-DF-OQ-Za-df-oq-z]441digit <- [_0-9A-DF-OQ-Za-df-oq-z]
437digit_int <- digit / [eEpP]442digit_int <- digit / [eEpP]
...@@ -440,6 +445,7 @@ NUMBERLITERAL...@@ -440,6 +445,7 @@ NUMBERLITERAL
440 <- [0-9] digit_int* '.' digit_float+ skip445 <- [0-9] digit_int* '.' digit_float+ skip
441 / [0-9] digit_float* skip446 / [0-9] digit_float* skip
442447
448
443STRINGLITERALSINGLE <- ["] string_char* ["] skip449STRINGLITERALSINGLE <- ["] string_char* ["] skip
444STRINGLITERAL450STRINGLITERAL
445 <- STRINGLITERALSINGLE451 <- STRINGLITERALSINGLE
lib/std/zig/parser_generated_oracle.zig+17-94
...@@ -2110,24 +2110,8 @@ const Parser = struct {...@@ -2110,24 +2110,8 @@ const Parser = struct {
2110 return blk_0: {2110 return blk_0: {
2111 const pos_0 = p.i;2111 const pos_0 = p.i;
2112 if ((p.i < p.source.len and switch (p.source[p.i]) {2112 if ((p.i < p.source.len and switch (p.source[p.i]) {
2113 ' '...'\xff',2113 ' '...'~',
2114 => blk_1: {2114 '\x80'...'\xff',
2115 p.i += 1;
2116 break :blk_1 true;
2117 },
2118 else => false,
2119 })) break :blk_0 true;
2120 p.i = pos_0;
2121 break :blk_0 false;
2122 };
2123 }
2124 pub fn parsehex(p: *Parser) bool {
2125 return blk_0: {
2126 const pos_0 = p.i;
2127 if ((p.i < p.source.len and switch (p.source[p.i]) {
2128 '0'...'9',
2129 'a'...'f',
2130 'A'...'F',
2131 => blk_1: {2115 => blk_1: {
2132 p.i += 1;2116 p.i += 1;
2133 break :blk_1 true;2117 break :blk_1 true;
...@@ -2138,87 +2122,21 @@ const Parser = struct {...@@ -2138,87 +2122,21 @@ const Parser = struct {
2138 break :blk_0 false;2122 break :blk_0 false;
2139 };2123 };
2140 }2124 }
2141 pub fn parsehex_(p: *Parser) bool {2125 pub fn parsechar_char(p: *Parser) bool {
2142 return blk_0: {
2143 const pos_0 = p.i;
2144 if ((blk_2: {
2145 if (std.mem.startsWith(u8, p.source[p.i..], "_")) {
2146 p.i += 1;
2147 break :blk_2 true;
2148 }
2149 break :blk_2 false;
2150 } or true) and p.parsehex()) break :blk_0 true;
2151 p.i = pos_0;
2152 break :blk_0 false;
2153 };
2154 }
2155 pub fn parsechar_escape(p: *Parser) bool {
2156 return blk_0: {2126 return blk_0: {
2157 const pos_0 = p.i;2127 const pos_0 = p.i;
2158 if (blk_1: {2128 if (blk_1: {
2159 if (std.mem.startsWith(u8, p.source[p.i..], "\\x")) {2129 if (std.mem.startsWith(u8, p.source[p.i..], "\\'")) {
2160 p.i += 2;2130 p.i += 2;
2161 break :blk_1 true;2131 break :blk_1 true;
2162 }2132 }
2163 break :blk_1 false;2133 break :blk_1 false;
2164 } and p.parsehex() and p.parsehex()) break :blk_0 true;
2165 p.i = pos_0;
2166 if (blk_1: {
2167 if (std.mem.startsWith(u8, p.source[p.i..], "\\u{")) {
2168 p.i += 3;
2169 break :blk_1 true;
2170 }
2171 break :blk_1 false;
2172 } and blk_1: {
2173 var match_1 = false;
2174 while (p.parsehex()) {
2175 match_1 = true;
2176 }
2177 break :blk_1 match_1;
2178 } and blk_1: {
2179 if (std.mem.startsWith(u8, p.source[p.i..], "}")) {
2180 p.i += 1;
2181 break :blk_1 true;
2182 }
2183 break :blk_1 false;
2184 }) break :blk_0 true;2134 }) break :blk_0 true;
2185 p.i = pos_0;2135 p.i = pos_0;
2186 if (blk_1: {
2187 if (std.mem.startsWith(u8, p.source[p.i..], "\\")) {
2188 p.i += 1;
2189 break :blk_1 true;
2190 }
2191 break :blk_1 false;
2192 } and (p.i < p.source.len and switch (p.source[p.i]) {
2193 'n'...'n',
2194 'r'...'r',
2195 '\\'...'\\',
2196 't'...'t',
2197 '\''...'\'',
2198 '"'...'"',
2199 => blk_1: {
2200 p.i += 1;
2201 break :blk_1 true;
2202 },
2203 else => false,
2204 })) break :blk_0 true;
2205 p.i = pos_0;
2206 break :blk_0 false;
2207 };
2208 }
2209 pub fn parsechar_char(p: *Parser) bool {
2210 return blk_0: {
2211 const pos_0 = p.i;
2212 if (p.parsemultibyte_utf8()) break :blk_0 true;
2213 p.i = pos_0;
2214 if (p.parsechar_escape()) break :blk_0 true;
2215 p.i = pos_0;
2216 if (blk_1: {2136 if (blk_1: {
2217 const pos_1 = p.i;2137 const pos_1 = p.i;
2218 const match_1 = (p.i < p.source.len and switch (p.source[p.i]) {2138 const match_1 = (p.i < p.source.len and switch (p.source[p.i]) {
2219 '\\'...'\\',
2220 '\''...'\'',2139 '\''...'\'',
2221 '\n'...'\n',
2222 => blk_2: {2140 => blk_2: {
2223 p.i += 1;2141 p.i += 1;
2224 break :blk_2 true;2142 break :blk_2 true;
...@@ -2227,7 +2145,7 @@ const Parser = struct {...@@ -2227,7 +2145,7 @@ const Parser = struct {
2227 });2145 });
2228 p.i = pos_1;2146 p.i = pos_1;
2229 break :blk_1 !match_1;2147 break :blk_1 !match_1;
2230 } and p.parsenon_control_ascii()) break :blk_0 true;2148 } and p.parsenon_control_utf8()) break :blk_0 true;
2231 p.i = pos_0;2149 p.i = pos_0;
2232 break :blk_0 false;2150 break :blk_0 false;
2233 };2151 };
...@@ -2235,16 +2153,18 @@ const Parser = struct {...@@ -2235,16 +2153,18 @@ const Parser = struct {
2235 pub fn parsestring_char(p: *Parser) bool {2153 pub fn parsestring_char(p: *Parser) bool {
2236 return blk_0: {2154 return blk_0: {
2237 const pos_0 = p.i;2155 const pos_0 = p.i;
2238 if (p.parsemultibyte_utf8()) break :blk_0 true;2156 if (blk_1: {
2239 p.i = pos_0;2157 if (std.mem.startsWith(u8, p.source[p.i..], "\\\"")) {
2240 if (p.parsechar_escape()) break :blk_0 true;2158 p.i += 2;
2159 break :blk_1 true;
2160 }
2161 break :blk_1 false;
2162 }) break :blk_0 true;
2241 p.i = pos_0;2163 p.i = pos_0;
2242 if (blk_1: {2164 if (blk_1: {
2243 const pos_1 = p.i;2165 const pos_1 = p.i;
2244 const match_1 = (p.i < p.source.len and switch (p.source[p.i]) {2166 const match_1 = (p.i < p.source.len and switch (p.source[p.i]) {
2245 '\\'...'\\',
2246 '"'...'"',2167 '"'...'"',
2247 '\n'...'\n',
2248 => blk_2: {2168 => blk_2: {
2249 p.i += 1;2169 p.i += 1;
2250 break :blk_2 true;2170 break :blk_2 true;
...@@ -2253,7 +2173,7 @@ const Parser = struct {...@@ -2253,7 +2173,7 @@ const Parser = struct {
2253 });2173 });
2254 p.i = pos_1;2174 p.i = pos_1;
2255 break :blk_1 !match_1;2175 break :blk_1 !match_1;
2256 } and p.parsenon_control_ascii()) break :blk_0 true;2176 } and p.parsenon_control_utf8()) break :blk_0 true;
2257 p.i = pos_0;2177 p.i = pos_0;
2258 break :blk_0 false;2178 break :blk_0 false;
2259 };2179 };
...@@ -2443,7 +2363,10 @@ const Parser = struct {...@@ -2443,7 +2363,10 @@ const Parser = struct {
2443 break :blk_1 true;2363 break :blk_1 true;
2444 },2364 },
2445 else => false,2365 else => false,
2446 }) and p.parsechar_char() and (p.i < p.source.len and switch (p.source[p.i]) {2366 }) and blk_1: {
2367 while (p.parsechar_char()) {}
2368 break :blk_1 true;
2369 } and (p.i < p.source.len and switch (p.source[p.i]) {
2447 '\''...'\'',2370 '\''...'\'',
2448 => blk_1: {2371 => blk_1: {
2449 p.i += 1;2372 p.i += 1;