| ... | @@ -7279,83 +7279,61 @@ test "zig fmt: fuzz" { | ... | @@ -7279,83 +7279,61 @@ test "zig fmt: fuzz" { |
| 7279 | try std.testing.fuzz({}, fuzzRender, .{}); | 7279 | try std.testing.fuzz({}, fuzzRender, .{}); |
| 7280 | } | 7280 | } |
| 7281 | | 7281 | |
| 7282 | fn parseTokens( | 7282 | fn isRewritable(source: []const u8, tokens: std.zig.Ast.TokenList.Slice) !bool { |
| 7283 | fba: Allocator, | | |
| 7284 | source: [:0]const u8, | | |
| 7285 | ) error{ SkipZigTest, OutOfMemory }!struct { | | |
| 7286 | toks: std.zig.Ast.TokenList, | | |
| 7287 | maybe_rewritable: bool, | | |
| 7288 | skip_idempotency: bool, | | |
| 7289 | } { | | |
| 7290 | @disableInstrumentation(); | 7283 | @disableInstrumentation(); |
| | 7284 | |
| 7291 | // Byte-order marker is stripped | 7285 | // Byte-order marker is stripped |
| 7292 | var maybe_rewritable = std.mem.startsWith(u8, source, "\xEF\xBB\xBF"); | 7286 | var maybe_rewritable = std.mem.startsWith(u8, source, "\xEF\xBB\xBF"); |
| 7293 | var skip_idempotency = false; // This should be able to be removed once all the bugs are fixed | 7287 | // The above variable can not yet be replaced by returns since error.SkipZigTest still needs to |
| 7294 | | 7288 | // be checked for. |
| 7295 | var tokens: std.zig.Ast.TokenList = .{}; | 7289 | |
| 7296 | try tokens.ensureTotalCapacity(fba, source.len / 2); | 7290 | for (0.., tokens.items(.tag), tokens.items(.start)) |i, tag, start| switch (tag) { |
| 7297 | var tokenizer: std.zig.Tokenizer = .init(source); | 7291 | // Extra colons can be removed |
| 7298 | while (true) { | 7292 | .keyword_asm, |
| 7299 | const tok = tokenizer.next(); | 7293 | // Qualifiers can be reordered |
| 7300 | switch (tok.tag) { | 7294 | // keyword_const is intentionally excluded since it is used in other contexts and |
| 7301 | .invalid, | 7295 | // having only one qualifier will never lead to reordering. |
| 7302 | .invalid_periodasterisks, | 7296 | .keyword_addrspace, |
| 7303 | => return error.SkipZigTest, | 7297 | .keyword_align, |
| 7304 | // Extra colons can be removed | 7298 | .keyword_allowzero, |
| 7305 | .keyword_asm, | 7299 | .keyword_callconv, |
| 7306 | // Qualifiers can be reordered | 7300 | .keyword_linksection, |
| 7307 | // keyword_const is intentionally excluded since it is used in other contexts and | 7301 | .keyword_volatile, |
| 7308 | // having only one qualifier will never lead to reordering. | 7302 | => maybe_rewritable = true, |
| 7309 | .keyword_addrspace, | 7303 | .builtin, |
| 7310 | .keyword_align, | 7304 | // Pointer casts can be reordered |
| 7311 | .keyword_allowzero, | 7305 | => for ([_][]const u8{ |
| 7312 | .keyword_callconv, | 7306 | "ptrCast", |
| 7313 | .keyword_linksection, | 7307 | "alignCast", |
| 7314 | .keyword_volatile, | 7308 | "addrSpaceCast", |
| 7315 | => maybe_rewritable = true, | 7309 | "constCast", |
| 7316 | .builtin, | 7310 | "volatileCast", |
| 7317 | // Pointer casts can be reordered | 7311 | }) |id| { |
| 7318 | => for ([_][]const u8{ | 7312 | if (std.mem.startsWith(u8, source[start + 1 ..], id)) { |
| 7319 | "ptrCast", | | |
| 7320 | "alignCast", | | |
| 7321 | "addrSpaceCast", | | |
| 7322 | "constCast", | | |
| 7323 | "volatileCast", | | |
| 7324 | }) |id| { | | |
| 7325 | if (std.mem.eql(u8, source[tok.loc.start + 1 .. tok.loc.end], id)) { | | |
| 7326 | maybe_rewritable = false; | | |
| 7327 | } | | |
| 7328 | }, | | |
| 7329 | // Quoted identifiers can be unquoted | | |
| 7330 | .identifier => maybe_rewritable = maybe_rewritable or source[tok.loc.start] == '@', | | |
| 7331 | else => {}, | | |
| 7332 | // #23754 | | |
| 7333 | .container_doc_comment, | | |
| 7334 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag), &.{.l_brace})) { | | |
| 7335 | return error.SkipZigTest; | | |
| 7336 | }, | | |
| 7337 | // #24507 | | |
| 7338 | .keyword_inline, | | |
| 7339 | .keyword_for, | | |
| 7340 | .keyword_while, | | |
| 7341 | .l_brace, | | |
| 7342 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag), &.{ .identifier, .colon })) { | | |
| 7343 | maybe_rewritable = true; | 7313 | maybe_rewritable = true; |
| 7344 | skip_idempotency = true; | 7314 | } |
| 7345 | }, | 7315 | }, |
| 7346 | } | 7316 | // Quoted identifiers can be unquoted |
| 7347 | try tokens.append(fba, .{ | 7317 | .identifier => if (source[start] == '@') { |
| 7348 | .tag = tok.tag, | 7318 | maybe_rewritable = true; |
| 7349 | .start = @intCast(tok.loc.start), | 7319 | }, |
| 7350 | }); | 7320 | else => {}, |
| 7351 | if (tok.tag == .eof) | 7321 | // #23754 |
| 7352 | break; | 7322 | .container_doc_comment, |
| 7353 | } | 7323 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag)[0..i], &.{.l_brace})) { |
| 7354 | return .{ | 7324 | return error.SkipZigTest; // Can cause I.B. |
| 7355 | .toks = tokens, | 7325 | }, |
| 7356 | .maybe_rewritable = maybe_rewritable, | 7326 | // #24507 |
| 7357 | .skip_idempotency = skip_idempotency, | 7327 | .keyword_inline, |
| | 7328 | .keyword_for, |
| | 7329 | .keyword_while, |
| | 7330 | .l_brace, |
| | 7331 | => if (std.mem.endsWith(Token.Tag, tokens.items(.tag)[0..i], &.{ .identifier, .colon })) { |
| | 7332 | return error.SkipZigTest; // Can cause I.B. due to double rendering of zig fmt on/off |
| | 7333 | }, |
| 7358 | }; | 7334 | }; |
| | 7335 | |
| | 7336 | return maybe_rewritable; |
| 7359 | } | 7337 | } |
| 7360 | | 7338 | |
| 7361 | /// Checks equivelence of non-whitespace characters. | 7339 | /// Checks equivelence of non-whitespace characters. |
| ... | @@ -7468,34 +7446,29 @@ fn reparseTokens( | ... | @@ -7468,34 +7446,29 @@ fn reparseTokens( |
| 7468 | fn fuzzRender(_: void, smith: *std.testing.Smith) !void { | 7446 | fn fuzzRender(_: void, smith: *std.testing.Smith) !void { |
| 7469 | @disableInstrumentation(); | 7447 | @disableInstrumentation(); |
| 7470 | | 7448 | |
| 7471 | var src_buf: [512]u8 = undefined; | 7449 | var ast_smith: std.zig.AstSmith = .init(smith); |
| 7472 | const src_len = smith.sliceWeighted(&src_buf, &.{ | 7450 | try ast_smith.generateSource(); |
| 7473 | .rangeLessThan(u32, 0, 32, 256), | | |
| 7474 | .rangeLessThan(u32, 32, 64, 64), | | |
| 7475 | .rangeLessThan(u32, 64, src_buf.len, 1), | | |
| 7476 | }, &.{ | | |
| 7477 | .rangeAtMost(u8, 0x20, 0x7e, 8), | | |
| 7478 | .value(u8, '\n', 32), | | |
| 7479 | .value(u8, '\t', 8), | | |
| 7480 | .value(u8, '\r', 4), | | |
| 7481 | .rangeAtMost(u8, 0x7f, 0xff, 1), | | |
| 7482 | }); | | |
| 7483 | src_buf[src_len] = 0; | | |
| 7484 | | | |
| 7485 | var fba_ctx = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem); | 7451 | var fba_ctx = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem); |
| 7486 | fuzzRenderInner(src_buf[0..src_len :0], fba_ctx.allocator()) catch |e| return switch (e) { | 7452 | var opt_rendered: ?[]const u8 = null; |
| 7487 | error.OutOfMemory => {}, | 7453 | fuzzRenderInner(&ast_smith, fba_ctx.allocator(), &opt_rendered) catch |e| switch (e) { |
| 7488 | else => e, | 7454 | error.SkipZigTest, error.OutOfMemory, error.WriteFailed => return error.SkipZigTest, |
| | 7455 | else => |failure| { |
| | 7456 | ast_smith.logSource(); |
| | 7457 | if (opt_rendered) |rendered| { |
| | 7458 | logRenderedSource(rendered); |
| | 7459 | } |
| | 7460 | return failure; |
| | 7461 | }, |
| 7489 | }; | 7462 | }; |
| 7490 | } | 7463 | } |
| 7491 | | 7464 | |
| 7492 | fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { | 7465 | fn fuzzRenderInner(ast_smith: *std.zig.AstSmith, fba: Allocator, opt_rendered: *?[]const u8) !void { |
| 7493 | @disableInstrumentation(); | 7466 | @disableInstrumentation(); |
| 7494 | | 7467 | |
| 7495 | const src_toks = try parseTokens(fba, source); | 7468 | const source = ast_smith.source(); |
| 7496 | const src_tree = try std.zig.Ast.parseTokens(fba, source, src_toks.toks.slice(), .zig); | 7469 | const src_rewritable = try isRewritable(source, ast_smith.tokens()); |
| 7497 | if (src_tree.errors.len != 0) | 7470 | const src_tree = try std.zig.Ast.parseTokens(fba, source, ast_smith.tokens(), .zig); |
| 7498 | return; | 7471 | std.debug.assert(src_tree.errors.len == 0); |
| 7499 | for (src_tree.nodes.items(.tag)) |tag| switch (tag) { | 7472 | for (src_tree.nodes.items(.tag)) |tag| switch (tag) { |
| 7500 | // #24507 (`switch(x) { inline for (a) |a| a => {} }` to | 7473 | // #24507 (`switch(x) { inline for (a) |a| a => {} }` to |
| 7501 | // `switch(x) { { inline for (a) |a| a => {} }` since | 7474 | // `switch(x) { { inline for (a) |a| a => {} }` since |
| ... | @@ -7511,15 +7484,16 @@ fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { | ... | @@ -7511,15 +7484,16 @@ fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { |
| 7511 | // list to save space which is useless for fixed buffer allocators. | 7484 | // list to save space which is useless for fixed buffer allocators. |
| 7512 | try rendered_w.writer.writeByte(0); | 7485 | try rendered_w.writer.writeByte(0); |
| 7513 | const rendered = rendered_w.written()[0 .. rendered_w.written().len - 1 :0]; | 7486 | const rendered = rendered_w.written()[0 .. rendered_w.written().len - 1 :0]; |
| | 7487 | opt_rendered.* = rendered; |
| 7514 | | 7488 | |
| 7515 | // First check that the non-whitespace characters match. This ensures that | 7489 | // First check that the non-whitespace characters match. This ensures that |
| 7516 | // identifier names, numbers, comments, et cetera are preserved. | 7490 | // identifier names, numbers, comments, et cetera are preserved. |
| 7517 | if (!src_toks.maybe_rewritable and isRewritten(source, rendered)) | 7491 | if (!src_rewritable and isRewritten(source, rendered)) |
| 7518 | return error.Rewritten; | 7492 | return error.Rewritten; |
| 7519 | // Next check that the tokens are the same since whitespace removal can change the tokens | 7493 | // Next check that the tokens are the same since whitespace removal can change the tokens |
| 7520 | const src_tags = src_toks.toks.items(.tag); | 7494 | const src_tags = ast_smith.tokens().items(.tag); |
| 7521 | const rendered_toks = try reparseTokens(fba, rendered, src_tags[0 .. src_tags.len - 1 :.eof]); | 7495 | const rendered_toks = try reparseTokens(fba, rendered, src_tags[0 .. src_tags.len - 1 :.eof]); |
| 7522 | if (!src_toks.maybe_rewritable and rendered_toks.rewritten) | 7496 | if (!src_rewritable and rendered_toks.rewritten) |
| 7523 | return error.Rewritten; | 7497 | return error.Rewritten; |
| 7524 | | 7498 | |
| 7525 | // Rerender the tree to check idempotency and that new commas | 7499 | // Rerender the tree to check idempotency and that new commas |
| ... | @@ -7527,10 +7501,39 @@ fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { | ... | @@ -7527,10 +7501,39 @@ fn fuzzRenderInner(source: [:0]const u8, fba: Allocator) !void { |
| 7527 | const rendered_tree = try std.zig.Ast.parseTokens(fba, rendered, rendered_toks.toks.slice(), .zig); | 7501 | const rendered_tree = try std.zig.Ast.parseTokens(fba, rendered, rendered_toks.toks.slice(), .zig); |
| 7528 | if (rendered_tree.errors.len != 0) | 7502 | if (rendered_tree.errors.len != 0) |
| 7529 | return error.Rewritten; | 7503 | return error.Rewritten; |
| 7530 | if (!src_toks.skip_idempotency) { | 7504 | var rerendered_w: std.Io.Writer.Allocating = .init(fba); |
| 7531 | var rerendered_w: std.Io.Writer.Allocating = .init(fba); | 7505 | try rerendered_w.ensureUnusedCapacity(source.len); |
| 7532 | try rerendered_w.ensureUnusedCapacity(source.len); | 7506 | try rendered_tree.render(fba, &rerendered_w.writer, .{}); |
| 7533 | try rendered_tree.render(fba, &rerendered_w.writer, .{}); | 7507 | try std.testing.expectEqualStrings(rendered, rerendered_w.written()); |
| 7534 | try std.testing.expectEqualStrings(rendered, rerendered_w.written()); | 7508 | } |
| 7535 | } | 7509 | |
| | 7510 | fn logRenderedSource(source: []const u8) void { |
| | 7511 | var buf: [256]u8 = undefined; |
| | 7512 | const ls = std.debug.lockStderr(&buf); |
| | 7513 | defer std.debug.unlockStderr(); |
| | 7514 | logRenderedSourceInner(source, ls.terminal()) catch {}; |
| | 7515 | } |
| | 7516 | |
| | 7517 | fn logRenderedSourceInner(source: []const u8, t: std.Io.Terminal) std.Io.Writer.Error!void { |
| | 7518 | const w = t.writer; |
| | 7519 | |
| | 7520 | t.setColor(.dim) catch {}; |
| | 7521 | try w.writeAll("=== Rendered Source ===\n"); |
| | 7522 | t.setColor(.reset) catch {}; |
| | 7523 | |
| | 7524 | for (0.., source) |i, c| switch (c) { |
| | 7525 | ' '...0x7e => try w.writeByte(c), |
| | 7526 | '\n' => { |
| | 7527 | if (i != 0 and source[i - 1] == ' ') { |
| | 7528 | try w.writeAll("⏎"); |
| | 7529 | } |
| | 7530 | try w.writeByte('\n'); |
| | 7531 | }, |
| | 7532 | else => { |
| | 7533 | t.setColor(.cyan) catch {}; |
| | 7534 | try w.print("\\x{x:0>2}", .{c}); |
| | 7535 | t.setColor(.reset) catch {}; |
| | 7536 | }, |
| | 7537 | }; |
| | 7538 | try w.writeAll("␃\n"); |
| 7536 | } | 7539 | } |
Kendall Condon <goon.pri.low@gmail.com> 2026-03-22 17:19:41-04:00
lib