authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-25 12:50:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-25 12:58:21-07:00
log73f77f30804f3a512fbb35caad3d87ff2778ecc0
treebcb3f80f6faea5847b83e6ed066c2fc3c4679306
parent49b3986417378278cef4157e71c38a817726ddc8

AstGen: fix O(N^2) perf for many decls with same parent

AstGen was calling findLineColumn() for every sibling Decl, using the parent Decl as the starting point for the search for newlines. This resulted in poor performance for large numbers of Decls with the same parent. The solution is simple: since AstGen progresses monotonically through the AST, keep a single cursor into the source file, and whenever line/column information is needed, advance the cursor. This guarantees O(N) on the number of bytes in the file. Perf: As an example I ran ast-check on zigwin32/win32/everything.zig (a 17 MiB file) in master branch, and after this commit. With master branch, I killed the process after 17 seconds out of boredom. With this commit, it completed in 300 milliseconds. Closes #9234

1 files changed, 50 insertions(+), 18 deletions(-)

src/AstGen.zig+50-18
...@@ -17,6 +17,16 @@ tree: *const ast.Tree,...@@ -17,6 +17,16 @@ tree: *const ast.Tree,
17instructions: std.MultiArrayList(Zir.Inst) = .{},17instructions: std.MultiArrayList(Zir.Inst) = .{},
18extra: ArrayListUnmanaged(u32) = .{},18extra: ArrayListUnmanaged(u32) = .{},
19string_bytes: ArrayListUnmanaged(u8) = .{},19string_bytes: ArrayListUnmanaged(u8) = .{},
20/// Tracks the current byte offset within the source file.
21/// Used to populate line deltas in the ZIR. AstGen maintains
22/// this "cursor" throughout the entire AST lowering process in order
23/// to avoid starting over the line/column scan for every declaration, which
24/// would be O(N^2).
25source_offset: u32 = 0,
26/// Tracks the current line of `source_offset`.
27source_line: u32 = 0,
28/// Tracks the current column of `source_offset`.
29source_column: u32 = 0,
20/// Used for temporary allocations; freed after AstGen is complete.30/// Used for temporary allocations; freed after AstGen is complete.
21/// The resulting ZIR code has no references to anything in this arena.31/// The resulting ZIR code has no references to anything in this arena.
22arena: *Allocator,32arena: *Allocator,
...@@ -2472,15 +2482,18 @@ fn emitDbgNode(gz: *GenZir, node: ast.Node.Index) !void {...@@ -2472,15 +2482,18 @@ fn emitDbgNode(gz: *GenZir, node: ast.Node.Index) !void {
24722482
2473 const astgen = gz.astgen;2483 const astgen = gz.astgen;
2474 const tree = astgen.tree;2484 const tree = astgen.tree;
2485 const source = tree.source;
2475 const token_starts = tree.tokens.items(.start);2486 const token_starts = tree.tokens.items(.start);
2476 const decl_start = token_starts[tree.firstToken(gz.decl_node_index)];
2477 const node_start = token_starts[tree.firstToken(node)];2487 const node_start = token_starts[tree.firstToken(node)];
2478 const source = tree.source[decl_start..node_start];2488
2479 const loc = std.zig.findLineColumn(source, source.len);2489 astgen.advanceSourceCursor(source, node_start);
2490 const line = @intCast(u32, astgen.source_line);
2491 const column = @intCast(u32, astgen.source_column);
2492
2480 _ = try gz.add(.{ .tag = .dbg_stmt, .data = .{2493 _ = try gz.add(.{ .tag = .dbg_stmt, .data = .{
2481 .dbg_stmt = .{2494 .dbg_stmt = .{
2482 .line = @intCast(u32, loc.line),2495 .line = line,
2483 .column = @intCast(u32, loc.column),2496 .column = column,
2484 },2497 },
2485 } });2498 } });
2486}2499}
...@@ -8564,12 +8577,13 @@ const GenZir = struct {...@@ -8564,12 +8577,13 @@ const GenZir = struct {
8564 fn calcLine(gz: GenZir, node: ast.Node.Index) u32 {8577 fn calcLine(gz: GenZir, node: ast.Node.Index) u32 {
8565 const astgen = gz.astgen;8578 const astgen = gz.astgen;
8566 const tree = astgen.tree;8579 const tree = astgen.tree;
8580 const source = tree.source;
8567 const token_starts = tree.tokens.items(.start);8581 const token_starts = tree.tokens.items(.start);
8568 const decl_start = token_starts[tree.firstToken(gz.decl_node_index)];
8569 const node_start = token_starts[tree.firstToken(node)];8582 const node_start = token_starts[tree.firstToken(node)];
8570 const source = tree.source[decl_start..node_start];8583
8571 const loc = std.zig.findLineColumn(source, source.len);8584 astgen.advanceSourceCursor(source, node_start);
8572 return @intCast(u32, gz.decl_line + loc.line);8585
8586 return @intCast(u32, gz.decl_line + astgen.source_line);
8573 }8587 }
85748588
8575 fn tokSrcLoc(gz: GenZir, token_index: ast.TokenIndex) LazySrcLoc {8589 fn tokSrcLoc(gz: GenZir, token_index: ast.TokenIndex) LazySrcLoc {
...@@ -8713,20 +8727,21 @@ const GenZir = struct {...@@ -8713,20 +8727,21 @@ const GenZir = struct {
8713 const node_tags = tree.nodes.items(.tag);8727 const node_tags = tree.nodes.items(.tag);
8714 const node_datas = tree.nodes.items(.data);8728 const node_datas = tree.nodes.items(.data);
8715 const token_starts = tree.tokens.items(.start);8729 const token_starts = tree.tokens.items(.start);
8716 const decl_start = token_starts[tree.firstToken(gz.decl_node_index)];
8717 const fn_decl = args.src_node;8730 const fn_decl = args.src_node;
8718 assert(node_tags[fn_decl] == .fn_decl or node_tags[fn_decl] == .test_decl);8731 assert(node_tags[fn_decl] == .fn_decl or node_tags[fn_decl] == .test_decl);
8719 const block = node_datas[fn_decl].rhs;8732 const block = node_datas[fn_decl].rhs;
8720 const lbrace_start = token_starts[tree.firstToken(block)];8733 const lbrace_start = token_starts[tree.firstToken(block)];
8721 const rbrace_start = token_starts[tree.lastToken(block)];8734 const rbrace_start = token_starts[tree.lastToken(block)];
8722 const lbrace_source = tree.source[decl_start..lbrace_start];8735
8723 const lbrace_loc = std.zig.findLineColumn(lbrace_source, lbrace_source.len);8736 astgen.advanceSourceCursor(tree.source, lbrace_start);
8724 const rbrace_source = tree.source[lbrace_start..rbrace_start];8737 const lbrace_line = @intCast(u32, astgen.source_line);
8725 const rbrace_loc = std.zig.findLineColumn(rbrace_source, rbrace_source.len);8738 const lbrace_column = @intCast(u32, astgen.source_column);
8726 const lbrace_line = @intCast(u32, lbrace_loc.line);8739
8727 const rbrace_line = lbrace_line + @intCast(u32, rbrace_loc.line);8740 astgen.advanceSourceCursor(tree.source, rbrace_start);
8728 const columns = @intCast(u32, lbrace_loc.column) |8741 const rbrace_line = @intCast(u32, astgen.source_line);
8729 (@intCast(u32, rbrace_loc.column) << 16);8742 const rbrace_column = @intCast(u32, astgen.source_column);
8743
8744 const columns = lbrace_column | (rbrace_column << 16);
8730 src_locs_buffer[0] = lbrace_line;8745 src_locs_buffer[0] = lbrace_line;
8731 src_locs_buffer[1] = rbrace_line;8746 src_locs_buffer[1] = rbrace_line;
8732 src_locs_buffer[2] = columns;8747 src_locs_buffer[2] = columns;
...@@ -9542,3 +9557,20 @@ fn declareNewName(...@@ -9542,3 +9557,20 @@ fn declareNewName(
9542 }9557 }
9543 }9558 }
9544}9559}
9560
9561fn advanceSourceCursor(astgen: *AstGen, source: []const u8, end: usize) void {
9562 var i = astgen.source_offset;
9563 var line = astgen.source_line;
9564 var column = astgen.source_column;
9565 while (i < end) : (i += 1) {
9566 if (source[i] == '\n') {
9567 line += 1;
9568 column = 0;
9569 } else {
9570 column += 1;
9571 }
9572 }
9573 astgen.source_offset = i;
9574 astgen.source_line = line;
9575 astgen.source_column = column;
9576}