authorgravatar for kappaloris@gmail.comLoris Cro <kappaloris@gmail.com> 2022-03-02 18:36:44+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-07-19 19:10:11-07:00
log028c8a3c91286fe42d915cfe35976e3392895447
tree70e03b02ed28fd07ea7ae28c5ffa5f9fa789e96c
parent580e633777e55bcfad6f039f8acfa338d686368e

autodoc: added support for same-file lazy resolution of decl paths


1 files changed, 178 insertions(+), 70 deletions(-)

src/Autodoc.zig+178-70
......@@ -9,11 +9,24 @@ const Ref = Zir.Inst.Ref;
99module: *Module,
1010doc_location: Compilation.EmitLoc,
1111arena: std.mem.Allocator,
12files: std.AutoHashMapUnmanaged(*File, DocData.AutodocFile) = .{},
12files: std.AutoHashMapUnmanaged(*File, usize) = .{},
1313types: std.ArrayListUnmanaged(DocData.Type) = .{},
1414decls: std.ArrayListUnmanaged(DocData.Decl) = .{},
1515ast_nodes: std.ArrayListUnmanaged(DocData.AstNode) = .{},
16comptimeExprs: std.ArrayListUnmanaged(DocData.ComptimeExpr) = .{},
16comptime_exprs: std.ArrayListUnmanaged(DocData.ComptimeExpr) = .{},
17decl_paths_pending_on_decls: std.AutoHashMapUnmanaged(
18 usize,
19 std.ArrayListUnmanaged(DeclPathResumeInfo),
20) = .{},
21decl_paths_pending_on_types: std.AutoHashMapUnmanaged(
22 usize,
23 std.ArrayListUnmanaged(DeclPathResumeInfo),
24) = .{},
25
26const DeclPathResumeInfo = struct {
27 file: *File,
28 path: []usize,
29};
1730
1831var arena_allocator: std.heap.ArenaAllocator = undefined;
1932pub fn init(m: *Module, doc_location: Compilation.EmitLoc) Autodoc {
......@@ -129,22 +142,23 @@ pub fn generateZirData(self: *Autodoc) !void {
129142
130143 var root_scope = Scope{ .parent = null };
131144 try self.ast_nodes.append(self.arena, .{ .name = "(root)" });
132 try self.files.put(self.arena, file, .{
133 .analyzed = false,
134 .root_struct = self.types.items.len,
135 });
145 try self.files.put(self.arena, file, self.types.items.len);
136146 const main_type_index = try self.walkInstruction(file, &root_scope, Zir.main_struct_inst);
137 self.files.getPtr(file).?.analyzed = true;
138147
139 // TODO: solve every single pending declpath whose analysis
140 // was delayed because of circular imports.
148 if (self.decl_paths_pending_on_decls.count() > 0) {
149 @panic("some decl paths were never fully analized (pending on decls)");
150 }
151
152 if (self.decl_paths_pending_on_types.count() > 0) {
153 @panic("some decl paths were never fully analized (pending on types)");
154 }
141155
142156 var data = DocData{
143157 .files = .{ .data = self.files },
144158 .types = self.types.items,
145159 .decls = self.decls.items,
146160 .astNodes = self.ast_nodes.items,
147 .comptimeExprs = self.comptimeExprs.items,
161 .comptimeExprs = self.comptime_exprs.items,
148162 };
149163
150164 data.packages[0].main = main_type_index.type;
......@@ -232,7 +246,7 @@ const DocData = struct {
232246 astNodes: []AstNode,
233247 files: struct {
234248 // this struct is a temporary hack to support json serialization
235 data: std.AutoHashMapUnmanaged(*File, AutodocFile),
249 data: std.AutoHashMapUnmanaged(*File, usize),
236250 pub fn jsonStringify(
237251 self: @This(),
238252 opt: std.json.StringifyOptions,
......@@ -248,7 +262,7 @@ const DocData = struct {
248262 if (options.whitespace) |ws| try ws.outputIndent(w);
249263 try w.print("\"{s}\": {d}", .{
250264 kv.key_ptr.*.sub_file_path,
251 kv.value_ptr.root_struct,
265 kv.value_ptr.*,
252266 });
253267 if (idx != self.data.count() - 1) try w.writeByte(',');
254268 try w.writeByte('\n');
......@@ -261,17 +275,17 @@ const DocData = struct {
261275 decls: []Decl,
262276 comptimeExprs: []ComptimeExpr,
263277
264 const AutodocFile = struct {
265 analyzed: bool, // omitted in json data
266 root_struct: usize, // index into `types`
267 };
268
269278 const DocTypeKinds = blk: {
270279 var info = @typeInfo(std.builtin.TypeId);
271 info.Enum.fields = info.Enum.fields ++ [1]std.builtin.TypeInfo.EnumField{
280 const original_len = info.Enum.fields.len;
281 info.Enum.fields = info.Enum.fields ++ [2]std.builtin.TypeInfo.EnumField{
272282 .{
273283 .name = "ComptimeExpr",
274 .value = info.Enum.fields.len,
284 .value = original_len,
285 },
286 .{
287 .name = "Unanalyzed",
288 .value = original_len + 1,
275289 },
276290 };
277291 break :blk @Type(info);
......@@ -298,6 +312,7 @@ const DocData = struct {
298312 value: WalkResult,
299313 // The index in astNodes of the `test declname { }` node
300314 decltest: ?usize = null,
315 _analyzed: bool, // omitted in json data
301316 };
302317
303318 const AstNode = struct {
......@@ -310,6 +325,7 @@ const DocData = struct {
310325 };
311326
312327 const Type = union(DocTypeKinds) {
328 Unanalyzed: void,
313329 Type: struct { name: []const u8 },
314330 Void: struct { name: []const u8 },
315331 Bool: struct { name: []const u8 },
......@@ -480,7 +496,7 @@ const DocData = struct {
480496 @"struct": Struct,
481497 bool: bool,
482498 type: usize, // index in `types`
483 declPath: []usize, // indices in `decls`
499 declPath: []usize, // indices in `decl`
484500 int: struct {
485501 typeRef: TypeRef,
486502 value: usize, // direct value
......@@ -584,16 +600,12 @@ fn walkInstruction(
584600 // importFile cannot error out since all files
585601 // are already loaded at this point
586602 const new_file = self.module.importFile(file, path) catch unreachable;
587
588603 const result = try self.files.getOrPut(self.arena, new_file.file);
589604 if (result.found_existing) {
590 return DocData.WalkResult{ .type = result.value_ptr.root_struct };
605 return DocData.WalkResult{ .type = result.value_ptr.* };
591606 }
592607
593 result.value_ptr.* = .{
594 .analyzed = false,
595 .root_struct = self.types.items.len,
596 };
608 result.value_ptr.* = self.types.items.len;
597609
598610 var new_scope = Scope{ .parent = null };
599611 const new_file_walk_result = self.walkInstruction(
......@@ -601,14 +613,12 @@ fn walkInstruction(
601613 &new_scope,
602614 Zir.main_struct_inst,
603615 );
604 // We re-access the hashmap in case it was modified
605 // by walkInstruction()
606 self.files.getPtr(new_file.file).?.analyzed = true;
616
607617 return new_file_walk_result;
608618 },
609619 .block => {
610 const res = DocData.WalkResult{ .comptimeExpr = self.comptimeExprs.items.len };
611 try self.comptimeExprs.append(self.arena, .{
620 const res = DocData.WalkResult{ .comptimeExpr = self.comptime_exprs.items.len };
621 try self.comptime_exprs.append(self.arena, .{
612622 .code = "if(banana) 1 else 0",
613623 .typeRef = .{ .type = 0 },
614624 });
......@@ -669,7 +679,7 @@ fn walkInstruction(
669679 // TODO: Actually, this is a good moment to check if
670680 // the result is indeed a type!!
671681 .comptimeExpr => {
672 self.comptimeExprs.items[operand.comptimeExpr].typeRef = dest_type_ref;
682 self.comptime_exprs.items[operand.comptimeExpr].typeRef = dest_type_ref;
673683 },
674684 .int => operand.int.typeRef = dest_type_ref,
675685 .@"struct" => operand.@"struct".typeRef = dest_type_ref,
......@@ -731,35 +741,7 @@ fn walkInstruction(
731741 // the analyzed data corresponding to the top-most decl of this path.
732742 // We are now going to reverse loop over `path` to resolve each name
733743 // to its corresponding index in `decls`.
734
735 var i: usize = path.items.len;
736 while (i > 1) {
737 i -= 1;
738 const parent = self.decls.items[path.items[i]];
739 const child_decl_name = file.zir.nullTerminatedString(path.items[i - 1]);
740 switch (parent.value) {
741 else => {
742 std.debug.print(
743 "TODO: handle `{s}`in walkInstruction.field_val\n",
744 .{@tagName(parent.value)},
745 );
746 unreachable;
747 },
748 .type => |t_index| {
749 const t_struct = self.types.items[t_index].Struct; // todo: support more types
750 for (t_struct.pubDecls) |d| {
751 // TODO: this could be improved a lot
752 // by having our own string table!
753 const decl = self.decls.items[d];
754 if (std.mem.eql(u8, decl.name, child_decl_name)) {
755 path.items[i - 1] = d;
756 continue;
757 }
758 }
759 },
760 }
761 }
762
744 try self.tryResolveDeclPath(file, path.items);
763745 return DocData.WalkResult{ .declPath = path.items };
764746 },
765747 .int_type => {
......@@ -841,6 +823,29 @@ fn walkInstruction(
841823 return DocData.WalkResult{ .type = self.types.items.len - 1 };
842824 },
843825 .extended => {
826 // TODO: this assumes that we always return a type when analyzing
827 // an extended instruction. Also we willingfully not reserve
828 // a slot for functions (handled right above) despite them
829 // being stored in `types`. The reason why we reserve a slot
830 // in here, is for decl paths and their resolution system.
831 const type_slot_index = self.types.items.len;
832 try self.types.append(self.arena, .{ .Unanalyzed = {} });
833
834 defer {
835 if (self.decl_paths_pending_on_types.get(type_slot_index)) |paths| {
836 for (paths.items) |resume_info| {
837 self.tryResolveDeclPath(resume_info.file, resume_info.path) catch {
838 @panic("Out of memory");
839 };
840 }
841
842 _ = self.decl_paths_pending_on_types.remove(type_slot_index);
843 // TODO: we should deallocate the arraylist that holds all the
844 // decl paths. not doing it now since it's arena-allocated
845 // anyway, but maybe we should put it elsewhere.
846 }
847 }
848
844849 const extended = data[inst_index].extended;
845850 switch (extended.opcode) {
846851 else => {
......@@ -898,6 +903,9 @@ fn walkInstruction(
898903 {
899904 var it = file.zir.declIterator(@intCast(u32, inst_index));
900905 try self.decls.resize(self.arena, decls_first_index + it.decls_len);
906 for (self.decls.items[decls_first_index..]) |*slot| {
907 slot._analyzed = false;
908 }
901909 var decls_slot_index = decls_first_index;
902910 while (it.next()) |d| : (decls_slot_index += 1) {
903911 const decl_name_index = file.zir.extra[d.sub_index + 5];
......@@ -937,7 +945,7 @@ fn walkInstruction(
937945
938946 self.ast_nodes.items[self_ast_node_index].fields = field_name_indexes.items;
939947
940 try self.types.append(self.arena, .{
948 self.types.items[type_slot_index] = .{
941949 .Union = .{
942950 .name = "todo_name",
943951 .src = self_ast_node_index,
......@@ -945,9 +953,9 @@ fn walkInstruction(
945953 .pubDecls = decl_indexes.items,
946954 .fields = field_type_refs.items,
947955 },
948 });
956 };
949957
950 return DocData.WalkResult{ .type = self.types.items.len - 1 };
958 return DocData.WalkResult{ .type = type_slot_index };
951959 },
952960 .enum_decl => {
953961 var scope: Scope = .{ .parent = parent_scope };
......@@ -998,6 +1006,9 @@ fn walkInstruction(
9981006 {
9991007 var it = file.zir.declIterator(@intCast(u32, inst_index));
10001008 try self.decls.resize(self.arena, decls_first_index + it.decls_len);
1009 for (self.decls.items[decls_first_index..]) |*slot| {
1010 slot._analyzed = false;
1011 }
10011012 var decls_slot_index = decls_first_index;
10021013 while (it.next()) |d| : (decls_slot_index += 1) {
10031014 const decl_name_index = file.zir.extra[d.sub_index + 5];
......@@ -1063,16 +1074,16 @@ fn walkInstruction(
10631074
10641075 self.ast_nodes.items[self_ast_node_index].fields = field_name_indexes.items;
10651076
1066 try self.types.append(self.arena, .{
1077 self.types.items[type_slot_index] = .{
10671078 .Enum = .{
10681079 .name = "todo_name",
10691080 .src = self_ast_node_index,
10701081 .privDecls = priv_decl_indexes.items,
10711082 .pubDecls = decl_indexes.items,
10721083 },
1073 });
1084 };
10741085
1075 return DocData.WalkResult{ .type = self.types.items.len - 1 };
1086 return DocData.WalkResult{ .type = type_slot_index };
10761087 },
10771088 .struct_decl => {
10781089 var scope: Scope = .{ .parent = parent_scope };
......@@ -1116,6 +1127,9 @@ fn walkInstruction(
11161127 {
11171128 var it = file.zir.declIterator(@intCast(u32, inst_index));
11181129 try self.decls.resize(self.arena, decls_first_index + it.decls_len);
1130 for (self.decls.items[decls_first_index..]) |*slot| {
1131 slot._analyzed = false;
1132 }
11191133 var decls_slot_index = decls_first_index;
11201134 while (it.next()) |d| : (decls_slot_index += 1) {
11211135 const decl_name_index = file.zir.extra[d.sub_index + 5];
......@@ -1149,7 +1163,7 @@ fn walkInstruction(
11491163
11501164 self.ast_nodes.items[self_ast_node_index].fields = field_name_indexes.items;
11511165
1152 try self.types.append(self.arena, .{
1166 self.types.items[type_slot_index] = .{
11531167 .Struct = .{
11541168 .name = "todo_name",
11551169 .src = self_ast_node_index,
......@@ -1157,9 +1171,9 @@ fn walkInstruction(
11571171 .pubDecls = decl_indexes.items,
11581172 .fields = field_type_refs.items,
11591173 },
1160 });
1174 };
11611175
1162 return DocData.WalkResult{ .type = self.types.items.len - 1 };
1176 return DocData.WalkResult{ .type = type_slot_index };
11631177 },
11641178 }
11651179 },
......@@ -1310,6 +1324,7 @@ fn walkDecls(
13101324 };
13111325 self.decls.items[decl_being_tested].decltest = ast_node_index;
13121326 self.decls.items[decls_slot_index] = .{
1327 ._analyzed = true,
13131328 .name = "test",
13141329 .src = ast_node_index,
13151330 .value = .{ .type = 0 },
......@@ -1368,17 +1383,110 @@ fn walkDecls(
13681383 // };
13691384
13701385 self.decls.items[decls_slot_index] = .{
1386 ._analyzed = true,
13711387 .name = name,
13721388 .src = ast_node_index,
13731389 // .typeRef = decl_type_ref,
13741390 .value = walk_result,
13751391 .kind = "const", // find where this information can be found
13761392 };
1393
1394 // Unblock any pending decl path that was waiting for this decl.
1395 if (self.decl_paths_pending_on_decls.get(decls_slot_index)) |paths| {
1396 for (paths.items) |resume_info| {
1397 try self.tryResolveDeclPath(resume_info.file, resume_info.path);
1398 }
1399
1400 _ = self.decl_paths_pending_on_decls.remove(decls_slot_index);
1401 // TODO: we should deallocate the arraylist that holds all the
1402 // decl paths. not doing it now since it's arena-allocated
1403 // anyway, but maybe we should put it elsewhere.
1404 }
13771405 }
13781406
13791407 return extra_index;
13801408}
13811409
1410/// An unresolved path has a decl index at its end, while every other element
1411/// is an index into the string table. Resolving means resolving iteratively
1412/// each string into a decl_index. If we encounter an unanalyzed decl during
1413/// the process, we append the unsolved sub-path to `self.decl_paths_pending_on_decls`
1414/// and bail out.
1415fn tryResolveDeclPath(
1416 self: *Autodoc,
1417 /// File from which the decl path originates.
1418 file: *File,
1419 path: []usize,
1420) !void {
1421 var i: usize = path.len;
1422 while (i > 1) {
1423 i -= 1;
1424 const decl_index = path[i];
1425 const string_index = path[i - 1];
1426
1427 const parent = self.decls.items[decl_index];
1428 if (!parent._analyzed) {
1429 const res = try self.decl_paths_pending_on_decls.getOrPut(self.arena, decl_index);
1430 if (!res.found_existing) res.value_ptr.* = .{};
1431 try res.value_ptr.*.append(self.arena, .{
1432 .file = file,
1433 .path = path[0 .. i + 1],
1434 });
1435 return;
1436 }
1437
1438 const child_decl_name = file.zir.nullTerminatedString(string_index);
1439 switch (parent.value) {
1440 else => {
1441 std.debug.panic(
1442 "TODO: handle `{s}`in walkInstruction.field_val\n",
1443 .{@tagName(parent.value)},
1444 );
1445 },
1446 .type => |t_index| switch (self.types.items[t_index]) {
1447 else => {
1448 std.debug.panic(
1449 "TODO: handle `{s}` in tryResolveDeclPath.type\n",
1450 .{@tagName(self.types.items[t_index])},
1451 );
1452 },
1453 .Unanalyzed => {
1454 const res = try self.decl_paths_pending_on_types.getOrPut(
1455 self.arena,
1456 t_index,
1457 );
1458 if (!res.found_existing) res.value_ptr.* = .{};
1459 try res.value_ptr.*.append(self.arena, .{
1460 .file = file,
1461 .path = path[0 .. i + 1],
1462 });
1463 return;
1464 },
1465 .Struct => |t_struct| {
1466 for (t_struct.pubDecls) |d| {
1467 // TODO: this could be improved a lot
1468 // by having our own string table!
1469 const decl = self.decls.items[d];
1470 if (std.mem.eql(u8, decl.name, child_decl_name)) {
1471 path[i - 1] = d;
1472 continue;
1473 }
1474 }
1475 for (t_struct.privDecls) |d| {
1476 // TODO: this could be improved a lot
1477 // by having our own string table!
1478 const decl = self.decls.items[d];
1479 if (std.mem.eql(u8, decl.name, child_decl_name)) {
1480 path[i - 1] = d;
1481 continue;
1482 }
1483 }
1484 },
1485 },
1486 }
1487 }
1488}
1489
13821490fn collectUnionFieldInfo(
13831491 self: *Autodoc,
13841492 file: *File,