authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-31 23:45:43-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-07 00:48:32-07:00
log107b27276602d1935a90ae97c57f640b090088b5
tree4606427ffab08b476973028265ee9d6c967304c6
parente0ffac4e3c2271a617616760f3084f5f01fb0785

fuzzer: share zig to html rendering with autodocs

9 files changed, 598 insertions(+), 434 deletions(-)

lib/compiler/std-docs.zig+4-5
......@@ -275,10 +275,6 @@ fn buildWasmBinary(
275275) ![]const u8 {
276276 const gpa = context.gpa;
277277
278 const main_src_path = try std.fs.path.join(arena, &.{
279 context.zig_lib_directory, "docs", "wasm", "main.zig",
280 });
281
282278 var argv: std.ArrayListUnmanaged([]const u8) = .{};
283279
284280 try argv.appendSlice(arena, &.{
......@@ -298,7 +294,10 @@ fn buildWasmBinary(
298294 "--name",
299295 "autodoc",
300296 "-rdynamic",
301 main_src_path,
297 "--dep",
298 "Walk",
299 try std.fmt.allocPrint(arena, "-Mroot={s}/docs/wasm/main.zig", .{context.zig_lib_directory}),
300 try std.fmt.allocPrint(arena, "-MWalk={s}/docs/wasm/Walk.zig", .{context.zig_lib_directory}),
302301 "--listen=-",
303302 });
304303
lib/docs/wasm/Decl.zig+9-9
......@@ -1,3 +1,12 @@
1const Decl = @This();
2const std = @import("std");
3const Ast = std.zig.Ast;
4const Walk = @import("Walk.zig");
5const gpa = std.heap.wasm_allocator;
6const assert = std.debug.assert;
7const log = std.log;
8const Oom = error{OutOfMemory};
9
110ast_node: Ast.Node.Index,
211file: Walk.File.Index,
312/// The decl whose namespace this is in.
......@@ -215,12 +224,3 @@ pub fn find(search_string: []const u8) Decl.Index {
215224 }
216225 return current_decl_index;
217226}
218
219const Decl = @This();
220const std = @import("std");
221const Ast = std.zig.Ast;
222const Walk = @import("Walk.zig");
223const gpa = std.heap.wasm_allocator;
224const assert = std.debug.assert;
225const log = std.log;
226const Oom = error{OutOfMemory};
lib/docs/wasm/Walk.zig+11-9
......@@ -1,4 +1,15 @@
11//! Find and annotate identifiers with links to their declarations.
2
3const Walk = @This();
4const std = @import("std");
5const Ast = std.zig.Ast;
6const assert = std.debug.assert;
7const log = std.log;
8const gpa = std.heap.wasm_allocator;
9const Oom = error{OutOfMemory};
10
11pub const Decl = @import("Decl.zig");
12
213pub var files: std.StringArrayHashMapUnmanaged(File) = .{};
314pub var decls: std.ArrayListUnmanaged(Decl) = .{};
415pub var modules: std.StringArrayHashMapUnmanaged(File.Index) = .{};
......@@ -1120,15 +1131,6 @@ pub fn isPrimitiveNonType(name: []const u8) bool {
11201131// try w.root();
11211132//}
11221133
1123const Walk = @This();
1124const std = @import("std");
1125const Ast = std.zig.Ast;
1126const assert = std.debug.assert;
1127const Decl = @import("Decl.zig");
1128const log = std.log;
1129const gpa = std.heap.wasm_allocator;
1130const Oom = error{OutOfMemory};
1131
11321134fn shrinkToFit(m: anytype) void {
11331135 m.shrinkAndFree(gpa, m.entries.len);
11341136}
lib/docs/wasm/html_render.zig created+388
......@@ -0,0 +1,388 @@
1const std = @import("std");
2const Ast = std.zig.Ast;
3const assert = std.debug.assert;
4
5const Walk = @import("Walk");
6const Decl = Walk.Decl;
7
8const gpa = std.heap.wasm_allocator;
9const Oom = error{OutOfMemory};
10
11/// Delete this to find out where URL escaping needs to be added.
12pub const missing_feature_url_escape = true;
13
14pub const RenderSourceOptions = struct {
15 skip_doc_comments: bool = false,
16 skip_comments: bool = false,
17 collapse_whitespace: bool = false,
18 fn_link: Decl.Index = .none,
19};
20
21pub fn fileSourceHtml(
22 file_index: Walk.File.Index,
23 out: *std.ArrayListUnmanaged(u8),
24 root_node: Ast.Node.Index,
25 options: RenderSourceOptions,
26) !void {
27 const ast = file_index.get_ast();
28 const file = file_index.get();
29
30 const g = struct {
31 var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
32 };
33
34 const token_tags = ast.tokens.items(.tag);
35 const token_starts = ast.tokens.items(.start);
36 const main_tokens = ast.nodes.items(.main_token);
37
38 const start_token = ast.firstToken(root_node);
39 const end_token = ast.lastToken(root_node) + 1;
40
41 var cursor: usize = token_starts[start_token];
42
43 var indent: usize = 0;
44 if (std.mem.lastIndexOf(u8, ast.source[0..cursor], "\n")) |newline_index| {
45 for (ast.source[newline_index + 1 .. cursor]) |c| {
46 if (c == ' ') {
47 indent += 1;
48 } else {
49 break;
50 }
51 }
52 }
53
54 for (
55 token_tags[start_token..end_token],
56 token_starts[start_token..end_token],
57 start_token..,
58 ) |tag, start, token_index| {
59 const between = ast.source[cursor..start];
60 if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
61 if (!options.skip_comments) {
62 try out.appendSlice(gpa, "<span class=\"tok-comment\">");
63 try appendUnindented(out, between, indent);
64 try out.appendSlice(gpa, "</span>");
65 }
66 } else if (between.len > 0) {
67 if (options.collapse_whitespace) {
68 if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
69 try out.append(gpa, ' ');
70 } else {
71 try appendUnindented(out, between, indent);
72 }
73 }
74 if (tag == .eof) break;
75 const slice = ast.tokenSlice(token_index);
76 cursor = start + slice.len;
77 switch (tag) {
78 .eof => unreachable,
79
80 .keyword_addrspace,
81 .keyword_align,
82 .keyword_and,
83 .keyword_asm,
84 .keyword_async,
85 .keyword_await,
86 .keyword_break,
87 .keyword_catch,
88 .keyword_comptime,
89 .keyword_const,
90 .keyword_continue,
91 .keyword_defer,
92 .keyword_else,
93 .keyword_enum,
94 .keyword_errdefer,
95 .keyword_error,
96 .keyword_export,
97 .keyword_extern,
98 .keyword_for,
99 .keyword_if,
100 .keyword_inline,
101 .keyword_noalias,
102 .keyword_noinline,
103 .keyword_nosuspend,
104 .keyword_opaque,
105 .keyword_or,
106 .keyword_orelse,
107 .keyword_packed,
108 .keyword_anyframe,
109 .keyword_pub,
110 .keyword_resume,
111 .keyword_return,
112 .keyword_linksection,
113 .keyword_callconv,
114 .keyword_struct,
115 .keyword_suspend,
116 .keyword_switch,
117 .keyword_test,
118 .keyword_threadlocal,
119 .keyword_try,
120 .keyword_union,
121 .keyword_unreachable,
122 .keyword_usingnamespace,
123 .keyword_var,
124 .keyword_volatile,
125 .keyword_allowzero,
126 .keyword_while,
127 .keyword_anytype,
128 .keyword_fn,
129 => {
130 try out.appendSlice(gpa, "<span class=\"tok-kw\">");
131 try appendEscaped(out, slice);
132 try out.appendSlice(gpa, "</span>");
133 },
134
135 .string_literal,
136 .char_literal,
137 .multiline_string_literal_line,
138 => {
139 try out.appendSlice(gpa, "<span class=\"tok-str\">");
140 try appendEscaped(out, slice);
141 try out.appendSlice(gpa, "</span>");
142 },
143
144 .builtin => {
145 try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
146 try appendEscaped(out, slice);
147 try out.appendSlice(gpa, "</span>");
148 },
149
150 .doc_comment,
151 .container_doc_comment,
152 => {
153 if (!options.skip_doc_comments) {
154 try out.appendSlice(gpa, "<span class=\"tok-comment\">");
155 try appendEscaped(out, slice);
156 try out.appendSlice(gpa, "</span>");
157 }
158 },
159
160 .identifier => i: {
161 if (options.fn_link != .none) {
162 const fn_link = options.fn_link.get();
163 const fn_token = main_tokens[fn_link.ast_node];
164 if (token_index == fn_token + 1) {
165 try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
166 _ = missing_feature_url_escape;
167 try fn_link.fqn(out);
168 try out.appendSlice(gpa, "\">");
169 try appendEscaped(out, slice);
170 try out.appendSlice(gpa, "</a>");
171 break :i;
172 }
173 }
174
175 if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
176 try out.appendSlice(gpa, "<span class=\"tok-fn\">");
177 try appendEscaped(out, slice);
178 try out.appendSlice(gpa, "</span>");
179 break :i;
180 }
181
182 if (Walk.isPrimitiveNonType(slice)) {
183 try out.appendSlice(gpa, "<span class=\"tok-null\">");
184 try appendEscaped(out, slice);
185 try out.appendSlice(gpa, "</span>");
186 break :i;
187 }
188
189 if (std.zig.primitives.isPrimitive(slice)) {
190 try out.appendSlice(gpa, "<span class=\"tok-type\">");
191 try appendEscaped(out, slice);
192 try out.appendSlice(gpa, "</span>");
193 break :i;
194 }
195
196 if (file.token_parents.get(token_index)) |field_access_node| {
197 g.field_access_buffer.clearRetainingCapacity();
198 try walkFieldAccesses(file_index, &g.field_access_buffer, field_access_node);
199 if (g.field_access_buffer.items.len > 0) {
200 try out.appendSlice(gpa, "<a href=\"#");
201 _ = missing_feature_url_escape;
202 try out.appendSlice(gpa, g.field_access_buffer.items);
203 try out.appendSlice(gpa, "\">");
204 try appendEscaped(out, slice);
205 try out.appendSlice(gpa, "</a>");
206 } else {
207 try appendEscaped(out, slice);
208 }
209 break :i;
210 }
211
212 {
213 g.field_access_buffer.clearRetainingCapacity();
214 try resolveIdentLink(file_index, &g.field_access_buffer, token_index);
215 if (g.field_access_buffer.items.len > 0) {
216 try out.appendSlice(gpa, "<a href=\"#");
217 _ = missing_feature_url_escape;
218 try out.appendSlice(gpa, g.field_access_buffer.items);
219 try out.appendSlice(gpa, "\">");
220 try appendEscaped(out, slice);
221 try out.appendSlice(gpa, "</a>");
222 break :i;
223 }
224 }
225
226 try appendEscaped(out, slice);
227 },
228
229 .number_literal => {
230 try out.appendSlice(gpa, "<span class=\"tok-number\">");
231 try appendEscaped(out, slice);
232 try out.appendSlice(gpa, "</span>");
233 },
234
235 .bang,
236 .pipe,
237 .pipe_pipe,
238 .pipe_equal,
239 .equal,
240 .equal_equal,
241 .equal_angle_bracket_right,
242 .bang_equal,
243 .l_paren,
244 .r_paren,
245 .semicolon,
246 .percent,
247 .percent_equal,
248 .l_brace,
249 .r_brace,
250 .l_bracket,
251 .r_bracket,
252 .period,
253 .period_asterisk,
254 .ellipsis2,
255 .ellipsis3,
256 .caret,
257 .caret_equal,
258 .plus,
259 .plus_plus,
260 .plus_equal,
261 .plus_percent,
262 .plus_percent_equal,
263 .plus_pipe,
264 .plus_pipe_equal,
265 .minus,
266 .minus_equal,
267 .minus_percent,
268 .minus_percent_equal,
269 .minus_pipe,
270 .minus_pipe_equal,
271 .asterisk,
272 .asterisk_equal,
273 .asterisk_asterisk,
274 .asterisk_percent,
275 .asterisk_percent_equal,
276 .asterisk_pipe,
277 .asterisk_pipe_equal,
278 .arrow,
279 .colon,
280 .slash,
281 .slash_equal,
282 .comma,
283 .ampersand,
284 .ampersand_equal,
285 .question_mark,
286 .angle_bracket_left,
287 .angle_bracket_left_equal,
288 .angle_bracket_angle_bracket_left,
289 .angle_bracket_angle_bracket_left_equal,
290 .angle_bracket_angle_bracket_left_pipe,
291 .angle_bracket_angle_bracket_left_pipe_equal,
292 .angle_bracket_right,
293 .angle_bracket_right_equal,
294 .angle_bracket_angle_bracket_right,
295 .angle_bracket_angle_bracket_right_equal,
296 .tilde,
297 => try appendEscaped(out, slice),
298
299 .invalid, .invalid_periodasterisks => return error.InvalidToken,
300 }
301 }
302}
303
304fn appendUnindented(out: *std.ArrayListUnmanaged(u8), s: []const u8, indent: usize) !void {
305 var it = std.mem.splitScalar(u8, s, '\n');
306 var is_first_line = true;
307 while (it.next()) |line| {
308 if (is_first_line) {
309 try appendEscaped(out, line);
310 is_first_line = false;
311 } else {
312 try out.appendSlice(gpa, "\n");
313 try appendEscaped(out, unindent(line, indent));
314 }
315 }
316}
317
318pub fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
319 for (s) |c| {
320 try out.ensureUnusedCapacity(gpa, 6);
321 switch (c) {
322 '&' => out.appendSliceAssumeCapacity("&amp;"),
323 '<' => out.appendSliceAssumeCapacity("&lt;"),
324 '>' => out.appendSliceAssumeCapacity("&gt;"),
325 '"' => out.appendSliceAssumeCapacity("&quot;"),
326 else => out.appendAssumeCapacity(c),
327 }
328 }
329}
330
331fn walkFieldAccesses(
332 file_index: Walk.File.Index,
333 out: *std.ArrayListUnmanaged(u8),
334 node: Ast.Node.Index,
335) Oom!void {
336 const ast = file_index.get_ast();
337 const node_tags = ast.nodes.items(.tag);
338 assert(node_tags[node] == .field_access);
339 const node_datas = ast.nodes.items(.data);
340 const main_tokens = ast.nodes.items(.main_token);
341 const object_node = node_datas[node].lhs;
342 const dot_token = main_tokens[node];
343 const field_ident = dot_token + 1;
344 switch (node_tags[object_node]) {
345 .identifier => {
346 const lhs_ident = main_tokens[object_node];
347 try resolveIdentLink(file_index, out, lhs_ident);
348 },
349 .field_access => {
350 try walkFieldAccesses(file_index, out, object_node);
351 },
352 else => {},
353 }
354 if (out.items.len > 0) {
355 try out.append(gpa, '.');
356 try out.appendSlice(gpa, ast.tokenSlice(field_ident));
357 }
358}
359
360fn resolveIdentLink(
361 file_index: Walk.File.Index,
362 out: *std.ArrayListUnmanaged(u8),
363 ident_token: Ast.TokenIndex,
364) Oom!void {
365 const decl_index = file_index.get().lookup_token(ident_token);
366 if (decl_index == .none) return;
367 try resolveDeclLink(decl_index, out);
368}
369
370fn unindent(s: []const u8, indent: usize) []const u8 {
371 var indent_idx: usize = 0;
372 for (s) |c| {
373 if (c == ' ' and indent_idx < indent) {
374 indent_idx += 1;
375 } else {
376 break;
377 }
378 }
379 return s[indent_idx..];
380}
381
382pub fn resolveDeclLink(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
383 const decl = decl_index.get();
384 switch (decl.categorize()) {
385 .alias => |alias_decl| try alias_decl.get().fqn(out),
386 else => try decl.fqn(out),
387 }
388}
lib/docs/wasm/main.zig+16-390
......@@ -1,15 +1,17 @@
1/// Delete this to find out where URL escaping needs to be added.
2const missing_feature_url_escape = true;
3
4const gpa = std.heap.wasm_allocator;
5
61const std = @import("std");
72const log = std.log;
83const assert = std.debug.assert;
94const Ast = std.zig.Ast;
10const Walk = @import("Walk.zig");
5const Walk = @import("Walk");
116const markdown = @import("markdown.zig");
12const Decl = @import("Decl.zig");
7const Decl = Walk.Decl;
8
9const fileSourceHtml = @import("html_render.zig").fileSourceHtml;
10const appendEscaped = @import("html_render.zig").appendEscaped;
11const resolveDeclLink = @import("html_render.zig").resolveDeclLink;
12const missing_feature_url_escape = @import("html_render.zig").missing_feature_url_escape;
13
14const gpa = std.heap.wasm_allocator;
1315
1416const js = struct {
1517 extern "js" fn log(ptr: [*]const u8, len: usize) void;
......@@ -439,7 +441,7 @@ fn decl_field_html_fallible(
439441 const decl = decl_index.get();
440442 const ast = decl.file.get_ast();
441443 try out.appendSlice(gpa, "<pre><code>");
442 try file_source_html(decl.file, out, field_node, .{});
444 try fileSourceHtml(decl.file, out, field_node, .{});
443445 try out.appendSlice(gpa, "</code></pre>");
444446
445447 const field = ast.fullContainerField(field_node).?;
......@@ -478,7 +480,7 @@ fn decl_param_html_fallible(
478480 try out.appendSlice(gpa, "<pre><code>");
479481 try appendEscaped(out, name);
480482 try out.appendSlice(gpa, ": ");
481 try file_source_html(decl.file, out, param_node, .{});
483 try fileSourceHtml(decl.file, out, param_node, .{});
482484 try out.appendSlice(gpa, "</code></pre>");
483485
484486 if (ast.tokens.items(.tag)[first_doc_comment] == .doc_comment) {
......@@ -506,7 +508,7 @@ export fn decl_fn_proto_html(decl_index: Decl.Index, linkify_fn_name: bool) Stri
506508 };
507509
508510 string_result.clearRetainingCapacity();
509 file_source_html(decl.file, &string_result, proto_node, .{
511 fileSourceHtml(decl.file, &string_result, proto_node, .{
510512 .skip_doc_comments = true,
511513 .skip_comments = true,
512514 .collapse_whitespace = true,
......@@ -521,7 +523,7 @@ export fn decl_source_html(decl_index: Decl.Index) String {
521523 const decl = decl_index.get();
522524
523525 string_result.clearRetainingCapacity();
524 file_source_html(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
526 fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
525527 fatal("unable to render source: {s}", .{@errorName(err)});
526528 };
527529 return String.init(string_result.items);
......@@ -533,7 +535,7 @@ export fn decl_doctest_html(decl_index: Decl.Index) String {
533535 return String.init("");
534536
535537 string_result.clearRetainingCapacity();
536 file_source_html(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {
538 fileSourceHtml(decl.file, &string_result, doctest_ast_node, .{}) catch |err| {
537539 fatal("unable to render source: {s}", .{@errorName(err)});
538540 };
539541 return String.init(string_result.items);
......@@ -691,7 +693,7 @@ fn render_docs(
691693 const content = doc.string(data.text.content);
692694 if (resolve_decl_path(r.context, content)) |resolved_decl_index| {
693695 g.link_buffer.clearRetainingCapacity();
694 try resolve_decl_link(resolved_decl_index, &g.link_buffer);
696 try resolveDeclLink(resolved_decl_index, &g.link_buffer);
695697
696698 try writer.writeAll("<a href=\"#");
697699 _ = missing_feature_url_escape;
......@@ -734,7 +736,7 @@ export fn decl_type_html(decl_index: Decl.Index) String {
734736 if (ast.fullVarDecl(decl.ast_node)) |var_decl| {
735737 if (var_decl.ast.type_node != 0) {
736738 string_result.appendSlice(gpa, "<code>") catch @panic("OOM");
737 file_source_html(decl.file, &string_result, var_decl.ast.type_node, .{
739 fileSourceHtml(decl.file, &string_result, var_decl.ast.type_node, .{
738740 .skip_comments = true,
739741 .collapse_whitespace = true,
740742 }) catch |e| {
......@@ -902,382 +904,6 @@ export fn namespace_members(parent: Decl.Index, include_private: bool) Slice(Dec
902904 return Slice(Decl.Index).init(g.members.items);
903905}
904906
905const RenderSourceOptions = struct {
906 skip_doc_comments: bool = false,
907 skip_comments: bool = false,
908 collapse_whitespace: bool = false,
909 fn_link: Decl.Index = .none,
910};
911
912fn file_source_html(
913 file_index: Walk.File.Index,
914 out: *std.ArrayListUnmanaged(u8),
915 root_node: Ast.Node.Index,
916 options: RenderSourceOptions,
917) !void {
918 const ast = file_index.get_ast();
919 const file = file_index.get();
920
921 const g = struct {
922 var field_access_buffer: std.ArrayListUnmanaged(u8) = .{};
923 };
924
925 const token_tags = ast.tokens.items(.tag);
926 const token_starts = ast.tokens.items(.start);
927 const main_tokens = ast.nodes.items(.main_token);
928
929 const start_token = ast.firstToken(root_node);
930 const end_token = ast.lastToken(root_node) + 1;
931
932 var cursor: usize = token_starts[start_token];
933
934 var indent: usize = 0;
935 if (std.mem.lastIndexOf(u8, ast.source[0..cursor], "\n")) |newline_index| {
936 for (ast.source[newline_index + 1 .. cursor]) |c| {
937 if (c == ' ') {
938 indent += 1;
939 } else {
940 break;
941 }
942 }
943 }
944
945 for (
946 token_tags[start_token..end_token],
947 token_starts[start_token..end_token],
948 start_token..,
949 ) |tag, start, token_index| {
950 const between = ast.source[cursor..start];
951 if (std.mem.trim(u8, between, " \t\r\n").len > 0) {
952 if (!options.skip_comments) {
953 try out.appendSlice(gpa, "<span class=\"tok-comment\">");
954 try appendUnindented(out, between, indent);
955 try out.appendSlice(gpa, "</span>");
956 }
957 } else if (between.len > 0) {
958 if (options.collapse_whitespace) {
959 if (out.items.len > 0 and out.items[out.items.len - 1] != ' ')
960 try out.append(gpa, ' ');
961 } else {
962 try appendUnindented(out, between, indent);
963 }
964 }
965 if (tag == .eof) break;
966 const slice = ast.tokenSlice(token_index);
967 cursor = start + slice.len;
968 switch (tag) {
969 .eof => unreachable,
970
971 .keyword_addrspace,
972 .keyword_align,
973 .keyword_and,
974 .keyword_asm,
975 .keyword_async,
976 .keyword_await,
977 .keyword_break,
978 .keyword_catch,
979 .keyword_comptime,
980 .keyword_const,
981 .keyword_continue,
982 .keyword_defer,
983 .keyword_else,
984 .keyword_enum,
985 .keyword_errdefer,
986 .keyword_error,
987 .keyword_export,
988 .keyword_extern,
989 .keyword_for,
990 .keyword_if,
991 .keyword_inline,
992 .keyword_noalias,
993 .keyword_noinline,
994 .keyword_nosuspend,
995 .keyword_opaque,
996 .keyword_or,
997 .keyword_orelse,
998 .keyword_packed,
999 .keyword_anyframe,
1000 .keyword_pub,
1001 .keyword_resume,
1002 .keyword_return,
1003 .keyword_linksection,
1004 .keyword_callconv,
1005 .keyword_struct,
1006 .keyword_suspend,
1007 .keyword_switch,
1008 .keyword_test,
1009 .keyword_threadlocal,
1010 .keyword_try,
1011 .keyword_union,
1012 .keyword_unreachable,
1013 .keyword_usingnamespace,
1014 .keyword_var,
1015 .keyword_volatile,
1016 .keyword_allowzero,
1017 .keyword_while,
1018 .keyword_anytype,
1019 .keyword_fn,
1020 => {
1021 try out.appendSlice(gpa, "<span class=\"tok-kw\">");
1022 try appendEscaped(out, slice);
1023 try out.appendSlice(gpa, "</span>");
1024 },
1025
1026 .string_literal,
1027 .char_literal,
1028 .multiline_string_literal_line,
1029 => {
1030 try out.appendSlice(gpa, "<span class=\"tok-str\">");
1031 try appendEscaped(out, slice);
1032 try out.appendSlice(gpa, "</span>");
1033 },
1034
1035 .builtin => {
1036 try out.appendSlice(gpa, "<span class=\"tok-builtin\">");
1037 try appendEscaped(out, slice);
1038 try out.appendSlice(gpa, "</span>");
1039 },
1040
1041 .doc_comment,
1042 .container_doc_comment,
1043 => {
1044 if (!options.skip_doc_comments) {
1045 try out.appendSlice(gpa, "<span class=\"tok-comment\">");
1046 try appendEscaped(out, slice);
1047 try out.appendSlice(gpa, "</span>");
1048 }
1049 },
1050
1051 .identifier => i: {
1052 if (options.fn_link != .none) {
1053 const fn_link = options.fn_link.get();
1054 const fn_token = main_tokens[fn_link.ast_node];
1055 if (token_index == fn_token + 1) {
1056 try out.appendSlice(gpa, "<a class=\"tok-fn\" href=\"#");
1057 _ = missing_feature_url_escape;
1058 try fn_link.fqn(out);
1059 try out.appendSlice(gpa, "\">");
1060 try appendEscaped(out, slice);
1061 try out.appendSlice(gpa, "</a>");
1062 break :i;
1063 }
1064 }
1065
1066 if (token_index > 0 and token_tags[token_index - 1] == .keyword_fn) {
1067 try out.appendSlice(gpa, "<span class=\"tok-fn\">");
1068 try appendEscaped(out, slice);
1069 try out.appendSlice(gpa, "</span>");
1070 break :i;
1071 }
1072
1073 if (Walk.isPrimitiveNonType(slice)) {
1074 try out.appendSlice(gpa, "<span class=\"tok-null\">");
1075 try appendEscaped(out, slice);
1076 try out.appendSlice(gpa, "</span>");
1077 break :i;
1078 }
1079
1080 if (std.zig.primitives.isPrimitive(slice)) {
1081 try out.appendSlice(gpa, "<span class=\"tok-type\">");
1082 try appendEscaped(out, slice);
1083 try out.appendSlice(gpa, "</span>");
1084 break :i;
1085 }
1086
1087 if (file.token_parents.get(token_index)) |field_access_node| {
1088 g.field_access_buffer.clearRetainingCapacity();
1089 try walk_field_accesses(file_index, &g.field_access_buffer, field_access_node);
1090 if (g.field_access_buffer.items.len > 0) {
1091 try out.appendSlice(gpa, "<a href=\"#");
1092 _ = missing_feature_url_escape;
1093 try out.appendSlice(gpa, g.field_access_buffer.items);
1094 try out.appendSlice(gpa, "\">");
1095 try appendEscaped(out, slice);
1096 try out.appendSlice(gpa, "</a>");
1097 } else {
1098 try appendEscaped(out, slice);
1099 }
1100 break :i;
1101 }
1102
1103 {
1104 g.field_access_buffer.clearRetainingCapacity();
1105 try resolve_ident_link(file_index, &g.field_access_buffer, token_index);
1106 if (g.field_access_buffer.items.len > 0) {
1107 try out.appendSlice(gpa, "<a href=\"#");
1108 _ = missing_feature_url_escape;
1109 try out.appendSlice(gpa, g.field_access_buffer.items);
1110 try out.appendSlice(gpa, "\">");
1111 try appendEscaped(out, slice);
1112 try out.appendSlice(gpa, "</a>");
1113 break :i;
1114 }
1115 }
1116
1117 try appendEscaped(out, slice);
1118 },
1119
1120 .number_literal => {
1121 try out.appendSlice(gpa, "<span class=\"tok-number\">");
1122 try appendEscaped(out, slice);
1123 try out.appendSlice(gpa, "</span>");
1124 },
1125
1126 .bang,
1127 .pipe,
1128 .pipe_pipe,
1129 .pipe_equal,
1130 .equal,
1131 .equal_equal,
1132 .equal_angle_bracket_right,
1133 .bang_equal,
1134 .l_paren,
1135 .r_paren,
1136 .semicolon,
1137 .percent,
1138 .percent_equal,
1139 .l_brace,
1140 .r_brace,
1141 .l_bracket,
1142 .r_bracket,
1143 .period,
1144 .period_asterisk,
1145 .ellipsis2,
1146 .ellipsis3,
1147 .caret,
1148 .caret_equal,
1149 .plus,
1150 .plus_plus,
1151 .plus_equal,
1152 .plus_percent,
1153 .plus_percent_equal,
1154 .plus_pipe,
1155 .plus_pipe_equal,
1156 .minus,
1157 .minus_equal,
1158 .minus_percent,
1159 .minus_percent_equal,
1160 .minus_pipe,
1161 .minus_pipe_equal,
1162 .asterisk,
1163 .asterisk_equal,
1164 .asterisk_asterisk,
1165 .asterisk_percent,
1166 .asterisk_percent_equal,
1167 .asterisk_pipe,
1168 .asterisk_pipe_equal,
1169 .arrow,
1170 .colon,
1171 .slash,
1172 .slash_equal,
1173 .comma,
1174 .ampersand,
1175 .ampersand_equal,
1176 .question_mark,
1177 .angle_bracket_left,
1178 .angle_bracket_left_equal,
1179 .angle_bracket_angle_bracket_left,
1180 .angle_bracket_angle_bracket_left_equal,
1181 .angle_bracket_angle_bracket_left_pipe,
1182 .angle_bracket_angle_bracket_left_pipe_equal,
1183 .angle_bracket_right,
1184 .angle_bracket_right_equal,
1185 .angle_bracket_angle_bracket_right,
1186 .angle_bracket_angle_bracket_right_equal,
1187 .tilde,
1188 => try appendEscaped(out, slice),
1189
1190 .invalid, .invalid_periodasterisks => return error.InvalidToken,
1191 }
1192 }
1193}
1194
1195fn unindent(s: []const u8, indent: usize) []const u8 {
1196 var indent_idx: usize = 0;
1197 for (s) |c| {
1198 if (c == ' ' and indent_idx < indent) {
1199 indent_idx += 1;
1200 } else {
1201 break;
1202 }
1203 }
1204 return s[indent_idx..];
1205}
1206
1207fn appendUnindented(out: *std.ArrayListUnmanaged(u8), s: []const u8, indent: usize) !void {
1208 var it = std.mem.splitScalar(u8, s, '\n');
1209 var is_first_line = true;
1210 while (it.next()) |line| {
1211 if (is_first_line) {
1212 try appendEscaped(out, line);
1213 is_first_line = false;
1214 } else {
1215 try out.appendSlice(gpa, "\n");
1216 try appendEscaped(out, unindent(line, indent));
1217 }
1218 }
1219}
1220
1221fn resolve_ident_link(
1222 file_index: Walk.File.Index,
1223 out: *std.ArrayListUnmanaged(u8),
1224 ident_token: Ast.TokenIndex,
1225) Oom!void {
1226 const decl_index = file_index.get().lookup_token(ident_token);
1227 if (decl_index == .none) return;
1228 try resolve_decl_link(decl_index, out);
1229}
1230
1231fn resolve_decl_link(decl_index: Decl.Index, out: *std.ArrayListUnmanaged(u8)) Oom!void {
1232 const decl = decl_index.get();
1233 switch (decl.categorize()) {
1234 .alias => |alias_decl| try alias_decl.get().fqn(out),
1235 else => try decl.fqn(out),
1236 }
1237}
1238
1239fn walk_field_accesses(
1240 file_index: Walk.File.Index,
1241 out: *std.ArrayListUnmanaged(u8),
1242 node: Ast.Node.Index,
1243) Oom!void {
1244 const ast = file_index.get_ast();
1245 const node_tags = ast.nodes.items(.tag);
1246 assert(node_tags[node] == .field_access);
1247 const node_datas = ast.nodes.items(.data);
1248 const main_tokens = ast.nodes.items(.main_token);
1249 const object_node = node_datas[node].lhs;
1250 const dot_token = main_tokens[node];
1251 const field_ident = dot_token + 1;
1252 switch (node_tags[object_node]) {
1253 .identifier => {
1254 const lhs_ident = main_tokens[object_node];
1255 try resolve_ident_link(file_index, out, lhs_ident);
1256 },
1257 .field_access => {
1258 try walk_field_accesses(file_index, out, object_node);
1259 },
1260 else => {},
1261 }
1262 if (out.items.len > 0) {
1263 try out.append(gpa, '.');
1264 try out.appendSlice(gpa, ast.tokenSlice(field_ident));
1265 }
1266}
1267
1268fn appendEscaped(out: *std.ArrayListUnmanaged(u8), s: []const u8) !void {
1269 for (s) |c| {
1270 try out.ensureUnusedCapacity(gpa, 6);
1271 switch (c) {
1272 '&' => out.appendSliceAssumeCapacity("&amp;"),
1273 '<' => out.appendSliceAssumeCapacity("&lt;"),
1274 '>' => out.appendSliceAssumeCapacity("&gt;"),
1275 '"' => out.appendSliceAssumeCapacity("&quot;"),
1276 else => out.appendAssumeCapacity(c),
1277 }
1278 }
1279}
1280
1281907fn count_scalar(haystack: []const u8, needle: u8) usize {
1282908 var total: usize = 0;
1283909 for (haystack) |elem| {
lib/fuzzer/index.html+59-1
......@@ -2,12 +2,56 @@
22<html>
33 <head>
44 <meta charset="utf-8">
5 <title>Zig Documentation</title>
5 <title>Zig Build System Interface</title>
66 <style type="text/css">
77 body {
88 font-family: system-ui, -apple-system, Roboto, "Segoe UI", sans-serif;
99 color: #000000;
1010 }
11 .hidden {
12 display: none;
13 }
14 table {
15 width: 100%;
16 }
17 a {
18 color: #2A6286;
19 }
20 pre{
21 font-family:"Source Code Pro",monospace;
22 font-size:1em;
23 background-color:#F5F5F5;
24 padding: 1em;
25 margin: 0;
26 overflow-x: auto;
27 }
28 :not(pre) > code {
29 white-space: break-spaces;
30 }
31 code {
32 font-family:"Source Code Pro",monospace;
33 font-size: 0.9em;
34 }
35 code a {
36 color: #000000;
37 }
38 kbd {
39 color: #000;
40 background-color: #fafbfc;
41 border-color: #d1d5da;
42 border-bottom-color: #c6cbd1;
43 box-shadow-color: #c6cbd1;
44 display: inline-block;
45 padding: 0.3em 0.2em;
46 font: 1.2em monospace;
47 line-height: 0.8em;
48 vertical-align: middle;
49 border: solid 1px;
50 border-radius: 3px;
51 box-shadow: inset 0 -1px 0;
52 cursor: default;
53 }
54
1155 .tok-kw {
1256 color: #333;
1357 font-weight: bold;
......@@ -42,6 +86,16 @@
4286 background-color: #111;
4387 color: #bbb;
4488 }
89 pre {
90 background-color: #222;
91 color: #ccc;
92 }
93 a {
94 color: #88f;
95 }
96 code a {
97 color: #ccc;
98 }
4599 .tok-kw {
46100 color: #eee;
47101 }
......@@ -70,6 +124,10 @@
70124 </style>
71125 </head>
72126 <body>
127 <div id="sectSource" class="hidden">
128 <h2>Source Code</h2>
129 <pre><code id="sourceText"></code></pre>
130 </div>
73131 <script src="main.js"></script>
74132 </body>
75133</html>
lib/fuzzer/main.js+48
......@@ -1,4 +1,7 @@
11(function() {
2 const domSectSource = document.getElementById("sectSource");
3 const domSourceText = document.getElementById("sourceText");
4
25 let wasm_promise = fetch("main.wasm");
36 let sources_promise = fetch("sources.tar").then(function(response) {
47 if (!response.ok) throw new Error("unable to download sources");
......@@ -30,11 +33,56 @@
3033 const wasm_array = new Uint8Array(wasm_exports.memory.buffer, ptr, js_array.length);
3134 wasm_array.set(js_array);
3235 wasm_exports.unpack(ptr, js_array.length);
36
37 render();
3338 });
3439 });
3540
41 function render() {
42 domSectSource.classList.add("hidden");
43
44 // TODO this is temporary debugging data
45 renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");
46 }
47
48 function renderSource(path) {
49 const decl_index = findFileRoot(path);
50 if (decl_index == null) throw new Error("file not found: " + path);
51
52 const h2 = domSectSource.children[0];
53 h2.innerText = path;
54 domSourceText.innerHTML = declSourceHtml(decl_index);
55
56 domSectSource.classList.remove("hidden");
57 }
58
59 function findFileRoot(path) {
60 setInputString(path);
61 const result = wasm_exports.find_file_root();
62 if (result === -1) return null;
63 return result;
64 }
65
3666 function decodeString(ptr, len) {
3767 if (len === 0) return "";
3868 return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
3969 }
70
71 function setInputString(s) {
72 const jsArray = text_encoder.encode(s);
73 const len = jsArray.length;
74 const ptr = wasm_exports.set_input_string(len);
75 const wasmArray = new Uint8Array(wasm_exports.memory.buffer, ptr, len);
76 wasmArray.set(jsArray);
77 }
78
79 function declSourceHtml(decl_index) {
80 return unwrapString(wasm_exports.decl_source_html(decl_index));
81 }
82
83 function unwrapString(bigint) {
84 const ptr = Number(bigint & 0xffffffffn);
85 const len = Number(bigint >> 32n);
86 return decodeString(ptr, len);
87 }
4088})();
lib/fuzzer/wasm/main.zig+44
......@@ -2,6 +2,8 @@ const std = @import("std");
22const assert = std.debug.assert;
33
44const Walk = @import("Walk");
5const Decl = Walk.Decl;
6const html_render = @import("html_render");
57
68const gpa = std.heap.wasm_allocator;
79const log = std.log;
......@@ -52,6 +54,48 @@ export fn unpack(tar_ptr: [*]u8, tar_len: usize) void {
5254 };
5355}
5456
57/// Set by `set_input_string`.
58var input_string: std.ArrayListUnmanaged(u8) = .{};
59var string_result: std.ArrayListUnmanaged(u8) = .{};
60
61export fn set_input_string(len: usize) [*]u8 {
62 input_string.resize(gpa, len) catch @panic("OOM");
63 return input_string.items.ptr;
64}
65
66/// Looks up the root struct decl corresponding to a file by path.
67/// Uses `input_string`.
68export fn find_file_root() Decl.Index {
69 const file: Walk.File.Index = @enumFromInt(Walk.files.getIndex(input_string.items) orelse return .none);
70 return file.findRootDecl();
71}
72
73export fn decl_source_html(decl_index: Decl.Index) String {
74 const decl = decl_index.get();
75
76 string_result.clearRetainingCapacity();
77 html_render.fileSourceHtml(decl.file, &string_result, decl.ast_node, .{}) catch |err| {
78 fatal("unable to render source: {s}", .{@errorName(err)});
79 };
80 return String.init(string_result.items);
81}
82
83const String = Slice(u8);
84
85fn Slice(T: type) type {
86 return packed struct(u64) {
87 ptr: u32,
88 len: u32,
89
90 fn init(s: []const T) @This() {
91 return .{
92 .ptr = @intFromPtr(s.ptr),
93 .len = s.len,
94 };
95 }
96 };
97}
98
5599fn unpackInner(tar_bytes: []u8) !void {
56100 var fbs = std.io.fixedBufferStream(tar_bytes);
57101 var file_name_buffer: [1024]u8 = undefined;
lib/std/Build/Fuzz.zig+19-20
......@@ -235,30 +235,29 @@ pub const WebServer = struct {
235235 .root_dir = ws.zig_lib_directory,
236236 .sub_path = "docs/wasm/Walk.zig",
237237 };
238 const html_render_src_path: Build.Cache.Path = .{
239 .root_dir = ws.zig_lib_directory,
240 .sub_path = "docs/wasm/html_render.zig",
241 };
238242
239243 var argv: std.ArrayListUnmanaged([]const u8) = .{};
240244
241245 try argv.appendSlice(arena, &.{
242 ws.zig_exe_path,
243 "build-exe",
244 "-fno-entry",
245 "-O",
246 @tagName(optimize_mode),
247 "-target",
248 "wasm32-freestanding",
249 "-mcpu",
250 "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext",
251 "--cache-dir",
252 ws.global_cache_directory.path orelse ".",
253 "--global-cache-dir",
254 ws.global_cache_directory.path orelse ".",
255 "--name",
256 "fuzzer",
257 "-rdynamic",
258 "--dep",
259 "Walk",
260 try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}),
261 try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}),
246 ws.zig_exe_path, "build-exe", //
247 "-fno-entry", //
248 "-O", @tagName(optimize_mode), //
249 "-target", "wasm32-freestanding", //
250 "-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //
251 "--cache-dir", ws.global_cache_directory.path orelse ".", //
252 "--global-cache-dir", ws.global_cache_directory.path orelse ".", //
253 "--name", "fuzzer", //
254 "-rdynamic", //
255 "--dep", "Walk", //
256 "--dep", "html_render", //
257 try std.fmt.allocPrint(arena, "-Mroot={}", .{main_src_path}), //
258 try std.fmt.allocPrint(arena, "-MWalk={}", .{walk_src_path}), //
259 "--dep", "Walk", //
260 try std.fmt.allocPrint(arena, "-Mhtml_render={}", .{html_render_src_path}), //
262261 "--listen=-",
263262 });
264263