1const builtin = @import("builtin");
2
3const std = @import("std");
4const Io = std.Io;
5const Dir = std.Io.Dir;
6const Path = std.Build.Cache.Path;
7const process = std.process;
8const Progress = std.Progress;
9const print = std.debug.print;
10const mem = std.mem;
11const testing = std.testing;
12const Allocator = std.mem.Allocator;
13const ArrayList = std.ArrayList;
14const fatal = std.process.fatal;
15const Writer = std.Io.Writer;
16
17const max_doc_file_size = 10 * 1024 * 1024;
18
19const obj_ext = builtin.object_format.fileExt(builtin.cpu.arch);
20
21const usage =
22 \\Usage: docgen [options] input output
23 \\
24 \\ Generates an HTML document from a docgen template.
25 \\
26 \\Options:
27 \\ --code-dir dir Path to directory containing code example outputs
28 \\ --grammar file Path to the PEG grammar definition
29 \\ -h, --help Print this help and exit
30 \\
31;
32
33pub fn main(init: std.process.Init) !void {
34 const arena = init.arena.allocator();
35 const io = init.io;
36
37 var args_it = try init.minimal.args.iterateAllocator(arena);
38 if (!args_it.skip()) @panic("expected self arg");
39
40 var opt_code_dir: ?[]const u8 = null;
41 var opt_grammar: ?[]const u8 = null;
42 var opt_input: ?[]const u8 = null;
43 var opt_output: ?[]const u8 = null;
44
45 while (args_it.next()) |arg| {
46 if (mem.startsWith(u8, arg, "-")) {
47 if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
48 try Io.File.stdout().writeStreamingAll(io, usage);
49 process.exit(0);
50 } else if (mem.eql(u8, arg, "--code-dir")) {
51 if (args_it.next()) |param| {
52 opt_code_dir = param;
53 } else {
54 fatal("expected parameter after --code-dir", .{});
55 }
56 } else if (mem.eql(u8, arg, "--grammar")) {
57 if (args_it.next()) |param| {
58 opt_grammar = param;
59 } else {
60 fatal("expected parameter after --grammar", .{});
61 }
62 } else {
63 fatal("unrecognized option: '{s}'", .{arg});
64 }
65 } else if (opt_input == null) {
66 opt_input = arg;
67 } else if (opt_output == null) {
68 opt_output = arg;
69 } else {
70 fatal("unexpected positional argument: '{s}'", .{arg});
71 }
72 }
73 const input_path = opt_input orelse fatal("missing input file", .{});
74 const output_path = opt_output orelse fatal("missing output file", .{});
75 const code_dir_path = opt_code_dir orelse fatal("missing --code-dir argument", .{});
76 const grammar_path = opt_grammar orelse fatal("missing --grammar argument", .{});
77
78 var in_file = try Dir.cwd().openFile(io, input_path, .{});
79 defer in_file.close(io);
80
81 var out_file = try Dir.cwd().createFile(io, output_path, .{});
82 defer out_file.close(io);
83 var out_file_buffer: [4096]u8 = undefined;
84 var out_file_writer = out_file.writer(io, &out_file_buffer);
85
86 var code_dir: Path = .{
87 .root_dir = .{
88 .handle = try Dir.cwd().openDir(io, code_dir_path, .{}),
89 .path = code_dir_path,
90 },
91 };
92 defer code_dir.root_dir.handle.close(io);
93
94 const grammar = try Dir.cwd().readFileAlloc(io, grammar_path, init.gpa, .limited(max_doc_file_size));
95 defer init.gpa.free(grammar);
96
97 var in_file_reader = in_file.reader(io, &.{});
98 const input_file_bytes = try in_file_reader.interface.allocRemaining(arena, .limited(max_doc_file_size));
99
100 var tokenizer = Tokenizer.init(input_path, input_file_bytes);
101 var toc = try genToc(arena, &tokenizer);
102
103 try genHtml(arena, io, &tokenizer, &toc, code_dir, grammar, &out_file_writer.interface);
104 try out_file_writer.end();
105}
106
107const Token = struct {
108 id: Id,
109 start: usize,
110 end: usize,
111
112 const Id = enum {
113 invalid,
114 content,
115 bracket_open,
116 tag_content,
117 separator,
118 bracket_close,
119 eof,
120 };
121};
122
123const Tokenizer = struct {
124 buffer: []const u8,
125 index: usize,
126 state: State,
127 source_file_name: []const u8,
128
129 const State = enum {
130 start,
131 l_bracket,
132 hash,
133 tag_name,
134 eof,
135 };
136
137 fn init(source_file_name: []const u8, buffer: []const u8) Tokenizer {
138 return Tokenizer{
139 .buffer = buffer,
140 .index = 0,
141 .state = .start,
142 .source_file_name = source_file_name,
143 };
144 }
145
146 fn next(self: *Tokenizer) Token {
147 var result = Token{
148 .id = .eof,
149 .start = self.index,
150 .end = undefined,
151 };
152 while (self.index < self.buffer.len) : (self.index += 1) {
153 const c = self.buffer[self.index];
154 switch (self.state) {
155 .start => switch (c) {
156 '{' => {
157 self.state = .l_bracket;
158 },
159 else => {
160 result.id = .content;
161 },
162 },
163 .l_bracket => switch (c) {
164 '#' => {
165 if (result.id != .eof) {
166 self.index -= 1;
167 self.state = .start;
168 break;
169 } else {
170 result.id = .bracket_open;
171 self.index += 1;
172 self.state = .tag_name;
173 break;
174 }
175 },
176 else => {
177 result.id = .content;
178 self.state = .start;
179 },
180 },
181 .tag_name => switch (c) {
182 '|' => {
183 if (result.id != .eof) {
184 break;
185 } else {
186 result.id = .separator;
187 self.index += 1;
188 break;
189 }
190 },
191 '#' => {
192 self.state = .hash;
193 },
194 else => {
195 result.id = .tag_content;
196 },
197 },
198 .hash => switch (c) {
199 '}' => {
200 if (result.id != .eof) {
201 self.index -= 1;
202 self.state = .tag_name;
203 break;
204 } else {
205 result.id = .bracket_close;
206 self.index += 1;
207 self.state = .start;
208 break;
209 }
210 },
211 else => {
212 result.id = .tag_content;
213 self.state = .tag_name;
214 },
215 },
216 .eof => unreachable,
217 }
218 } else {
219 switch (self.state) {
220 .start, .l_bracket, .eof => {},
221 else => {
222 result.id = .invalid;
223 },
224 }
225 self.state = .eof;
226 }
227 result.end = self.index;
228 return result;
229 }
230
231 const Location = struct {
232 line: usize,
233 column: usize,
234 line_start: usize,
235 line_end: usize,
236 };
237
238 fn getTokenLocation(self: *Tokenizer, token: Token) Location {
239 var loc = Location{
240 .line = 0,
241 .column = 0,
242 .line_start = 0,
243 .line_end = 0,
244 };
245 for (self.buffer, 0..) |c, i| {
246 if (i == token.start) {
247 loc.line_end = i;
248 while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
249 return loc;
250 }
251 if (c == '\n') {
252 loc.line += 1;
253 loc.column = 0;
254 loc.line_start = i + 1;
255 } else {
256 loc.column += 1;
257 }
258 }
259 return loc;
260 }
261};
262
263fn parseError(tokenizer: *Tokenizer, token: Token, comptime fmt: []const u8, args: anytype) anyerror {
264 const loc = tokenizer.getTokenLocation(token);
265 const args_prefix = .{ tokenizer.source_file_name, loc.line + 1, loc.column + 1 };
266 print("{s}:{d}:{d}: error: " ++ fmt ++ "\n", args_prefix ++ args);
267 if (loc.line_start <= loc.line_end) {
268 print("{s}\n", .{tokenizer.buffer[loc.line_start..loc.line_end]});
269 {
270 var i: usize = 0;
271 while (i < loc.column) : (i += 1) {
272 print(" ", .{});
273 }
274 }
275 {
276 const caret_count = @min(token.end, loc.line_end) - token.start;
277 var i: usize = 0;
278 while (i < caret_count) : (i += 1) {
279 print("~", .{});
280 }
281 }
282 print("\n", .{});
283 }
284 return error.ParseError;
285}
286
287fn assertToken(tokenizer: *Tokenizer, token: Token, id: Token.Id) !void {
288 if (token.id != id) {
289 return parseError(tokenizer, token, "expected {s}, found {s}", .{ @tagName(id), @tagName(token.id) });
290 }
291}
292
293fn eatToken(tokenizer: *Tokenizer, id: Token.Id) !Token {
294 const token = tokenizer.next();
295 try assertToken(tokenizer, token, id);
296 return token;
297}
298
299const HeaderOpen = struct {
300 name: []const u8,
301 url: []const u8,
302 n: usize,
303};
304
305const SeeAlsoItem = struct {
306 name: []const u8,
307 token: Token,
308};
309
310const Code = struct {
311 name: []const u8,
312 token: Token,
313};
314
315const Link = struct {
316 url: []const u8,
317 name: []const u8,
318 token: Token,
319};
320
321const SyntaxBlock = struct {
322 source_type: SourceType,
323 name: []const u8,
324 source_token: Token,
325
326 const SourceType = enum {
327 zig,
328 c,
329 peg,
330 javascript,
331 };
332};
333
334const Node = union(enum) {
335 Content: []const u8,
336 Nav,
337 Builtin: Token,
338 HeaderOpen: HeaderOpen,
339 SeeAlso: []const SeeAlsoItem,
340 Code: Code,
341 Grammar,
342 Link: Link,
343 InlineSyntax: Token,
344 Shell: Token,
345 SyntaxBlock: SyntaxBlock,
346};
347
348const Toc = struct {
349 nodes: []Node,
350 toc: []u8,
351 urls: std.StringHashMap(Token),
352};
353
354const Action = enum {
355 open,
356 close,
357};
358
359fn genToc(gpa: Allocator, tokenizer: *Tokenizer) !Toc {
360 var urls = std.StringHashMap(Token).init(gpa);
361 errdefer urls.deinit();
362
363 var header_stack_size: usize = 0;
364 var last_action: Action = .open;
365 var last_columns: ?u8 = null;
366
367 var toc_buf: Writer.Allocating = .init(gpa);
368 defer toc_buf.deinit();
369
370 const toc = &toc_buf.writer;
371
372 var nodes = std.array_list.Managed(Node).init(gpa);
373 defer nodes.deinit();
374
375 try toc.writeByte('\n');
376
377 while (true) {
378 const token = tokenizer.next();
379 switch (token.id) {
380 .eof => {
381 if (header_stack_size != 0) {
382 return parseError(tokenizer, token, "unbalanced headers", .{});
383 }
384 try toc.writeAll(" </ul>\n");
385 break;
386 },
387 .content => {
388 try nodes.append(Node{ .Content = tokenizer.buffer[token.start..token.end] });
389 },
390 .bracket_open => {
391 const tag_token = try eatToken(tokenizer, .tag_content);
392 const tag_name = tokenizer.buffer[tag_token.start..tag_token.end];
393
394 if (mem.eql(u8, tag_name, "nav")) {
395 _ = try eatToken(tokenizer, .bracket_close);
396
397 try nodes.append(Node.Nav);
398 } else if (mem.eql(u8, tag_name, "builtin")) {
399 _ = try eatToken(tokenizer, .bracket_close);
400 try nodes.append(Node{ .Builtin = tag_token });
401 } else if (mem.eql(u8, tag_name, "header_open")) {
402 _ = try eatToken(tokenizer, .separator);
403 const content_token = try eatToken(tokenizer, .tag_content);
404 const content = tokenizer.buffer[content_token.start..content_token.end];
405 var columns: ?u8 = null;
406 while (true) {
407 const bracket_tok = tokenizer.next();
408 switch (bracket_tok.id) {
409 .bracket_close => break,
410 .separator => continue,
411 .tag_content => {
412 const param = tokenizer.buffer[bracket_tok.start..bracket_tok.end];
413 if (mem.eql(u8, param, "2col")) {
414 columns = 2;
415 } else {
416 return parseError(
417 tokenizer,
418 bracket_tok,
419 "unrecognized header_open param: {s}",
420 .{param},
421 );
422 }
423 },
424 else => return parseError(tokenizer, bracket_tok, "invalid header_open token", .{}),
425 }
426 }
427
428 header_stack_size += 1;
429
430 const urlized = try urlize(gpa, content);
431 try nodes.append(Node{
432 .HeaderOpen = HeaderOpen{
433 .name = content,
434 .url = urlized,
435 .n = header_stack_size + 1, // highest-level section headers start at h2
436 },
437 });
438 if (try urls.fetchPut(urlized, tag_token)) |kv| {
439 parseError(tokenizer, tag_token, "duplicate header url: #{s}", .{urlized}) catch {};
440 parseError(tokenizer, kv.value, "other tag here", .{}) catch {};
441 return error.ParseError;
442 }
443 if (last_action == .open) {
444 try toc.writeByte('\n');
445 try toc.splatByteAll(' ', header_stack_size * 4);
446 if (last_columns) |n| {
447 try toc.print("<ul style=\"columns: {d}\">\n", .{n});
448 } else {
449 try toc.writeAll("<ul>\n");
450 }
451 } else {
452 last_action = .open;
453 }
454 last_columns = columns;
455 try toc.splatByteAll(' ', 4 + header_stack_size * 4);
456 try toc.print("<li><a id=\"toc-{s}\" href=\"#{s}\">{s}</a>", .{ urlized, urlized, content });
457 } else if (mem.eql(u8, tag_name, "header_close")) {
458 if (header_stack_size == 0) {
459 return parseError(tokenizer, tag_token, "unbalanced close header", .{});
460 }
461 header_stack_size -= 1;
462 _ = try eatToken(tokenizer, .bracket_close);
463
464 if (last_action == .close) {
465 try toc.splatByteAll(' ', 8 + header_stack_size * 4);
466 try toc.writeAll("</ul></li>\n");
467 } else {
468 try toc.writeAll("</li>\n");
469 last_action = .close;
470 }
471 } else if (mem.eql(u8, tag_name, "see_also")) {
472 var list = std.array_list.Managed(SeeAlsoItem).init(gpa);
473 errdefer list.deinit();
474
475 while (true) {
476 const see_also_tok = tokenizer.next();
477 switch (see_also_tok.id) {
478 .tag_content => {
479 const content = tokenizer.buffer[see_also_tok.start..see_also_tok.end];
480 try list.append(SeeAlsoItem{
481 .name = content,
482 .token = see_also_tok,
483 });
484 },
485 .separator => {},
486 .bracket_close => {
487 try nodes.ensureUnusedCapacity(1);
488 nodes.appendAssumeCapacity(.{ .SeeAlso = try list.toOwnedSlice() });
489 break;
490 },
491 else => return parseError(tokenizer, see_also_tok, "invalid see_also token", .{}),
492 }
493 }
494 } else if (mem.eql(u8, tag_name, "link")) {
495 _ = try eatToken(tokenizer, .separator);
496 const name_tok = try eatToken(tokenizer, .tag_content);
497 const name = tokenizer.buffer[name_tok.start..name_tok.end];
498
499 const url_name = blk: {
500 const tok = tokenizer.next();
501 switch (tok.id) {
502 .bracket_close => break :blk name,
503 .separator => {
504 const explicit_text = try eatToken(tokenizer, .tag_content);
505 _ = try eatToken(tokenizer, .bracket_close);
506 break :blk tokenizer.buffer[explicit_text.start..explicit_text.end];
507 },
508 else => return parseError(tokenizer, tok, "invalid link token", .{}),
509 }
510 };
511
512 try nodes.append(Node{
513 .Link = Link{
514 .url = try urlize(gpa, url_name),
515 .name = name,
516 .token = name_tok,
517 },
518 });
519 } else if (mem.eql(u8, tag_name, "code")) {
520 _ = try eatToken(tokenizer, .separator);
521 const name_tok = try eatToken(tokenizer, .tag_content);
522 _ = try eatToken(tokenizer, .bracket_close);
523 try nodes.append(.{
524 .Code = .{
525 .name = tokenizer.buffer[name_tok.start..name_tok.end],
526 .token = name_tok,
527 },
528 });
529 } else if (mem.eql(u8, tag_name, "grammar")) {
530 _ = try eatToken(tokenizer, .bracket_close);
531 try nodes.append(.Grammar);
532 } else if (mem.eql(u8, tag_name, "syntax")) {
533 _ = try eatToken(tokenizer, .bracket_close);
534 const content_tok = try eatToken(tokenizer, .content);
535 _ = try eatToken(tokenizer, .bracket_open);
536 const end_syntax_tag = try eatToken(tokenizer, .tag_content);
537 const end_tag_name = tokenizer.buffer[end_syntax_tag.start..end_syntax_tag.end];
538 if (!mem.eql(u8, end_tag_name, "endsyntax")) {
539 return parseError(
540 tokenizer,
541 end_syntax_tag,
542 "invalid token inside syntax: {s}",
543 .{end_tag_name},
544 );
545 }
546 _ = try eatToken(tokenizer, .bracket_close);
547 try nodes.append(Node{ .InlineSyntax = content_tok });
548 } else if (mem.eql(u8, tag_name, "shell_samp")) {
549 _ = try eatToken(tokenizer, .bracket_close);
550 const content_tok = try eatToken(tokenizer, .content);
551 _ = try eatToken(tokenizer, .bracket_open);
552 const end_syntax_tag = try eatToken(tokenizer, .tag_content);
553 const end_tag_name = tokenizer.buffer[end_syntax_tag.start..end_syntax_tag.end];
554 if (!mem.eql(u8, end_tag_name, "end_shell_samp")) {
555 return parseError(
556 tokenizer,
557 end_syntax_tag,
558 "invalid token inside syntax: {s}",
559 .{end_tag_name},
560 );
561 }
562 _ = try eatToken(tokenizer, .bracket_close);
563 try nodes.append(Node{ .Shell = content_tok });
564 } else if (mem.eql(u8, tag_name, "syntax_block")) {
565 _ = try eatToken(tokenizer, .separator);
566 const source_type_tok = try eatToken(tokenizer, .tag_content);
567 var name: []const u8 = "sample_code";
568 const maybe_sep = tokenizer.next();
569 switch (maybe_sep.id) {
570 .separator => {
571 const name_tok = try eatToken(tokenizer, .tag_content);
572 name = tokenizer.buffer[name_tok.start..name_tok.end];
573 _ = try eatToken(tokenizer, .bracket_close);
574 },
575 .bracket_close => {},
576 else => return parseError(tokenizer, token, "invalid token", .{}),
577 }
578 const source_type_str = tokenizer.buffer[source_type_tok.start..source_type_tok.end];
579 var source_type: SyntaxBlock.SourceType = undefined;
580 if (mem.eql(u8, source_type_str, "zig")) {
581 source_type = SyntaxBlock.SourceType.zig;
582 } else if (mem.eql(u8, source_type_str, "c")) {
583 source_type = SyntaxBlock.SourceType.c;
584 } else if (mem.eql(u8, source_type_str, "peg")) {
585 source_type = SyntaxBlock.SourceType.peg;
586 } else if (mem.eql(u8, source_type_str, "javascript")) {
587 source_type = SyntaxBlock.SourceType.javascript;
588 } else {
589 return parseError(tokenizer, source_type_tok, "unrecognized code kind: {s}", .{source_type_str});
590 }
591 const source_token = while (true) {
592 const content_tok = try eatToken(tokenizer, .content);
593 _ = try eatToken(tokenizer, .bracket_open);
594 const end_code_tag = try eatToken(tokenizer, .tag_content);
595 const end_tag_name = tokenizer.buffer[end_code_tag.start..end_code_tag.end];
596 if (mem.eql(u8, end_tag_name, "end_syntax_block")) {
597 _ = try eatToken(tokenizer, .bracket_close);
598 break content_tok;
599 } else {
600 return parseError(
601 tokenizer,
602 end_code_tag,
603 "invalid token inside code_begin: {s}",
604 .{end_tag_name},
605 );
606 }
607 _ = try eatToken(tokenizer, .bracket_close);
608 };
609 try nodes.append(Node{ .SyntaxBlock = SyntaxBlock{ .source_type = source_type, .name = name, .source_token = source_token } });
610 } else {
611 return parseError(tokenizer, tag_token, "unrecognized tag name: {s}", .{tag_name});
612 }
613 },
614 else => return parseError(tokenizer, token, "invalid token", .{}),
615 }
616 }
617
618 const nodes_slice = try nodes.toOwnedSlice();
619 errdefer gpa.free(nodes_slice);
620 const toc_slice = try toc_buf.toOwnedSlice();
621 errdefer gpa.free(toc_slice);
622
623 return .{
624 .nodes = nodes_slice,
625 .toc = toc_slice,
626 .urls = urls,
627 };
628}
629
630fn urlize(gpa: Allocator, input: []const u8) ![]u8 {
631 var buf: ArrayList(u8) = .empty;
632 defer buf.deinit(gpa);
633
634 for (input) |c| {
635 switch (c) {
636 'a'...'z', 'A'...'Z', '_', '-', '0'...'9' => {
637 try buf.append(gpa, c);
638 },
639 ' ' => {
640 try buf.append(gpa, '-');
641 },
642 else => {},
643 }
644 }
645 return try buf.toOwnedSlice(gpa);
646}
647
648fn escapeHtml(gpa: Allocator, input: []const u8) ![]u8 {
649 var buf: std.Io.Writer.Allocating = .init(gpa);
650 defer buf.deinit(gpa);
651
652 try writeEscaped(&buf.writer, input);
653 return try buf.toOwnedSlice();
654}
655
656fn writeEscaped(out: *Writer, input: []const u8) !void {
657 for (input) |c| {
658 try switch (c) {
659 '&' => out.writeAll("&amp;"),
660 '<' => out.writeAll("&lt;"),
661 '>' => out.writeAll("&gt;"),
662 '"' => out.writeAll("&quot;"),
663 else => out.writeByte(c),
664 };
665 }
666}
667
668// Returns true if number is in slice.
669fn in(slice: []const u8, number: u8) bool {
670 for (slice) |n| {
671 if (number == n) return true;
672 }
673 return false;
674}
675
676const builtin_types = [_][]const u8{
677 "f16", "f32", "f64", "f80", "f128",
678 "c_longdouble", "c_short", "c_ushort", "c_int", "c_uint",
679 "c_long", "c_ulong", "c_longlong", "c_ulonglong", "c_char",
680 "anyopaque", "void", "bool", "isize", "usize",
681 "noreturn", "type", "anyerror", "comptime_int", "comptime_float",
682};
683
684fn isType(name: []const u8) bool {
685 for (builtin_types) |t| {
686 if (mem.eql(u8, t, name))
687 return true;
688 }
689 return false;
690}
691
692fn writeEscapedLines(out: *Writer, text: []const u8) !void {
693 return writeEscaped(out, text);
694}
695
696fn tokenizeAndPrintRaw(
697 allocator: Allocator,
698 docgen_tokenizer: *Tokenizer,
699 out: *Writer,
700 source_token: Token,
701 raw_src: []const u8,
702) !void {
703 const src_non_terminated = mem.trim(u8, raw_src, " \r\n");
704 const src = try allocator.dupeSentinel(u8, src_non_terminated, 0);
705
706 try out.writeAll("<code>");
707 var tokenizer = std.zig.Tokenizer.init(src);
708 var index: usize = 0;
709 var next_tok_is_fn = false;
710 while (true) {
711 const prev_tok_was_fn = next_tok_is_fn;
712 next_tok_is_fn = false;
713
714 const token = tokenizer.next();
715 if (mem.find(u8, src[index..token.loc.start], "//")) |comment_start_off| {
716 // render one comment
717 const comment_start = index + comment_start_off;
718 const comment_end_off = mem.find(u8, src[comment_start..token.loc.start], "\n");
719 const comment_end = if (comment_end_off) |o| comment_start + o else token.loc.start;
720
721 try writeEscapedLines(out, src[index..comment_start]);
722 try out.writeAll("<span class=\"tok-comment\">");
723 try writeEscaped(out, src[comment_start..comment_end]);
724 try out.writeAll("</span>");
725 index = comment_end;
726 tokenizer.index = index;
727 continue;
728 }
729
730 try writeEscapedLines(out, src[index..token.loc.start]);
731 switch (token.tag) {
732 .eof => break,
733
734 .keyword_addrspace,
735 .keyword_align,
736 .keyword_and,
737 .keyword_asm,
738 .keyword_break,
739 .keyword_catch,
740 .keyword_comptime,
741 .keyword_const,
742 .keyword_continue,
743 .keyword_defer,
744 .keyword_else,
745 .keyword_enum,
746 .keyword_errdefer,
747 .keyword_error,
748 .keyword_export,
749 .keyword_extern,
750 .keyword_for,
751 .keyword_if,
752 .keyword_inline,
753 .keyword_noalias,
754 .keyword_noinline,
755 .keyword_nosuspend,
756 .keyword_opaque,
757 .keyword_or,
758 .keyword_orelse,
759 .keyword_packed,
760 .keyword_anyframe,
761 .keyword_pub,
762 .keyword_resume,
763 .keyword_return,
764 .keyword_linksection,
765 .keyword_callconv,
766 .keyword_struct,
767 .keyword_suspend,
768 .keyword_switch,
769 .keyword_test,
770 .keyword_threadlocal,
771 .keyword_try,
772 .keyword_union,
773 .keyword_unreachable,
774 .keyword_var,
775 .keyword_volatile,
776 .keyword_allowzero,
777 .keyword_while,
778 .keyword_anytype,
779 => {
780 try out.writeAll("<span class=\"tok-kw\">");
781 try writeEscaped(out, src[token.loc.start..token.loc.end]);
782 try out.writeAll("</span>");
783 },
784
785 .keyword_fn => {
786 try out.writeAll("<span class=\"tok-kw\">");
787 try writeEscaped(out, src[token.loc.start..token.loc.end]);
788 try out.writeAll("</span>");
789 next_tok_is_fn = true;
790 },
791
792 .string_literal,
793 .multiline_string_literal_line,
794 .char_literal,
795 => {
796 try out.writeAll("<span class=\"tok-str\">");
797 try writeEscaped(out, src[token.loc.start..token.loc.end]);
798 try out.writeAll("</span>");
799 },
800
801 .builtin => {
802 try out.writeAll("<span class=\"tok-builtin\">");
803 try writeEscaped(out, src[token.loc.start..token.loc.end]);
804 try out.writeAll("</span>");
805 },
806
807 .doc_comment,
808 .container_doc_comment,
809 => {
810 try out.writeAll("<span class=\"tok-comment\">");
811 try writeEscaped(out, src[token.loc.start..token.loc.end]);
812 try out.writeAll("</span>");
813 },
814
815 .identifier => {
816 const tok_bytes = src[token.loc.start..token.loc.end];
817 if (mem.eql(u8, tok_bytes, "undefined") or
818 mem.eql(u8, tok_bytes, "null") or
819 mem.eql(u8, tok_bytes, "true") or
820 mem.eql(u8, tok_bytes, "false"))
821 {
822 try out.writeAll("<span class=\"tok-null\">");
823 try writeEscaped(out, tok_bytes);
824 try out.writeAll("</span>");
825 } else if (prev_tok_was_fn) {
826 try out.writeAll("<span class=\"tok-fn\">");
827 try writeEscaped(out, tok_bytes);
828 try out.writeAll("</span>");
829 } else {
830 const is_int = blk: {
831 if (src[token.loc.start] != 'i' and src[token.loc.start] != 'u')
832 break :blk false;
833 var i = token.loc.start + 1;
834 if (i == token.loc.end)
835 break :blk false;
836 while (i != token.loc.end) : (i += 1) {
837 if (src[i] < '0' or src[i] > '9')
838 break :blk false;
839 }
840 break :blk true;
841 };
842 if (is_int or isType(tok_bytes)) {
843 try out.writeAll("<span class=\"tok-type\">");
844 try writeEscaped(out, tok_bytes);
845 try out.writeAll("</span>");
846 } else {
847 try writeEscaped(out, tok_bytes);
848 }
849 }
850 },
851
852 .number_literal => {
853 try out.writeAll("<span class=\"tok-number\">");
854 try writeEscaped(out, src[token.loc.start..token.loc.end]);
855 try out.writeAll("</span>");
856 },
857
858 .bang,
859 .pipe,
860 .pipe_pipe,
861 .pipe_equal,
862 .equal,
863 .equal_equal,
864 .equal_angle_bracket_right,
865 .bang_equal,
866 .l_paren,
867 .r_paren,
868 .semicolon,
869 .percent,
870 .percent_equal,
871 .l_brace,
872 .r_brace,
873 .l_bracket,
874 .r_bracket,
875 .period,
876 .period_asterisk,
877 .ellipsis2,
878 .ellipsis3,
879 .caret,
880 .caret_equal,
881 .plus,
882 .plus_plus,
883 .plus_equal,
884 .plus_percent,
885 .plus_percent_equal,
886 .plus_pipe,
887 .plus_pipe_equal,
888 .minus,
889 .minus_equal,
890 .minus_percent,
891 .minus_percent_equal,
892 .minus_pipe,
893 .minus_pipe_equal,
894 .asterisk,
895 .asterisk_equal,
896 .asterisk_percent,
897 .asterisk_percent_equal,
898 .asterisk_pipe,
899 .asterisk_pipe_equal,
900 .arrow,
901 .colon,
902 .slash,
903 .slash_equal,
904 .comma,
905 .ampersand,
906 .ampersand_equal,
907 .question_mark,
908 .angle_bracket_left,
909 .angle_bracket_left_equal,
910 .angle_bracket_angle_bracket_left,
911 .angle_bracket_angle_bracket_left_equal,
912 .angle_bracket_angle_bracket_left_pipe,
913 .angle_bracket_angle_bracket_left_pipe_equal,
914 .angle_bracket_right,
915 .angle_bracket_right_equal,
916 .angle_bracket_angle_bracket_right,
917 .angle_bracket_angle_bracket_right_equal,
918 .tilde,
919 => try writeEscaped(out, src[token.loc.start..token.loc.end]),
920
921 .invalid => return parseError(
922 docgen_tokenizer,
923 source_token,
924 "syntax error",
925 .{},
926 ),
927 }
928 index = token.loc.end;
929 }
930 try out.writeAll("</code>");
931}
932
933fn tokenizeAndPrint(
934 allocator: Allocator,
935 docgen_tokenizer: *Tokenizer,
936 out: *Writer,
937 source_token: Token,
938) !void {
939 const raw_src = docgen_tokenizer.buffer[source_token.start..source_token.end];
940 return tokenizeAndPrintRaw(allocator, docgen_tokenizer, out, source_token, raw_src);
941}
942
943fn printSourceBlock(allocator: Allocator, docgen_tokenizer: *Tokenizer, out: *Writer, syntax_block: SyntaxBlock, content: ?[]const u8) !void {
944 const source_type = @tagName(syntax_block.source_type);
945
946 try out.print("<figure><figcaption class=\"{s}-cap\"><cite class=\"file\">{s}</cite></figcaption><pre>", .{ source_type, syntax_block.name });
947 switch (syntax_block.source_type) {
948 .zig => try tokenizeAndPrint(allocator, docgen_tokenizer, out, syntax_block.source_token),
949 else => {
950 const raw_source = content orelse docgen_tokenizer.buffer[syntax_block.source_token.start..syntax_block.source_token.end];
951 const trimmed_raw_source = mem.trim(u8, raw_source, " \r\n");
952
953 try out.writeAll("<code>");
954 try writeEscapedLines(out, trimmed_raw_source);
955 try out.writeAll("</code>");
956 },
957 }
958 try out.writeAll("</pre></figure>");
959}
960
961fn printShell(out: *Writer, shell_content: []const u8, escape: bool) !void {
962 const trimmed_shell_content = mem.trim(u8, shell_content, " \r\n");
963 try out.writeAll("<figure><figcaption class=\"shell-cap\">Shell</figcaption><pre><samp>");
964 var cmd_cont: bool = false;
965 var iter = std.mem.splitScalar(u8, trimmed_shell_content, '\n');
966 while (iter.next()) |orig_line| {
967 const line = mem.trimEnd(u8, orig_line, " \r");
968 if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] != '\\') {
969 try out.writeAll("$ <kbd>");
970 const s = std.mem.trimStart(u8, line[1..], " ");
971 if (escape) {
972 try writeEscaped(out, s);
973 } else {
974 try out.writeAll(s);
975 }
976 try out.writeAll("</kbd>" ++ "\n");
977 } else if (!cmd_cont and line.len > 1 and mem.eql(u8, line[0..2], "$ ") and line[line.len - 1] == '\\') {
978 try out.writeAll("$ <kbd>");
979 const s = std.mem.trimStart(u8, line[1..], " ");
980 if (escape) {
981 try writeEscaped(out, s);
982 } else {
983 try out.writeAll(s);
984 }
985 try out.writeAll("\n");
986 cmd_cont = true;
987 } else if (line.len > 0 and line[line.len - 1] != '\\' and cmd_cont) {
988 if (escape) {
989 try writeEscaped(out, line);
990 } else {
991 try out.writeAll(line);
992 }
993 try out.writeAll("</kbd>" ++ "\n");
994 cmd_cont = false;
995 } else {
996 if (escape) {
997 try writeEscaped(out, line);
998 } else {
999 try out.writeAll(line);
1000 }
1001 try out.writeAll("\n");
1002 }
1003 }
1004
1005 try out.writeAll("</samp></pre></figure>");
1006}
1007
1008fn genHtml(
1009 allocator: Allocator,
1010 io: Io,
1011 tokenizer: *Tokenizer,
1012 toc: *Toc,
1013 code_dir: Path,
1014 grammar: []const u8,
1015 out: *Writer,
1016) !void {
1017 for (toc.nodes) |node| {
1018 switch (node) {
1019 .Content => |data| {
1020 try out.writeAll(data);
1021 },
1022 .Link => |info| {
1023 if (!toc.urls.contains(info.url)) {
1024 return parseError(tokenizer, info.token, "url not found: {s}", .{info.url});
1025 }
1026 try out.print("<a href=\"#{s}\">{s}</a>", .{ info.url, info.name });
1027 },
1028 .Nav => {
1029 try out.writeAll(toc.toc);
1030 },
1031 .Builtin => |tok| {
1032 try out.writeAll("<figure><figcaption class=\"zig-cap\"><cite>@import(\"builtin\")</cite></figcaption><pre>");
1033 const builtin_code = @embedFile("builtin"); // 😎
1034 try tokenizeAndPrintRaw(allocator, tokenizer, out, tok, builtin_code);
1035 try out.writeAll("</pre></figure>");
1036 },
1037 .HeaderOpen => |info| {
1038 try out.print(
1039 "<h{d} id=\"{s}\"><a href=\"#toc-{s}\">{s}</a> <a class=\"hdr\" href=\"#{s}\">§</a></h{d}>\n",
1040 .{ info.n, info.url, info.url, info.name, info.url, info.n },
1041 );
1042 },
1043 .SeeAlso => |items| {
1044 try out.writeAll("<p>See also:</p><ul>\n");
1045 for (items) |item| {
1046 const url = try urlize(allocator, item.name);
1047 if (!toc.urls.contains(url)) {
1048 return parseError(tokenizer, item.token, "url not found: {s}", .{url});
1049 }
1050 try out.print("<li><a href=\"#{s}\">{s}</a></li>\n", .{ url, item.name });
1051 }
1052 try out.writeAll("</ul>\n");
1053 },
1054 .InlineSyntax => |content_tok| {
1055 try tokenizeAndPrint(allocator, tokenizer, out, content_tok);
1056 },
1057 .Shell => |content_tok| {
1058 const raw_shell_content = tokenizer.buffer[content_tok.start..content_tok.end];
1059 try printShell(out, raw_shell_content, true);
1060 },
1061 .SyntaxBlock => |syntax_block| {
1062 try printSourceBlock(allocator, tokenizer, out, syntax_block, null);
1063 },
1064 .Code => |code| {
1065 const out_basename = try std.fmt.allocPrint(allocator, "{s}.out", .{
1066 Dir.path.stem(code.name),
1067 });
1068 defer allocator.free(out_basename);
1069
1070 const out_path: Path = .{
1071 .root_dir = code_dir.root_dir,
1072 .sub_path = out_basename,
1073 };
1074
1075 const contents = out_path.root_dir.handle.readFileAlloc(io, out_path.sub_path, allocator, .unlimited) catch |err| {
1076 return parseError(tokenizer, code.token, "failed opening {f}: {t}", .{ out_path, err });
1077 };
1078 defer allocator.free(contents);
1079
1080 try out.writeAll(contents);
1081 },
1082 .Grammar => {
1083 try printSourceBlock(allocator, tokenizer, out, .{
1084 .source_type = .peg,
1085 .name = "grammar.peg",
1086 .source_token = undefined,
1087 }, grammar);
1088 },
1089 }
1090 }
1091}