authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-01-17 01:50:35-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-01-17 01:50:35-05:00
log1eda7e0fde88ac06eaf595f74157d1e6214c1b3d
tree967645940c9b7b25b4d8dfbfa68678cb3734de41
parent5aefabe045ca9c6f961bb5354961fe483085f145

docgen: support executing exe code examples

See #465

5 files changed, 171 insertions(+), 17 deletions(-)

build.zig+3
......@@ -15,8 +15,10 @@ pub fn build(b: &Builder) -> %void {
1515
1616 var docgen_exe = b.addExecutable("docgen", "doc/docgen.zig");
1717
18 const rel_zig_exe = try os.path.relative(b.allocator, b.build_root, b.zig_exe);
1819 var docgen_cmd = b.addCommand(null, b.env_map, [][]const u8 {
1920 docgen_exe.getOutputPath(),
21 rel_zig_exe,
2022 "doc/langref.html.in",
2123 os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable,
2224 });
......@@ -24,6 +26,7 @@ pub fn build(b: &Builder) -> %void {
2426
2527 var docgen_home_cmd = b.addCommand(null, b.env_map, [][]const u8 {
2628 docgen_exe.getOutputPath(),
29 rel_zig_exe,
2730 "doc/home.html.in",
2831 os.path.join(b.allocator, b.cache_root, "home.html") catch unreachable,
2932 });
doc/docgen.zig+142-7
......@@ -4,7 +4,9 @@ const os = std.os;
44const warn = std.debug.warn;
55const mem = std.mem;
66
7pub const max_doc_file_size = 10 * 1024 * 1024;
7const max_doc_file_size = 10 * 1024 * 1024;
8
9const exe_ext = std.build.Target(std.build.Target.Native).exeFileExt();
810
911pub fn main() -> %void {
1012 // TODO use a more general purpose allocator here
......@@ -16,6 +18,9 @@ pub fn main() -> %void {
1618
1719 if (!args_it.skip()) @panic("expected self arg");
1820
21 const zig_exe = try (args_it.next(allocator) ?? @panic("expected zig exe arg"));
22 defer allocator.free(zig_exe);
23
1924 const in_file_name = try (args_it.next(allocator) ?? @panic("expected input arg"));
2025 defer allocator.free(in_file_name);
2126
......@@ -38,7 +43,7 @@ pub fn main() -> %void {
3843 var tokenizer = Tokenizer.init(in_file_name, input_file_bytes);
3944 var toc = try genToc(allocator, &tokenizer);
4045
41 try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream);
46 try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream, zig_exe);
4247 try buffered_out_stream.flush();
4348}
4449
......@@ -246,11 +251,24 @@ const SeeAlsoItem = struct {
246251 token: Token,
247252};
248253
254const Code = struct {
255 id: Id,
256 name: []const u8,
257 source_token: Token,
258
259 const Id = enum {
260 Test,
261 Exe,
262 Error,
263 };
264};
265
249266const Node = union(enum) {
250267 Content: []const u8,
251268 Nav,
252269 HeaderOpen: HeaderOpen,
253270 SeeAlso: []const SeeAlsoItem,
271 Code: Code,
254272};
255273
256274const Toc = struct {
......@@ -300,14 +318,14 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {
300318 const tag_name = tokenizer.buffer[tag_token.start..tag_token.end];
301319
302320 if (mem.eql(u8, tag_name, "nav")) {
303 _ = eatToken(tokenizer, Token.Id.BracketClose);
321 _ = try eatToken(tokenizer, Token.Id.BracketClose);
304322
305323 try nodes.append(Node.Nav);
306324 } else if (mem.eql(u8, tag_name, "header_open")) {
307 _ = eatToken(tokenizer, Token.Id.Separator);
325 _ = try eatToken(tokenizer, Token.Id.Separator);
308326 const content_token = try eatToken(tokenizer, Token.Id.TagContent);
309327 const content = tokenizer.buffer[content_token.start..content_token.end];
310 _ = eatToken(tokenizer, Token.Id.BracketClose);
328 _ = try eatToken(tokenizer, Token.Id.BracketClose);
311329
312330 header_stack_size += 1;
313331
......@@ -336,7 +354,7 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {
336354 return parseError(tokenizer, tag_token, "unbalanced close header");
337355 }
338356 header_stack_size -= 1;
339 _ = eatToken(tokenizer, Token.Id.BracketClose);
357 _ = try eatToken(tokenizer, Token.Id.BracketClose);
340358
341359 if (last_action == Action.Close) {
342360 try toc.writeByteNTimes(' ', 8 + header_stack_size * 4);
......@@ -367,6 +385,44 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {
367385 else => return parseError(tokenizer, see_also_tok, "invalid see_also token"),
368386 }
369387 }
388 } else if (mem.eql(u8, tag_name, "code_begin")) {
389 _ = try eatToken(tokenizer, Token.Id.Separator);
390 const code_kind_tok = try eatToken(tokenizer, Token.Id.TagContent);
391 var name: []const u8 = "test";
392 const maybe_sep = tokenizer.next();
393 switch (maybe_sep.id) {
394 Token.Id.Separator => {
395 const name_tok = try eatToken(tokenizer, Token.Id.TagContent);
396 name = tokenizer.buffer[name_tok.start..name_tok.end];
397 _ = try eatToken(tokenizer, Token.Id.BracketClose);
398 },
399 Token.Id.BracketClose => {},
400 else => return parseError(tokenizer, token, "invalid token"),
401 }
402 const code_kind_str = tokenizer.buffer[code_kind_tok.start..code_kind_tok.end];
403 var code_kind_id: Code.Id = undefined;
404 if (mem.eql(u8, code_kind_str, "exe")) {
405 code_kind_id = Code.Id.Exe;
406 } else if (mem.eql(u8, code_kind_str, "test")) {
407 code_kind_id = Code.Id.Test;
408 } else if (mem.eql(u8, code_kind_str, "error")) {
409 code_kind_id = Code.Id.Error;
410 } else {
411 return parseError(tokenizer, code_kind_tok, "unrecognized code kind: {}", code_kind_str);
412 }
413 const source_token = try eatToken(tokenizer, Token.Id.Content);
414 _ = try eatToken(tokenizer, Token.Id.BracketOpen);
415 const end_code_tag = try eatToken(tokenizer, Token.Id.TagContent);
416 const end_tag_name = tokenizer.buffer[end_code_tag.start..end_code_tag.end];
417 if (!mem.eql(u8, end_tag_name, "code_end")) {
418 return parseError(tokenizer, end_code_tag, "expected code_end token");
419 }
420 _ = try eatToken(tokenizer, Token.Id.BracketClose);
421 try nodes.append(Node {.Code = Code{
422 .id = code_kind_id,
423 .name = name,
424 .source_token = source_token,
425 }});
370426 } else {
371427 return parseError(tokenizer, tag_token, "unrecognized tag name: {}", tag_name);
372428 }
......@@ -402,7 +458,27 @@ fn urlize(allocator: &mem.Allocator, input: []const u8) -> %[]u8 {
402458 return buf.toOwnedSlice();
403459}
404460
405fn genHtml(allocator: &mem.Allocator, tokenizer: &Tokenizer, toc: &Toc, out: &io.OutStream) -> %void {
461fn escapeHtml(allocator: &mem.Allocator, input: []const u8) -> %[]u8 {
462 var buf = try std.Buffer.initSize(allocator, 0);
463 defer buf.deinit();
464
465 var buf_adapter = io.BufferOutStream.init(&buf);
466 var out = &buf_adapter.stream;
467 for (input) |c| {
468 try switch (c) {
469 '&' => out.write("&amp;"),
470 '<' => out.write("&lt;"),
471 '>' => out.write("&gt;"),
472 '"' => out.write("&quot;"),
473 else => out.writeByte(c),
474 };
475 }
476 return buf.toOwnedSlice();
477}
478
479error ExampleFailedToCompile;
480
481fn genHtml(allocator: &mem.Allocator, tokenizer: &Tokenizer, toc: &Toc, out: &io.OutStream, zig_exe: []const u8) -> %void {
406482 for (toc.nodes) |node| {
407483 switch (node) {
408484 Node.Content => |data| {
......@@ -425,6 +501,65 @@ fn genHtml(allocator: &mem.Allocator, tokenizer: &Tokenizer, toc: &Toc, out: &io
425501 }
426502 try out.write("</ul>\n");
427503 },
504 Node.Code => |code| {
505 const raw_source = tokenizer.buffer[code.source_token.start..code.source_token.end];
506 const trimmed_raw_source = mem.trim(u8, raw_source, " \n");
507 const escaped_source = try escapeHtml(allocator, trimmed_raw_source);
508 try out.print("<pre><code class=\"zig\">{}</code></pre>", escaped_source);
509 const tmp_dir_name = "docgen_tmp";
510 try os.makePath(allocator, tmp_dir_name);
511 const name_plus_ext = try std.fmt.allocPrint(allocator, "{}.zig", code.name);
512 const name_plus_bin_ext = try std.fmt.allocPrint(allocator, "{}{}", code.name, exe_ext);
513 const tmp_source_file_name = try os.path.join(allocator, tmp_dir_name, name_plus_ext);
514 const tmp_bin_file_name = try os.path.join(allocator, tmp_dir_name, name_plus_bin_ext);
515 try io.writeFile(tmp_source_file_name, trimmed_raw_source, null);
516
517 switch (code.id) {
518 Code.Id.Exe => {
519 {
520 const args = [][]const u8 {zig_exe, "build-exe", tmp_source_file_name, "--output", tmp_bin_file_name};
521 const result = try os.ChildProcess.exec(allocator, args, null, null, max_doc_file_size);
522 switch (result.term) {
523 os.ChildProcess.Term.Exited => |exit_code| {
524 if (exit_code != 0) {
525 warn("{}\nThe following command exited with code {}:\n", result.stderr, exit_code);
526 for (args) |arg| warn("{} ", arg) else warn("\n");
527 return parseError(tokenizer, code.source_token, "example failed to compile");
528 }
529 },
530 else => {
531 warn("{}\nThe following command crashed:\n", result.stderr);
532 for (args) |arg| warn("{} ", arg) else warn("\n");
533 return parseError(tokenizer, code.source_token, "example failed to compile");
534 },
535 }
536 }
537 const args = [][]const u8 {tmp_bin_file_name};
538 const result = try os.ChildProcess.exec(allocator, args, null, null, max_doc_file_size);
539 switch (result.term) {
540 os.ChildProcess.Term.Exited => |exit_code| {
541 if (exit_code != 0) {
542 warn("The following command exited with code {}:\n", exit_code);
543 for (args) |arg| warn("{} ", arg) else warn("\n");
544 return parseError(tokenizer, code.source_token, "example exited with code {}", exit_code);
545 }
546 },
547 else => {
548 warn("The following command crashed:\n");
549 for (args) |arg| warn("{} ", arg) else warn("\n");
550 return parseError(tokenizer, code.source_token, "example crashed");
551 },
552 }
553 try out.print("<pre><code class=\"sh\">$ zig build-exe {}.zig\n$ ./{}\n{}{}</code></pre>\n", code.name, code.name, result.stderr, result.stdout);
554 },
555 Code.Id.Test => {
556 @panic("TODO");
557 },
558 Code.Id.Error => {
559 @panic("TODO");
560 },
561 }
562 },
428563 }
429564 }
430565
doc/langref.html.in+11-9
......@@ -55,28 +55,30 @@
5555 </p>
5656 {#header_close#}
5757 {#header_open|Hello World#}
58 <pre><code class="zig">const std = @import("std");
5958
60pub fn main() -&gt; %void {
59 {#code_begin|exe|hello#}
60const std = @import("std");
61
62pub fn main() -> %void {
6163 // If this program is run without stdout attached, exit with an error.
6264 var stdout_file = try std.io.getStdOut();
6365 // If this program encounters pipe failure when printing to stdout, exit
6466 // with an error.
6567 try stdout_file.write("Hello, world!\n");
66}</code></pre>
67 <pre><code class="sh">$ zig build-exe hello.zig
68$ ./hello
69Hello, world!</code></pre>
68}
69 {#code_end#}
7070 <p>
7171 Usually you don't want to write to stdout. You want to write to stderr. And you
7272 don't care if it fails. It's more like a <em>warning message</em> that you want
7373 to emit. For that you can use a simpler API:
7474 </p>
75 <pre><code class="zig">const warn = @import("std").debug.warn;
75 {#code_begin|exe|hello#}
76const warn = @import("std").debug.warn;
7677
77pub fn main() -&gt; %void {
78pub fn main() -> %void {
7879 warn("Hello, world!\n");
79}</code></pre>
80}
81 {#code_end#}
8082 {#see_also|Values|@import|Errors|Root Source File#}
8183 {#header_close#}
8284 {#header_open|Source Encoding#}
std/build.zig+1-1
......@@ -760,7 +760,7 @@ const CrossTarget = struct {
760760 environ: builtin.Environ,
761761};
762762
763const Target = union(enum) {
763pub const Target = union(enum) {
764764 Native: void,
765765 Cross: CrossTarget,
766766
std/mem.zig+14
......@@ -203,6 +203,20 @@ pub fn dupe(allocator: &Allocator, comptime T: type, m: []const T) -> %[]T {
203203 return new_buf;
204204}
205205
206/// Remove values from the beginning and end of a slice.
207pub fn trim(comptime T: type, slice: []const T, values_to_strip: []const T) -> []const T {
208 var begin: usize = 0;
209 var end: usize = slice.len;
210 while (begin < end and indexOfScalar(T, values_to_strip, slice[begin]) != null) : (begin += 1) {}
211 while (end > begin and indexOfScalar(T, values_to_strip, slice[end - 1]) != null) : (end -= 1) {}
212 return slice[begin..end];
213}
214
215test "mem.trim" {
216 assert(eql(u8, trim(u8, " foo\n ", " \n"), "foo"));
217 assert(eql(u8, trim(u8, "foo", " \n"), "foo"));
218}
219
206220/// Linear search for the index of a scalar value inside a slice.
207221pub fn indexOfScalar(comptime T: type, slice: []const T, value: T) -> ?usize {
208222 return indexOfScalarPos(T, slice, 0, value);