authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-01-17 01:50:35-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-01-17 01:50:35-05:00
log1eda7e0fde88ac06eaf595f74157d1e6214c1b3d
tree967645940c9b7b25b4d8dfbfa68678cb3734de41
parent5aefabe045ca9c6f961bb5354961fe483085f145

docgen: support executing exe code examples

See #465

5 files changed, 171 insertions(+), 17 deletions(-)

build.zig+3
...@@ -15,8 +15,10 @@ pub fn build(b: &Builder) -> %void {...@@ -15,8 +15,10 @@ pub fn build(b: &Builder) -> %void {
1515
16 var docgen_exe = b.addExecutable("docgen", "doc/docgen.zig");16 var docgen_exe = b.addExecutable("docgen", "doc/docgen.zig");
1717
18 const rel_zig_exe = try os.path.relative(b.allocator, b.build_root, b.zig_exe);
18 var docgen_cmd = b.addCommand(null, b.env_map, [][]const u8 {19 var docgen_cmd = b.addCommand(null, b.env_map, [][]const u8 {
19 docgen_exe.getOutputPath(),20 docgen_exe.getOutputPath(),
21 rel_zig_exe,
20 "doc/langref.html.in",22 "doc/langref.html.in",
21 os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable,23 os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable,
22 });24 });
...@@ -24,6 +26,7 @@ pub fn build(b: &Builder) -> %void {...@@ -24,6 +26,7 @@ pub fn build(b: &Builder) -> %void {
2426
25 var docgen_home_cmd = b.addCommand(null, b.env_map, [][]const u8 {27 var docgen_home_cmd = b.addCommand(null, b.env_map, [][]const u8 {
26 docgen_exe.getOutputPath(),28 docgen_exe.getOutputPath(),
29 rel_zig_exe,
27 "doc/home.html.in",30 "doc/home.html.in",
28 os.path.join(b.allocator, b.cache_root, "home.html") catch unreachable,31 os.path.join(b.allocator, b.cache_root, "home.html") catch unreachable,
29 });32 });
doc/docgen.zig+142-7
...@@ -4,7 +4,9 @@ const os = std.os;...@@ -4,7 +4,9 @@ const os = std.os;
4const warn = std.debug.warn;4const warn = std.debug.warn;
5const mem = std.mem;5const mem = std.mem;
66
7pub const max_doc_file_size = 10 * 1024 * 1024;7const max_doc_file_size = 10 * 1024 * 1024;
8
9const exe_ext = std.build.Target(std.build.Target.Native).exeFileExt();
810
9pub fn main() -> %void {11pub fn main() -> %void {
10 // TODO use a more general purpose allocator here12 // TODO use a more general purpose allocator here
...@@ -16,6 +18,9 @@ pub fn main() -> %void {...@@ -16,6 +18,9 @@ pub fn main() -> %void {
1618
17 if (!args_it.skip()) @panic("expected self arg");19 if (!args_it.skip()) @panic("expected self arg");
1820
21 const zig_exe = try (args_it.next(allocator) ?? @panic("expected zig exe arg"));
22 defer allocator.free(zig_exe);
23
19 const in_file_name = try (args_it.next(allocator) ?? @panic("expected input arg"));24 const in_file_name = try (args_it.next(allocator) ?? @panic("expected input arg"));
20 defer allocator.free(in_file_name);25 defer allocator.free(in_file_name);
2126
...@@ -38,7 +43,7 @@ pub fn main() -> %void {...@@ -38,7 +43,7 @@ pub fn main() -> %void {
38 var tokenizer = Tokenizer.init(in_file_name, input_file_bytes);43 var tokenizer = Tokenizer.init(in_file_name, input_file_bytes);
39 var toc = try genToc(allocator, &tokenizer);44 var toc = try genToc(allocator, &tokenizer);
4045
41 try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream);46 try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream, zig_exe);
42 try buffered_out_stream.flush();47 try buffered_out_stream.flush();
43}48}
4449
...@@ -246,11 +251,24 @@ const SeeAlsoItem = struct {...@@ -246,11 +251,24 @@ const SeeAlsoItem = struct {
246 token: Token,251 token: Token,
247};252};
248253
254const Code = struct {
255 id: Id,
256 name: []const u8,
257 source_token: Token,
258
259 const Id = enum {
260 Test,
261 Exe,
262 Error,
263 };
264};
265
249const Node = union(enum) {266const Node = union(enum) {
250 Content: []const u8,267 Content: []const u8,
251 Nav,268 Nav,
252 HeaderOpen: HeaderOpen,269 HeaderOpen: HeaderOpen,
253 SeeAlso: []const SeeAlsoItem,270 SeeAlso: []const SeeAlsoItem,
271 Code: Code,
254};272};
255273
256const Toc = struct {274const Toc = struct {
...@@ -300,14 +318,14 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {...@@ -300,14 +318,14 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {
300 const tag_name = tokenizer.buffer[tag_token.start..tag_token.end];318 const tag_name = tokenizer.buffer[tag_token.start..tag_token.end];
301319
302 if (mem.eql(u8, tag_name, "nav")) {320 if (mem.eql(u8, tag_name, "nav")) {
303 _ = eatToken(tokenizer, Token.Id.BracketClose);321 _ = try eatToken(tokenizer, Token.Id.BracketClose);
304322
305 try nodes.append(Node.Nav);323 try nodes.append(Node.Nav);
306 } else if (mem.eql(u8, tag_name, "header_open")) {324 } else if (mem.eql(u8, tag_name, "header_open")) {
307 _ = eatToken(tokenizer, Token.Id.Separator);325 _ = try eatToken(tokenizer, Token.Id.Separator);
308 const content_token = try eatToken(tokenizer, Token.Id.TagContent);326 const content_token = try eatToken(tokenizer, Token.Id.TagContent);
309 const content = tokenizer.buffer[content_token.start..content_token.end];327 const content = tokenizer.buffer[content_token.start..content_token.end];
310 _ = eatToken(tokenizer, Token.Id.BracketClose);328 _ = try eatToken(tokenizer, Token.Id.BracketClose);
311329
312 header_stack_size += 1;330 header_stack_size += 1;
313331
...@@ -336,7 +354,7 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {...@@ -336,7 +354,7 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {
336 return parseError(tokenizer, tag_token, "unbalanced close header");354 return parseError(tokenizer, tag_token, "unbalanced close header");
337 }355 }
338 header_stack_size -= 1;356 header_stack_size -= 1;
339 _ = eatToken(tokenizer, Token.Id.BracketClose);357 _ = try eatToken(tokenizer, Token.Id.BracketClose);
340358
341 if (last_action == Action.Close) {359 if (last_action == Action.Close) {
342 try toc.writeByteNTimes(' ', 8 + header_stack_size * 4);360 try toc.writeByteNTimes(' ', 8 + header_stack_size * 4);
...@@ -367,6 +385,44 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {...@@ -367,6 +385,44 @@ fn genToc(allocator: &mem.Allocator, tokenizer: &Tokenizer) -> %Toc {
367 else => return parseError(tokenizer, see_also_tok, "invalid see_also token"),385 else => return parseError(tokenizer, see_also_tok, "invalid see_also token"),
368 }386 }
369 }387 }
388 } else if (mem.eql(u8, tag_name, "code_begin")) {
389 _ = try eatToken(tokenizer, Token.Id.Separator);
390 const code_kind_tok = try eatToken(tokenizer, Token.Id.TagContent);
391 var name: []const u8 = "test";
392 const maybe_sep = tokenizer.next();
393 switch (maybe_sep.id) {
394 Token.Id.Separator => {
395 const name_tok = try eatToken(tokenizer, Token.Id.TagContent);
396 name = tokenizer.buffer[name_tok.start..name_tok.end];
397 _ = try eatToken(tokenizer, Token.Id.BracketClose);
398 },
399 Token.Id.BracketClose => {},
400 else => return parseError(tokenizer, token, "invalid token"),
401 }
402 const code_kind_str = tokenizer.buffer[code_kind_tok.start..code_kind_tok.end];
403 var code_kind_id: Code.Id = undefined;
404 if (mem.eql(u8, code_kind_str, "exe")) {
405 code_kind_id = Code.Id.Exe;
406 } else if (mem.eql(u8, code_kind_str, "test")) {
407 code_kind_id = Code.Id.Test;
408 } else if (mem.eql(u8, code_kind_str, "error")) {
409 code_kind_id = Code.Id.Error;
410 } else {
411 return parseError(tokenizer, code_kind_tok, "unrecognized code kind: {}", code_kind_str);
412 }
413 const source_token = try eatToken(tokenizer, Token.Id.Content);
414 _ = try eatToken(tokenizer, Token.Id.BracketOpen);
415 const end_code_tag = try eatToken(tokenizer, Token.Id.TagContent);
416 const end_tag_name = tokenizer.buffer[end_code_tag.start..end_code_tag.end];
417 if (!mem.eql(u8, end_tag_name, "code_end")) {
418 return parseError(tokenizer, end_code_tag, "expected code_end token");
419 }
420 _ = try eatToken(tokenizer, Token.Id.BracketClose);
421 try nodes.append(Node {.Code = Code{
422 .id = code_kind_id,
423 .name = name,
424 .source_token = source_token,
425 }});
370 } else {426 } else {
371 return parseError(tokenizer, tag_token, "unrecognized tag name: {}", tag_name);427 return parseError(tokenizer, tag_token, "unrecognized tag name: {}", tag_name);
372 }428 }
...@@ -402,7 +458,27 @@ fn urlize(allocator: &mem.Allocator, input: []const u8) -> %[]u8 {...@@ -402,7 +458,27 @@ fn urlize(allocator: &mem.Allocator, input: []const u8) -> %[]u8 {
402 return buf.toOwnedSlice();458 return buf.toOwnedSlice();
403}459}
404460
405fn genHtml(allocator: &mem.Allocator, tokenizer: &Tokenizer, toc: &Toc, out: &io.OutStream) -> %void {461fn escapeHtml(allocator: &mem.Allocator, input: []const u8) -> %[]u8 {
462 var buf = try std.Buffer.initSize(allocator, 0);
463 defer buf.deinit();
464
465 var buf_adapter = io.BufferOutStream.init(&buf);
466 var out = &buf_adapter.stream;
467 for (input) |c| {
468 try switch (c) {
469 '&' => out.write("&amp;"),
470 '<' => out.write("&lt;"),
471 '>' => out.write("&gt;"),
472 '"' => out.write("&quot;"),
473 else => out.writeByte(c),
474 };
475 }
476 return buf.toOwnedSlice();
477}
478
479error ExampleFailedToCompile;
480
481fn genHtml(allocator: &mem.Allocator, tokenizer: &Tokenizer, toc: &Toc, out: &io.OutStream, zig_exe: []const u8) -> %void {
406 for (toc.nodes) |node| {482 for (toc.nodes) |node| {
407 switch (node) {483 switch (node) {
408 Node.Content => |data| {484 Node.Content => |data| {
...@@ -425,6 +501,65 @@ fn genHtml(allocator: &mem.Allocator, tokenizer: &Tokenizer, toc: &Toc, out: &io...@@ -425,6 +501,65 @@ fn genHtml(allocator: &mem.Allocator, tokenizer: &Tokenizer, toc: &Toc, out: &io
425 }501 }
426 try out.write("</ul>\n");502 try out.write("</ul>\n");
427 },503 },
504 Node.Code => |code| {
505 const raw_source = tokenizer.buffer[code.source_token.start..code.source_token.end];
506 const trimmed_raw_source = mem.trim(u8, raw_source, " \n");
507 const escaped_source = try escapeHtml(allocator, trimmed_raw_source);
508 try out.print("<pre><code class=\"zig\">{}</code></pre>", escaped_source);
509 const tmp_dir_name = "docgen_tmp";
510 try os.makePath(allocator, tmp_dir_name);
511 const name_plus_ext = try std.fmt.allocPrint(allocator, "{}.zig", code.name);
512 const name_plus_bin_ext = try std.fmt.allocPrint(allocator, "{}{}", code.name, exe_ext);
513 const tmp_source_file_name = try os.path.join(allocator, tmp_dir_name, name_plus_ext);
514 const tmp_bin_file_name = try os.path.join(allocator, tmp_dir_name, name_plus_bin_ext);
515 try io.writeFile(tmp_source_file_name, trimmed_raw_source, null);
516
517 switch (code.id) {
518 Code.Id.Exe => {
519 {
520 const args = [][]const u8 {zig_exe, "build-exe", tmp_source_file_name, "--output", tmp_bin_file_name};
521 const result = try os.ChildProcess.exec(allocator, args, null, null, max_doc_file_size);
522 switch (result.term) {
523 os.ChildProcess.Term.Exited => |exit_code| {
524 if (exit_code != 0) {
525 warn("{}\nThe following command exited with code {}:\n", result.stderr, exit_code);
526 for (args) |arg| warn("{} ", arg) else warn("\n");
527 return parseError(tokenizer, code.source_token, "example failed to compile");
528 }
529 },
530 else => {
531 warn("{}\nThe following command crashed:\n", result.stderr);
532 for (args) |arg| warn("{} ", arg) else warn("\n");
533 return parseError(tokenizer, code.source_token, "example failed to compile");
534 },
535 }
536 }
537 const args = [][]const u8 {tmp_bin_file_name};
538 const result = try os.ChildProcess.exec(allocator, args, null, null, max_doc_file_size);
539 switch (result.term) {
540 os.ChildProcess.Term.Exited => |exit_code| {
541 if (exit_code != 0) {
542 warn("The following command exited with code {}:\n", exit_code);
543 for (args) |arg| warn("{} ", arg) else warn("\n");
544 return parseError(tokenizer, code.source_token, "example exited with code {}", exit_code);
545 }
546 },
547 else => {
548 warn("The following command crashed:\n");
549 for (args) |arg| warn("{} ", arg) else warn("\n");
550 return parseError(tokenizer, code.source_token, "example crashed");
551 },
552 }
553 try out.print("<pre><code class=\"sh\">$ zig build-exe {}.zig\n$ ./{}\n{}{}</code></pre>\n", code.name, code.name, result.stderr, result.stdout);
554 },
555 Code.Id.Test => {
556 @panic("TODO");
557 },
558 Code.Id.Error => {
559 @panic("TODO");
560 },
561 }
562 },
428 }563 }
429 }564 }
430565
doc/langref.html.in+11-9
...@@ -55,28 +55,30 @@...@@ -55,28 +55,30 @@
55 </p>55 </p>
56 {#header_close#}56 {#header_close#}
57 {#header_open|Hello World#}57 {#header_open|Hello World#}
58 <pre><code class="zig">const std = @import("std");
5958
60pub fn main() -&gt; %void {59 {#code_begin|exe|hello#}
60const std = @import("std");
61
62pub fn main() -> %void {
61 // If this program is run without stdout attached, exit with an error.63 // If this program is run without stdout attached, exit with an error.
62 var stdout_file = try std.io.getStdOut();64 var stdout_file = try std.io.getStdOut();
63 // If this program encounters pipe failure when printing to stdout, exit65 // If this program encounters pipe failure when printing to stdout, exit
64 // with an error.66 // with an error.
65 try stdout_file.write("Hello, world!\n");67 try stdout_file.write("Hello, world!\n");
66}</code></pre>68}
67 <pre><code class="sh">$ zig build-exe hello.zig69 {#code_end#}
68$ ./hello
69Hello, world!</code></pre>
70 <p>70 <p>
71 Usually you don't want to write to stdout. You want to write to stderr. And you71 Usually you don't want to write to stdout. You want to write to stderr. And you
72 don't care if it fails. It's more like a <em>warning message</em> that you want72 don't care if it fails. It's more like a <em>warning message</em> that you want
73 to emit. For that you can use a simpler API:73 to emit. For that you can use a simpler API:
74 </p>74 </p>
75 <pre><code class="zig">const warn = @import("std").debug.warn;75 {#code_begin|exe|hello#}
76const warn = @import("std").debug.warn;
7677
77pub fn main() -&gt; %void {78pub fn main() -> %void {
78 warn("Hello, world!\n");79 warn("Hello, world!\n");
79}</code></pre>80}
81 {#code_end#}
80 {#see_also|Values|@import|Errors|Root Source File#}82 {#see_also|Values|@import|Errors|Root Source File#}
81 {#header_close#}83 {#header_close#}
82 {#header_open|Source Encoding#}84 {#header_open|Source Encoding#}
std/build.zig+1-1
...@@ -760,7 +760,7 @@ const CrossTarget = struct {...@@ -760,7 +760,7 @@ const CrossTarget = struct {
760 environ: builtin.Environ,760 environ: builtin.Environ,
761};761};
762762
763const Target = union(enum) {763pub const Target = union(enum) {
764 Native: void,764 Native: void,
765 Cross: CrossTarget,765 Cross: CrossTarget,
766766
std/mem.zig+14
...@@ -203,6 +203,20 @@ pub fn dupe(allocator: &Allocator, comptime T: type, m: []const T) -> %[]T {...@@ -203,6 +203,20 @@ pub fn dupe(allocator: &Allocator, comptime T: type, m: []const T) -> %[]T {
203 return new_buf;203 return new_buf;
204}204}
205205
206/// Remove values from the beginning and end of a slice.
207pub fn trim(comptime T: type, slice: []const T, values_to_strip: []const T) -> []const T {
208 var begin: usize = 0;
209 var end: usize = slice.len;
210 while (begin < end and indexOfScalar(T, values_to_strip, slice[begin]) != null) : (begin += 1) {}
211 while (end > begin and indexOfScalar(T, values_to_strip, slice[end - 1]) != null) : (end -= 1) {}
212 return slice[begin..end];
213}
214
215test "mem.trim" {
216 assert(eql(u8, trim(u8, " foo\n ", " \n"), "foo"));
217 assert(eql(u8, trim(u8, "foo", " \n"), "foo"));
218}
219
206/// Linear search for the index of a scalar value inside a slice.220/// Linear search for the index of a scalar value inside a slice.
207pub fn indexOfScalar(comptime T: type, slice: []const T, value: T) -> ?usize {221pub fn indexOfScalar(comptime T: type, slice: []const T, value: T) -> ?usize {
208 return indexOfScalarPos(T, slice, 0, value);222 return indexOfScalarPos(T, slice, 0, value);