authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-09-18 18:36:39-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-09-18 18:36:39-04:00
log21328e0036db94c75083ab8e43ff9195000343f0
treecb6ddae8aaf1d46c7fb33941c9a2ee64cd1e5ba0
parent345f8db1c49efe3d9ca9859a296f24e728a3b43a
signaturelock-open Commit is signed but in an unrecognized format.

zig fmt: handle shebang lines

closes #1546

5 files changed, 46 insertions(+), 6 deletions(-)

std/zig/ast.zig+2
...@@ -469,6 +469,7 @@ pub const Node = struct {...@@ -469,6 +469,7 @@ pub const Node = struct {
469 doc_comments: ?*DocComment,469 doc_comments: ?*DocComment,
470 decls: DeclList,470 decls: DeclList,
471 eof_token: TokenIndex,471 eof_token: TokenIndex,
472 shebang: ?TokenIndex,
472473
473 pub const DeclList = SegmentedList(*Node, 4);474 pub const DeclList = SegmentedList(*Node, 4);
474475
...@@ -480,6 +481,7 @@ pub const Node = struct {...@@ -480,6 +481,7 @@ pub const Node = struct {
480 }481 }
481482
482 pub fn firstToken(self: *const Root) TokenIndex {483 pub fn firstToken(self: *const Root) TokenIndex {
484 if (self.shebang) |shebang| return shebang;
483 return if (self.decls.len == 0) self.eof_token else (self.decls.at(0).*).firstToken();485 return if (self.decls.len == 0) self.eof_token else (self.decls.at(0).*).firstToken();
484 }486 }
485487
std/zig/parse.zig+10
...@@ -21,6 +21,7 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree {...@@ -21,6 +21,7 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree {
21 .base = ast.Node{ .id = ast.Node.Id.Root },21 .base = ast.Node{ .id = ast.Node.Id.Root },
22 .decls = ast.Node.Root.DeclList.init(arena),22 .decls = ast.Node.Root.DeclList.init(arena),
23 .doc_comments = null,23 .doc_comments = null,
24 .shebang = null,
24 // initialized when we get the eof token25 // initialized when we get the eof token
25 .eof_token = undefined,26 .eof_token = undefined,
26 });27 });
...@@ -41,6 +42,15 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree {...@@ -41,6 +42,15 @@ pub fn parse(allocator: *mem.Allocator, source: []const u8) !ast.Tree {
41 }42 }
42 var tok_it = tree.tokens.iterator(0);43 var tok_it = tree.tokens.iterator(0);
4344
45 // skip over shebang line
46 shebang: {
47 const shebang_tok_index = tok_it.index;
48 const shebang_tok_ptr = tok_it.peek() orelse break :shebang;
49 if (shebang_tok_ptr.id != Token.Id.ShebangLine) break :shebang;
50 root_node.shebang = shebang_tok_index;
51 _ = tok_it.next();
52 }
53
44 // skip over line comments at the top of the file54 // skip over line comments at the top of the file
45 while (true) {55 while (true) {
46 const next_tok = tok_it.peek() orelse break;56 const next_tok = tok_it.peek() orelse break;
std/zig/parser_test.zig+8
...@@ -1,3 +1,11 @@...@@ -1,3 +1,11 @@
1test "zig fmt: shebang line" {
2 try testCanonical(
3 \\#!/usr/bin/env zig
4 \\pub fn main() void {}
5 \\
6 );
7}
8
1test "zig fmt: correctly move doc comments on struct fields" {9test "zig fmt: correctly move doc comments on struct fields" {
2 try testTransform(10 try testTransform(
3 \\pub const section_64 = extern struct {11 \\pub const section_64 = extern struct {
std/zig/render.zig+7-1
...@@ -67,8 +67,14 @@ fn renderRoot(...@@ -67,8 +67,14 @@ fn renderRoot(
67 stream: var,67 stream: var,
68 tree: *ast.Tree,68 tree: *ast.Tree,
69) (@typeOf(stream).Child.Error || Error)!void {69) (@typeOf(stream).Child.Error || Error)!void {
70 // render all the line comments at the beginning of the file
71 var tok_it = tree.tokens.iterator(0);70 var tok_it = tree.tokens.iterator(0);
71
72 // render the shebang line
73 if (tree.root_node.shebang) |shebang| {
74 try stream.write(tree.tokenSlice(shebang));
75 }
76
77 // render all the line comments at the beginning of the file
72 while (tok_it.next()) |token| {78 while (tok_it.next()) |token| {
73 if (token.id != Token.Id.LineComment) break;79 if (token.id != Token.Id.LineComment) break;
74 try stream.print("{}\n", mem.trimRight(u8, tree.tokenSlicePtr(token), " "));80 try stream.print("{}\n", mem.trimRight(u8, tree.tokenSlicePtr(token), " "));
std/zig/tokenizer.zig+19-5
...@@ -145,6 +145,7 @@ pub const Token = struct {...@@ -145,6 +145,7 @@ pub const Token = struct {
145 LineComment,145 LineComment,
146 DocComment,146 DocComment,
147 BracketStarBracket,147 BracketStarBracket,
148 ShebangLine,
148 Keyword_align,149 Keyword_align,
149 Keyword_and,150 Keyword_and,
150 Keyword_asm,151 Keyword_asm,
...@@ -208,11 +209,24 @@ pub const Tokenizer = struct {...@@ -208,11 +209,24 @@ pub const Tokenizer = struct {
208 }209 }
209210
210 pub fn init(buffer: []const u8) Tokenizer {211 pub fn init(buffer: []const u8) Tokenizer {
211 return Tokenizer{212 if (mem.startsWith(u8, buffer, "#!")) {
212 .buffer = buffer,213 const src_start = if (mem.indexOfScalar(u8, buffer, '\n')) |i| i + 1 else buffer.len;
213 .index = 0,214 return Tokenizer{
214 .pending_invalid_token = null,215 .buffer = buffer,
215 };216 .index = src_start,
217 .pending_invalid_token = Token{
218 .id = Token.Id.ShebangLine,
219 .start = 0,
220 .end = src_start,
221 },
222 };
223 } else {
224 return Tokenizer{
225 .buffer = buffer,
226 .index = 0,
227 .pending_invalid_token = null,
228 };
229 }
216 }230 }
217231
218 const State = enum {232 const State = enum {