authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-04 10:27:38+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-04 10:27:38+02:00
log020724cfa0677bf42f48957d0ca7a474f6fe31e0
tree6c9763a6bac0a5daed355ae758de6109addd729d
parent09cf82361999c1a22819467e02fc68fd22ca188e

std.zig.tokenizer Tokens now don't contain a line and column field.

* Instead, this information is optained by asking the tokenizer. * getTokenLocation takes a start_index, so relative loc can be optained

2 files changed, 25 insertions(+), 42 deletions(-)

std/zig/parser.zig+10-13
......@@ -1571,12 +1571,12 @@ pub const Parser = struct {
15711571 }
15721572
15731573 fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) {
1574 const loc = self.tokenizer.getTokenLocation(token);
1575 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args);
1574 const loc = self.tokenizer.getTokenLocation(0, token);
1575 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
15761576 warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
15771577 {
15781578 var i: usize = 0;
1579 while (i < token.column) : (i += 1) {
1579 while (i < loc.column) : (i += 1) {
15801580 warn(" ");
15811581 }
15821582 }
......@@ -1679,9 +1679,8 @@ pub const Parser = struct {
16791679 try stack.append(RenderState {
16801680 .Text = blk: {
16811681 const prev_node = root_node.decls.at(i - 1);
1682 const prev_line_index = prev_node.lastToken().line;
1683 const this_line_index = decl.firstToken().line;
1684 if (this_line_index - prev_line_index >= 2) {
1682 const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, decl.firstToken());
1683 if (loc.line >= 2) {
16851684 break :blk "\n\n";
16861685 }
16871686 break :blk "\n";
......@@ -1858,10 +1857,9 @@ pub const Parser = struct {
18581857 try stack.append(RenderState {
18591858 .Text = blk: {
18601859 if (i != 0) {
1861 const prev_statement_node = block.statements.items[i - 1];
1862 const prev_line_index = prev_statement_node.lastToken().line;
1863 const this_line_index = statement_node.firstToken().line;
1864 if (this_line_index - prev_line_index >= 2) {
1860 const prev_node = block.statements.items[i - 1];
1861 const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, statement_node.firstToken());
1862 if (loc.line >= 2) {
18651863 break :blk "\n\n";
18661864 }
18671865 }
......@@ -2083,9 +2081,8 @@ pub const Parser = struct {
20832081 .Text = blk: {
20842082 if (i != 0) {
20852083 const prev_node = fields_and_decls[i - 1];
2086 const prev_line_index = prev_node.lastToken().line;
2087 const this_line_index = node.firstToken().line;
2088 if (this_line_index - prev_line_index >= 2) {
2084 const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken());
2085 if (loc.line >= 2) {
20892086 break :blk "\n\n";
20902087 }
20912088 }
std/zig/tokenizer.zig+15-29
......@@ -5,8 +5,6 @@ pub const Token = struct {
55 id: Id,
66 start: usize,
77 end: usize,
8 line: usize,
9 column: usize,
108
119 const KeywordId = struct {
1210 bytes: []const u8,
......@@ -180,28 +178,34 @@ pub const Token = struct {
180178pub const Tokenizer = struct {
181179 buffer: []const u8,
182180 index: usize,
183 line: usize,
184 column: usize,
185181 pending_invalid_token: ?Token,
186182
187 pub const LineLocation = struct {
183 pub const Location = struct {
184 line: usize,
185 column: usize,
188186 line_start: usize,
189187 line_end: usize,
190188 };
191189
192 pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation {
193 var loc = LineLocation {
194 .line_start = 0,
190 pub fn getTokenLocation(self: &Tokenizer, start_index: usize, token: &const Token) Location {
191 var loc = Location {
192 .line = 0,
193 .column = 0,
194 .line_start = start_index,
195195 .line_end = self.buffer.len,
196196 };
197 for (self.buffer) |c, i| {
198 if (i == token.start) {
199 loc.line_end = i;
197 for (self.buffer[start_index..]) |c, i| {
198 if (i + start_index == token.start) {
199 loc.line_end = i + start_index;
200200 while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
201201 return loc;
202202 }
203203 if (c == '\n') {
204 loc.line += 1;
205 loc.column = 0;
204206 loc.line_start = i + 1;
207 } else {
208 loc.column += 1;
205209 }
206210 }
207211 return loc;
......@@ -216,8 +220,6 @@ pub const Tokenizer = struct {
216220 return Tokenizer {
217221 .buffer = buffer,
218222 .index = 0,
219 .line = 0,
220 .column = 0,
221223 .pending_invalid_token = null,
222224 };
223225 }
......@@ -277,8 +279,6 @@ pub const Tokenizer = struct {
277279 .id = Token.Id.Eof,
278280 .start = self.index,
279281 .end = undefined,
280 .line = self.line,
281 .column = self.column,
282282 };
283283 while (self.index < self.buffer.len) : (self.index += 1) {
284284 const c = self.buffer[self.index];
......@@ -286,12 +286,9 @@ pub const Tokenizer = struct {
286286 State.Start => switch (c) {
287287 ' ' => {
288288 result.start = self.index + 1;
289 result.column += 1;
290289 },
291290 '\n' => {
292291 result.start = self.index + 1;
293 result.line += 1;
294 result.column = 0;
295292 },
296293 'c' => {
297294 state = State.C;
......@@ -977,15 +974,6 @@ pub const Tokenizer = struct {
977974 }
978975 }
979976
980 for (self.buffer[start_index..self.index]) |c| {
981 if (c == '\n') {
982 self.line += 1;
983 self.column = 0;
984 } else {
985 self.column += 1;
986 }
987 }
988
989977 if (result.id == Token.Id.Eof) {
990978 if (self.pending_invalid_token) |token| {
991979 self.pending_invalid_token = null;
......@@ -1009,8 +997,6 @@ pub const Tokenizer = struct {
1009997 .id = Token.Id.Invalid,
1010998 .start = self.index,
1011999 .end = self.index + invalid_length,
1012 .line = self.line,
1013 .column = self.column,
10141000 };
10151001 }
10161002