authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 11:18:18+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 11:18:18+02:00
log22e38ffb54775c2e523665ac739f2a895d8757ad
tree79ab74c2127736ac330e01fb58221cd1bd756dd0
parenta2330d0ea366e0ced7fa917d3bf1ccd022e932c3

std.zig.tokenizer fixed tokens having wrong column and line

1 files changed, 12 insertions(+), 9 deletions(-)

std/zig/tokenizer.zig+12-9
...@@ -271,6 +271,7 @@ pub const Tokenizer = struct {...@@ -271,6 +271,7 @@ pub const Tokenizer = struct {
271 self.pending_invalid_token = null;271 self.pending_invalid_token = null;
272 return token;272 return token;
273 }273 }
274 const start_index = self.index;
274 var state = State.Start;275 var state = State.Start;
275 var result = Token {276 var result = Token {
276 .id = Token.Id.Eof,277 .id = Token.Id.Eof,
...@@ -279,7 +280,7 @@ pub const Tokenizer = struct {...@@ -279,7 +280,7 @@ pub const Tokenizer = struct {
279 .line = self.line,280 .line = self.line,
280 .column = self.column,281 .column = self.column,
281 };282 };
282 while (self.index < self.buffer.len) {283 while (self.index < self.buffer.len) : (self.index += 1) {
283 const c = self.buffer[self.index];284 const c = self.buffer[self.index];
284 switch (state) {285 switch (state) {
285 State.Start => switch (c) {286 State.Start => switch (c) {
...@@ -877,14 +878,6 @@ pub const Tokenizer = struct {...@@ -877,14 +878,6 @@ pub const Tokenizer = struct {
877 else => break,878 else => break,
878 },879 },
879 }880 }
880
881 self.index += 1;
882 if (c == '\n') {
883 self.line += 1;
884 self.column = 0;
885 } else {
886 self.column += 1;
887 }
888 } else if (self.index == self.buffer.len) {881 } else if (self.index == self.buffer.len) {
889 switch (state) {882 switch (state) {
890 State.Start,883 State.Start,
...@@ -983,6 +976,16 @@ pub const Tokenizer = struct {...@@ -983,6 +976,16 @@ pub const Tokenizer = struct {
983 },976 },
984 }977 }
985 }978 }
979
980 for (self.buffer[start_index..self.index]) |c| {
981 if (c == '\n') {
982 self.line += 1;
983 self.column = 0;
984 } else {
985 self.column += 1;
986 }
987 }
988
986 if (result.id == Token.Id.Eof) {989 if (result.id == Token.Id.Eof) {
987 if (self.pending_invalid_token) |token| {990 if (self.pending_invalid_token) |token| {
988 self.pending_invalid_token = null;991 self.pending_invalid_token = null;