Commit 5c6cd5e2c9e8b2d0feb0026bad7c201035a175b4 | zig

author	LemonBoy <thatlemon@gmail.com> 2020-09-27 17:17:27+02:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-09-28 14:16:26-04:00
log	5c6cd5e2c9e8b2d0feb0026bad7c201035a175b4
tree	78552e0b970107df097bc7699e0be5996aa2831b
parent	e60939bfaafc9e6b3ccdc172009b950fc7a3eab1

stage{1,2}: Fix parsing of range literals

stage1 was unable to parse ranges whose starting point was written in
binary/octal as the first dot in '...' was incorrectly interpreted as
decimal point.

stage2 forgot to reset the literal type to IntegerLiteral when it
discovered the dot was not a decimal point.

I've only stumbled across this bug because zig fmt keeps formatting the
ranges without any space around the ...

2 files changed, 12 insertions(+), 3 deletions(-)

lib
- std
  - zig
    tokenizer.zig
src
- tokenizer.cpp

lib/std/zig/tokenizer.zig+9

...	...	@@ -1195,6 +1195,7 @@ pub const Tokenizer = struct {
1195	1195	},
1196	1196	.num_dot_hex => switch (c) {
1197	1197	'.' => {
	1198	result.id = .IntegerLiteral;
1198	1199	self.index -= 1;
1199	1200	state = .start;
1200	1201	break;
...	...	@@ -1758,6 +1759,14 @@ test "correctly parse pointer assignment" {
1758	1759	});
1759	1760	}
1760	1761
	1762	test "tokenizer - range literals" {
	1763	testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
	1764	testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
	1765	testTokenize("0x00...0x09", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
	1766	testTokenize("0b00...0b11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
	1767	testTokenize("0o00...0o11", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
	1768	}
	1769
1761	1770	test "tokenizer - number literals decimal" {
1762	1771	testTokenize("0", &[_]Token.Id{.IntegerLiteral});
1763	1772	testTokenize("1", &[_]Token.Id{.IntegerLiteral});

src/tokenizer.cpp+3-3

...	...	@@ -1225,9 +1225,6 @@ void tokenize(Buf buf, Tokenization out) {
1225	1225	invalid_char_error(&t, c);
1226	1226	break;
1227	1227	}
1228		if (t.radix != 16 && t.radix != 10) {
1229		invalid_char_error(&t, c);
1230		}
1231	1228	t.state = TokenizeStateNumberDot;
1232	1229	break;
1233	1230	}
...	...	@@ -1281,6 +1278,9 @@ void tokenize(Buf buf, Tokenization out) {
1281	1278	t.state = TokenizeStateStart;
1282	1279	continue;
1283	1280	}
	1281	if (t.radix != 16 && t.radix != 10) {
	1282	invalid_char_error(&t, c);
	1283	}
1284	1284	t.pos -= 1;
1285	1285	t.state = TokenizeStateFloatFractionNoUnderscore;
1286	1286	assert(t.cur_tok->id == TokenIdIntLiteral);