authorgravatar for wink@saville.comWink Saville <wink@saville.com> 2018-09-23 15:53:52-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-09-24 19:28:46-04:00
log0e6c18c8207cab5d3946f119a51d62f91c1b9028
treeb1e18987d751ac82beaaecb544753cb260c40595
parent4241cd666dbf5117a6c32357c091ca3bc7a0fcd0

Remove StrLitKind enum

I was looking at the tokenizer specifically fn testTokenize and the this statement looked odd: if (@TagType(Token.Id)(token.id) != @TagType(Token.Id)(expected_token_id)) { I then saw the TODO and thought I'd remove StrLitKind figuring that would make testTokenize simpler. It did so I thought I'd prepare this PR. The tests are still working and stage2 zig seems to work, it compiles and I was able to use the fmt command.

3 files changed, 25 insertions(+), 40 deletions(-)

std/zig/ast.zig+2-2
......@@ -211,7 +211,7 @@ pub const Error = union(enum) {
211211
212212 pub const ExpectedToken = struct {
213213 token: TokenIndex,
214 expected_id: @TagType(Token.Id),
214 expected_id: Token.Id,
215215
216216 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
217217 const token_name = @tagName(tokens.at(self.token).id);
......@@ -221,7 +221,7 @@ pub const Error = union(enum) {
221221
222222 pub const ExpectedCommaOrEnd = struct {
223223 token: TokenIndex,
224 end_id: @TagType(Token.Id),
224 end_id: Token.Id,
225225
226226 pub fn render(self: *const ExpectedCommaOrEnd, tokens: *Tree.TokenList, stream: var) !void {
227227 const token_name = @tagName(tokens.at(self.token).id);
std/zig/parse.zig+12-12
......@@ -2846,12 +2846,12 @@ const ContainerKindCtx = struct {
28462846};
28472847
28482848const ExpectTokenSave = struct {
2849 id: @TagType(Token.Id),
2849 id: Token.Id,
28502850 ptr: *TokenIndex,
28512851};
28522852
28532853const OptionalTokenSave = struct {
2854 id: @TagType(Token.Id),
2854 id: Token.Id,
28552855 ptr: *?TokenIndex,
28562856};
28572857
......@@ -3066,9 +3066,9 @@ const State = union(enum) {
30663066 Identifier: OptionalCtx,
30673067 ErrorTag: **ast.Node,
30683068
3069 IfToken: @TagType(Token.Id),
3069 IfToken: Token.Id,
30703070 IfTokenSave: ExpectTokenSave,
3071 ExpectToken: @TagType(Token.Id),
3071 ExpectToken: Token.Id,
30723072 ExpectTokenSave: ExpectTokenSave,
30733073 OptionalTokenSave: OptionalTokenSave,
30743074};
......@@ -3243,7 +3243,7 @@ const ExpectCommaOrEndResult = union(enum) {
32433243 parse_error: Error,
32443244};
32453245
3246fn expectCommaOrEnd(tok_it: *ast.Tree.TokenList.Iterator, tree: *ast.Tree, end: @TagType(Token.Id)) ExpectCommaOrEndResult {
3246fn expectCommaOrEnd(tok_it: *ast.Tree.TokenList.Iterator, tree: *ast.Tree, end: Token.Id) ExpectCommaOrEndResult {
32473247 const token = nextToken(tok_it, tree);
32483248 const token_index = token.index;
32493249 const token_ptr = token.ptr;
......@@ -3288,7 +3288,7 @@ fn tokenIdToAssignment(id: *const Token.Id) ?ast.Node.InfixOp.Op {
32883288 };
32893289}
32903290
3291fn tokenIdToUnwrapExpr(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3291fn tokenIdToUnwrapExpr(id: Token.Id) ?ast.Node.InfixOp.Op {
32923292 return switch (id) {
32933293 Token.Id.Keyword_catch => ast.Node.InfixOp.Op{ .Catch = null },
32943294 Token.Id.Keyword_orelse => ast.Node.InfixOp.Op{ .UnwrapOptional = void{} },
......@@ -3296,7 +3296,7 @@ fn tokenIdToUnwrapExpr(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
32963296 };
32973297}
32983298
3299fn tokenIdToComparison(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3299fn tokenIdToComparison(id: Token.Id) ?ast.Node.InfixOp.Op {
33003300 return switch (id) {
33013301 Token.Id.BangEqual => ast.Node.InfixOp.Op{ .BangEqual = void{} },
33023302 Token.Id.EqualEqual => ast.Node.InfixOp.Op{ .EqualEqual = void{} },
......@@ -3308,7 +3308,7 @@ fn tokenIdToComparison(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
33083308 };
33093309}
33103310
3311fn tokenIdToBitShift(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3311fn tokenIdToBitShift(id: Token.Id) ?ast.Node.InfixOp.Op {
33123312 return switch (id) {
33133313 Token.Id.AngleBracketAngleBracketLeft => ast.Node.InfixOp.Op{ .BitShiftLeft = void{} },
33143314 Token.Id.AngleBracketAngleBracketRight => ast.Node.InfixOp.Op{ .BitShiftRight = void{} },
......@@ -3316,7 +3316,7 @@ fn tokenIdToBitShift(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
33163316 };
33173317}
33183318
3319fn tokenIdToAddition(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3319fn tokenIdToAddition(id: Token.Id) ?ast.Node.InfixOp.Op {
33203320 return switch (id) {
33213321 Token.Id.Minus => ast.Node.InfixOp.Op{ .Sub = void{} },
33223322 Token.Id.MinusPercent => ast.Node.InfixOp.Op{ .SubWrap = void{} },
......@@ -3327,7 +3327,7 @@ fn tokenIdToAddition(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
33273327 };
33283328}
33293329
3330fn tokenIdToMultiply(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3330fn tokenIdToMultiply(id: Token.Id) ?ast.Node.InfixOp.Op {
33313331 return switch (id) {
33323332 Token.Id.Slash => ast.Node.InfixOp.Op{ .Div = void{} },
33333333 Token.Id.Asterisk => ast.Node.InfixOp.Op{ .Mult = void{} },
......@@ -3339,7 +3339,7 @@ fn tokenIdToMultiply(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
33393339 };
33403340}
33413341
3342fn tokenIdToPrefixOp(id: @TagType(Token.Id)) ?ast.Node.PrefixOp.Op {
3342fn tokenIdToPrefixOp(id: Token.Id) ?ast.Node.PrefixOp.Op {
33433343 return switch (id) {
33443344 Token.Id.Bang => ast.Node.PrefixOp.Op{ .BoolNot = void{} },
33453345 Token.Id.Tilde => ast.Node.PrefixOp.Op{ .BitNot = void{} },
......@@ -3374,7 +3374,7 @@ fn createToCtxLiteral(arena: *mem.Allocator, opt_ctx: *const OptionalCtx, compti
33743374 return node;
33753375}
33763376
3377fn eatToken(tok_it: *ast.Tree.TokenList.Iterator, tree: *ast.Tree, id: @TagType(Token.Id)) ?TokenIndex {
3377fn eatToken(tok_it: *ast.Tree.TokenList.Iterator, tree: *ast.Tree, id: Token.Id) ?TokenIndex {
33783378 const token = tok_it.peek().?;
33793379
33803380 if (token.id == id) {
std/zig/tokenizer.zig+11-26
......@@ -73,17 +73,11 @@ pub const Token = struct {
7373 return null;
7474 }
7575
76 /// TODO remove this enum
77 const StrLitKind = enum {
78 Normal,
79 C,
80 };
81
82 pub const Id = union(enum) {
76 pub const Id = enum {
8377 Invalid,
8478 Identifier,
85 StringLiteral: StrLitKind,
86 MultilineStringLiteralLine: StrLitKind,
79 StringLiteral,
80 MultilineStringLiteralLine,
8781 CharLiteral,
8882 Eof,
8983 Builtin,
......@@ -311,7 +305,7 @@ pub const Tokenizer = struct {
311305 },
312306 '"' => {
313307 state = State.StringLiteral;
314 result.id = Token.Id{ .StringLiteral = Token.StrLitKind.Normal };
308 result.id = Token.Id.StringLiteral;
315309 },
316310 '\'' => {
317311 state = State.CharLiteral;
......@@ -390,7 +384,7 @@ pub const Tokenizer = struct {
390384 },
391385 '\\' => {
392386 state = State.Backslash;
393 result.id = Token.Id{ .MultilineStringLiteralLine = Token.StrLitKind.Normal };
387 result.id = Token.Id.MultilineStringLiteralLine;
394388 },
395389 '{' => {
396390 result.id = Token.Id.LBrace;
......@@ -591,11 +585,11 @@ pub const Tokenizer = struct {
591585 State.C => switch (c) {
592586 '\\' => {
593587 state = State.Backslash;
594 result.id = Token.Id{ .MultilineStringLiteralLine = Token.StrLitKind.C };
588 result.id = Token.Id.MultilineStringLiteralLine;
595589 },
596590 '"' => {
597591 state = State.StringLiteral;
598 result.id = Token.Id{ .StringLiteral = Token.StrLitKind.C };
592 result.id = Token.Id.StringLiteral;
599593 },
600594 'a'...'z', 'A'...'Z', '_', '0'...'9' => {
601595 state = State.Identifier;
......@@ -1218,7 +1212,7 @@ test "tokenizer - invalid token characters" {
12181212
12191213test "tokenizer - invalid literal/comment characters" {
12201214 testTokenize("\"\x00\"", []Token.Id{
1221 Token.Id{ .StringLiteral = Token.StrLitKind.Normal },
1215 Token.Id.StringLiteral,
12221216 Token.Id.Invalid,
12231217 });
12241218 testTokenize("//\x00", []Token.Id{
......@@ -1304,7 +1298,7 @@ test "tokenizer - string identifier and builtin fns" {
13041298 Token.Id.Equal,
13051299 Token.Id.Builtin,
13061300 Token.Id.LParen,
1307 Token.Id{ .StringLiteral = Token.StrLitKind.Normal },
1301 Token.Id.StringLiteral,
13081302 Token.Id.RParen,
13091303 Token.Id.Semicolon,
13101304 });
......@@ -1344,17 +1338,8 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
13441338 var tokenizer = Tokenizer.init(source);
13451339 for (expected_tokens) |expected_token_id| {
13461340 const token = tokenizer.next();
1347 if (@TagType(Token.Id)(token.id) != @TagType(Token.Id)(expected_token_id)) {
1348 std.debug.panic("expected {}, found {}\n", @tagName(@TagType(Token.Id)(expected_token_id)), @tagName(@TagType(Token.Id)(token.id)));
1349 }
1350 switch (expected_token_id) {
1351 Token.Id.StringLiteral => |expected_kind| {
1352 std.debug.assert(expected_kind == switch (token.id) {
1353 Token.Id.StringLiteral => |kind| kind,
1354 else => unreachable,
1355 });
1356 },
1357 else => {},
1341 if (token.id != expected_token_id) {
1342 std.debug.panic("expected {}, found {}\n", @tagName(expected_token_id), @tagName(token.id));
13581343 }
13591344 }
13601345 const last_token = tokenizer.next();