authorgravatar for wink@saville.comWink Saville <wink@saville.com> 2018-09-23 15:53:52-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-09-24 19:28:46-04:00
log0e6c18c8207cab5d3946f119a51d62f91c1b9028
treeb1e18987d751ac82beaaecb544753cb260c40595
parent4241cd666dbf5117a6c32357c091ca3bc7a0fcd0

Remove StrLitKind enum

I was looking at the tokenizer specifically fn testTokenize and the this statement looked odd: if (@TagType(Token.Id)(token.id) != @TagType(Token.Id)(expected_token_id)) { I then saw the TODO and thought I'd remove StrLitKind figuring that would make testTokenize simpler. It did so I thought I'd prepare this PR. The tests are still working and stage2 zig seems to work, it compiles and I was able to use the fmt command.

3 files changed, 25 insertions(+), 40 deletions(-)

std/zig/ast.zig+2-2
...@@ -211,7 +211,7 @@ pub const Error = union(enum) {...@@ -211,7 +211,7 @@ pub const Error = union(enum) {
211211
212 pub const ExpectedToken = struct {212 pub const ExpectedToken = struct {
213 token: TokenIndex,213 token: TokenIndex,
214 expected_id: @TagType(Token.Id),214 expected_id: Token.Id,
215215
216 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {216 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
217 const token_name = @tagName(tokens.at(self.token).id);217 const token_name = @tagName(tokens.at(self.token).id);
...@@ -221,7 +221,7 @@ pub const Error = union(enum) {...@@ -221,7 +221,7 @@ pub const Error = union(enum) {
221221
222 pub const ExpectedCommaOrEnd = struct {222 pub const ExpectedCommaOrEnd = struct {
223 token: TokenIndex,223 token: TokenIndex,
224 end_id: @TagType(Token.Id),224 end_id: Token.Id,
225225
226 pub fn render(self: *const ExpectedCommaOrEnd, tokens: *Tree.TokenList, stream: var) !void {226 pub fn render(self: *const ExpectedCommaOrEnd, tokens: *Tree.TokenList, stream: var) !void {
227 const token_name = @tagName(tokens.at(self.token).id);227 const token_name = @tagName(tokens.at(self.token).id);
std/zig/parse.zig+12-12
...@@ -2846,12 +2846,12 @@ const ContainerKindCtx = struct {...@@ -2846,12 +2846,12 @@ const ContainerKindCtx = struct {
2846};2846};
28472847
2848const ExpectTokenSave = struct {2848const ExpectTokenSave = struct {
2849 id: @TagType(Token.Id),2849 id: Token.Id,
2850 ptr: *TokenIndex,2850 ptr: *TokenIndex,
2851};2851};
28522852
2853const OptionalTokenSave = struct {2853const OptionalTokenSave = struct {
2854 id: @TagType(Token.Id),2854 id: Token.Id,
2855 ptr: *?TokenIndex,2855 ptr: *?TokenIndex,
2856};2856};
28572857
...@@ -3066,9 +3066,9 @@ const State = union(enum) {...@@ -3066,9 +3066,9 @@ const State = union(enum) {
3066 Identifier: OptionalCtx,3066 Identifier: OptionalCtx,
3067 ErrorTag: **ast.Node,3067 ErrorTag: **ast.Node,
30683068
3069 IfToken: @TagType(Token.Id),3069 IfToken: Token.Id,
3070 IfTokenSave: ExpectTokenSave,3070 IfTokenSave: ExpectTokenSave,
3071 ExpectToken: @TagType(Token.Id),3071 ExpectToken: Token.Id,
3072 ExpectTokenSave: ExpectTokenSave,3072 ExpectTokenSave: ExpectTokenSave,
3073 OptionalTokenSave: OptionalTokenSave,3073 OptionalTokenSave: OptionalTokenSave,
3074};3074};
...@@ -3243,7 +3243,7 @@ const ExpectCommaOrEndResult = union(enum) {...@@ -3243,7 +3243,7 @@ const ExpectCommaOrEndResult = union(enum) {
3243 parse_error: Error,3243 parse_error: Error,
3244};3244};
32453245
3246fn expectCommaOrEnd(tok_it: *ast.Tree.TokenList.Iterator, tree: *ast.Tree, end: @TagType(Token.Id)) ExpectCommaOrEndResult {3246fn expectCommaOrEnd(tok_it: *ast.Tree.TokenList.Iterator, tree: *ast.Tree, end: Token.Id) ExpectCommaOrEndResult {
3247 const token = nextToken(tok_it, tree);3247 const token = nextToken(tok_it, tree);
3248 const token_index = token.index;3248 const token_index = token.index;
3249 const token_ptr = token.ptr;3249 const token_ptr = token.ptr;
...@@ -3288,7 +3288,7 @@ fn tokenIdToAssignment(id: *const Token.Id) ?ast.Node.InfixOp.Op {...@@ -3288,7 +3288,7 @@ fn tokenIdToAssignment(id: *const Token.Id) ?ast.Node.InfixOp.Op {
3288 };3288 };
3289}3289}
32903290
3291fn tokenIdToUnwrapExpr(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {3291fn tokenIdToUnwrapExpr(id: Token.Id) ?ast.Node.InfixOp.Op {
3292 return switch (id) {3292 return switch (id) {
3293 Token.Id.Keyword_catch => ast.Node.InfixOp.Op{ .Catch = null },3293 Token.Id.Keyword_catch => ast.Node.InfixOp.Op{ .Catch = null },
3294 Token.Id.Keyword_orelse => ast.Node.InfixOp.Op{ .UnwrapOptional = void{} },3294 Token.Id.Keyword_orelse => ast.Node.InfixOp.Op{ .UnwrapOptional = void{} },
...@@ -3296,7 +3296,7 @@ fn tokenIdToUnwrapExpr(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {...@@ -3296,7 +3296,7 @@ fn tokenIdToUnwrapExpr(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3296 };3296 };
3297}3297}
32983298
3299fn tokenIdToComparison(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {3299fn tokenIdToComparison(id: Token.Id) ?ast.Node.InfixOp.Op {
3300 return switch (id) {3300 return switch (id) {
3301 Token.Id.BangEqual => ast.Node.InfixOp.Op{ .BangEqual = void{} },3301 Token.Id.BangEqual => ast.Node.InfixOp.Op{ .BangEqual = void{} },
3302 Token.Id.EqualEqual => ast.Node.InfixOp.Op{ .EqualEqual = void{} },3302 Token.Id.EqualEqual => ast.Node.InfixOp.Op{ .EqualEqual = void{} },
...@@ -3308,7 +3308,7 @@ fn tokenIdToComparison(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {...@@ -3308,7 +3308,7 @@ fn tokenIdToComparison(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3308 };3308 };
3309}3309}
33103310
3311fn tokenIdToBitShift(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {3311fn tokenIdToBitShift(id: Token.Id) ?ast.Node.InfixOp.Op {
3312 return switch (id) {3312 return switch (id) {
3313 Token.Id.AngleBracketAngleBracketLeft => ast.Node.InfixOp.Op{ .BitShiftLeft = void{} },3313 Token.Id.AngleBracketAngleBracketLeft => ast.Node.InfixOp.Op{ .BitShiftLeft = void{} },
3314 Token.Id.AngleBracketAngleBracketRight => ast.Node.InfixOp.Op{ .BitShiftRight = void{} },3314 Token.Id.AngleBracketAngleBracketRight => ast.Node.InfixOp.Op{ .BitShiftRight = void{} },
...@@ -3316,7 +3316,7 @@ fn tokenIdToBitShift(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {...@@ -3316,7 +3316,7 @@ fn tokenIdToBitShift(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3316 };3316 };
3317}3317}
33183318
3319fn tokenIdToAddition(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {3319fn tokenIdToAddition(id: Token.Id) ?ast.Node.InfixOp.Op {
3320 return switch (id) {3320 return switch (id) {
3321 Token.Id.Minus => ast.Node.InfixOp.Op{ .Sub = void{} },3321 Token.Id.Minus => ast.Node.InfixOp.Op{ .Sub = void{} },
3322 Token.Id.MinusPercent => ast.Node.InfixOp.Op{ .SubWrap = void{} },3322 Token.Id.MinusPercent => ast.Node.InfixOp.Op{ .SubWrap = void{} },
...@@ -3327,7 +3327,7 @@ fn tokenIdToAddition(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {...@@ -3327,7 +3327,7 @@ fn tokenIdToAddition(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3327 };3327 };
3328}3328}
33293329
3330fn tokenIdToMultiply(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {3330fn tokenIdToMultiply(id: Token.Id) ?ast.Node.InfixOp.Op {
3331 return switch (id) {3331 return switch (id) {
3332 Token.Id.Slash => ast.Node.InfixOp.Op{ .Div = void{} },3332 Token.Id.Slash => ast.Node.InfixOp.Op{ .Div = void{} },
3333 Token.Id.Asterisk => ast.Node.InfixOp.Op{ .Mult = void{} },3333 Token.Id.Asterisk => ast.Node.InfixOp.Op{ .Mult = void{} },
...@@ -3339,7 +3339,7 @@ fn tokenIdToMultiply(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {...@@ -3339,7 +3339,7 @@ fn tokenIdToMultiply(id: @TagType(Token.Id)) ?ast.Node.InfixOp.Op {
3339 };3339 };
3340}3340}
33413341
3342fn tokenIdToPrefixOp(id: @TagType(Token.Id)) ?ast.Node.PrefixOp.Op {3342fn tokenIdToPrefixOp(id: Token.Id) ?ast.Node.PrefixOp.Op {
3343 return switch (id) {3343 return switch (id) {
3344 Token.Id.Bang => ast.Node.PrefixOp.Op{ .BoolNot = void{} },3344 Token.Id.Bang => ast.Node.PrefixOp.Op{ .BoolNot = void{} },
3345 Token.Id.Tilde => ast.Node.PrefixOp.Op{ .BitNot = void{} },3345 Token.Id.Tilde => ast.Node.PrefixOp.Op{ .BitNot = void{} },
...@@ -3374,7 +3374,7 @@ fn createToCtxLiteral(arena: *mem.Allocator, opt_ctx: *const OptionalCtx, compti...@@ -3374,7 +3374,7 @@ fn createToCtxLiteral(arena: *mem.Allocator, opt_ctx: *const OptionalCtx, compti
3374 return node;3374 return node;
3375}3375}
33763376
3377fn eatToken(tok_it: *ast.Tree.TokenList.Iterator, tree: *ast.Tree, id: @TagType(Token.Id)) ?TokenIndex {3377fn eatToken(tok_it: *ast.Tree.TokenList.Iterator, tree: *ast.Tree, id: Token.Id) ?TokenIndex {
3378 const token = tok_it.peek().?;3378 const token = tok_it.peek().?;
33793379
3380 if (token.id == id) {3380 if (token.id == id) {
std/zig/tokenizer.zig+11-26
...@@ -73,17 +73,11 @@ pub const Token = struct {...@@ -73,17 +73,11 @@ pub const Token = struct {
73 return null;73 return null;
74 }74 }
7575
76 /// TODO remove this enum76 pub const Id = enum {
77 const StrLitKind = enum {
78 Normal,
79 C,
80 };
81
82 pub const Id = union(enum) {
83 Invalid,77 Invalid,
84 Identifier,78 Identifier,
85 StringLiteral: StrLitKind,79 StringLiteral,
86 MultilineStringLiteralLine: StrLitKind,80 MultilineStringLiteralLine,
87 CharLiteral,81 CharLiteral,
88 Eof,82 Eof,
89 Builtin,83 Builtin,
...@@ -311,7 +305,7 @@ pub const Tokenizer = struct {...@@ -311,7 +305,7 @@ pub const Tokenizer = struct {
311 },305 },
312 '"' => {306 '"' => {
313 state = State.StringLiteral;307 state = State.StringLiteral;
314 result.id = Token.Id{ .StringLiteral = Token.StrLitKind.Normal };308 result.id = Token.Id.StringLiteral;
315 },309 },
316 '\'' => {310 '\'' => {
317 state = State.CharLiteral;311 state = State.CharLiteral;
...@@ -390,7 +384,7 @@ pub const Tokenizer = struct {...@@ -390,7 +384,7 @@ pub const Tokenizer = struct {
390 },384 },
391 '\\' => {385 '\\' => {
392 state = State.Backslash;386 state = State.Backslash;
393 result.id = Token.Id{ .MultilineStringLiteralLine = Token.StrLitKind.Normal };387 result.id = Token.Id.MultilineStringLiteralLine;
394 },388 },
395 '{' => {389 '{' => {
396 result.id = Token.Id.LBrace;390 result.id = Token.Id.LBrace;
...@@ -591,11 +585,11 @@ pub const Tokenizer = struct {...@@ -591,11 +585,11 @@ pub const Tokenizer = struct {
591 State.C => switch (c) {585 State.C => switch (c) {
592 '\\' => {586 '\\' => {
593 state = State.Backslash;587 state = State.Backslash;
594 result.id = Token.Id{ .MultilineStringLiteralLine = Token.StrLitKind.C };588 result.id = Token.Id.MultilineStringLiteralLine;
595 },589 },
596 '"' => {590 '"' => {
597 state = State.StringLiteral;591 state = State.StringLiteral;
598 result.id = Token.Id{ .StringLiteral = Token.StrLitKind.C };592 result.id = Token.Id.StringLiteral;
599 },593 },
600 'a'...'z', 'A'...'Z', '_', '0'...'9' => {594 'a'...'z', 'A'...'Z', '_', '0'...'9' => {
601 state = State.Identifier;595 state = State.Identifier;
...@@ -1218,7 +1212,7 @@ test "tokenizer - invalid token characters" {...@@ -1218,7 +1212,7 @@ test "tokenizer - invalid token characters" {
12181212
1219test "tokenizer - invalid literal/comment characters" {1213test "tokenizer - invalid literal/comment characters" {
1220 testTokenize("\"\x00\"", []Token.Id{1214 testTokenize("\"\x00\"", []Token.Id{
1221 Token.Id{ .StringLiteral = Token.StrLitKind.Normal },1215 Token.Id.StringLiteral,
1222 Token.Id.Invalid,1216 Token.Id.Invalid,
1223 });1217 });
1224 testTokenize("//\x00", []Token.Id{1218 testTokenize("//\x00", []Token.Id{
...@@ -1304,7 +1298,7 @@ test "tokenizer - string identifier and builtin fns" {...@@ -1304,7 +1298,7 @@ test "tokenizer - string identifier and builtin fns" {
1304 Token.Id.Equal,1298 Token.Id.Equal,
1305 Token.Id.Builtin,1299 Token.Id.Builtin,
1306 Token.Id.LParen,1300 Token.Id.LParen,
1307 Token.Id{ .StringLiteral = Token.StrLitKind.Normal },1301 Token.Id.StringLiteral,
1308 Token.Id.RParen,1302 Token.Id.RParen,
1309 Token.Id.Semicolon,1303 Token.Id.Semicolon,
1310 });1304 });
...@@ -1344,17 +1338,8 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {...@@ -1344,17 +1338,8 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
1344 var tokenizer = Tokenizer.init(source);1338 var tokenizer = Tokenizer.init(source);
1345 for (expected_tokens) |expected_token_id| {1339 for (expected_tokens) |expected_token_id| {
1346 const token = tokenizer.next();1340 const token = tokenizer.next();
1347 if (@TagType(Token.Id)(token.id) != @TagType(Token.Id)(expected_token_id)) {1341 if (token.id != expected_token_id) {
1348 std.debug.panic("expected {}, found {}\n", @tagName(@TagType(Token.Id)(expected_token_id)), @tagName(@TagType(Token.Id)(token.id)));1342 std.debug.panic("expected {}, found {}\n", @tagName(expected_token_id), @tagName(token.id));
1349 }
1350 switch (expected_token_id) {
1351 Token.Id.StringLiteral => |expected_kind| {
1352 std.debug.assert(expected_kind == switch (token.id) {
1353 Token.Id.StringLiteral => |kind| kind,
1354 else => unreachable,
1355 });
1356 },
1357 else => {},
1358 }1343 }
1359 }1344 }
1360 const last_token = tokenizer.next();1345 const last_token = tokenizer.next();