authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-29 22:31:17+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-29 22:31:17+02:00
log530f79576923989928afeeb8b50b7e40b95a529f
treebfb369729475fce98d03cb25b7229725b2eed0fe
parentb80398b3558be09d0950415b0a8feeb09bc2f55d

std.zig.parser now supports all infix operators

3 files changed, 535 insertions(+), 62 deletions(-)

std/zig/ast.zig+60-1
......@@ -20,6 +20,7 @@ pub const Node = struct {
2020 IntegerLiteral,
2121 FloatLiteral,
2222 StringLiteral,
23 UndefinedLiteral,
2324 BuiltinCall,
2425 LineComment,
2526 TestDecl,
......@@ -38,6 +39,7 @@ pub const Node = struct {
3839 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
3940 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
4041 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
42 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),
4143 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
4244 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),
4345 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index),
......@@ -57,6 +59,7 @@ pub const Node = struct {
5759 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
5860 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
5961 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
62 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),
6063 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
6164 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),
6265 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(),
......@@ -76,6 +79,7 @@ pub const Node = struct {
7679 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
7780 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
7881 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
82 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),
7983 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
8084 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),
8185 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(),
......@@ -309,9 +313,47 @@ pub const NodeInfixOp = struct {
309313 rhs: &Node,
310314
311315 const InfixOp = enum {
312 EqualEqual,
316 Add,
317 AddWrap,
318 ArrayCat,
319 ArrayMult,
320 Assign,
321 AssignBitAnd,
322 AssignBitOr,
323 AssignBitShiftLeft,
324 AssignBitShiftRight,
325 AssignBitXor,
326 AssignDiv,
327 AssignMinus,
328 AssignMinusWrap,
329 AssignMod,
330 AssignPlus,
331 AssignPlusWrap,
332 AssignTimes,
333 AssignTimesWarp,
313334 BangEqual,
335 BitAnd,
336 BitOr,
337 BitShiftLeft,
338 BitShiftRight,
339 BitXor,
340 BoolAnd,
341 BoolOr,
342 Div,
343 EqualEqual,
344 ErrorUnion,
345 GreaterOrEqual,
346 GreaterThan,
347 LessOrEqual,
348 LessThan,
349 MergeErrorSets,
350 Mod,
351 Mult,
352 MultWrap,
314353 Period,
354 Sub,
355 SubWrap,
356 UnwrapMaybe,
315357 };
316358
317359 pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
......@@ -464,6 +506,23 @@ pub const NodeStringLiteral = struct {
464506 }
465507};
466508
509pub const NodeUndefinedLiteral = struct {
510 base: Node,
511 token: Token,
512
513 pub fn iterate(self: &NodeUndefinedLiteral, index: usize) ?&Node {
514 return null;
515 }
516
517 pub fn firstToken(self: &NodeUndefinedLiteral) Token {
518 return self.token;
519 }
520
521 pub fn lastToken(self: &NodeUndefinedLiteral) Token {
522 return self.token;
523 }
524};
525
467526pub const NodeLineComment = struct {
468527 base: Node,
469528 lines: ArrayList(Token),
std/zig/parser.zig+193-56
......@@ -371,6 +371,13 @@ pub const Parser = struct {
371371 try stack.append(State.AfterOperand);
372372 continue;
373373 },
374 Token.Id.Keyword_undefined => {
375 try stack.append(State {
376 .Operand = &(try self.createUndefined(arena, token)).base
377 });
378 try stack.append(State.AfterOperand);
379 continue;
380 },
374381 Token.Id.Builtin => {
375382 const node = try arena.create(ast.NodeBuiltinCall);
376383 *node = ast.NodeBuiltinCall {
......@@ -414,56 +421,41 @@ pub const Parser = struct {
414421 // or a postfix operator (like () or {}),
415422 // otherwise this expression is done (like on a ; or else).
416423 var token = self.getNextToken();
417 switch (token.id) {
418 Token.Id.EqualEqual => {
419 try stack.append(State {
420 .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.EqualEqual)
421 });
422 try stack.append(State.ExpectOperand);
423 continue;
424 },
425 Token.Id.BangEqual => {
424 if (tokenIdToInfixOp(token.id)) |infix_id| {
426425 try stack.append(State {
427 .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BangEqual)
426 .InfixOp = try self.createInfixOp(arena, token, infix_id)
428427 });
429428 try stack.append(State.ExpectOperand);
430429 continue;
431 },
432 Token.Id.Period => {
433 try stack.append(State {
434 .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period)
435 });
436 try stack.append(State.ExpectOperand);
437 continue;
438 },
439 else => {
440 // no postfix/infix operator after this operand.
441 self.putBackToken(token);
442 // reduce the stack
443 var expression: &ast.Node = stack.pop().Operand;
444 while (true) {
445 switch (stack.pop()) {
446 State.Expression => |dest_ptr| {
447 // we're done
448 try dest_ptr.store(expression);
449 break;
450 },
451 State.InfixOp => |infix_op| {
452 infix_op.rhs = expression;
453 infix_op.lhs = stack.pop().Operand;
454 expression = &infix_op.base;
455 continue;
456 },
457 State.PrefixOp => |prefix_op| {
458 prefix_op.rhs = expression;
459 expression = &prefix_op.base;
460 continue;
461 },
462 else => unreachable,
463 }
430
431 // TODO: Parse postfix operator
432 } else {
433 // no postfix/infix operator after this operand.
434 self.putBackToken(token);
435 // reduce the stack
436 var expression: &ast.Node = stack.pop().Operand;
437 while (true) {
438 switch (stack.pop()) {
439 State.Expression => |dest_ptr| {
440 // we're done
441 try dest_ptr.store(expression);
442 break;
443 },
444 State.InfixOp => |infix_op| {
445 infix_op.rhs = expression;
446 infix_op.lhs = stack.pop().Operand;
447 expression = &infix_op.base;
448 continue;
449 },
450 State.PrefixOp => |prefix_op| {
451 prefix_op.rhs = expression;
452 expression = &prefix_op.base;
453 continue;
454 },
455 else => unreachable,
464456 }
465 continue;
466 },
457 }
458 continue;
467459 }
468460 },
469461
......@@ -706,6 +698,53 @@ pub const Parser = struct {
706698 }
707699 }
708700
701 fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp {
702 return switch (*id) {
703 Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd,
704 Token.Id.AmpersandEqual => ast.NodeInfixOp.InfixOp.AssignBitAnd,
705 Token.Id.AngleBracketAngleBracketLeft => ast.NodeInfixOp.InfixOp.BitShiftLeft,
706 Token.Id.AngleBracketAngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftLeft,
707 Token.Id.AngleBracketAngleBracketRight => ast.NodeInfixOp.InfixOp.BitShiftRight,
708 Token.Id.AngleBracketAngleBracketRightEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftRight,
709 Token.Id.AngleBracketLeft => ast.NodeInfixOp.InfixOp.LessThan,
710 Token.Id.AngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.LessOrEqual,
711 Token.Id.AngleBracketRight => ast.NodeInfixOp.InfixOp.GreaterThan,
712 Token.Id.AngleBracketRightEqual => ast.NodeInfixOp.InfixOp.GreaterOrEqual,
713 Token.Id.Asterisk => ast.NodeInfixOp.InfixOp.Mult,
714 Token.Id.AsteriskAsterisk => ast.NodeInfixOp.InfixOp.ArrayMult,
715 Token.Id.AsteriskEqual => ast.NodeInfixOp.InfixOp.AssignTimes,
716 Token.Id.AsteriskPercent => ast.NodeInfixOp.InfixOp.MultWrap,
717 Token.Id.AsteriskPercentEqual => ast.NodeInfixOp.InfixOp.AssignTimesWarp,
718 Token.Id.Bang => ast.NodeInfixOp.InfixOp.ErrorUnion,
719 Token.Id.BangEqual => ast.NodeInfixOp.InfixOp.BangEqual,
720 Token.Id.Caret => ast.NodeInfixOp.InfixOp.BitXor,
721 Token.Id.CaretEqual => ast.NodeInfixOp.InfixOp.AssignBitXor,
722 Token.Id.Equal => ast.NodeInfixOp.InfixOp.Assign,
723 Token.Id.EqualEqual => ast.NodeInfixOp.InfixOp.EqualEqual,
724 Token.Id.Keyword_and => ast.NodeInfixOp.InfixOp.BoolAnd,
725 Token.Id.Keyword_or => ast.NodeInfixOp.InfixOp.BoolOr,
726 Token.Id.Minus => ast.NodeInfixOp.InfixOp.Sub,
727 Token.Id.MinusEqual => ast.NodeInfixOp.InfixOp.AssignMinus,
728 Token.Id.MinusPercent => ast.NodeInfixOp.InfixOp.SubWrap,
729 Token.Id.MinusPercentEqual => ast.NodeInfixOp.InfixOp.AssignMinusWrap,
730 Token.Id.Percent => ast.NodeInfixOp.InfixOp.Mod,
731 Token.Id.PercentEqual => ast.NodeInfixOp.InfixOp.AssignMod,
732 Token.Id.Period => ast.NodeInfixOp.InfixOp.Period,
733 Token.Id.Pipe => ast.NodeInfixOp.InfixOp.BitOr,
734 Token.Id.PipeEqual => ast.NodeInfixOp.InfixOp.AssignBitOr,
735 Token.Id.PipePipe => ast.NodeInfixOp.InfixOp.MergeErrorSets,
736 Token.Id.Plus => ast.NodeInfixOp.InfixOp.Add,
737 Token.Id.PlusEqual => ast.NodeInfixOp.InfixOp.AssignPlus,
738 Token.Id.PlusPercent => ast.NodeInfixOp.InfixOp.AddWrap,
739 Token.Id.PlusPercentEqual => ast.NodeInfixOp.InfixOp.AssignPlusWrap,
740 Token.Id.PlusPlus => ast.NodeInfixOp.InfixOp.ArrayCat,
741 Token.Id.QuestionMarkQuestionMark => ast.NodeInfixOp.InfixOp.UnwrapMaybe,
742 Token.Id.Slash => ast.NodeInfixOp.InfixOp.Div,
743 Token.Id.SlashEqual => ast.NodeInfixOp.InfixOp.AssignDiv,
744 else => null,
745 };
746 }
747
709748 fn initNode(self: &Parser, id: ast.Node.Id) ast.Node {
710749 if (self.pending_line_comment_node) |comment_node| {
711750 self.pending_line_comment_node = null;
......@@ -867,6 +906,16 @@ pub const Parser = struct {
867906 return node;
868907 }
869908
909 fn createUndefined(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeUndefinedLiteral {
910 const node = try arena.create(ast.NodeUndefinedLiteral);
911
912 *node = ast.NodeUndefinedLiteral {
913 .base = self.initNode(ast.Node.Id.UndefinedLiteral),
914 .token = *token,
915 };
916 return node;
917 }
918
870919 fn createAttachIdentifier(self: &Parser, arena: &mem.Allocator, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier {
871920 const node = try self.createIdentifier(arena, name_token);
872921 try dest_ptr.store(&node.base);
......@@ -1173,17 +1222,51 @@ pub const Parser = struct {
11731222 ast.Node.Id.InfixOp => {
11741223 const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base);
11751224 try stack.append(RenderState { .Expression = prefix_op_node.rhs });
1176 switch (prefix_op_node.op) {
1177 ast.NodeInfixOp.InfixOp.EqualEqual => {
1178 try stack.append(RenderState { .Text = " == "});
1179 },
1180 ast.NodeInfixOp.InfixOp.BangEqual => {
1181 try stack.append(RenderState { .Text = " != "});
1182 },
1183 ast.NodeInfixOp.InfixOp.Period => {
1184 try stack.append(RenderState { .Text = "."});
1185 },
1186 }
1225 const text = switch (prefix_op_node.op) {
1226 ast.NodeInfixOp.InfixOp.Add => " + ",
1227 ast.NodeInfixOp.InfixOp.AddWrap => " +% ",
1228 ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ",
1229 ast.NodeInfixOp.InfixOp.ArrayMult => " ** ",
1230 ast.NodeInfixOp.InfixOp.Assign => " = ",
1231 ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ",
1232 ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ",
1233 ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ",
1234 ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ",
1235 ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ",
1236 ast.NodeInfixOp.InfixOp.AssignDiv => " /= ",
1237 ast.NodeInfixOp.InfixOp.AssignMinus => " -= ",
1238 ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ",
1239 ast.NodeInfixOp.InfixOp.AssignMod => " %= ",
1240 ast.NodeInfixOp.InfixOp.AssignPlus => " += ",
1241 ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ",
1242 ast.NodeInfixOp.InfixOp.AssignTimes => " *= ",
1243 ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ",
1244 ast.NodeInfixOp.InfixOp.BangEqual => " != ",
1245 ast.NodeInfixOp.InfixOp.BitAnd => " & ",
1246 ast.NodeInfixOp.InfixOp.BitOr => " | ",
1247 ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ",
1248 ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ",
1249 ast.NodeInfixOp.InfixOp.BitXor => " ^ ",
1250 ast.NodeInfixOp.InfixOp.BoolAnd => " and ",
1251 ast.NodeInfixOp.InfixOp.BoolOr => " or ",
1252 ast.NodeInfixOp.InfixOp.Div => " / ",
1253 ast.NodeInfixOp.InfixOp.EqualEqual => " == ",
1254 ast.NodeInfixOp.InfixOp.ErrorUnion => "!",
1255 ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ",
1256 ast.NodeInfixOp.InfixOp.GreaterThan => " > ",
1257 ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ",
1258 ast.NodeInfixOp.InfixOp.LessThan => " < ",
1259 ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ",
1260 ast.NodeInfixOp.InfixOp.Mod => " % ",
1261 ast.NodeInfixOp.InfixOp.Mult => " * ",
1262 ast.NodeInfixOp.InfixOp.MultWrap => " *% ",
1263 ast.NodeInfixOp.InfixOp.Period => ".",
1264 ast.NodeInfixOp.InfixOp.Sub => " - ",
1265 ast.NodeInfixOp.InfixOp.SubWrap => " -% ",
1266 ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ",
1267 };
1268
1269 try stack.append(RenderState { .Text = text });
11871270 try stack.append(RenderState { .Expression = prefix_op_node.lhs });
11881271 },
11891272 ast.Node.Id.PrefixOp => {
......@@ -1224,6 +1307,10 @@ pub const Parser = struct {
12241307 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);
12251308 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));
12261309 },
1310 ast.Node.Id.UndefinedLiteral => {
1311 const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base);
1312 try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token));
1313 },
12271314 ast.Node.Id.BuiltinCall => {
12281315 const builtin_call = @fieldParentPtr(ast.NodeBuiltinCall, "base", base);
12291316 try stream.print("{}(", self.tokenizer.getTokenSlice(builtin_call.builtin_token));
......@@ -1473,4 +1560,54 @@ test "zig fmt" {
14731560 \\}
14741561 \\
14751562 );
1563
1564 try testCanonical(
1565 \\test "operators" {
1566 \\ var i = undefined;
1567 \\ i = 2;
1568 \\ i *= 2;
1569 \\ i |= 2;
1570 \\ i ^= 2;
1571 \\ i <<= 2;
1572 \\ i >>= 2;
1573 \\ i &= 2;
1574 \\ i *= 2;
1575 \\ i *%= 2;
1576 \\ i -= 2;
1577 \\ i -%= 2;
1578 \\ i += 2;
1579 \\ i +%= 2;
1580 \\ i /= 2;
1581 \\ i %= 2;
1582 \\ _ = i == i;
1583 \\ _ = i != i;
1584 \\ _ = i != i;
1585 \\ _ = i.i;
1586 \\ _ = i || i;
1587 \\ _ = i!i;
1588 \\ _ = i ** i;
1589 \\ _ = i ++ i;
1590 \\ _ = i ?? i;
1591 \\ _ = i % i;
1592 \\ _ = i / i;
1593 \\ _ = i *% i;
1594 \\ _ = i * i;
1595 \\ _ = i -% i;
1596 \\ _ = i - i;
1597 \\ _ = i +% i;
1598 \\ _ = i + i;
1599 \\ _ = i << i;
1600 \\ _ = i >> i;
1601 \\ _ = i & i;
1602 \\ _ = i ^ i;
1603 \\ _ = i | i;
1604 \\ _ = i >= i;
1605 \\ _ = i <= i;
1606 \\ _ = i > i;
1607 \\ _ = i < i;
1608 \\ _ = i and i;
1609 \\ _ = i or i;
1610 \\}
1611 \\
1612 );
14761613}
std/zig/tokenizer.zig+282-5
......@@ -77,6 +77,7 @@ pub const Token = struct {
7777 Builtin,
7878 Bang,
7979 Pipe,
80 PipePipe,
8081 PipeEqual,
8182 Equal,
8283 EqualEqual,
......@@ -85,18 +86,45 @@ pub const Token = struct {
8586 RParen,
8687 Semicolon,
8788 Percent,
89 PercentEqual,
8890 LBrace,
8991 RBrace,
9092 Period,
9193 Ellipsis2,
9294 Ellipsis3,
95 Caret,
96 CaretEqual,
97 Plus,
98 PlusPlus,
99 PlusEqual,
100 PlusPercent,
101 PlusPercentEqual,
93102 Minus,
103 MinusEqual,
104 MinusPercent,
105 MinusPercentEqual,
106 Asterisk,
107 AsteriskEqual,
108 AsteriskAsterisk,
109 AsteriskPercent,
110 AsteriskPercentEqual,
94111 Arrow,
95112 Colon,
96113 Slash,
114 SlashEqual,
97115 Comma,
98116 Ampersand,
99117 AmpersandEqual,
118 QuestionMark,
119 QuestionMarkQuestionMark,
120 AngleBracketLeft,
121 AngleBracketLeftEqual,
122 AngleBracketAngleBracketLeft,
123 AngleBracketAngleBracketLeftEqual,
124 AngleBracketRight,
125 AngleBracketRightEqual,
126 AngleBracketAngleBracketRight,
127 AngleBracketAngleBracketRightEqual,
100128 IntegerLiteral,
101129 FloatLiteral,
102130 LineComment,
......@@ -200,6 +228,9 @@ pub const Tokenizer = struct {
200228 Bang,
201229 Pipe,
202230 Minus,
231 MinusPercent,
232 Asterisk,
233 AsteriskPercent,
203234 Slash,
204235 LineComment,
205236 Zero,
......@@ -210,6 +241,15 @@ pub const Tokenizer = struct {
210241 FloatExponentUnsigned,
211242 FloatExponentNumber,
212243 Ampersand,
244 Caret,
245 Percent,
246 QuestionMark,
247 Plus,
248 PlusPercent,
249 AngleBracketLeft,
250 AngleBracketAngleBracketLeft,
251 AngleBracketRight,
252 AngleBracketAngleBracketRight,
213253 Period,
214254 Period2,
215255 SawAtSign,
......@@ -291,9 +331,25 @@ pub const Tokenizer = struct {
291331 break;
292332 },
293333 '%' => {
294 result.id = Token.Id.Percent;
295 self.index += 1;
296 break;
334 state = State.Percent;
335 },
336 '*' => {
337 state = State.Asterisk;
338 },
339 '+' => {
340 state = State.Plus;
341 },
342 '?' => {
343 state = State.QuestionMark;
344 },
345 '<' => {
346 state = State.AngleBracketLeft;
347 },
348 '>' => {
349 state = State.AngleBracketRight;
350 },
351 '^' => {
352 state = State.Caret;
297353 },
298354 '{' => {
299355 result.id = Token.Id.LBrace;
......@@ -356,6 +412,107 @@ pub const Tokenizer = struct {
356412 break;
357413 },
358414 },
415
416 State.Asterisk => switch (c) {
417 '=' => {
418 result.id = Token.Id.AsteriskEqual;
419 self.index += 1;
420 break;
421 },
422 '*' => {
423 result.id = Token.Id.AsteriskAsterisk;
424 self.index += 1;
425 break;
426 },
427 '%' => {
428 state = State.AsteriskPercent;
429 },
430 else => {
431 result.id = Token.Id.Asterisk;
432 break;
433 }
434 },
435
436 State.AsteriskPercent => switch (c) {
437 '=' => {
438 result.id = Token.Id.AsteriskPercentEqual;
439 self.index += 1;
440 break;
441 },
442 else => {
443 result.id = Token.Id.AsteriskPercent;
444 break;
445 }
446 },
447
448 State.QuestionMark => switch (c) {
449 '?' => {
450 result.id = Token.Id.QuestionMarkQuestionMark;
451 self.index += 1;
452 break;
453 },
454 else => {
455 result.id = Token.Id.QuestionMark;
456 break;
457 },
458 },
459
460 State.Percent => switch (c) {
461 '=' => {
462 result.id = Token.Id.PercentEqual;
463 self.index += 1;
464 break;
465 },
466 else => {
467 result.id = Token.Id.Percent;
468 break;
469 },
470 },
471
472 State.Plus => switch (c) {
473 '=' => {
474 result.id = Token.Id.PlusEqual;
475 self.index += 1;
476 break;
477 },
478 '+' => {
479 result.id = Token.Id.PlusPlus;
480 self.index += 1;
481 break;
482 },
483 '%' => {
484 state = State.PlusPercent;
485 },
486 else => {
487 result.id = Token.Id.Plus;
488 break;
489 },
490 },
491
492 State.PlusPercent => switch (c) {
493 '=' => {
494 result.id = Token.Id.PlusPercentEqual;
495 self.index += 1;
496 break;
497 },
498 else => {
499 result.id = Token.Id.PlusPercent;
500 break;
501 },
502 },
503
504 State.Caret => switch (c) {
505 '=' => {
506 result.id = Token.Id.CaretEqual;
507 self.index += 1;
508 break;
509 },
510 else => {
511 result.id = Token.Id.Caret;
512 break;
513 }
514 },
515
359516 State.Identifier => switch (c) {
360517 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
361518 else => {
......@@ -417,6 +574,11 @@ pub const Tokenizer = struct {
417574 self.index += 1;
418575 break;
419576 },
577 '|' => {
578 result.id = Token.Id.PipePipe;
579 self.index += 1;
580 break;
581 },
420582 else => {
421583 result.id = Token.Id.Pipe;
422584 break;
......@@ -441,12 +603,86 @@ pub const Tokenizer = struct {
441603 self.index += 1;
442604 break;
443605 },
606 '=' => {
607 result.id = Token.Id.MinusEqual;
608 self.index += 1;
609 break;
610 },
611 '%' => {
612 state = State.MinusPercent;
613 },
444614 else => {
445615 result.id = Token.Id.Minus;
446616 break;
447617 },
448618 },
449619
620 State.MinusPercent => switch (c) {
621 '=' => {
622 result.id = Token.Id.MinusPercentEqual;
623 self.index += 1;
624 break;
625 },
626 else => {
627 result.id = Token.Id.MinusPercent;
628 break;
629 }
630 },
631
632 State.AngleBracketLeft => switch (c) {
633 '<' => {
634 state = State.AngleBracketAngleBracketLeft;
635 },
636 '=' => {
637 result.id = Token.Id.AngleBracketLeftEqual;
638 self.index += 1;
639 break;
640 },
641 else => {
642 result.id = Token.Id.AngleBracketLeft;
643 break;
644 },
645 },
646
647 State.AngleBracketAngleBracketLeft => switch (c) {
648 '=' => {
649 result.id = Token.Id.AngleBracketAngleBracketLeftEqual;
650 self.index += 1;
651 break;
652 },
653 else => {
654 result.id = Token.Id.AngleBracketAngleBracketLeft;
655 break;
656 },
657 },
658
659 State.AngleBracketRight => switch (c) {
660 '>' => {
661 state = State.AngleBracketAngleBracketRight;
662 },
663 '=' => {
664 result.id = Token.Id.AngleBracketRightEqual;
665 self.index += 1;
666 break;
667 },
668 else => {
669 result.id = Token.Id.AngleBracketRight;
670 break;
671 },
672 },
673
674 State.AngleBracketAngleBracketRight => switch (c) {
675 '=' => {
676 result.id = Token.Id.AngleBracketAngleBracketRightEqual;
677 self.index += 1;
678 break;
679 },
680 else => {
681 result.id = Token.Id.AngleBracketAngleBracketRight;
682 break;
683 },
684 },
685
450686 State.Period => switch (c) {
451687 '.' => {
452688 state = State.Period2;
......@@ -474,6 +710,11 @@ pub const Tokenizer = struct {
474710 result.id = Token.Id.LineComment;
475711 state = State.LineComment;
476712 },
713 '=' => {
714 result.id = Token.Id.SlashEqual;
715 self.index += 1;
716 break;
717 },
477718 else => {
478719 result.id = Token.Id.Slash;
479720 break;
......@@ -609,6 +850,42 @@ pub const Tokenizer = struct {
609850 State.Pipe => {
610851 result.id = Token.Id.Pipe;
611852 },
853 State.AngleBracketAngleBracketRight => {
854 result.id = Token.Id.AngleBracketAngleBracketRight;
855 },
856 State.AngleBracketRight => {
857 result.id = Token.Id.AngleBracketRight;
858 },
859 State.AngleBracketAngleBracketLeft => {
860 result.id = Token.Id.AngleBracketAngleBracketLeft;
861 },
862 State.AngleBracketLeft => {
863 result.id = Token.Id.AngleBracketLeft;
864 },
865 State.PlusPercent => {
866 result.id = Token.Id.PlusPercent;
867 },
868 State.Plus => {
869 result.id = Token.Id.Plus;
870 },
871 State.QuestionMark => {
872 result.id = Token.Id.QuestionMark;
873 },
874 State.Percent => {
875 result.id = Token.Id.Percent;
876 },
877 State.Caret => {
878 result.id = Token.Id.Caret;
879 },
880 State.AsteriskPercent => {
881 result.id = Token.Id.AsteriskPercent;
882 },
883 State.Asterisk => {
884 result.id = Token.Id.Asterisk;
885 },
886 State.MinusPercent => {
887 result.id = Token.Id.MinusPercent;
888 },
612889 }
613890 }
614891 if (result.id == Token.Id.Eof) {
......@@ -752,8 +1029,8 @@ test "tokenizer - string identifier and builtin fns" {
7521029
7531030test "tokenizer - pipe and then invalid" {
7541031 testTokenize("||=", []Token.Id{
755 Token.Id.Pipe,
756 Token.Id.PipeEqual,
1032 Token.Id.PipePipe,
1033 Token.Id.Equal,
7571034 });
7581035}
7591036