authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-29 22:31:17+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-29 22:31:17+02:00
log530f79576923989928afeeb8b50b7e40b95a529f
treebfb369729475fce98d03cb25b7229725b2eed0fe
parentb80398b3558be09d0950415b0a8feeb09bc2f55d

std.zig.parser now supports all infix operators

3 files changed, 535 insertions(+), 62 deletions(-)

std/zig/ast.zig+60-1
...@@ -20,6 +20,7 @@ pub const Node = struct {...@@ -20,6 +20,7 @@ pub const Node = struct {
20 IntegerLiteral,20 IntegerLiteral,
21 FloatLiteral,21 FloatLiteral,
22 StringLiteral,22 StringLiteral,
23 UndefinedLiteral,
23 BuiltinCall,24 BuiltinCall,
24 LineComment,25 LineComment,
25 TestDecl,26 TestDecl,
...@@ -38,6 +39,7 @@ pub const Node = struct {...@@ -38,6 +39,7 @@ pub const Node = struct {
38 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),39 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
39 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),40 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
40 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),41 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
42 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),
41 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),43 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
42 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),44 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),
43 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index),45 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index),
...@@ -57,6 +59,7 @@ pub const Node = struct {...@@ -57,6 +59,7 @@ pub const Node = struct {
57 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),59 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
58 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),60 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
59 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),61 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
62 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),
60 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),63 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
61 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),64 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),
62 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(),65 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(),
...@@ -76,6 +79,7 @@ pub const Node = struct {...@@ -76,6 +79,7 @@ pub const Node = struct {
76 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),79 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
77 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),80 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
78 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),81 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
82 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),
79 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),83 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
80 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),84 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),
81 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(),85 Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(),
...@@ -309,9 +313,47 @@ pub const NodeInfixOp = struct {...@@ -309,9 +313,47 @@ pub const NodeInfixOp = struct {
309 rhs: &Node,313 rhs: &Node,
310314
311 const InfixOp = enum {315 const InfixOp = enum {
312 EqualEqual,316 Add,
317 AddWrap,
318 ArrayCat,
319 ArrayMult,
320 Assign,
321 AssignBitAnd,
322 AssignBitOr,
323 AssignBitShiftLeft,
324 AssignBitShiftRight,
325 AssignBitXor,
326 AssignDiv,
327 AssignMinus,
328 AssignMinusWrap,
329 AssignMod,
330 AssignPlus,
331 AssignPlusWrap,
332 AssignTimes,
333 AssignTimesWarp,
313 BangEqual,334 BangEqual,
335 BitAnd,
336 BitOr,
337 BitShiftLeft,
338 BitShiftRight,
339 BitXor,
340 BoolAnd,
341 BoolOr,
342 Div,
343 EqualEqual,
344 ErrorUnion,
345 GreaterOrEqual,
346 GreaterThan,
347 LessOrEqual,
348 LessThan,
349 MergeErrorSets,
350 Mod,
351 Mult,
352 MultWrap,
314 Period,353 Period,
354 Sub,
355 SubWrap,
356 UnwrapMaybe,
315 };357 };
316358
317 pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {359 pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
...@@ -464,6 +506,23 @@ pub const NodeStringLiteral = struct {...@@ -464,6 +506,23 @@ pub const NodeStringLiteral = struct {
464 }506 }
465};507};
466508
509pub const NodeUndefinedLiteral = struct {
510 base: Node,
511 token: Token,
512
513 pub fn iterate(self: &NodeUndefinedLiteral, index: usize) ?&Node {
514 return null;
515 }
516
517 pub fn firstToken(self: &NodeUndefinedLiteral) Token {
518 return self.token;
519 }
520
521 pub fn lastToken(self: &NodeUndefinedLiteral) Token {
522 return self.token;
523 }
524};
525
467pub const NodeLineComment = struct {526pub const NodeLineComment = struct {
468 base: Node,527 base: Node,
469 lines: ArrayList(Token),528 lines: ArrayList(Token),
std/zig/parser.zig+193-56
...@@ -371,6 +371,13 @@ pub const Parser = struct {...@@ -371,6 +371,13 @@ pub const Parser = struct {
371 try stack.append(State.AfterOperand);371 try stack.append(State.AfterOperand);
372 continue;372 continue;
373 },373 },
374 Token.Id.Keyword_undefined => {
375 try stack.append(State {
376 .Operand = &(try self.createUndefined(arena, token)).base
377 });
378 try stack.append(State.AfterOperand);
379 continue;
380 },
374 Token.Id.Builtin => {381 Token.Id.Builtin => {
375 const node = try arena.create(ast.NodeBuiltinCall);382 const node = try arena.create(ast.NodeBuiltinCall);
376 *node = ast.NodeBuiltinCall {383 *node = ast.NodeBuiltinCall {
...@@ -414,56 +421,41 @@ pub const Parser = struct {...@@ -414,56 +421,41 @@ pub const Parser = struct {
414 // or a postfix operator (like () or {}),421 // or a postfix operator (like () or {}),
415 // otherwise this expression is done (like on a ; or else).422 // otherwise this expression is done (like on a ; or else).
416 var token = self.getNextToken();423 var token = self.getNextToken();
417 switch (token.id) {424 if (tokenIdToInfixOp(token.id)) |infix_id| {
418 Token.Id.EqualEqual => {
419 try stack.append(State {
420 .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.EqualEqual)
421 });
422 try stack.append(State.ExpectOperand);
423 continue;
424 },
425 Token.Id.BangEqual => {
426 try stack.append(State {425 try stack.append(State {
427 .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BangEqual)426 .InfixOp = try self.createInfixOp(arena, token, infix_id)
428 });427 });
429 try stack.append(State.ExpectOperand);428 try stack.append(State.ExpectOperand);
430 continue;429 continue;
431 },430
432 Token.Id.Period => {431 // TODO: Parse postfix operator
433 try stack.append(State {432 } else {
434 .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period)433 // no postfix/infix operator after this operand.
435 });434 self.putBackToken(token);
436 try stack.append(State.ExpectOperand);435 // reduce the stack
437 continue;436 var expression: &ast.Node = stack.pop().Operand;
438 },437 while (true) {
439 else => {438 switch (stack.pop()) {
440 // no postfix/infix operator after this operand.439 State.Expression => |dest_ptr| {
441 self.putBackToken(token);440 // we're done
442 // reduce the stack441 try dest_ptr.store(expression);
443 var expression: &ast.Node = stack.pop().Operand;442 break;
444 while (true) {443 },
445 switch (stack.pop()) {444 State.InfixOp => |infix_op| {
446 State.Expression => |dest_ptr| {445 infix_op.rhs = expression;
447 // we're done446 infix_op.lhs = stack.pop().Operand;
448 try dest_ptr.store(expression);447 expression = &infix_op.base;
449 break;448 continue;
450 },449 },
451 State.InfixOp => |infix_op| {450 State.PrefixOp => |prefix_op| {
452 infix_op.rhs = expression;451 prefix_op.rhs = expression;
453 infix_op.lhs = stack.pop().Operand;452 expression = &prefix_op.base;
454 expression = &infix_op.base;453 continue;
455 continue;454 },
456 },455 else => unreachable,
457 State.PrefixOp => |prefix_op| {
458 prefix_op.rhs = expression;
459 expression = &prefix_op.base;
460 continue;
461 },
462 else => unreachable,
463 }
464 }456 }
465 continue;457 }
466 },458 continue;
467 }459 }
468 },460 },
469461
...@@ -706,6 +698,53 @@ pub const Parser = struct {...@@ -706,6 +698,53 @@ pub const Parser = struct {
706 }698 }
707 }699 }
708700
701 fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp {
702 return switch (*id) {
703 Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd,
704 Token.Id.AmpersandEqual => ast.NodeInfixOp.InfixOp.AssignBitAnd,
705 Token.Id.AngleBracketAngleBracketLeft => ast.NodeInfixOp.InfixOp.BitShiftLeft,
706 Token.Id.AngleBracketAngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftLeft,
707 Token.Id.AngleBracketAngleBracketRight => ast.NodeInfixOp.InfixOp.BitShiftRight,
708 Token.Id.AngleBracketAngleBracketRightEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftRight,
709 Token.Id.AngleBracketLeft => ast.NodeInfixOp.InfixOp.LessThan,
710 Token.Id.AngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.LessOrEqual,
711 Token.Id.AngleBracketRight => ast.NodeInfixOp.InfixOp.GreaterThan,
712 Token.Id.AngleBracketRightEqual => ast.NodeInfixOp.InfixOp.GreaterOrEqual,
713 Token.Id.Asterisk => ast.NodeInfixOp.InfixOp.Mult,
714 Token.Id.AsteriskAsterisk => ast.NodeInfixOp.InfixOp.ArrayMult,
715 Token.Id.AsteriskEqual => ast.NodeInfixOp.InfixOp.AssignTimes,
716 Token.Id.AsteriskPercent => ast.NodeInfixOp.InfixOp.MultWrap,
717 Token.Id.AsteriskPercentEqual => ast.NodeInfixOp.InfixOp.AssignTimesWarp,
718 Token.Id.Bang => ast.NodeInfixOp.InfixOp.ErrorUnion,
719 Token.Id.BangEqual => ast.NodeInfixOp.InfixOp.BangEqual,
720 Token.Id.Caret => ast.NodeInfixOp.InfixOp.BitXor,
721 Token.Id.CaretEqual => ast.NodeInfixOp.InfixOp.AssignBitXor,
722 Token.Id.Equal => ast.NodeInfixOp.InfixOp.Assign,
723 Token.Id.EqualEqual => ast.NodeInfixOp.InfixOp.EqualEqual,
724 Token.Id.Keyword_and => ast.NodeInfixOp.InfixOp.BoolAnd,
725 Token.Id.Keyword_or => ast.NodeInfixOp.InfixOp.BoolOr,
726 Token.Id.Minus => ast.NodeInfixOp.InfixOp.Sub,
727 Token.Id.MinusEqual => ast.NodeInfixOp.InfixOp.AssignMinus,
728 Token.Id.MinusPercent => ast.NodeInfixOp.InfixOp.SubWrap,
729 Token.Id.MinusPercentEqual => ast.NodeInfixOp.InfixOp.AssignMinusWrap,
730 Token.Id.Percent => ast.NodeInfixOp.InfixOp.Mod,
731 Token.Id.PercentEqual => ast.NodeInfixOp.InfixOp.AssignMod,
732 Token.Id.Period => ast.NodeInfixOp.InfixOp.Period,
733 Token.Id.Pipe => ast.NodeInfixOp.InfixOp.BitOr,
734 Token.Id.PipeEqual => ast.NodeInfixOp.InfixOp.AssignBitOr,
735 Token.Id.PipePipe => ast.NodeInfixOp.InfixOp.MergeErrorSets,
736 Token.Id.Plus => ast.NodeInfixOp.InfixOp.Add,
737 Token.Id.PlusEqual => ast.NodeInfixOp.InfixOp.AssignPlus,
738 Token.Id.PlusPercent => ast.NodeInfixOp.InfixOp.AddWrap,
739 Token.Id.PlusPercentEqual => ast.NodeInfixOp.InfixOp.AssignPlusWrap,
740 Token.Id.PlusPlus => ast.NodeInfixOp.InfixOp.ArrayCat,
741 Token.Id.QuestionMarkQuestionMark => ast.NodeInfixOp.InfixOp.UnwrapMaybe,
742 Token.Id.Slash => ast.NodeInfixOp.InfixOp.Div,
743 Token.Id.SlashEqual => ast.NodeInfixOp.InfixOp.AssignDiv,
744 else => null,
745 };
746 }
747
709 fn initNode(self: &Parser, id: ast.Node.Id) ast.Node {748 fn initNode(self: &Parser, id: ast.Node.Id) ast.Node {
710 if (self.pending_line_comment_node) |comment_node| {749 if (self.pending_line_comment_node) |comment_node| {
711 self.pending_line_comment_node = null;750 self.pending_line_comment_node = null;
...@@ -867,6 +906,16 @@ pub const Parser = struct {...@@ -867,6 +906,16 @@ pub const Parser = struct {
867 return node;906 return node;
868 }907 }
869908
909 fn createUndefined(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeUndefinedLiteral {
910 const node = try arena.create(ast.NodeUndefinedLiteral);
911
912 *node = ast.NodeUndefinedLiteral {
913 .base = self.initNode(ast.Node.Id.UndefinedLiteral),
914 .token = *token,
915 };
916 return node;
917 }
918
870 fn createAttachIdentifier(self: &Parser, arena: &mem.Allocator, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier {919 fn createAttachIdentifier(self: &Parser, arena: &mem.Allocator, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier {
871 const node = try self.createIdentifier(arena, name_token);920 const node = try self.createIdentifier(arena, name_token);
872 try dest_ptr.store(&node.base);921 try dest_ptr.store(&node.base);
...@@ -1173,17 +1222,51 @@ pub const Parser = struct {...@@ -1173,17 +1222,51 @@ pub const Parser = struct {
1173 ast.Node.Id.InfixOp => {1222 ast.Node.Id.InfixOp => {
1174 const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base);1223 const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base);
1175 try stack.append(RenderState { .Expression = prefix_op_node.rhs });1224 try stack.append(RenderState { .Expression = prefix_op_node.rhs });
1176 switch (prefix_op_node.op) {1225 const text = switch (prefix_op_node.op) {
1177 ast.NodeInfixOp.InfixOp.EqualEqual => {1226 ast.NodeInfixOp.InfixOp.Add => " + ",
1178 try stack.append(RenderState { .Text = " == "});1227 ast.NodeInfixOp.InfixOp.AddWrap => " +% ",
1179 },1228 ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ",
1180 ast.NodeInfixOp.InfixOp.BangEqual => {1229 ast.NodeInfixOp.InfixOp.ArrayMult => " ** ",
1181 try stack.append(RenderState { .Text = " != "});1230 ast.NodeInfixOp.InfixOp.Assign => " = ",
1182 },1231 ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ",
1183 ast.NodeInfixOp.InfixOp.Period => {1232 ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ",
1184 try stack.append(RenderState { .Text = "."});1233 ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ",
1185 },1234 ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ",
1186 }1235 ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ",
1236 ast.NodeInfixOp.InfixOp.AssignDiv => " /= ",
1237 ast.NodeInfixOp.InfixOp.AssignMinus => " -= ",
1238 ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ",
1239 ast.NodeInfixOp.InfixOp.AssignMod => " %= ",
1240 ast.NodeInfixOp.InfixOp.AssignPlus => " += ",
1241 ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ",
1242 ast.NodeInfixOp.InfixOp.AssignTimes => " *= ",
1243 ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ",
1244 ast.NodeInfixOp.InfixOp.BangEqual => " != ",
1245 ast.NodeInfixOp.InfixOp.BitAnd => " & ",
1246 ast.NodeInfixOp.InfixOp.BitOr => " | ",
1247 ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ",
1248 ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ",
1249 ast.NodeInfixOp.InfixOp.BitXor => " ^ ",
1250 ast.NodeInfixOp.InfixOp.BoolAnd => " and ",
1251 ast.NodeInfixOp.InfixOp.BoolOr => " or ",
1252 ast.NodeInfixOp.InfixOp.Div => " / ",
1253 ast.NodeInfixOp.InfixOp.EqualEqual => " == ",
1254 ast.NodeInfixOp.InfixOp.ErrorUnion => "!",
1255 ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ",
1256 ast.NodeInfixOp.InfixOp.GreaterThan => " > ",
1257 ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ",
1258 ast.NodeInfixOp.InfixOp.LessThan => " < ",
1259 ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ",
1260 ast.NodeInfixOp.InfixOp.Mod => " % ",
1261 ast.NodeInfixOp.InfixOp.Mult => " * ",
1262 ast.NodeInfixOp.InfixOp.MultWrap => " *% ",
1263 ast.NodeInfixOp.InfixOp.Period => ".",
1264 ast.NodeInfixOp.InfixOp.Sub => " - ",
1265 ast.NodeInfixOp.InfixOp.SubWrap => " -% ",
1266 ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ",
1267 };
1268
1269 try stack.append(RenderState { .Text = text });
1187 try stack.append(RenderState { .Expression = prefix_op_node.lhs });1270 try stack.append(RenderState { .Expression = prefix_op_node.lhs });
1188 },1271 },
1189 ast.Node.Id.PrefixOp => {1272 ast.Node.Id.PrefixOp => {
...@@ -1224,6 +1307,10 @@ pub const Parser = struct {...@@ -1224,6 +1307,10 @@ pub const Parser = struct {
1224 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);1307 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);
1225 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));1308 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));
1226 },1309 },
1310 ast.Node.Id.UndefinedLiteral => {
1311 const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base);
1312 try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token));
1313 },
1227 ast.Node.Id.BuiltinCall => {1314 ast.Node.Id.BuiltinCall => {
1228 const builtin_call = @fieldParentPtr(ast.NodeBuiltinCall, "base", base);1315 const builtin_call = @fieldParentPtr(ast.NodeBuiltinCall, "base", base);
1229 try stream.print("{}(", self.tokenizer.getTokenSlice(builtin_call.builtin_token));1316 try stream.print("{}(", self.tokenizer.getTokenSlice(builtin_call.builtin_token));
...@@ -1473,4 +1560,54 @@ test "zig fmt" {...@@ -1473,4 +1560,54 @@ test "zig fmt" {
1473 \\}1560 \\}
1474 \\1561 \\
1475 );1562 );
1563
1564 try testCanonical(
1565 \\test "operators" {
1566 \\ var i = undefined;
1567 \\ i = 2;
1568 \\ i *= 2;
1569 \\ i |= 2;
1570 \\ i ^= 2;
1571 \\ i <<= 2;
1572 \\ i >>= 2;
1573 \\ i &= 2;
1574 \\ i *= 2;
1575 \\ i *%= 2;
1576 \\ i -= 2;
1577 \\ i -%= 2;
1578 \\ i += 2;
1579 \\ i +%= 2;
1580 \\ i /= 2;
1581 \\ i %= 2;
1582 \\ _ = i == i;
1583 \\ _ = i != i;
1584 \\ _ = i != i;
1585 \\ _ = i.i;
1586 \\ _ = i || i;
1587 \\ _ = i!i;
1588 \\ _ = i ** i;
1589 \\ _ = i ++ i;
1590 \\ _ = i ?? i;
1591 \\ _ = i % i;
1592 \\ _ = i / i;
1593 \\ _ = i *% i;
1594 \\ _ = i * i;
1595 \\ _ = i -% i;
1596 \\ _ = i - i;
1597 \\ _ = i +% i;
1598 \\ _ = i + i;
1599 \\ _ = i << i;
1600 \\ _ = i >> i;
1601 \\ _ = i & i;
1602 \\ _ = i ^ i;
1603 \\ _ = i | i;
1604 \\ _ = i >= i;
1605 \\ _ = i <= i;
1606 \\ _ = i > i;
1607 \\ _ = i < i;
1608 \\ _ = i and i;
1609 \\ _ = i or i;
1610 \\}
1611 \\
1612 );
1476}1613}
std/zig/tokenizer.zig+282-5
...@@ -77,6 +77,7 @@ pub const Token = struct {...@@ -77,6 +77,7 @@ pub const Token = struct {
77 Builtin,77 Builtin,
78 Bang,78 Bang,
79 Pipe,79 Pipe,
80 PipePipe,
80 PipeEqual,81 PipeEqual,
81 Equal,82 Equal,
82 EqualEqual,83 EqualEqual,
...@@ -85,18 +86,45 @@ pub const Token = struct {...@@ -85,18 +86,45 @@ pub const Token = struct {
85 RParen,86 RParen,
86 Semicolon,87 Semicolon,
87 Percent,88 Percent,
89 PercentEqual,
88 LBrace,90 LBrace,
89 RBrace,91 RBrace,
90 Period,92 Period,
91 Ellipsis2,93 Ellipsis2,
92 Ellipsis3,94 Ellipsis3,
95 Caret,
96 CaretEqual,
97 Plus,
98 PlusPlus,
99 PlusEqual,
100 PlusPercent,
101 PlusPercentEqual,
93 Minus,102 Minus,
103 MinusEqual,
104 MinusPercent,
105 MinusPercentEqual,
106 Asterisk,
107 AsteriskEqual,
108 AsteriskAsterisk,
109 AsteriskPercent,
110 AsteriskPercentEqual,
94 Arrow,111 Arrow,
95 Colon,112 Colon,
96 Slash,113 Slash,
114 SlashEqual,
97 Comma,115 Comma,
98 Ampersand,116 Ampersand,
99 AmpersandEqual,117 AmpersandEqual,
118 QuestionMark,
119 QuestionMarkQuestionMark,
120 AngleBracketLeft,
121 AngleBracketLeftEqual,
122 AngleBracketAngleBracketLeft,
123 AngleBracketAngleBracketLeftEqual,
124 AngleBracketRight,
125 AngleBracketRightEqual,
126 AngleBracketAngleBracketRight,
127 AngleBracketAngleBracketRightEqual,
100 IntegerLiteral,128 IntegerLiteral,
101 FloatLiteral,129 FloatLiteral,
102 LineComment,130 LineComment,
...@@ -200,6 +228,9 @@ pub const Tokenizer = struct {...@@ -200,6 +228,9 @@ pub const Tokenizer = struct {
200 Bang,228 Bang,
201 Pipe,229 Pipe,
202 Minus,230 Minus,
231 MinusPercent,
232 Asterisk,
233 AsteriskPercent,
203 Slash,234 Slash,
204 LineComment,235 LineComment,
205 Zero,236 Zero,
...@@ -210,6 +241,15 @@ pub const Tokenizer = struct {...@@ -210,6 +241,15 @@ pub const Tokenizer = struct {
210 FloatExponentUnsigned,241 FloatExponentUnsigned,
211 FloatExponentNumber,242 FloatExponentNumber,
212 Ampersand,243 Ampersand,
244 Caret,
245 Percent,
246 QuestionMark,
247 Plus,
248 PlusPercent,
249 AngleBracketLeft,
250 AngleBracketAngleBracketLeft,
251 AngleBracketRight,
252 AngleBracketAngleBracketRight,
213 Period,253 Period,
214 Period2,254 Period2,
215 SawAtSign,255 SawAtSign,
...@@ -291,9 +331,25 @@ pub const Tokenizer = struct {...@@ -291,9 +331,25 @@ pub const Tokenizer = struct {
291 break;331 break;
292 },332 },
293 '%' => {333 '%' => {
294 result.id = Token.Id.Percent;334 state = State.Percent;
295 self.index += 1;335 },
296 break;336 '*' => {
337 state = State.Asterisk;
338 },
339 '+' => {
340 state = State.Plus;
341 },
342 '?' => {
343 state = State.QuestionMark;
344 },
345 '<' => {
346 state = State.AngleBracketLeft;
347 },
348 '>' => {
349 state = State.AngleBracketRight;
350 },
351 '^' => {
352 state = State.Caret;
297 },353 },
298 '{' => {354 '{' => {
299 result.id = Token.Id.LBrace;355 result.id = Token.Id.LBrace;
...@@ -356,6 +412,107 @@ pub const Tokenizer = struct {...@@ -356,6 +412,107 @@ pub const Tokenizer = struct {
356 break;412 break;
357 },413 },
358 },414 },
415
416 State.Asterisk => switch (c) {
417 '=' => {
418 result.id = Token.Id.AsteriskEqual;
419 self.index += 1;
420 break;
421 },
422 '*' => {
423 result.id = Token.Id.AsteriskAsterisk;
424 self.index += 1;
425 break;
426 },
427 '%' => {
428 state = State.AsteriskPercent;
429 },
430 else => {
431 result.id = Token.Id.Asterisk;
432 break;
433 }
434 },
435
436 State.AsteriskPercent => switch (c) {
437 '=' => {
438 result.id = Token.Id.AsteriskPercentEqual;
439 self.index += 1;
440 break;
441 },
442 else => {
443 result.id = Token.Id.AsteriskPercent;
444 break;
445 }
446 },
447
448 State.QuestionMark => switch (c) {
449 '?' => {
450 result.id = Token.Id.QuestionMarkQuestionMark;
451 self.index += 1;
452 break;
453 },
454 else => {
455 result.id = Token.Id.QuestionMark;
456 break;
457 },
458 },
459
460 State.Percent => switch (c) {
461 '=' => {
462 result.id = Token.Id.PercentEqual;
463 self.index += 1;
464 break;
465 },
466 else => {
467 result.id = Token.Id.Percent;
468 break;
469 },
470 },
471
472 State.Plus => switch (c) {
473 '=' => {
474 result.id = Token.Id.PlusEqual;
475 self.index += 1;
476 break;
477 },
478 '+' => {
479 result.id = Token.Id.PlusPlus;
480 self.index += 1;
481 break;
482 },
483 '%' => {
484 state = State.PlusPercent;
485 },
486 else => {
487 result.id = Token.Id.Plus;
488 break;
489 },
490 },
491
492 State.PlusPercent => switch (c) {
493 '=' => {
494 result.id = Token.Id.PlusPercentEqual;
495 self.index += 1;
496 break;
497 },
498 else => {
499 result.id = Token.Id.PlusPercent;
500 break;
501 },
502 },
503
504 State.Caret => switch (c) {
505 '=' => {
506 result.id = Token.Id.CaretEqual;
507 self.index += 1;
508 break;
509 },
510 else => {
511 result.id = Token.Id.Caret;
512 break;
513 }
514 },
515
359 State.Identifier => switch (c) {516 State.Identifier => switch (c) {
360 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},517 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
361 else => {518 else => {
...@@ -417,6 +574,11 @@ pub const Tokenizer = struct {...@@ -417,6 +574,11 @@ pub const Tokenizer = struct {
417 self.index += 1;574 self.index += 1;
418 break;575 break;
419 },576 },
577 '|' => {
578 result.id = Token.Id.PipePipe;
579 self.index += 1;
580 break;
581 },
420 else => {582 else => {
421 result.id = Token.Id.Pipe;583 result.id = Token.Id.Pipe;
422 break;584 break;
...@@ -441,12 +603,86 @@ pub const Tokenizer = struct {...@@ -441,12 +603,86 @@ pub const Tokenizer = struct {
441 self.index += 1;603 self.index += 1;
442 break;604 break;
443 },605 },
606 '=' => {
607 result.id = Token.Id.MinusEqual;
608 self.index += 1;
609 break;
610 },
611 '%' => {
612 state = State.MinusPercent;
613 },
444 else => {614 else => {
445 result.id = Token.Id.Minus;615 result.id = Token.Id.Minus;
446 break;616 break;
447 },617 },
448 },618 },
449619
620 State.MinusPercent => switch (c) {
621 '=' => {
622 result.id = Token.Id.MinusPercentEqual;
623 self.index += 1;
624 break;
625 },
626 else => {
627 result.id = Token.Id.MinusPercent;
628 break;
629 }
630 },
631
632 State.AngleBracketLeft => switch (c) {
633 '<' => {
634 state = State.AngleBracketAngleBracketLeft;
635 },
636 '=' => {
637 result.id = Token.Id.AngleBracketLeftEqual;
638 self.index += 1;
639 break;
640 },
641 else => {
642 result.id = Token.Id.AngleBracketLeft;
643 break;
644 },
645 },
646
647 State.AngleBracketAngleBracketLeft => switch (c) {
648 '=' => {
649 result.id = Token.Id.AngleBracketAngleBracketLeftEqual;
650 self.index += 1;
651 break;
652 },
653 else => {
654 result.id = Token.Id.AngleBracketAngleBracketLeft;
655 break;
656 },
657 },
658
659 State.AngleBracketRight => switch (c) {
660 '>' => {
661 state = State.AngleBracketAngleBracketRight;
662 },
663 '=' => {
664 result.id = Token.Id.AngleBracketRightEqual;
665 self.index += 1;
666 break;
667 },
668 else => {
669 result.id = Token.Id.AngleBracketRight;
670 break;
671 },
672 },
673
674 State.AngleBracketAngleBracketRight => switch (c) {
675 '=' => {
676 result.id = Token.Id.AngleBracketAngleBracketRightEqual;
677 self.index += 1;
678 break;
679 },
680 else => {
681 result.id = Token.Id.AngleBracketAngleBracketRight;
682 break;
683 },
684 },
685
450 State.Period => switch (c) {686 State.Period => switch (c) {
451 '.' => {687 '.' => {
452 state = State.Period2;688 state = State.Period2;
...@@ -474,6 +710,11 @@ pub const Tokenizer = struct {...@@ -474,6 +710,11 @@ pub const Tokenizer = struct {
474 result.id = Token.Id.LineComment;710 result.id = Token.Id.LineComment;
475 state = State.LineComment;711 state = State.LineComment;
476 },712 },
713 '=' => {
714 result.id = Token.Id.SlashEqual;
715 self.index += 1;
716 break;
717 },
477 else => {718 else => {
478 result.id = Token.Id.Slash;719 result.id = Token.Id.Slash;
479 break;720 break;
...@@ -609,6 +850,42 @@ pub const Tokenizer = struct {...@@ -609,6 +850,42 @@ pub const Tokenizer = struct {
609 State.Pipe => {850 State.Pipe => {
610 result.id = Token.Id.Pipe;851 result.id = Token.Id.Pipe;
611 },852 },
853 State.AngleBracketAngleBracketRight => {
854 result.id = Token.Id.AngleBracketAngleBracketRight;
855 },
856 State.AngleBracketRight => {
857 result.id = Token.Id.AngleBracketRight;
858 },
859 State.AngleBracketAngleBracketLeft => {
860 result.id = Token.Id.AngleBracketAngleBracketLeft;
861 },
862 State.AngleBracketLeft => {
863 result.id = Token.Id.AngleBracketLeft;
864 },
865 State.PlusPercent => {
866 result.id = Token.Id.PlusPercent;
867 },
868 State.Plus => {
869 result.id = Token.Id.Plus;
870 },
871 State.QuestionMark => {
872 result.id = Token.Id.QuestionMark;
873 },
874 State.Percent => {
875 result.id = Token.Id.Percent;
876 },
877 State.Caret => {
878 result.id = Token.Id.Caret;
879 },
880 State.AsteriskPercent => {
881 result.id = Token.Id.AsteriskPercent;
882 },
883 State.Asterisk => {
884 result.id = Token.Id.Asterisk;
885 },
886 State.MinusPercent => {
887 result.id = Token.Id.MinusPercent;
888 },
612 }889 }
613 }890 }
614 if (result.id == Token.Id.Eof) {891 if (result.id == Token.Id.Eof) {
...@@ -752,8 +1029,8 @@ test "tokenizer - string identifier and builtin fns" {...@@ -752,8 +1029,8 @@ test "tokenizer - string identifier and builtin fns" {
7521029
753test "tokenizer - pipe and then invalid" {1030test "tokenizer - pipe and then invalid" {
754 testTokenize("||=", []Token.Id{1031 testTokenize("||=", []Token.Id{
755 Token.Id.Pipe,1032 Token.Id.PipePipe,
756 Token.Id.PipeEqual,1033 Token.Id.Equal,
757 });1034 });
758}1035}
7591036