| ... | ... | @@ -994,94 +994,101 @@ void tokenize(Buf *buf, Tokenization *out) { |
| 994 | 994 | break; |
| 995 | 995 | } |
| 996 | 996 | if (t.state != TokenizeStateError) { |
| 997 | | t.pos = -1; |
| 997 | if (t.tokens->length > 0) { |
| 998 | Token *last_token = &t.tokens->last(); |
| 999 | t.line = last_token->start_line; |
| 1000 | t.column = last_token->start_column; |
| 1001 | t.pos = last_token->start_pos; |
| 1002 | } else { |
| 1003 | t.pos = 0; |
| 1004 | } |
| 998 | 1005 | begin_token(&t, TokenIdEof); |
| 999 | 1006 | end_token(&t); |
| 1000 | 1007 | assert(!t.cur_tok); |
| 1001 | 1008 | } |
| 1002 | 1009 | } |
| 1003 | 1010 | |
| 1004 | | static const char * token_name(Token *token) { |
| 1005 | | switch (token->id) { |
| 1011 | const char * token_name(TokenId id) { |
| 1012 | switch (id) { |
| 1006 | 1013 | case TokenIdEof: return "EOF"; |
| 1007 | 1014 | case TokenIdSymbol: return "Symbol"; |
| 1008 | | case TokenIdKeywordFn: return "Fn"; |
| 1009 | | case TokenIdKeywordConst: return "Const"; |
| 1010 | | case TokenIdKeywordVar: return "Var"; |
| 1011 | | case TokenIdKeywordReturn: return "Return"; |
| 1012 | | case TokenIdKeywordExtern: return "Extern"; |
| 1013 | | case TokenIdKeywordPub: return "Pub"; |
| 1014 | | case TokenIdKeywordExport: return "Export"; |
| 1015 | | case TokenIdKeywordAs: return "As"; |
| 1016 | | case TokenIdKeywordUse: return "Use"; |
| 1017 | | case TokenIdKeywordTrue: return "True"; |
| 1018 | | case TokenIdKeywordFalse: return "False"; |
| 1019 | | case TokenIdKeywordIf: return "If"; |
| 1020 | | case TokenIdKeywordElse: return "Else"; |
| 1021 | | case TokenIdKeywordGoto: return "Goto"; |
| 1022 | | case TokenIdKeywordVolatile: return "Volatile"; |
| 1023 | | case TokenIdKeywordAsm: return "Asm"; |
| 1024 | | case TokenIdKeywordStruct: return "Struct"; |
| 1025 | | case TokenIdKeywordEnum: return "Enum"; |
| 1026 | | case TokenIdKeywordWhile: return "While"; |
| 1027 | | case TokenIdKeywordContinue: return "Continue"; |
| 1028 | | case TokenIdKeywordBreak: return "Break"; |
| 1029 | | case TokenIdKeywordNull: return "Null"; |
| 1030 | | case TokenIdKeywordNoAlias: return "NoAlias"; |
| 1031 | | case TokenIdLParen: return "LParen"; |
| 1032 | | case TokenIdRParen: return "RParen"; |
| 1033 | | case TokenIdComma: return "Comma"; |
| 1034 | | case TokenIdStar: return "Star"; |
| 1035 | | case TokenIdLBrace: return "LBrace"; |
| 1036 | | case TokenIdRBrace: return "RBrace"; |
| 1037 | | case TokenIdLBracket: return "LBracket"; |
| 1038 | | case TokenIdRBracket: return "RBracket"; |
| 1015 | case TokenIdKeywordFn: return "fn"; |
| 1016 | case TokenIdKeywordConst: return "const"; |
| 1017 | case TokenIdKeywordVar: return "var"; |
| 1018 | case TokenIdKeywordReturn: return "return"; |
| 1019 | case TokenIdKeywordExtern: return "extern"; |
| 1020 | case TokenIdKeywordPub: return "pub"; |
| 1021 | case TokenIdKeywordExport: return "export"; |
| 1022 | case TokenIdKeywordAs: return "as"; |
| 1023 | case TokenIdKeywordUse: return "use"; |
| 1024 | case TokenIdKeywordTrue: return "true"; |
| 1025 | case TokenIdKeywordFalse: return "false"; |
| 1026 | case TokenIdKeywordIf: return "if"; |
| 1027 | case TokenIdKeywordElse: return "else"; |
| 1028 | case TokenIdKeywordGoto: return "goto"; |
| 1029 | case TokenIdKeywordVolatile: return "volatile"; |
| 1030 | case TokenIdKeywordAsm: return "asm"; |
| 1031 | case TokenIdKeywordStruct: return "struct"; |
| 1032 | case TokenIdKeywordEnum: return "enum"; |
| 1033 | case TokenIdKeywordWhile: return "while"; |
| 1034 | case TokenIdKeywordContinue: return "continue"; |
| 1035 | case TokenIdKeywordBreak: return "break"; |
| 1036 | case TokenIdKeywordNull: return "null"; |
| 1037 | case TokenIdKeywordNoAlias: return "noalias"; |
| 1038 | case TokenIdLParen: return "("; |
| 1039 | case TokenIdRParen: return ")"; |
| 1040 | case TokenIdComma: return ","; |
| 1041 | case TokenIdStar: return "*"; |
| 1042 | case TokenIdLBrace: return "{"; |
| 1043 | case TokenIdRBrace: return "}"; |
| 1044 | case TokenIdLBracket: return "["; |
| 1045 | case TokenIdRBracket: return "]"; |
| 1039 | 1046 | case TokenIdStringLiteral: return "StringLiteral"; |
| 1040 | 1047 | case TokenIdCharLiteral: return "CharLiteral"; |
| 1041 | | case TokenIdSemicolon: return "Semicolon"; |
| 1048 | case TokenIdSemicolon: return ";"; |
| 1042 | 1049 | case TokenIdNumberLiteral: return "NumberLiteral"; |
| 1043 | | case TokenIdPlus: return "Plus"; |
| 1044 | | case TokenIdColon: return "Colon"; |
| 1045 | | case TokenIdArrow: return "Arrow"; |
| 1046 | | case TokenIdFatArrow: return "FatArrow"; |
| 1047 | | case TokenIdDash: return "Dash"; |
| 1048 | | case TokenIdNumberSign: return "NumberSign"; |
| 1049 | | case TokenIdBinOr: return "BinOr"; |
| 1050 | | case TokenIdAmpersand: return "Ampersand"; |
| 1051 | | case TokenIdBinXor: return "BinXor"; |
| 1052 | | case TokenIdBoolOr: return "BoolOr"; |
| 1053 | | case TokenIdBoolAnd: return "BoolAnd"; |
| 1054 | | case TokenIdEq: return "Eq"; |
| 1055 | | case TokenIdTimesEq: return "TimesEq"; |
| 1056 | | case TokenIdDivEq: return "DivEq"; |
| 1057 | | case TokenIdModEq: return "ModEq"; |
| 1058 | | case TokenIdPlusEq: return "PlusEq"; |
| 1059 | | case TokenIdMinusEq: return "MinusEq"; |
| 1060 | | case TokenIdBitShiftLeftEq: return "BitShiftLeftEq"; |
| 1061 | | case TokenIdBitShiftRightEq: return "BitShiftRightEq"; |
| 1062 | | case TokenIdBitAndEq: return "BitAndEq"; |
| 1063 | | case TokenIdBitXorEq: return "BitXorEq"; |
| 1064 | | case TokenIdBitOrEq: return "BitOrEq"; |
| 1065 | | case TokenIdBoolAndEq: return "BoolAndEq"; |
| 1066 | | case TokenIdBoolOrEq: return "BoolOrEq"; |
| 1067 | | case TokenIdBang: return "Bang"; |
| 1068 | | case TokenIdTilde: return "Tilde"; |
| 1069 | | case TokenIdCmpEq: return "CmpEq"; |
| 1070 | | case TokenIdCmpNotEq: return "CmpNotEq"; |
| 1071 | | case TokenIdCmpLessThan: return "CmpLessThan"; |
| 1072 | | case TokenIdCmpGreaterThan: return "CmpGreaterThan"; |
| 1073 | | case TokenIdCmpLessOrEq: return "CmpLessOrEq"; |
| 1074 | | case TokenIdCmpGreaterOrEq: return "CmpGreaterOrEq"; |
| 1075 | | case TokenIdBitShiftLeft: return "BitShiftLeft"; |
| 1076 | | case TokenIdBitShiftRight: return "BitShiftRight"; |
| 1077 | | case TokenIdSlash: return "Slash"; |
| 1078 | | case TokenIdPercent: return "Percent"; |
| 1079 | | case TokenIdDot: return "Dot"; |
| 1080 | | case TokenIdEllipsis: return "Ellipsis"; |
| 1081 | | case TokenIdMaybe: return "Maybe"; |
| 1082 | | case TokenIdDoubleQuestion: return "DoubleQuestion"; |
| 1083 | | case TokenIdMaybeAssign: return "MaybeAssign"; |
| 1084 | | case TokenIdAtSign: return "AtSign"; |
| 1050 | case TokenIdPlus: return "+"; |
| 1051 | case TokenIdColon: return ":"; |
| 1052 | case TokenIdArrow: return "->"; |
| 1053 | case TokenIdFatArrow: return "=>"; |
| 1054 | case TokenIdDash: return "-"; |
| 1055 | case TokenIdNumberSign: return "#"; |
| 1056 | case TokenIdBinOr: return "|"; |
| 1057 | case TokenIdAmpersand: return "&"; |
| 1058 | case TokenIdBinXor: return "^"; |
| 1059 | case TokenIdBoolOr: return "||"; |
| 1060 | case TokenIdBoolAnd: return "&&"; |
| 1061 | case TokenIdEq: return "="; |
| 1062 | case TokenIdTimesEq: return "*="; |
| 1063 | case TokenIdDivEq: return "/="; |
| 1064 | case TokenIdModEq: return "%="; |
| 1065 | case TokenIdPlusEq: return "+="; |
| 1066 | case TokenIdMinusEq: return "-="; |
| 1067 | case TokenIdBitShiftLeftEq: return "<<="; |
| 1068 | case TokenIdBitShiftRightEq: return ">>="; |
| 1069 | case TokenIdBitAndEq: return "&="; |
| 1070 | case TokenIdBitXorEq: return "^="; |
| 1071 | case TokenIdBitOrEq: return "|="; |
| 1072 | case TokenIdBoolAndEq: return "&&="; |
| 1073 | case TokenIdBoolOrEq: return "||="; |
| 1074 | case TokenIdBang: return "!"; |
| 1075 | case TokenIdTilde: return "~"; |
| 1076 | case TokenIdCmpEq: return "=="; |
| 1077 | case TokenIdCmpNotEq: return "!="; |
| 1078 | case TokenIdCmpLessThan: return "<"; |
| 1079 | case TokenIdCmpGreaterThan: return ">"; |
| 1080 | case TokenIdCmpLessOrEq: return "<="; |
| 1081 | case TokenIdCmpGreaterOrEq: return ">="; |
| 1082 | case TokenIdBitShiftLeft: return "<<"; |
| 1083 | case TokenIdBitShiftRight: return ">>"; |
| 1084 | case TokenIdSlash: return "/"; |
| 1085 | case TokenIdPercent: return "%"; |
| 1086 | case TokenIdDot: return "."; |
| 1087 | case TokenIdEllipsis: return "..."; |
| 1088 | case TokenIdMaybe: return "?"; |
| 1089 | case TokenIdDoubleQuestion: return "??"; |
| 1090 | case TokenIdMaybeAssign: return "?="; |
| 1091 | case TokenIdAtSign: return "@"; |
| 1085 | 1092 | } |
| 1086 | 1093 | return "(invalid token)"; |
| 1087 | 1094 | } |
| ... | ... | @@ -1089,7 +1096,7 @@ static const char * token_name(Token *token) { |
| 1089 | 1096 | void print_tokens(Buf *buf, ZigList<Token> *tokens) { |
| 1090 | 1097 | for (int i = 0; i < tokens->length; i += 1) { |
| 1091 | 1098 | Token *token = &tokens->at(i); |
| 1092 | | fprintf(stderr, "%s ", token_name(token)); |
| 1099 | fprintf(stderr, "%s ", token_name(token->id)); |
| 1093 | 1100 | if (token->start_pos >= 0) { |
| 1094 | 1101 | fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stderr); |
| 1095 | 1102 | } |