| ... | @@ -994,94 +994,101 @@ void tokenize(Buf *buf, Tokenization *out) { | ... | @@ -994,94 +994,101 @@ void tokenize(Buf *buf, Tokenization *out) { |
| 994 | break; | 994 | break; |
| 995 | } | 995 | } |
| 996 | if (t.state != TokenizeStateError) { | 996 | if (t.state != TokenizeStateError) { |
| 997 | t.pos = -1; | 997 | if (t.tokens->length > 0) { |
| | 998 | Token *last_token = &t.tokens->last(); |
| | 999 | t.line = last_token->start_line; |
| | 1000 | t.column = last_token->start_column; |
| | 1001 | t.pos = last_token->start_pos; |
| | 1002 | } else { |
| | 1003 | t.pos = 0; |
| | 1004 | } |
| 998 | begin_token(&t, TokenIdEof); | 1005 | begin_token(&t, TokenIdEof); |
| 999 | end_token(&t); | 1006 | end_token(&t); |
| 1000 | assert(!t.cur_tok); | 1007 | assert(!t.cur_tok); |
| 1001 | } | 1008 | } |
| 1002 | } | 1009 | } |
| 1003 | | 1010 | |
| 1004 | static const char * token_name(Token *token) { | 1011 | const char * token_name(TokenId id) { |
| 1005 | switch (token->id) { | 1012 | switch (id) { |
| 1006 | case TokenIdEof: return "EOF"; | 1013 | case TokenIdEof: return "EOF"; |
| 1007 | case TokenIdSymbol: return "Symbol"; | 1014 | case TokenIdSymbol: return "Symbol"; |
| 1008 | case TokenIdKeywordFn: return "Fn"; | 1015 | case TokenIdKeywordFn: return "fn"; |
| 1009 | case TokenIdKeywordConst: return "Const"; | 1016 | case TokenIdKeywordConst: return "const"; |
| 1010 | case TokenIdKeywordVar: return "Var"; | 1017 | case TokenIdKeywordVar: return "var"; |
| 1011 | case TokenIdKeywordReturn: return "Return"; | 1018 | case TokenIdKeywordReturn: return "return"; |
| 1012 | case TokenIdKeywordExtern: return "Extern"; | 1019 | case TokenIdKeywordExtern: return "extern"; |
| 1013 | case TokenIdKeywordPub: return "Pub"; | 1020 | case TokenIdKeywordPub: return "pub"; |
| 1014 | case TokenIdKeywordExport: return "Export"; | 1021 | case TokenIdKeywordExport: return "export"; |
| 1015 | case TokenIdKeywordAs: return "As"; | 1022 | case TokenIdKeywordAs: return "as"; |
| 1016 | case TokenIdKeywordUse: return "Use"; | 1023 | case TokenIdKeywordUse: return "use"; |
| 1017 | case TokenIdKeywordTrue: return "True"; | 1024 | case TokenIdKeywordTrue: return "true"; |
| 1018 | case TokenIdKeywordFalse: return "False"; | 1025 | case TokenIdKeywordFalse: return "false"; |
| 1019 | case TokenIdKeywordIf: return "If"; | 1026 | case TokenIdKeywordIf: return "if"; |
| 1020 | case TokenIdKeywordElse: return "Else"; | 1027 | case TokenIdKeywordElse: return "else"; |
| 1021 | case TokenIdKeywordGoto: return "Goto"; | 1028 | case TokenIdKeywordGoto: return "goto"; |
| 1022 | case TokenIdKeywordVolatile: return "Volatile"; | 1029 | case TokenIdKeywordVolatile: return "volatile"; |
| 1023 | case TokenIdKeywordAsm: return "Asm"; | 1030 | case TokenIdKeywordAsm: return "asm"; |
| 1024 | case TokenIdKeywordStruct: return "Struct"; | 1031 | case TokenIdKeywordStruct: return "struct"; |
| 1025 | case TokenIdKeywordEnum: return "Enum"; | 1032 | case TokenIdKeywordEnum: return "enum"; |
| 1026 | case TokenIdKeywordWhile: return "While"; | 1033 | case TokenIdKeywordWhile: return "while"; |
| 1027 | case TokenIdKeywordContinue: return "Continue"; | 1034 | case TokenIdKeywordContinue: return "continue"; |
| 1028 | case TokenIdKeywordBreak: return "Break"; | 1035 | case TokenIdKeywordBreak: return "break"; |
| 1029 | case TokenIdKeywordNull: return "Null"; | 1036 | case TokenIdKeywordNull: return "null"; |
| 1030 | case TokenIdKeywordNoAlias: return "NoAlias"; | 1037 | case TokenIdKeywordNoAlias: return "noalias"; |
| 1031 | case TokenIdLParen: return "LParen"; | 1038 | case TokenIdLParen: return "("; |
| 1032 | case TokenIdRParen: return "RParen"; | 1039 | case TokenIdRParen: return ")"; |
| 1033 | case TokenIdComma: return "Comma"; | 1040 | case TokenIdComma: return ","; |
| 1034 | case TokenIdStar: return "Star"; | 1041 | case TokenIdStar: return "*"; |
| 1035 | case TokenIdLBrace: return "LBrace"; | 1042 | case TokenIdLBrace: return "{"; |
| 1036 | case TokenIdRBrace: return "RBrace"; | 1043 | case TokenIdRBrace: return "}"; |
| 1037 | case TokenIdLBracket: return "LBracket"; | 1044 | case TokenIdLBracket: return "["; |
| 1038 | case TokenIdRBracket: return "RBracket"; | 1045 | case TokenIdRBracket: return "]"; |
| 1039 | case TokenIdStringLiteral: return "StringLiteral"; | 1046 | case TokenIdStringLiteral: return "StringLiteral"; |
| 1040 | case TokenIdCharLiteral: return "CharLiteral"; | 1047 | case TokenIdCharLiteral: return "CharLiteral"; |
| 1041 | case TokenIdSemicolon: return "Semicolon"; | 1048 | case TokenIdSemicolon: return ";"; |
| 1042 | case TokenIdNumberLiteral: return "NumberLiteral"; | 1049 | case TokenIdNumberLiteral: return "NumberLiteral"; |
| 1043 | case TokenIdPlus: return "Plus"; | 1050 | case TokenIdPlus: return "+"; |
| 1044 | case TokenIdColon: return "Colon"; | 1051 | case TokenIdColon: return ":"; |
| 1045 | case TokenIdArrow: return "Arrow"; | 1052 | case TokenIdArrow: return "->"; |
| 1046 | case TokenIdFatArrow: return "FatArrow"; | 1053 | case TokenIdFatArrow: return "=>"; |
| 1047 | case TokenIdDash: return "Dash"; | 1054 | case TokenIdDash: return "-"; |
| 1048 | case TokenIdNumberSign: return "NumberSign"; | 1055 | case TokenIdNumberSign: return "#"; |
| 1049 | case TokenIdBinOr: return "BinOr"; | 1056 | case TokenIdBinOr: return "|"; |
| 1050 | case TokenIdAmpersand: return "Ampersand"; | 1057 | case TokenIdAmpersand: return "&"; |
| 1051 | case TokenIdBinXor: return "BinXor"; | 1058 | case TokenIdBinXor: return "^"; |
| 1052 | case TokenIdBoolOr: return "BoolOr"; | 1059 | case TokenIdBoolOr: return "||"; |
| 1053 | case TokenIdBoolAnd: return "BoolAnd"; | 1060 | case TokenIdBoolAnd: return "&&"; |
| 1054 | case TokenIdEq: return "Eq"; | 1061 | case TokenIdEq: return "="; |
| 1055 | case TokenIdTimesEq: return "TimesEq"; | 1062 | case TokenIdTimesEq: return "*="; |
| 1056 | case TokenIdDivEq: return "DivEq"; | 1063 | case TokenIdDivEq: return "/="; |
| 1057 | case TokenIdModEq: return "ModEq"; | 1064 | case TokenIdModEq: return "%="; |
| 1058 | case TokenIdPlusEq: return "PlusEq"; | 1065 | case TokenIdPlusEq: return "+="; |
| 1059 | case TokenIdMinusEq: return "MinusEq"; | 1066 | case TokenIdMinusEq: return "-="; |
| 1060 | case TokenIdBitShiftLeftEq: return "BitShiftLeftEq"; | 1067 | case TokenIdBitShiftLeftEq: return "<<="; |
| 1061 | case TokenIdBitShiftRightEq: return "BitShiftRightEq"; | 1068 | case TokenIdBitShiftRightEq: return ">>="; |
| 1062 | case TokenIdBitAndEq: return "BitAndEq"; | 1069 | case TokenIdBitAndEq: return "&="; |
| 1063 | case TokenIdBitXorEq: return "BitXorEq"; | 1070 | case TokenIdBitXorEq: return "^="; |
| 1064 | case TokenIdBitOrEq: return "BitOrEq"; | 1071 | case TokenIdBitOrEq: return "|="; |
| 1065 | case TokenIdBoolAndEq: return "BoolAndEq"; | 1072 | case TokenIdBoolAndEq: return "&&="; |
| 1066 | case TokenIdBoolOrEq: return "BoolOrEq"; | 1073 | case TokenIdBoolOrEq: return "||="; |
| 1067 | case TokenIdBang: return "Bang"; | 1074 | case TokenIdBang: return "!"; |
| 1068 | case TokenIdTilde: return "Tilde"; | 1075 | case TokenIdTilde: return "~"; |
| 1069 | case TokenIdCmpEq: return "CmpEq"; | 1076 | case TokenIdCmpEq: return "=="; |
| 1070 | case TokenIdCmpNotEq: return "CmpNotEq"; | 1077 | case TokenIdCmpNotEq: return "!="; |
| 1071 | case TokenIdCmpLessThan: return "CmpLessThan"; | 1078 | case TokenIdCmpLessThan: return "<"; |
| 1072 | case TokenIdCmpGreaterThan: return "CmpGreaterThan"; | 1079 | case TokenIdCmpGreaterThan: return ">"; |
| 1073 | case TokenIdCmpLessOrEq: return "CmpLessOrEq"; | 1080 | case TokenIdCmpLessOrEq: return "<="; |
| 1074 | case TokenIdCmpGreaterOrEq: return "CmpGreaterOrEq"; | 1081 | case TokenIdCmpGreaterOrEq: return ">="; |
| 1075 | case TokenIdBitShiftLeft: return "BitShiftLeft"; | 1082 | case TokenIdBitShiftLeft: return "<<"; |
| 1076 | case TokenIdBitShiftRight: return "BitShiftRight"; | 1083 | case TokenIdBitShiftRight: return ">>"; |
| 1077 | case TokenIdSlash: return "Slash"; | 1084 | case TokenIdSlash: return "/"; |
| 1078 | case TokenIdPercent: return "Percent"; | 1085 | case TokenIdPercent: return "%"; |
| 1079 | case TokenIdDot: return "Dot"; | 1086 | case TokenIdDot: return "."; |
| 1080 | case TokenIdEllipsis: return "Ellipsis"; | 1087 | case TokenIdEllipsis: return "..."; |
| 1081 | case TokenIdMaybe: return "Maybe"; | 1088 | case TokenIdMaybe: return "?"; |
| 1082 | case TokenIdDoubleQuestion: return "DoubleQuestion"; | 1089 | case TokenIdDoubleQuestion: return "??"; |
| 1083 | case TokenIdMaybeAssign: return "MaybeAssign"; | 1090 | case TokenIdMaybeAssign: return "?="; |
| 1084 | case TokenIdAtSign: return "AtSign"; | 1091 | case TokenIdAtSign: return "@"; |
| 1085 | } | 1092 | } |
| 1086 | return "(invalid token)"; | 1093 | return "(invalid token)"; |
| 1087 | } | 1094 | } |
| ... | @@ -1089,7 +1096,7 @@ static const char * token_name(Token *token) { | ... | @@ -1089,7 +1096,7 @@ static const char * token_name(Token *token) { |
| 1089 | void print_tokens(Buf *buf, ZigList<Token> *tokens) { | 1096 | void print_tokens(Buf *buf, ZigList<Token> *tokens) { |
| 1090 | for (int i = 0; i < tokens->length; i += 1) { | 1097 | for (int i = 0; i < tokens->length; i += 1) { |
| 1091 | Token *token = &tokens->at(i); | 1098 | Token *token = &tokens->at(i); |
| 1092 | fprintf(stderr, "%s ", token_name(token)); | 1099 | fprintf(stderr, "%s ", token_name(token->id)); |
| 1093 | if (token->start_pos >= 0) { | 1100 | if (token->start_pos >= 0) { |
| 1094 | fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stderr); | 1101 | fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stderr); |
| 1095 | } | 1102 | } |