authorgravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-23 09:21:34+10:00
committergravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-23 09:21:34+10:00
log2d18178c27060ff9b9b4a5b56941617dc47868d0
tree35f02fd50f5508f06ef1defcc20fbe69fc4be017
parent8de45e51438d7f593825ae2134ab7dfc46f3bab2

minor fixes and more tests for _ separators

* Make the tokenizer spit out an Invalid token on the first invalid character found in the number literal. * More parsing and tokenizer tests for number literals * fix invalid switch statement in ir.zig

3 files changed, 242 insertions(+), 112 deletions(-)

lib/std/zig/parser_test.zig+69
...@@ -2800,6 +2800,75 @@ test "zig fmt: extern without container keyword returns error" {...@@ -2800,6 +2800,75 @@ test "zig fmt: extern without container keyword returns error" {
2800 );2800 );
2801}2801}
28022802
2803test "zig fmt: integer literals with underscore separators" {
2804 try testTransform(
2805 \\const
2806 \\ x =
2807 \\ 1_234_567
2808 \\ +(0b0_1-0o7_0+0xff_FF ) + 0_0;
2809 ,
2810 \\const x = 1_234_567 + (0b0_1 - 0o7_0 + 0xff_FF) + 0_0;
2811 \\
2812 );
2813}
2814
2815test "zig fmt: hex literals with underscore separators" {
2816 try testTransform(
2817 \\pub fn orMask(a: [ 1_000 ]u64, b: [ 1_000] u64) [1_000]u64 {
2818 \\ var c: [1_000]u64 = [1]u64{ 0xFFFF_FFFF_FFFF_FFFF}**1_000;
2819 \\ for (c [ 0_0 .. ]) |_, i| {
2820 \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA;
2821 \\ }
2822 \\ return c;
2823 \\}
2824 \\
2825 \\
2826 ,
2827 \\pub fn orMask(a: [1_000]u64, b: [1_000]u64) [1_000]u64 {
2828 \\ var c: [1_000]u64 = [1]u64{0xFFFF_FFFF_FFFF_FFFF} ** 1_000;
2829 \\ for (c[0_0..]) |_, i| {
2830 \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA;
2831 \\ }
2832 \\ return c;
2833 \\}
2834 \\
2835 );
2836}
2837
2838test "zig fmt: decimal float literals with underscore separators" {
2839 try testTransform(
2840 \\pub fn main() void {
2841 \\ const a:f64=(10.0e-0+(10.e+0))+10_00.00_00e-2+00_00.00_10e+4;
2842 \\ const b:f64=010.0--0_10.+0_1_0.0_0+1e2;
2843 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2844 \\}
2845 ,
2846 \\pub fn main() void {
2847 \\ const a: f64 = (10.0e-0 + (10.e+0)) + 10_00.00_00e-2 + 00_00.00_10e+4;
2848 \\ const b: f64 = 010.0 - -0_10. + 0_1_0.0_0 + 1e2;
2849 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2850 \\}
2851 \\
2852 );
2853}
2854
2855test "zig fmt: hexadeciaml float literals with underscore separators" {
2856 try testTransform(
2857 \\pub fn main() void {
2858 \\ const a: f64 = (0x10.0p-0+(0x10.p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16;
2859 \\ const b: f64 = 0x0010.0--0x00_10.+0x10.00+0x1p4;
2860 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2861 \\}
2862 ,
2863 \\pub fn main() void {
2864 \\ const a: f64 = (0x10.0p-0 + (0x10.p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16;
2865 \\ const b: f64 = 0x0010.0 - -0x00_10. + 0x10.00 + 0x1p4;
2866 \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b });
2867 \\}
2868 \\
2869 );
2870}
2871
2803const std = @import("std");2872const std = @import("std");
2804const mem = std.mem;2873const mem = std.mem;
2805const warn = std.debug.warn;2874const warn = std.debug.warn;
lib/std/zig/tokenizer.zig+168-107
...@@ -418,6 +418,10 @@ pub const Tokenizer = struct {...@@ -418,6 +418,10 @@ pub const Tokenizer = struct {
418 SawAtSign,418 SawAtSign,
419 };419 };
420420
421 fn isIdentifierChar(char: u8) bool {
422 return std.ascii.isAlNum(char) or char == '_';
423 }
424
421 pub fn next(self: *Tokenizer) Token {425 pub fn next(self: *Tokenizer) Token {
422 if (self.pending_invalid_token) |token| {426 if (self.pending_invalid_token) |token| {
423 self.pending_invalid_token = null;427 self.pending_invalid_token = null;
...@@ -1063,11 +1067,17 @@ pub const Tokenizer = struct {...@@ -1063,11 +1067,17 @@ pub const Tokenizer = struct {
1063 'x' => {1067 'x' => {
1064 state = State.IntegerLiteralHexNoUnderscore;1068 state = State.IntegerLiteralHexNoUnderscore;
1065 },1069 },
1066 else => {1070 '0'...'9', '_', '.', 'e', 'E' => {
1067 // reinterpret as a normal number1071 // reinterpret as a decimal number
1068 self.index -= 1;1072 self.index -= 1;
1069 state = State.IntegerLiteralDec;1073 state = State.IntegerLiteralDec;
1070 },1074 },
1075 else => {
1076 if (isIdentifierChar(c)) {
1077 result.id = Token.Id.Invalid;
1078 }
1079 break;
1080 },
1071 },1081 },
1072 State.IntegerLiteralBinNoUnderscore => switch (c) {1082 State.IntegerLiteralBinNoUnderscore => switch (c) {
1073 '0'...'1' => {1083 '0'...'1' => {
...@@ -1075,6 +1085,7 @@ pub const Tokenizer = struct {...@@ -1075,6 +1085,7 @@ pub const Tokenizer = struct {
1075 },1085 },
1076 else => {1086 else => {
1077 result.id = Token.Id.Invalid;1087 result.id = Token.Id.Invalid;
1088 break;
1078 },1089 },
1079 },1090 },
1080 State.IntegerLiteralBin => switch (c) {1091 State.IntegerLiteralBin => switch (c) {
...@@ -1082,10 +1093,12 @@ pub const Tokenizer = struct {...@@ -1082,10 +1093,12 @@ pub const Tokenizer = struct {
1082 state = State.IntegerLiteralBinNoUnderscore;1093 state = State.IntegerLiteralBinNoUnderscore;
1083 },1094 },
1084 '0'...'1' => {},1095 '0'...'1' => {},
1085 '2'...'9', 'a'...'z', 'A'...'Z' => {1096 else => {
1086 result.id = Token.Id.Invalid;1097 if (isIdentifierChar(c)) {
1098 result.id = Token.Id.Invalid;
1099 }
1100 break;
1087 },1101 },
1088 else => break,
1089 },1102 },
1090 State.IntegerLiteralOctNoUnderscore => switch (c) {1103 State.IntegerLiteralOctNoUnderscore => switch (c) {
1091 '0'...'7' => {1104 '0'...'7' => {
...@@ -1093,6 +1106,7 @@ pub const Tokenizer = struct {...@@ -1093,6 +1106,7 @@ pub const Tokenizer = struct {
1093 },1106 },
1094 else => {1107 else => {
1095 result.id = Token.Id.Invalid;1108 result.id = Token.Id.Invalid;
1109 break;
1096 },1110 },
1097 },1111 },
1098 State.IntegerLiteralOct => switch (c) {1112 State.IntegerLiteralOct => switch (c) {
...@@ -1100,10 +1114,12 @@ pub const Tokenizer = struct {...@@ -1100,10 +1114,12 @@ pub const Tokenizer = struct {
1100 state = State.IntegerLiteralOctNoUnderscore;1114 state = State.IntegerLiteralOctNoUnderscore;
1101 },1115 },
1102 '0'...'7' => {},1116 '0'...'7' => {},
1103 '8'...'9', 'a'...'z', 'A'...'Z' => {1117 else => {
1104 result.id = Token.Id.Invalid;1118 if (isIdentifierChar(c)) {
1119 result.id = Token.Id.Invalid;
1120 }
1121 break;
1105 },1122 },
1106 else => break,
1107 },1123 },
1108 State.IntegerLiteralDecNoUnderscore => switch (c) {1124 State.IntegerLiteralDecNoUnderscore => switch (c) {
1109 '0'...'9' => {1125 '0'...'9' => {
...@@ -1111,6 +1127,7 @@ pub const Tokenizer = struct {...@@ -1111,6 +1127,7 @@ pub const Tokenizer = struct {
1111 },1127 },
1112 else => {1128 else => {
1113 result.id = Token.Id.Invalid;1129 result.id = Token.Id.Invalid;
1130 break;
1114 },1131 },
1115 },1132 },
1116 State.IntegerLiteralDec => switch (c) {1133 State.IntegerLiteralDec => switch (c) {
...@@ -1126,10 +1143,12 @@ pub const Tokenizer = struct {...@@ -1126,10 +1143,12 @@ pub const Tokenizer = struct {
1126 result.id = Token.Id.FloatLiteral;1143 result.id = Token.Id.FloatLiteral;
1127 },1144 },
1128 '0'...'9' => {},1145 '0'...'9' => {},
1129 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {1146 else => {
1130 result.id = Token.Id.Invalid;1147 if (isIdentifierChar(c)) {
1148 result.id = Token.Id.Invalid;
1149 }
1150 break;
1131 },1151 },
1132 else => break,
1133 },1152 },
1134 State.IntegerLiteralHexNoUnderscore => switch (c) {1153 State.IntegerLiteralHexNoUnderscore => switch (c) {
1135 '0'...'9', 'a'...'f', 'A'...'F' => {1154 '0'...'9', 'a'...'f', 'A'...'F' => {
...@@ -1137,6 +1156,7 @@ pub const Tokenizer = struct {...@@ -1137,6 +1156,7 @@ pub const Tokenizer = struct {
1137 },1156 },
1138 else => {1157 else => {
1139 result.id = Token.Id.Invalid;1158 result.id = Token.Id.Invalid;
1159 break;
1140 },1160 },
1141 },1161 },
1142 State.IntegerLiteralHex => switch (c) {1162 State.IntegerLiteralHex => switch (c) {
...@@ -1152,10 +1172,12 @@ pub const Tokenizer = struct {...@@ -1152,10 +1172,12 @@ pub const Tokenizer = struct {
1152 result.id = Token.Id.FloatLiteral;1172 result.id = Token.Id.FloatLiteral;
1153 },1173 },
1154 '0'...'9', 'a'...'f', 'A'...'F' => {},1174 '0'...'9', 'a'...'f', 'A'...'F' => {},
1155 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {1175 else => {
1156 result.id = Token.Id.Invalid;1176 if (isIdentifierChar(c)) {
1177 result.id = Token.Id.Invalid;
1178 }
1179 break;
1157 },1180 },
1158 else => break,
1159 },1181 },
1160 State.NumberDotDec => switch (c) {1182 State.NumberDotDec => switch (c) {
1161 '.' => {1183 '.' => {
...@@ -1166,10 +1188,15 @@ pub const Tokenizer = struct {...@@ -1166,10 +1188,15 @@ pub const Tokenizer = struct {
1166 'e', 'E' => {1188 'e', 'E' => {
1167 state = State.FloatExponentUnsigned;1189 state = State.FloatExponentUnsigned;
1168 },1190 },
1169 else => {1191 '0'...'9' => {
1170 self.index -= 1;
1171 result.id = Token.Id.FloatLiteral;1192 result.id = Token.Id.FloatLiteral;
1172 state = State.FloatFractionDecNoUnderscore;1193 state = State.FloatFractionDec;
1194 },
1195 else => {
1196 if (isIdentifierChar(c)) {
1197 result.id = Token.Id.Invalid;
1198 }
1199 break;
1173 },1200 },
1174 },1201 },
1175 State.NumberDotHex => switch (c) {1202 State.NumberDotHex => switch (c) {
...@@ -1181,10 +1208,15 @@ pub const Tokenizer = struct {...@@ -1181,10 +1208,15 @@ pub const Tokenizer = struct {
1181 'p', 'P' => {1208 'p', 'P' => {
1182 state = State.FloatExponentUnsigned;1209 state = State.FloatExponentUnsigned;
1183 },1210 },
1184 else => {1211 '0'...'9', 'a'...'f', 'A'...'F' => {
1185 self.index -= 1;
1186 result.id = Token.Id.FloatLiteral;1212 result.id = Token.Id.FloatLiteral;
1187 state = State.FloatFractionHexNoUnderscore;1213 state = State.FloatFractionHex;
1214 },
1215 else => {
1216 if (isIdentifierChar(c)) {
1217 result.id = Token.Id.Invalid;
1218 }
1219 break;
1188 },1220 },
1189 },1221 },
1190 State.FloatFractionDecNoUnderscore => switch (c) {1222 State.FloatFractionDecNoUnderscore => switch (c) {
...@@ -1193,6 +1225,7 @@ pub const Tokenizer = struct {...@@ -1193,6 +1225,7 @@ pub const Tokenizer = struct {
1193 },1225 },
1194 else => {1226 else => {
1195 result.id = Token.Id.Invalid;1227 result.id = Token.Id.Invalid;
1228 break;
1196 },1229 },
1197 },1230 },
1198 State.FloatFractionDec => switch (c) {1231 State.FloatFractionDec => switch (c) {
...@@ -1203,10 +1236,12 @@ pub const Tokenizer = struct {...@@ -1203,10 +1236,12 @@ pub const Tokenizer = struct {
1203 state = State.FloatExponentUnsigned;1236 state = State.FloatExponentUnsigned;
1204 },1237 },
1205 '0'...'9' => {},1238 '0'...'9' => {},
1206 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => {1239 else => {
1207 result.id = Token.Id.Invalid;1240 if (isIdentifierChar(c)) {
1241 result.id = Token.Id.Invalid;
1242 }
1243 break;
1208 },1244 },
1209 else => break,
1210 },1245 },
1211 State.FloatFractionHexNoUnderscore => switch (c) {1246 State.FloatFractionHexNoUnderscore => switch (c) {
1212 '0'...'9', 'a'...'f', 'A'...'F' => {1247 '0'...'9', 'a'...'f', 'A'...'F' => {
...@@ -1214,6 +1249,7 @@ pub const Tokenizer = struct {...@@ -1214,6 +1249,7 @@ pub const Tokenizer = struct {
1214 },1249 },
1215 else => {1250 else => {
1216 result.id = Token.Id.Invalid;1251 result.id = Token.Id.Invalid;
1252 break;
1217 },1253 },
1218 },1254 },
1219 State.FloatFractionHex => switch (c) {1255 State.FloatFractionHex => switch (c) {
...@@ -1224,10 +1260,12 @@ pub const Tokenizer = struct {...@@ -1224,10 +1260,12 @@ pub const Tokenizer = struct {
1224 state = State.FloatExponentUnsigned;1260 state = State.FloatExponentUnsigned;
1225 },1261 },
1226 '0'...'9', 'a'...'f', 'A'...'F' => {},1262 '0'...'9', 'a'...'f', 'A'...'F' => {},
1227 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => {1263 else => {
1228 result.id = Token.Id.Invalid;1264 if (isIdentifierChar(c)) {
1265 result.id = Token.Id.Invalid;
1266 }
1267 break;
1229 },1268 },
1230 else => break,
1231 },1269 },
1232 State.FloatExponentUnsigned => switch (c) {1270 State.FloatExponentUnsigned => switch (c) {
1233 '+', '-' => {1271 '+', '-' => {
...@@ -1245,6 +1283,7 @@ pub const Tokenizer = struct {...@@ -1245,6 +1283,7 @@ pub const Tokenizer = struct {
1245 },1283 },
1246 else => {1284 else => {
1247 result.id = Token.Id.Invalid;1285 result.id = Token.Id.Invalid;
1286 break;
1248 },1287 },
1249 },1288 },
1250 State.FloatExponentNumber => switch (c) {1289 State.FloatExponentNumber => switch (c) {
...@@ -1252,10 +1291,12 @@ pub const Tokenizer = struct {...@@ -1252,10 +1291,12 @@ pub const Tokenizer = struct {
1252 state = State.FloatExponentNumberNoUnderscore;1291 state = State.FloatExponentNumberNoUnderscore;
1253 },1292 },
1254 '0'...'9' => {},1293 '0'...'9' => {},
1255 'a'...'z', 'A'...'Z' => {1294 else => {
1256 result.id = Token.Id.Invalid;1295 if (isIdentifierChar(c)) {
1296 result.id = Token.Id.Invalid;
1297 }
1298 break;
1257 },1299 },
1258 else => break,
1259 },1300 },
1260 }1301 }
1261 } else if (self.index == self.buffer.len) {1302 } else if (self.index == self.buffer.len) {
...@@ -1706,11 +1747,11 @@ test "tokenizer - number literals decimal" {...@@ -1706,11 +1747,11 @@ test "tokenizer - number literals decimal" {
1706 testTokenize("7", &[_]Token.Id{.IntegerLiteral});1747 testTokenize("7", &[_]Token.Id{.IntegerLiteral});
1707 testTokenize("8", &[_]Token.Id{.IntegerLiteral});1748 testTokenize("8", &[_]Token.Id{.IntegerLiteral});
1708 testTokenize("9", &[_]Token.Id{.IntegerLiteral});1749 testTokenize("9", &[_]Token.Id{.IntegerLiteral});
1709 testTokenize("0a", &[_]Token.Id{.Invalid});1750 testTokenize("0a", &[_]Token.Id{ .Invalid, .Identifier });
1710 testTokenize("9b", &[_]Token.Id{.Invalid});1751 testTokenize("9b", &[_]Token.Id{ .Invalid, .Identifier });
1711 testTokenize("1z", &[_]Token.Id{.Invalid});1752 testTokenize("1z", &[_]Token.Id{ .Invalid, .Identifier });
1712 testTokenize("1z_1", &[_]Token.Id{.Invalid});1753 testTokenize("1z_1", &[_]Token.Id{ .Invalid, .Identifier });
1713 testTokenize("9z3", &[_]Token.Id{.Invalid});1754 testTokenize("9z3", &[_]Token.Id{ .Invalid, .Identifier });
17141755
1715 testTokenize("0_0", &[_]Token.Id{.IntegerLiteral});1756 testTokenize("0_0", &[_]Token.Id{.IntegerLiteral});
1716 testTokenize("0001", &[_]Token.Id{.IntegerLiteral});1757 testTokenize("0001", &[_]Token.Id{.IntegerLiteral});
...@@ -1720,15 +1761,17 @@ test "tokenizer - number literals decimal" {...@@ -1720,15 +1761,17 @@ test "tokenizer - number literals decimal" {
17201761
1721 testTokenize("00_", &[_]Token.Id{.Invalid});1762 testTokenize("00_", &[_]Token.Id{.Invalid});
1722 testTokenize("0_0_", &[_]Token.Id{.Invalid});1763 testTokenize("0_0_", &[_]Token.Id{.Invalid});
1723 testTokenize("0__0", &[_]Token.Id{.Invalid});1764 testTokenize("0__0", &[_]Token.Id{ .Invalid, .Identifier });
1724 testTokenize("0_0f", &[_]Token.Id{.Invalid});1765 testTokenize("0_0f", &[_]Token.Id{ .Invalid, .Identifier });
1725 testTokenize("0_0_f", &[_]Token.Id{.Invalid});1766 testTokenize("0_0_f", &[_]Token.Id{ .Invalid, .Identifier });
1726 testTokenize("1_,", &[_]Token.Id{.Invalid});1767 testTokenize("0_0_f_00", &[_]Token.Id{ .Invalid, .Identifier });
1768 testTokenize("1_,", &[_]Token.Id{ .Invalid, .Comma });
17271769
1728 testTokenize("1.", &[_]Token.Id{.FloatLiteral});1770 testTokenize("1.", &[_]Token.Id{.FloatLiteral});
1729 testTokenize("0.0", &[_]Token.Id{.FloatLiteral});1771 testTokenize("0.0", &[_]Token.Id{.FloatLiteral});
1730 testTokenize("1.0", &[_]Token.Id{.FloatLiteral});1772 testTokenize("1.0", &[_]Token.Id{.FloatLiteral});
1731 testTokenize("10.0", &[_]Token.Id{.FloatLiteral});1773 testTokenize("10.0", &[_]Token.Id{.FloatLiteral});
1774 testTokenize("0e0", &[_]Token.Id{.FloatLiteral});
1732 testTokenize("1e0", &[_]Token.Id{.FloatLiteral});1775 testTokenize("1e0", &[_]Token.Id{.FloatLiteral});
1733 testTokenize("1e100", &[_]Token.Id{.FloatLiteral});1776 testTokenize("1e100", &[_]Token.Id{.FloatLiteral});
1734 testTokenize("1.e100", &[_]Token.Id{.FloatLiteral});1777 testTokenize("1.e100", &[_]Token.Id{.FloatLiteral});
...@@ -1736,34 +1779,47 @@ test "tokenizer - number literals decimal" {...@@ -1736,34 +1779,47 @@ test "tokenizer - number literals decimal" {
1736 testTokenize("1.0e+100", &[_]Token.Id{.FloatLiteral});1779 testTokenize("1.0e+100", &[_]Token.Id{.FloatLiteral});
1737 testTokenize("1.0e-100", &[_]Token.Id{.FloatLiteral});1780 testTokenize("1.0e-100", &[_]Token.Id{.FloatLiteral});
1738 testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{.FloatLiteral});1781 testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{.FloatLiteral});
1782 testTokenize("1.+", &[_]Token.Id{ .FloatLiteral, .Plus });
17391783
1740 testTokenize("1e", &[_]Token.Id{.Invalid});1784 testTokenize("1e", &[_]Token.Id{.Invalid});
1741 testTokenize("1.0e1f0", &[_]Token.Id{.Invalid});1785 testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid, .Identifier });
1742 testTokenize("1.0p100", &[_]Token.Id{.Invalid});1786 testTokenize("1.0p100", &[_]Token.Id{ .Invalid, .Identifier });
1743 testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Minus, .IntegerLiteral });1787 testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Identifier, .Minus, .IntegerLiteral });
1744 testTokenize("1.0p1f0", &[_]Token.Id{.Invalid});1788 testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid, .Identifier });
1745 testTokenize("1.0_,", &[_]Token.Id{.Invalid});1789 testTokenize("1.0_,", &[_]Token.Id{ .Invalid, .Comma });
1746 testTokenize("1.0e,", &[_]Token.Id{.Invalid});1790 testTokenize("1_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral });
1791 testTokenize("1._", &[_]Token.Id{ .Invalid, .Identifier });
1792 testTokenize("1.a", &[_]Token.Id{ .Invalid, .Identifier });
1793 testTokenize("1.z", &[_]Token.Id{ .Invalid, .Identifier });
1794 testTokenize("1._0", &[_]Token.Id{ .Invalid, .Identifier });
1795 testTokenize("1._+", &[_]Token.Id{ .Invalid, .Identifier, .Plus });
1796 testTokenize("1._e", &[_]Token.Id{ .Invalid, .Identifier });
1797 testTokenize("1.0e", &[_]Token.Id{.Invalid});
1798 testTokenize("1.0e,", &[_]Token.Id{ .Invalid, .Comma });
1799 testTokenize("1.0e_", &[_]Token.Id{ .Invalid, .Identifier });
1800 testTokenize("1.0e+_", &[_]Token.Id{ .Invalid, .Identifier });
1801 testTokenize("1.0e-_", &[_]Token.Id{ .Invalid, .Identifier });
1802 testTokenize("1.0e0_+", &[_]Token.Id{ .Invalid, .Plus });
1747}1803}
17481804
1749test "tokenizer - number literals binary" {1805test "tokenizer - number literals binary" {
1750 testTokenize("0b0", &[_]Token.Id{.IntegerLiteral});1806 testTokenize("0b0", &[_]Token.Id{.IntegerLiteral});
1751 testTokenize("0b1", &[_]Token.Id{.IntegerLiteral});1807 testTokenize("0b1", &[_]Token.Id{.IntegerLiteral});
1752 testTokenize("0b2", &[_]Token.Id{.Invalid});1808 testTokenize("0b2", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1753 testTokenize("0b3", &[_]Token.Id{.Invalid});1809 testTokenize("0b3", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1754 testTokenize("0b4", &[_]Token.Id{.Invalid});1810 testTokenize("0b4", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1755 testTokenize("0b5", &[_]Token.Id{.Invalid});1811 testTokenize("0b5", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1756 testTokenize("0b6", &[_]Token.Id{.Invalid});1812 testTokenize("0b6", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1757 testTokenize("0b7", &[_]Token.Id{.Invalid});1813 testTokenize("0b7", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1758 testTokenize("0b8", &[_]Token.Id{.Invalid});1814 testTokenize("0b8", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1759 testTokenize("0b9", &[_]Token.Id{.Invalid});1815 testTokenize("0b9", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1760 testTokenize("0ba", &[_]Token.Id{.Invalid});1816 testTokenize("0ba", &[_]Token.Id{ .Invalid, .Identifier });
1761 testTokenize("0bb", &[_]Token.Id{.Invalid});1817 testTokenize("0bb", &[_]Token.Id{ .Invalid, .Identifier });
1762 testTokenize("0bc", &[_]Token.Id{.Invalid});1818 testTokenize("0bc", &[_]Token.Id{ .Invalid, .Identifier });
1763 testTokenize("0bd", &[_]Token.Id{.Invalid});1819 testTokenize("0bd", &[_]Token.Id{ .Invalid, .Identifier });
1764 testTokenize("0be", &[_]Token.Id{.Invalid});1820 testTokenize("0be", &[_]Token.Id{ .Invalid, .Identifier });
1765 testTokenize("0bf", &[_]Token.Id{.Invalid});1821 testTokenize("0bf", &[_]Token.Id{ .Invalid, .Identifier });
1766 testTokenize("0bz", &[_]Token.Id{.Invalid});1822 testTokenize("0bz", &[_]Token.Id{ .Invalid, .Identifier });
17671823
1768 testTokenize("0b0000_0000", &[_]Token.Id{.IntegerLiteral});1824 testTokenize("0b0000_0000", &[_]Token.Id{.IntegerLiteral});
1769 testTokenize("0b1111_1111", &[_]Token.Id{.IntegerLiteral});1825 testTokenize("0b1111_1111", &[_]Token.Id{.IntegerLiteral});
...@@ -1772,17 +1828,17 @@ test "tokenizer - number literals binary" {...@@ -1772,17 +1828,17 @@ test "tokenizer - number literals binary" {
1772 testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period });1828 testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period });
1773 testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });1829 testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
17741830
1775 testTokenize("0B0", &[_]Token.Id{.Invalid});1831 testTokenize("0B0", &[_]Token.Id{ .Invalid, .Identifier });
1776 testTokenize("0b_", &[_]Token.Id{.Invalid});1832 testTokenize("0b_", &[_]Token.Id{ .Invalid, .Identifier });
1777 testTokenize("0b_0", &[_]Token.Id{.Invalid});1833 testTokenize("0b_0", &[_]Token.Id{ .Invalid, .Identifier });
1778 testTokenize("0b1_", &[_]Token.Id{.Invalid});1834 testTokenize("0b1_", &[_]Token.Id{.Invalid});
1779 testTokenize("0b0__1", &[_]Token.Id{.Invalid});1835 testTokenize("0b0__1", &[_]Token.Id{ .Invalid, .Identifier });
1780 testTokenize("0b0_1_", &[_]Token.Id{.Invalid});1836 testTokenize("0b0_1_", &[_]Token.Id{.Invalid});
1781 testTokenize("0b1e", &[_]Token.Id{.Invalid});1837 testTokenize("0b1e", &[_]Token.Id{ .Invalid, .Identifier });
1782 testTokenize("0b1p", &[_]Token.Id{.Invalid});1838 testTokenize("0b1p", &[_]Token.Id{ .Invalid, .Identifier });
1783 testTokenize("0b1e0", &[_]Token.Id{.Invalid});1839 testTokenize("0b1e0", &[_]Token.Id{ .Invalid, .Identifier });
1784 testTokenize("0b1p0", &[_]Token.Id{.Invalid});1840 testTokenize("0b1p0", &[_]Token.Id{ .Invalid, .Identifier });
1785 testTokenize("0b1_,", &[_]Token.Id{.Invalid});1841 testTokenize("0b1_,", &[_]Token.Id{ .Invalid, .Comma });
1786}1842}
17871843
1788test "tokenizer - number literals octal" {1844test "tokenizer - number literals octal" {
...@@ -1794,15 +1850,15 @@ test "tokenizer - number literals octal" {...@@ -1794,15 +1850,15 @@ test "tokenizer - number literals octal" {
1794 testTokenize("0o5", &[_]Token.Id{.IntegerLiteral});1850 testTokenize("0o5", &[_]Token.Id{.IntegerLiteral});
1795 testTokenize("0o6", &[_]Token.Id{.IntegerLiteral});1851 testTokenize("0o6", &[_]Token.Id{.IntegerLiteral});
1796 testTokenize("0o7", &[_]Token.Id{.IntegerLiteral});1852 testTokenize("0o7", &[_]Token.Id{.IntegerLiteral});
1797 testTokenize("0o8", &[_]Token.Id{.Invalid});1853 testTokenize("0o8", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1798 testTokenize("0o9", &[_]Token.Id{.Invalid});1854 testTokenize("0o9", &[_]Token.Id{ .Invalid, .IntegerLiteral });
1799 testTokenize("0oa", &[_]Token.Id{.Invalid});1855 testTokenize("0oa", &[_]Token.Id{ .Invalid, .Identifier });
1800 testTokenize("0ob", &[_]Token.Id{.Invalid});1856 testTokenize("0ob", &[_]Token.Id{ .Invalid, .Identifier });
1801 testTokenize("0oc", &[_]Token.Id{.Invalid});1857 testTokenize("0oc", &[_]Token.Id{ .Invalid, .Identifier });
1802 testTokenize("0od", &[_]Token.Id{.Invalid});1858 testTokenize("0od", &[_]Token.Id{ .Invalid, .Identifier });
1803 testTokenize("0oe", &[_]Token.Id{.Invalid});1859 testTokenize("0oe", &[_]Token.Id{ .Invalid, .Identifier });
1804 testTokenize("0of", &[_]Token.Id{.Invalid});1860 testTokenize("0of", &[_]Token.Id{ .Invalid, .Identifier });
1805 testTokenize("0oz", &[_]Token.Id{.Invalid});1861 testTokenize("0oz", &[_]Token.Id{ .Invalid, .Identifier });
18061862
1807 testTokenize("0o01234567", &[_]Token.Id{.IntegerLiteral});1863 testTokenize("0o01234567", &[_]Token.Id{.IntegerLiteral});
1808 testTokenize("0o0123_4567", &[_]Token.Id{.IntegerLiteral});1864 testTokenize("0o0123_4567", &[_]Token.Id{.IntegerLiteral});
...@@ -1811,17 +1867,17 @@ test "tokenizer - number literals octal" {...@@ -1811,17 +1867,17 @@ test "tokenizer - number literals octal" {
1811 testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period });1867 testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period });
1812 testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });1868 testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral });
18131869
1814 testTokenize("0O0", &[_]Token.Id{.Invalid});1870 testTokenize("0O0", &[_]Token.Id{ .Invalid, .Identifier });
1815 testTokenize("0o_", &[_]Token.Id{.Invalid});1871 testTokenize("0o_", &[_]Token.Id{ .Invalid, .Identifier });
1816 testTokenize("0o_0", &[_]Token.Id{.Invalid});1872 testTokenize("0o_0", &[_]Token.Id{ .Invalid, .Identifier });
1817 testTokenize("0o1_", &[_]Token.Id{.Invalid});1873 testTokenize("0o1_", &[_]Token.Id{.Invalid});
1818 testTokenize("0o0__1", &[_]Token.Id{.Invalid});1874 testTokenize("0o0__1", &[_]Token.Id{ .Invalid, .Identifier });
1819 testTokenize("0o0_1_", &[_]Token.Id{.Invalid});1875 testTokenize("0o0_1_", &[_]Token.Id{.Invalid});
1820 testTokenize("0o1e", &[_]Token.Id{.Invalid});1876 testTokenize("0o1e", &[_]Token.Id{ .Invalid, .Identifier });
1821 testTokenize("0o1p", &[_]Token.Id{.Invalid});1877 testTokenize("0o1p", &[_]Token.Id{ .Invalid, .Identifier });
1822 testTokenize("0o1e0", &[_]Token.Id{.Invalid});1878 testTokenize("0o1e0", &[_]Token.Id{ .Invalid, .Identifier });
1823 testTokenize("0o1p0", &[_]Token.Id{.Invalid});1879 testTokenize("0o1p0", &[_]Token.Id{ .Invalid, .Identifier });
1824 testTokenize("0o_,", &[_]Token.Id{.Invalid});1880 testTokenize("0o_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma });
1825}1881}
18261882
1827test "tokenizer - number literals hexadeciaml" {1883test "tokenizer - number literals hexadeciaml" {
...@@ -1847,21 +1903,21 @@ test "tokenizer - number literals hexadeciaml" {...@@ -1847,21 +1903,21 @@ test "tokenizer - number literals hexadeciaml" {
1847 testTokenize("0xD", &[_]Token.Id{.IntegerLiteral});1903 testTokenize("0xD", &[_]Token.Id{.IntegerLiteral});
1848 testTokenize("0xE", &[_]Token.Id{.IntegerLiteral});1904 testTokenize("0xE", &[_]Token.Id{.IntegerLiteral});
1849 testTokenize("0xF", &[_]Token.Id{.IntegerLiteral});1905 testTokenize("0xF", &[_]Token.Id{.IntegerLiteral});
1850 testTokenize("0x0z", &[_]Token.Id{.Invalid});1906 testTokenize("0x0z", &[_]Token.Id{ .Invalid, .Identifier });
1851 testTokenize("0xz", &[_]Token.Id{.Invalid});1907 testTokenize("0xz", &[_]Token.Id{ .Invalid, .Identifier });
18521908
1853 testTokenize("0x0123456789ABCDEF", &[_]Token.Id{.IntegerLiteral});1909 testTokenize("0x0123456789ABCDEF", &[_]Token.Id{.IntegerLiteral});
1854 testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{.IntegerLiteral});1910 testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{.IntegerLiteral});
1855 testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{.IntegerLiteral});1911 testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{.IntegerLiteral});
1856 testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{.IntegerLiteral});1912 testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{.IntegerLiteral});
18571913
1858 testTokenize("0X0", &[_]Token.Id{.Invalid});1914 testTokenize("0X0", &[_]Token.Id{ .Invalid, .Identifier });
1859 testTokenize("0x_", &[_]Token.Id{.Invalid});1915 testTokenize("0x_", &[_]Token.Id{ .Invalid, .Identifier });
1860 testTokenize("0x_1", &[_]Token.Id{.Invalid});1916 testTokenize("0x_1", &[_]Token.Id{ .Invalid, .Identifier });
1861 testTokenize("0x1_", &[_]Token.Id{.Invalid});1917 testTokenize("0x1_", &[_]Token.Id{.Invalid});
1862 testTokenize("0x0__1", &[_]Token.Id{.Invalid});1918 testTokenize("0x0__1", &[_]Token.Id{ .Invalid, .Identifier });
1863 testTokenize("0x0_1_", &[_]Token.Id{.Invalid});1919 testTokenize("0x0_1_", &[_]Token.Id{.Invalid});
1864 testTokenize("0x_,", &[_]Token.Id{.Invalid});1920 testTokenize("0x_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma });
18651921
1866 testTokenize("0x1.", &[_]Token.Id{.FloatLiteral});1922 testTokenize("0x1.", &[_]Token.Id{.FloatLiteral});
1867 testTokenize("0x1.0", &[_]Token.Id{.FloatLiteral});1923 testTokenize("0x1.0", &[_]Token.Id{.FloatLiteral});
...@@ -1872,10 +1928,12 @@ test "tokenizer - number literals hexadeciaml" {...@@ -1872,10 +1928,12 @@ test "tokenizer - number literals hexadeciaml" {
1872 testTokenize("0xF.FP0", &[_]Token.Id{.FloatLiteral});1928 testTokenize("0xF.FP0", &[_]Token.Id{.FloatLiteral});
1873 testTokenize("0x1p0", &[_]Token.Id{.FloatLiteral});1929 testTokenize("0x1p0", &[_]Token.Id{.FloatLiteral});
1874 testTokenize("0xfp0", &[_]Token.Id{.FloatLiteral});1930 testTokenize("0xfp0", &[_]Token.Id{.FloatLiteral});
1931 testTokenize("0x1.+0xF.", &[_]Token.Id{ .FloatLiteral, .Plus, .FloatLiteral });
18751932
1876 testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{.FloatLiteral});1933 testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{.FloatLiteral});
1877 testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{.FloatLiteral});1934 testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{.FloatLiteral});
1878 testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{.FloatLiteral});1935 testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{.FloatLiteral});
1936 testTokenize("0x0p0", &[_]Token.Id{.FloatLiteral});
1879 testTokenize("0x0.0p0", &[_]Token.Id{.FloatLiteral});1937 testTokenize("0x0.0p0", &[_]Token.Id{.FloatLiteral});
1880 testTokenize("0xff.ffp10", &[_]Token.Id{.FloatLiteral});1938 testTokenize("0xff.ffp10", &[_]Token.Id{.FloatLiteral});
1881 testTokenize("0xff.ffP10", &[_]Token.Id{.FloatLiteral});1939 testTokenize("0xff.ffP10", &[_]Token.Id{.FloatLiteral});
...@@ -1888,21 +1946,24 @@ test "tokenizer - number literals hexadeciaml" {...@@ -1888,21 +1946,24 @@ test "tokenizer - number literals hexadeciaml" {
1888 testTokenize("0x1e", &[_]Token.Id{.IntegerLiteral});1946 testTokenize("0x1e", &[_]Token.Id{.IntegerLiteral});
1889 testTokenize("0x1e0", &[_]Token.Id{.IntegerLiteral});1947 testTokenize("0x1e0", &[_]Token.Id{.IntegerLiteral});
1890 testTokenize("0x1p", &[_]Token.Id{.Invalid});1948 testTokenize("0x1p", &[_]Token.Id{.Invalid});
1891 testTokenize("0xfp0z1", &[_]Token.Id{.Invalid});1949 testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid, .Identifier });
1892 testTokenize("0xff.ffpff", &[_]Token.Id{.Invalid});1950 testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid, .Identifier });
1893 testTokenize("0x0_.0", &[_]Token.Id{.Invalid});1951 testTokenize("0x0.p", &[_]Token.Id{.Invalid});
1894 testTokenize("0x0._0", &[_]Token.Id{.Invalid});1952 testTokenize("0x0.z", &[_]Token.Id{ .Invalid, .Identifier });
1953 testTokenize("0x0._", &[_]Token.Id{ .Invalid, .Identifier });
1954 testTokenize("0x0_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral });
1955 testTokenize("0x0_.0.0", &[_]Token.Id{ .Invalid, .Period, .FloatLiteral });
1956 testTokenize("0x0._0", &[_]Token.Id{ .Invalid, .Identifier });
1895 testTokenize("0x0.0_", &[_]Token.Id{.Invalid});1957 testTokenize("0x0.0_", &[_]Token.Id{.Invalid});
1896 testTokenize("0x0_p0", &[_]Token.Id{.Invalid});1958 testTokenize("0x0_p0", &[_]Token.Id{ .Invalid, .Identifier });
1897 testTokenize("0x0_.p0", &[_]Token.Id{.Invalid});1959 testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid, .Period, .Identifier });
1898 testTokenize("0x0._p0", &[_]Token.Id{.Invalid});1960 testTokenize("0x0._p0", &[_]Token.Id{ .Invalid, .Identifier });
1899 testTokenize("0x0.0_p0", &[_]Token.Id{.Invalid});1961 testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid, .Identifier });
1900 testTokenize("0x0._0p0", &[_]Token.Id{.Invalid});1962 testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid, .Identifier });
1901 testTokenize("0x0.0_p0", &[_]Token.Id{.Invalid});1963 testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid, .Identifier });
1902 testTokenize("0x0.0p_0", &[_]Token.Id{.Invalid});1964 testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid, .Identifier });
1903 testTokenize("0x0.0p+_0", &[_]Token.Id{.Invalid});1965 testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid, .Identifier });
1904 testTokenize("0x0.0p-_0", &[_]Token.Id{.Invalid});1966 testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid, .Eof });
1905 testTokenize("0x0.0p0_", &[_]Token.Id{.Invalid});
1906}1967}
19071968
1908fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {1969fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
src-self-hosted/ir.zig+5-5
...@@ -1312,15 +1312,15 @@ pub const Builder = struct {...@@ -1312,15 +1312,15 @@ pub const Builder = struct {
1312 var rest: []const u8 = undefined;1312 var rest: []const u8 = undefined;
1313 if (int_token.len >= 3 and int_token[0] == '0') {1313 if (int_token.len >= 3 and int_token[0] == '0') {
1314 rest = int_token[2..];1314 rest = int_token[2..];
1315 base = switch (int_token[1]) {1315 switch (int_token[1]) {
1316 'b' => 2,1316 'b' => base = 2,
1317 'o' => 8,1317 'o' => base = 8,
1318 'x' => 16,1318 'x' => base = 16,
1319 else => {1319 else => {
1320 base = 10;1320 base = 10;
1321 rest = int_token;1321 rest = int_token;
1322 },1322 },
1323 };1323 }
1324 } else {1324 } else {
1325 base = 10;1325 base = 10;
1326 rest = int_token;1326 rest = int_token;