authorgravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-15 11:37:36+10:00
committergravatar for 57862114+momumi@users.noreply.github.commomumi <57862114+momumi@users.noreply.github.com> 2020-03-15 12:38:35+10:00
log7aac21c6f59b70deea6ced617f7b6a550e92bab4
treedf02c3553e9c5d66a858c73eaed7b25d723676df
parenteb4d313dbc406b37f6bfdd98988c88c3b8ed542e

allow `_` separators in number literals (stage 1)

* Underscores `_` may be placed between two digits in a int/float literal * Consecutive underscores are not allowed * Fixed parsing bug in exponents of hexadecimal float literals. Exponents should always be base 10, but hex characters would be parsed inside the exponent and everything after them would be ignored. eg: `0x1.0p1ab1` would be parsed as `0x1.0p1`.

6 files changed, 297 insertions(+), 92 deletions(-)

doc/langref.html.in+11
...@@ -885,6 +885,12 @@ const hex_int = 0xff;...@@ -885,6 +885,12 @@ const hex_int = 0xff;
885const another_hex_int = 0xFF;885const another_hex_int = 0xFF;
886const octal_int = 0o755;886const octal_int = 0o755;
887const binary_int = 0b11110000;887const binary_int = 0b11110000;
888
889// underscores may be placed between two digits as a visual separator
890const one_billion = 1_000_000_000;
891const binary_mask = 0b1_1111_1111;
892const permissions = 0o7_5_5;
893const big_address = 0xFF80_0000_0000_0000;
888 {#code_end#}894 {#code_end#}
889 {#header_close#}895 {#header_close#}
890 {#header_open|Runtime Integer Values#}896 {#header_open|Runtime Integer Values#}
...@@ -947,6 +953,11 @@ const yet_another = 123.0e+77;...@@ -947,6 +953,11 @@ const yet_another = 123.0e+77;
947const hex_floating_point = 0x103.70p-5;953const hex_floating_point = 0x103.70p-5;
948const another_hex_float = 0x103.70;954const another_hex_float = 0x103.70;
949const yet_another_hex_float = 0x103.70P-5;955const yet_another_hex_float = 0x103.70P-5;
956
957// underscores may be placed between two digits as a visual separator
958const lightspeed = 299_792_458.000_000;
959const nanosecond = 0.000_000_001;
960const more_hex = 0x1234_5678.9ABC_CDEFp-10;
950 {#code_end#}961 {#code_end#}
951 <p>962 <p>
952 There is no syntax for NaN, infinity, or negative infinity. For these special values,963 There is no syntax for NaN, infinity, or negative infinity. For these special values,
lib/std/special/compiler_rt/floatundisf.zig+19-19
...@@ -69,23 +69,23 @@ test "floatundisf" {...@@ -69,23 +69,23 @@ test "floatundisf" {
69 test__floatundisf(0, 0.0);69 test__floatundisf(0, 0.0);
70 test__floatundisf(1, 1.0);70 test__floatundisf(1, 1.0);
71 test__floatundisf(2, 2.0);71 test__floatundisf(2, 2.0);
72 test__floatundisf(0x7FFFFF8000000000, 0x1.FFFFFEp+62F);72 test__floatundisf(0x7FFFFF8000000000, 0x1.FFFFFEp+62);
73 test__floatundisf(0x7FFFFF0000000000, 0x1.FFFFFCp+62F);73 test__floatundisf(0x7FFFFF0000000000, 0x1.FFFFFCp+62);
74 test__floatundisf(0x8000008000000000, 0x1p+63F);74 test__floatundisf(0x8000008000000000, 0x1p+63);
75 test__floatundisf(0x8000010000000000, 0x1.000002p+63F);75 test__floatundisf(0x8000010000000000, 0x1.000002p+63);
76 test__floatundisf(0x8000000000000000, 0x1p+63F);76 test__floatundisf(0x8000000000000000, 0x1p+63);
77 test__floatundisf(0x8000000000000001, 0x1p+63F);77 test__floatundisf(0x8000000000000001, 0x1p+63);
78 test__floatundisf(0xFFFFFFFFFFFFFFFE, 0x1p+64F);78 test__floatundisf(0xFFFFFFFFFFFFFFFE, 0x1p+64);
79 test__floatundisf(0xFFFFFFFFFFFFFFFF, 0x1p+64F);79 test__floatundisf(0xFFFFFFFFFFFFFFFF, 0x1p+64);
80 test__floatundisf(0x0007FB72E8000000, 0x1.FEDCBAp+50F);80 test__floatundisf(0x0007FB72E8000000, 0x1.FEDCBAp+50);
81 test__floatundisf(0x0007FB72EA000000, 0x1.FEDCBAp+50F);81 test__floatundisf(0x0007FB72EA000000, 0x1.FEDCBAp+50);
82 test__floatundisf(0x0007FB72EB000000, 0x1.FEDCBAp+50F);82 test__floatundisf(0x0007FB72EB000000, 0x1.FEDCBAp+50);
83 test__floatundisf(0x0007FB72EBFFFFFF, 0x1.FEDCBAp+50F);83 test__floatundisf(0x0007FB72EBFFFFFF, 0x1.FEDCBAp+50);
84 test__floatundisf(0x0007FB72EC000000, 0x1.FEDCBCp+50F);84 test__floatundisf(0x0007FB72EC000000, 0x1.FEDCBCp+50);
85 test__floatundisf(0x0007FB72E8000001, 0x1.FEDCBAp+50F);85 test__floatundisf(0x0007FB72E8000001, 0x1.FEDCBAp+50);
86 test__floatundisf(0x0007FB72E6000000, 0x1.FEDCBAp+50F);86 test__floatundisf(0x0007FB72E6000000, 0x1.FEDCBAp+50);
87 test__floatundisf(0x0007FB72E7000000, 0x1.FEDCBAp+50F);87 test__floatundisf(0x0007FB72E7000000, 0x1.FEDCBAp+50);
88 test__floatundisf(0x0007FB72E7FFFFFF, 0x1.FEDCBAp+50F);88 test__floatundisf(0x0007FB72E7FFFFFF, 0x1.FEDCBAp+50);
89 test__floatundisf(0x0007FB72E4000001, 0x1.FEDCBAp+50F);89 test__floatundisf(0x0007FB72E4000001, 0x1.FEDCBAp+50);
90 test__floatundisf(0x0007FB72E4000000, 0x1.FEDCB8p+50F);90 test__floatundisf(0x0007FB72E4000000, 0x1.FEDCB8p+50);
91}91}
src/parse_f128.c+62-17
...@@ -165,22 +165,36 @@ static long long scanexp(struct MuslFILE *f, int pok)...@@ -165,22 +165,36 @@ static long long scanexp(struct MuslFILE *f, int pok)
165 int x;165 int x;
166 long long y;166 long long y;
167 int neg = 0;167 int neg = 0;
168 168
169 c = shgetc(f);169 c = shgetc(f);
170 if (c=='+' || c=='-') {170 if (c=='+' || c=='-') {
171 neg = (c=='-');171 neg = (c=='-');
172 c = shgetc(f);172 c = shgetc(f);
173 if (c-'0'>=10U && pok) shunget(f);173 if (c-'0'>=10U && pok) shunget(f);
174 }174 }
175 if (c-'0'>=10U) {175 if (c-'0'>=10U && c!='_') {
176 shunget(f);176 shunget(f);
177 return LLONG_MIN;177 return LLONG_MIN;
178 }178 }
179 for (x=0; c-'0'<10U && x<INT_MAX/10; c = shgetc(f))179 for (x=0; ; c = shgetc(f)) {
180 x = 10*x + c-'0';180 if (c=='_') {
181 for (y=x; c-'0'<10U && y<LLONG_MAX/100; c = shgetc(f))181 continue;
182 y = 10*y + c-'0';182 } else if (c-'0'<10U && x<INT_MAX/10) {
183 for (; c-'0'<10U; c = shgetc(f));183 x = 10*x + c-'0';
184 } else {
185 break;
186 }
187 }
188 for (y=x; ; c = shgetc(f)) {
189 if (c=='_') {
190 continue;
191 } else if (c-'0'<10U && y<LLONG_MAX/100) {
192 y = 10*y + c-'0';
193 } else {
194 break;
195 }
196 }
197 for (; c-'0'<10U || c=='_'; c = shgetc(f));
184 shunget(f);198 shunget(f);
185 return neg ? -y : y;199 return neg ? -y : y;
186}200}
...@@ -450,16 +464,36 @@ static float128_t decfloat(struct MuslFILE *f, int c, int bits, int emin, int si...@@ -450,16 +464,36 @@ static float128_t decfloat(struct MuslFILE *f, int c, int bits, int emin, int si
450 j=0;464 j=0;
451 k=0;465 k=0;
452466
453 /* Don't let leading zeros consume buffer space */467 /* Don't let leading zeros/underscores consume buffer space */
454 for (; c=='0'; c = shgetc(f)) gotdig=1;468 for (; ; c = shgetc(f)) {
469 if (c=='_') {
470 continue;
471 } else if (c=='0') {
472 gotdig=1;
473 } else {
474 break;
475 }
476 }
477
455 if (c=='.') {478 if (c=='.') {
456 gotrad = 1;479 gotrad = 1;
457 for (c = shgetc(f); c=='0'; c = shgetc(f)) gotdig=1, lrp--;480 for (c = shgetc(f); ; c = shgetc(f)) {
481 if (c == '_') {
482 continue;
483 } else if (c=='0') {
484 gotdig=1;
485 lrp--;
486 } else {
487 break;
488 }
489 }
458 }490 }
459491
460 x[0] = 0;492 x[0] = 0;
461 for (; c-'0'<10U || c=='.'; c = shgetc(f)) {493 for (; c-'0'<10U || c=='.' || c=='_'; c = shgetc(f)) {
462 if (c == '.') {494 if (c == '_') {
495 continue;
496 } else if (c == '.') {
463 if (gotrad) break;497 if (gotrad) break;
464 gotrad = 1;498 gotrad = 1;
465 lrp = dc;499 lrp = dc;
...@@ -773,18 +807,29 @@ static float128_t hexfloat(struct MuslFILE *f, int bits, int emin, int sign, int...@@ -773,18 +807,29 @@ static float128_t hexfloat(struct MuslFILE *f, int bits, int emin, int sign, int
773807
774 c = shgetc(f);808 c = shgetc(f);
775809
776 /* Skip leading zeros */810 /* Skip leading zeros/underscores */
777 for (; c=='0'; c = shgetc(f)) gotdig = 1;811 for (; c=='0' || c=='_'; c = shgetc(f)) gotdig = 1;
778812
779 if (c=='.') {813 if (c=='.') {
780 gotrad = 1;814 gotrad = 1;
781 c = shgetc(f);815 c = shgetc(f);
782 /* Count zeros after the radix point before significand */816 /* Count zeros after the radix point before significand */
783 for (rp=0; c=='0'; c = shgetc(f), rp--) gotdig = 1;817 for (rp=0; ; c = shgetc(f)) {
818 if (c == '_') {
819 continue;
820 } else if (c == '0') {
821 gotdig = 1;
822 rp--;
823 } else {
824 break;
825 }
826 }
784 }827 }
785828
786 for (; c-'0'<10U || (c|32)-'a'<6U || c=='.'; c = shgetc(f)) {829 for (; c-'0'<10U || (c|32)-'a'<6U || c=='.' || c=='_'; c = shgetc(f)) {
787 if (c=='.') {830 if (c=='_') {
831 continue;
832 } else if (c=='.') {
788 if (gotrad) break;833 if (gotrad) break;
789 rp = dc;834 rp = dc;
790 gotrad = 1;835 gotrad = 1;
src/tokenizer.cpp+81-56
...@@ -177,10 +177,13 @@ enum TokenizeState {...@@ -177,10 +177,13 @@ enum TokenizeState {
177 TokenizeStateSymbol,177 TokenizeStateSymbol,
178 TokenizeStateZero, // "0", which might lead to "0x"178 TokenizeStateZero, // "0", which might lead to "0x"
179 TokenizeStateNumber, // "123", "0x123"179 TokenizeStateNumber, // "123", "0x123"
180 TokenizeStateNumberNoUnderscore, // "12_", "0x12_" next char must be digit
180 TokenizeStateNumberDot,181 TokenizeStateNumberDot,
181 TokenizeStateFloatFraction, // "123.456", "0x123.456"182 TokenizeStateFloatFraction, // "123.456", "0x123.456"
183 TokenizeStateFloatFractionNoUnderscore, // "123.45_", "0x123.45_"
182 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"184 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
183 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"185 TokenizeStateFloatExponentNumber, // "123.456e7", "123.456e+7", "123.456e-7"
186 TokenizeStateFloatExponentNumberNoUnderscore, // "123.456e7_", "123.456e+7_", "123.456e-7_"
184 TokenizeStateString,187 TokenizeStateString,
185 TokenizeStateStringEscape,188 TokenizeStateStringEscape,
186 TokenizeStateStringEscapeUnicodeStart,189 TokenizeStateStringEscapeUnicodeStart,
...@@ -233,14 +236,10 @@ struct Tokenize {...@@ -233,14 +236,10 @@ struct Tokenize {
233 Token *cur_tok;236 Token *cur_tok;
234 Tokenization *out;237 Tokenization *out;
235 uint32_t radix;238 uint32_t radix;
236 int32_t exp_add_amt;239 bool is_trailing_underscore;
237 bool is_exp_negative;
238 size_t char_code_index;240 size_t char_code_index;
239 bool unicode;241 bool unicode;
240 uint32_t char_code;242 uint32_t char_code;
241 int exponent_in_bin_or_dec;
242 BigInt specified_exponent;
243 BigInt significand;
244 size_t remaining_code_units;243 size_t remaining_code_units;
245};244};
246245
...@@ -426,20 +425,16 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -426,20 +425,16 @@ void tokenize(Buf *buf, Tokenization *out) {
426 case '0':425 case '0':
427 t.state = TokenizeStateZero;426 t.state = TokenizeStateZero;
428 begin_token(&t, TokenIdIntLiteral);427 begin_token(&t, TokenIdIntLiteral);
428 t.is_trailing_underscore = false;
429 t.radix = 10;429 t.radix = 10;
430 t.exp_add_amt = 1;
431 t.exponent_in_bin_or_dec = 0;
432 bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, 0);430 bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, 0);
433 bigint_init_unsigned(&t.specified_exponent, 0);
434 break;431 break;
435 case DIGIT_NON_ZERO:432 case DIGIT_NON_ZERO:
436 t.state = TokenizeStateNumber;433 t.state = TokenizeStateNumber;
437 begin_token(&t, TokenIdIntLiteral);434 begin_token(&t, TokenIdIntLiteral);
435 t.is_trailing_underscore = false;
438 t.radix = 10;436 t.radix = 10;
439 t.exp_add_amt = 1;
440 t.exponent_in_bin_or_dec = 0;
441 bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, get_digit_value(c));437 bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, get_digit_value(c));
442 bigint_init_unsigned(&t.specified_exponent, 0);
443 break;438 break;
444 case '"':439 case '"':
445 begin_token(&t, TokenIdStringLiteral);440 begin_token(&t, TokenIdStringLiteral);
...@@ -1189,17 +1184,15 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1189,17 +1184,15 @@ void tokenize(Buf *buf, Tokenization *out) {
1189 switch (c) {1184 switch (c) {
1190 case 'b':1185 case 'b':
1191 t.radix = 2;1186 t.radix = 2;
1192 t.state = TokenizeStateNumber;1187 t.state = TokenizeStateNumberNoUnderscore;
1193 break;1188 break;
1194 case 'o':1189 case 'o':
1195 t.radix = 8;1190 t.radix = 8;
1196 t.exp_add_amt = 3;1191 t.state = TokenizeStateNumberNoUnderscore;
1197 t.state = TokenizeStateNumber;
1198 break;1192 break;
1199 case 'x':1193 case 'x':
1200 t.radix = 16;1194 t.radix = 16;
1201 t.exp_add_amt = 4;1195 t.state = TokenizeStateNumberNoUnderscore;
1202 t.state = TokenizeStateNumber;
1203 break;1196 break;
1204 default:1197 default:
1205 // reinterpret as normal number1198 // reinterpret as normal number
...@@ -1208,9 +1201,27 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1208,9 +1201,27 @@ void tokenize(Buf *buf, Tokenization *out) {
1208 continue;1201 continue;
1209 }1202 }
1210 break;1203 break;
1204 case TokenizeStateNumberNoUnderscore:
1205 if (c == '_') {
1206 invalid_char_error(&t, c);
1207 break;
1208 } else if (get_digit_value(c) < t.radix) {
1209 t.is_trailing_underscore = false;
1210 t.state = TokenizeStateNumber;
1211 }
1212 // fall through
1211 case TokenizeStateNumber:1213 case TokenizeStateNumber:
1212 {1214 {
1215 if (c == '_') {
1216 t.is_trailing_underscore = true;
1217 t.state = TokenizeStateNumberNoUnderscore;
1218 break;
1219 }
1213 if (c == '.') {1220 if (c == '.') {
1221 if (t.is_trailing_underscore) {
1222 invalid_char_error(&t, c);
1223 break;
1224 }
1214 if (t.radix != 16 && t.radix != 10) {1225 if (t.radix != 16 && t.radix != 10) {
1215 invalid_char_error(&t, c);1226 invalid_char_error(&t, c);
1216 }1227 }
...@@ -1222,13 +1233,18 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1222,13 +1233,18 @@ void tokenize(Buf *buf, Tokenization *out) {
1222 invalid_char_error(&t, c);1233 invalid_char_error(&t, c);
1223 }1234 }
1224 t.state = TokenizeStateFloatExponentUnsigned;1235 t.state = TokenizeStateFloatExponentUnsigned;
1236 t.radix = 10; // exponent is always base 10
1225 assert(t.cur_tok->id == TokenIdIntLiteral);1237 assert(t.cur_tok->id == TokenIdIntLiteral);
1226 bigint_init_bigint(&t.significand, &t.cur_tok->data.int_lit.bigint);
1227 set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);1238 set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);
1228 break;1239 break;
1229 }1240 }
1230 uint32_t digit_value = get_digit_value(c);1241 uint32_t digit_value = get_digit_value(c);
1231 if (digit_value >= t.radix) {1242 if (digit_value >= t.radix) {
1243 if (t.is_trailing_underscore) {
1244 invalid_char_error(&t, c);
1245 break;
1246 }
1247
1232 if (is_symbol_char(c)) {1248 if (is_symbol_char(c)) {
1233 invalid_char_error(&t, c);1249 invalid_char_error(&t, c);
1234 }1250 }
...@@ -1259,20 +1275,37 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1259,20 +1275,37 @@ void tokenize(Buf *buf, Tokenization *out) {
1259 continue;1275 continue;
1260 }1276 }
1261 t.pos -= 1;1277 t.pos -= 1;
1262 t.state = TokenizeStateFloatFraction;1278 t.state = TokenizeStateFloatFractionNoUnderscore;
1263 assert(t.cur_tok->id == TokenIdIntLiteral);1279 assert(t.cur_tok->id == TokenIdIntLiteral);
1264 bigint_init_bigint(&t.significand, &t.cur_tok->data.int_lit.bigint);
1265 set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);1280 set_token_id(&t, t.cur_tok, TokenIdFloatLiteral);
1266 continue;1281 continue;
1267 }1282 }
1283 case TokenizeStateFloatFractionNoUnderscore:
1284 if (c == '_') {
1285 invalid_char_error(&t, c);
1286 } else if (get_digit_value(c) < t.radix) {
1287 t.is_trailing_underscore = false;
1288 t.state = TokenizeStateFloatFraction;
1289 }
1290 // fall through
1268 case TokenizeStateFloatFraction:1291 case TokenizeStateFloatFraction:
1269 {1292 {
1293 if (c == '_') {
1294 t.is_trailing_underscore = true;
1295 t.state = TokenizeStateFloatFractionNoUnderscore;
1296 break;
1297 }
1270 if (is_exponent_signifier(c, t.radix)) {1298 if (is_exponent_signifier(c, t.radix)) {
1271 t.state = TokenizeStateFloatExponentUnsigned;1299 t.state = TokenizeStateFloatExponentUnsigned;
1300 t.radix = 10; // exponent is always base 10
1272 break;1301 break;
1273 }1302 }
1274 uint32_t digit_value = get_digit_value(c);1303 uint32_t digit_value = get_digit_value(c);
1275 if (digit_value >= t.radix) {1304 if (digit_value >= t.radix) {
1305 if (t.is_trailing_underscore) {
1306 invalid_char_error(&t, c);
1307 break;
1308 }
1276 if (is_symbol_char(c)) {1309 if (is_symbol_char(c)) {
1277 invalid_char_error(&t, c);1310 invalid_char_error(&t, c);
1278 }1311 }
...@@ -1282,46 +1315,47 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1282,46 +1315,47 @@ void tokenize(Buf *buf, Tokenization *out) {
1282 t.state = TokenizeStateStart;1315 t.state = TokenizeStateStart;
1283 continue;1316 continue;
1284 }1317 }
1285 t.exponent_in_bin_or_dec -= t.exp_add_amt;
1286 if (t.radix == 10) {
1287 // For now we use strtod to parse decimal floats, so we just have to get to the
1288 // end of the token.
1289 break;
1290 }
1291 BigInt digit_value_bi;
1292 bigint_init_unsigned(&digit_value_bi, digit_value);
12931318
1294 BigInt radix_bi;1319 // we use parse_f128 to generate the float literal, so just
1295 bigint_init_unsigned(&radix_bi, t.radix);1320 // need to get to the end of the token
1296
1297 BigInt multiplied;
1298 bigint_mul(&multiplied, &t.significand, &radix_bi);
1299
1300 bigint_add(&t.significand, &multiplied, &digit_value_bi);
1301 break;
1302 }1321 }
1322 break;
1303 case TokenizeStateFloatExponentUnsigned:1323 case TokenizeStateFloatExponentUnsigned:
1304 switch (c) {1324 switch (c) {
1305 case '+':1325 case '+':
1306 t.is_exp_negative = false;1326 t.state = TokenizeStateFloatExponentNumberNoUnderscore;
1307 t.state = TokenizeStateFloatExponentNumber;
1308 break;1327 break;
1309 case '-':1328 case '-':
1310 t.is_exp_negative = true;1329 t.state = TokenizeStateFloatExponentNumberNoUnderscore;
1311 t.state = TokenizeStateFloatExponentNumber;
1312 break;1330 break;
1313 default:1331 default:
1314 // reinterpret as normal exponent number1332 // reinterpret as normal exponent number
1315 t.pos -= 1;1333 t.pos -= 1;
1316 t.is_exp_negative = false;1334 t.state = TokenizeStateFloatExponentNumberNoUnderscore;
1317 t.state = TokenizeStateFloatExponentNumber;
1318 continue;1335 continue;
1319 }1336 }
1320 break;1337 break;
1338 case TokenizeStateFloatExponentNumberNoUnderscore:
1339 if (c == '_') {
1340 invalid_char_error(&t, c);
1341 } else if (get_digit_value(c) < t.radix) {
1342 t.is_trailing_underscore = false;
1343 t.state = TokenizeStateFloatExponentNumber;
1344 }
1345 // fall through
1321 case TokenizeStateFloatExponentNumber:1346 case TokenizeStateFloatExponentNumber:
1322 {1347 {
1348 if (c == '_') {
1349 t.is_trailing_underscore = true;
1350 t.state = TokenizeStateFloatExponentNumberNoUnderscore;
1351 break;
1352 }
1323 uint32_t digit_value = get_digit_value(c);1353 uint32_t digit_value = get_digit_value(c);
1324 if (digit_value >= t.radix) {1354 if (digit_value >= t.radix) {
1355 if (t.is_trailing_underscore) {
1356 invalid_char_error(&t, c);
1357 break;
1358 }
1325 if (is_symbol_char(c)) {1359 if (is_symbol_char(c)) {
1326 invalid_char_error(&t, c);1360 invalid_char_error(&t, c);
1327 }1361 }
...@@ -1331,21 +1365,9 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1331,21 +1365,9 @@ void tokenize(Buf *buf, Tokenization *out) {
1331 t.state = TokenizeStateStart;1365 t.state = TokenizeStateStart;
1332 continue;1366 continue;
1333 }1367 }
1334 if (t.radix == 10) {
1335 // For now we use strtod to parse decimal floats, so we just have to get to the
1336 // end of the token.
1337 break;
1338 }
1339 BigInt digit_value_bi;
1340 bigint_init_unsigned(&digit_value_bi, digit_value);
1341
1342 BigInt radix_bi;
1343 bigint_init_unsigned(&radix_bi, 10);
1344
1345 BigInt multiplied;
1346 bigint_mul(&multiplied, &t.specified_exponent, &radix_bi);
13471368
1348 bigint_add(&t.specified_exponent, &multiplied, &digit_value_bi);1369 // we use parse_f128 to generate the float literal, so just
1370 // need to get to the end of the token
1349 }1371 }
1350 break;1372 break;
1351 case TokenizeStateSawDash:1373 case TokenizeStateSawDash:
...@@ -1399,6 +1421,9 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1399,6 +1421,9 @@ void tokenize(Buf *buf, Tokenization *out) {
1399 case TokenizeStateStart:1421 case TokenizeStateStart:
1400 case TokenizeStateError:1422 case TokenizeStateError:
1401 break;1423 break;
1424 case TokenizeStateNumberNoUnderscore:
1425 case TokenizeStateFloatFractionNoUnderscore:
1426 case TokenizeStateFloatExponentNumberNoUnderscore:
1402 case TokenizeStateNumberDot:1427 case TokenizeStateNumberDot:
1403 tokenize_error(&t, "unterminated number literal");1428 tokenize_error(&t, "unterminated number literal");
1404 break;1429 break;
test/compile_errors.zig+96
...@@ -389,6 +389,102 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {...@@ -389,6 +389,102 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
389 "tmp.zig:5:29: error: invalid token: '.'",389 "tmp.zig:5:29: error: invalid token: '.'",
390 });390 });
391391
392 cases.add("invalid underscore placement in float literal - 1",
393 \\fn main() void {
394 \\ var bad: f128 = 0._0;
395 \\})
396 , &[_][]const u8{
397 "tmp.zig:2:23: error: invalid character: '_'",
398 });
399
400 cases.add("invalid underscore placement in float literal - 2",
401 \\fn main() void {
402 \\ var bad: f128 = 0_.0;
403 \\})
404 , &[_][]const u8{
405 "tmp.zig:2:23: error: invalid character: '.'",
406 });
407
408 cases.add("invalid underscore placement in float literal - 3",
409 \\fn main() void {
410 \\ var bad: f128 = 0.0_;
411 \\})
412 , &[_][]const u8{
413 "tmp.zig:2:25: error: invalid character: ';'",
414 });
415
416 cases.add("invalid underscore placement in float literal - 4",
417 \\fn main() void {
418 \\ var bad: f128 = 1.0e_1;
419 \\})
420 , &[_][]const u8{
421 "tmp.zig:2:25: error: invalid character: '_'",
422 });
423
424 cases.add("invalid underscore placement in float literal - 5",
425 \\fn main() void {
426 \\ var bad: f128 = 1.0e+_1;
427 \\})
428 , &[_][]const u8{
429 "tmp.zig:2:26: error: invalid character: '_'",
430 });
431
432 cases.add("invalid underscore placement in float literal - 6",
433 \\fn main() void {
434 \\ var bad: f128 = 1.0e-_1;
435 \\})
436 , &[_][]const u8{
437 "tmp.zig:2:26: error: invalid character: '_'",
438 });
439
440 cases.add("invalid underscore placement in float literal - 7",
441 \\fn main() void {
442 \\ var bad: f128 = 1.0e-1_;
443 \\})
444 , &[_][]const u8{
445 "tmp.zig:2:28: error: invalid character: ';'",
446 });
447
448 cases.add("invalid underscore placement in float literal - 9",
449 \\fn main() void {
450 \\ var bad: f128 = 1__0.0e-1;
451 \\})
452 , &[_][]const u8{
453 "tmp.zig:2:23: error: invalid character: '_'",
454 });
455
456 cases.add("invalid underscore placement in float literal - 10",
457 \\fn main() void {
458 \\ var bad: f128 = 1.0__0e-1;
459 \\})
460 , &[_][]const u8{
461 "tmp.zig:2:25: error: invalid character: '_'",
462 });
463
464 cases.add("invalid underscore placement in float literal - 11",
465 \\fn main() void {
466 \\ var bad: f128 = 1.0e-1__0;
467 \\})
468 , &[_][]const u8{
469 "tmp.zig:2:28: error: invalid character: '_'",
470 });
471
472 cases.add("invalid underscore placement in float literal - 12",
473 \\fn main() void {
474 \\ var bad: f128 = 0_x0.0;
475 \\})
476 , &[_][]const u8{
477 "tmp.zig:2:23: error: invalid character: 'x'",
478 });
479
480 cases.add("invalid underscore placement in float literal - 13",
481 \\fn main() void {
482 \\ var bad: f128 = 0x_0.0;
483 \\})
484 , &[_][]const u8{
485 "tmp.zig:2:23: error: invalid character: '_'",
486 });
487
392 cases.add("var args without c calling conv",488 cases.add("var args without c calling conv",
393 \\fn foo(args: ...) void {}489 \\fn foo(args: ...) void {}
394 \\comptime {490 \\comptime {
test/stage1/behavior/math.zig+28
...@@ -411,6 +411,34 @@ test "quad hex float literal parsing accurate" {...@@ -411,6 +411,34 @@ test "quad hex float literal parsing accurate" {
411 comptime S.doTheTest();411 comptime S.doTheTest();
412}412}
413413
414test "underscore separator parsing" {
415 expect(0_0_0_0 == 0);
416 expect(1_234_567 == 1234567);
417 expect(001_234_567 == 1234567);
418 expect(0_0_1_2_3_4_5_6_7 == 1234567);
419
420 expect(0b0_0_0_0 == 0);
421 expect(0b1010_1010 == 0b10101010);
422 expect(0b0000_1010_1010 == 0b10101010);
423 expect(0b1_0_1_0_1_0_1_0 == 0b10101010);
424
425 expect(0o0_0_0_0 == 0);
426 expect(0o1010_1010 == 0o10101010);
427 expect(0o0000_1010_1010 == 0o10101010);
428 expect(0o1_0_1_0_1_0_1_0 == 0o10101010);
429
430 expect(0x0_0_0_0 == 0);
431 expect(0x1010_1010 == 0x10101010);
432 expect(0x0000_1010_1010 == 0x10101010);
433 expect(0x1_0_1_0_1_0_1_0 == 0x10101010);
434
435 expect(123_456.789_000e1_0 == 123456.789000e10);
436 expect(0_1_2_3_4_5_6.7_8_9_0_0_0e0_0_1_0 == 123456.789000e10);
437
438 expect(0x1234_5678.9ABC_DEF0p-1_0 == 0x12345678.9ABCDEF0p-10);
439 expect(0x1_2_3_4_5_6_7_8.9_A_B_C_D_E_F_0p-0_0_0_1_0 == 0x12345678.9ABCDEF0p-10);
440}
441
414test "hex float literal within range" {442test "hex float literal within range" {
415 const a = 0x1.0p16383;443 const a = 0x1.0p16383;
416 const b = 0x0.1p16387;444 const b = 0x0.1p16387;