authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2024-06-14 13:09:55+12:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2024-06-15 18:23:06+02:00
log1b728e1834672848b12d10dd992b34141a38fc05
tree11bd57e9184c281e73aa6398c82bb5d4d577e2ad
parentffb1a6d9a75a7c8804c0e9db0e23ac54265f447c

std.float.parseFloat: fix large hex-float parsing

There were two primary issues at play here: 1. The hex float prefix was not handled correctly when the stream was reset for the fallback parsing path, which occured when the mantissa was longer max mantissa digits. 2. The implied exponent was not adjusted for hex-floats in this branch. Additionally, some of the float parsing routines have been condensed, making use of comptime. closes #20275

4 files changed, 32 insertions(+), 40 deletions(-)

lib/std/fmt/parse_float.zig+7-1
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1const std = @import("../std.zig");1const std = @import("std");
2const math = std.math;2const math = std.math;
3const testing = std.testing;3const testing = std.testing;
4const expect = testing.expect;4const expect = testing.expect;
...@@ -151,6 +151,12 @@ test "#11169" {...@@ -151,6 +151,12 @@ test "#11169" {
151 try expectEqual(try parseFloat(f128, "9007199254740993.0"), 9007199254740993.0);151 try expectEqual(try parseFloat(f128, "9007199254740993.0"), 9007199254740993.0);
152}152}
153153
154test "many_digits hex" {
155 const a: f32 = try std.fmt.parseFloat(f32, "0xffffffffffffffff.0p0");
156 const b: f32 = @floatCast(try std.fmt.parseFloat(f128, "0xffffffffffffffff.0p0"));
157 try std.testing.expectEqual(a, b);
158}
159
154test "hex.special" {160test "hex.special" {
155 try testing.expect(math.isNan(try parseFloat(f32, "nAn")));161 try testing.expect(math.isNan(try parseFloat(f32, "nAn")));
156 try testing.expect(math.isPositiveInf(try parseFloat(f32, "iNf")));162 try testing.expect(math.isPositiveInf(try parseFloat(f32, "iNf")));
lib/std/fmt/parse_float/FloatStream.zig+6-24
...@@ -48,30 +48,16 @@ pub fn isEmpty(self: FloatStream) bool {...@@ -48,30 +48,16 @@ pub fn isEmpty(self: FloatStream) bool {
48 return !self.hasLen(1);48 return !self.hasLen(1);
49}49}
5050
51pub fn firstIs(self: FloatStream, c: u8) bool {51pub fn firstIs(self: FloatStream, comptime cs: []const u8) bool {
52 if (self.first()) |ok| {52 if (self.first()) |ok| {
53 return ok == c;53 inline for (cs) |c| if (ok == c) return true;
54 }54 }
55 return false;55 return false;
56}56}
5757
58pub fn firstIsLower(self: FloatStream, c: u8) bool {58pub fn firstIsLower(self: FloatStream, comptime cs: []const u8) bool {
59 if (self.first()) |ok| {59 if (self.first()) |ok| {
60 return ok | 0x20 == c;60 inline for (cs) |c| if (ok | 0x20 == c) return true;
61 }
62 return false;
63}
64
65pub fn firstIs2(self: FloatStream, c1: u8, c2: u8) bool {
66 if (self.first()) |ok| {
67 return ok == c1 or ok == c2;
68 }
69 return false;
70}
71
72pub fn firstIs3(self: FloatStream, c1: u8, c2: u8, c3: u8) bool {
73 if (self.first()) |ok| {
74 return ok == c1 or ok == c2 or ok == c3;
75 }61 }
76 return false;62 return false;
77}63}
...@@ -89,12 +75,8 @@ pub fn advance(self: *FloatStream, n: usize) void {...@@ -89,12 +75,8 @@ pub fn advance(self: *FloatStream, n: usize) void {
89 self.offset += n;75 self.offset += n;
90}76}
9177
92pub fn skipChars(self: *FloatStream, c: u8) void {78pub fn skipChars(self: *FloatStream, comptime cs: []const u8) void {
93 while (self.firstIs(c)) : (self.advance(1)) {}79 while (self.firstIs(cs)) : (self.advance(1)) {}
94}
95
96pub fn skipChars2(self: *FloatStream, c1: u8, c2: u8) void {
97 while (self.firstIs2(c1, c2)) : (self.advance(1)) {}
98}80}
9981
100pub fn readU64Unchecked(self: FloatStream) u64 {82pub fn readU64Unchecked(self: FloatStream) u64 {
lib/std/fmt/parse_float/decimal.zig+6-6
...@@ -241,18 +241,18 @@ pub fn Decimal(comptime T: type) type {...@@ -241,18 +241,18 @@ pub fn Decimal(comptime T: type) type {
241 var d = Self.new();241 var d = Self.new();
242 var stream = FloatStream.init(s);242 var stream = FloatStream.init(s);
243243
244 stream.skipChars2('0', '_');244 stream.skipChars("0_");
245 while (stream.scanDigit(10)) |digit| {245 while (stream.scanDigit(10)) |digit| {
246 d.tryAddDigit(digit);246 d.tryAddDigit(digit);
247 }247 }
248248
249 if (stream.firstIs('.')) {249 if (stream.firstIs(".")) {
250 stream.advance(1);250 stream.advance(1);
251 const marker = stream.offsetTrue();251 const marker = stream.offsetTrue();
252252
253 // Skip leading zeroes253 // Skip leading zeroes
254 if (d.num_digits == 0) {254 if (d.num_digits == 0) {
255 stream.skipChars('0');255 stream.skipChars("0");
256 }256 }
257257
258 while (stream.hasLen(8) and d.num_digits + 8 < max_digits) {258 while (stream.hasLen(8) and d.num_digits + 8 < max_digits) {
...@@ -292,13 +292,13 @@ pub fn Decimal(comptime T: type) type {...@@ -292,13 +292,13 @@ pub fn Decimal(comptime T: type) type {
292 d.num_digits = max_digits;292 d.num_digits = max_digits;
293 }293 }
294 }294 }
295 if (stream.firstIsLower('e')) {295 if (stream.firstIsLower("e")) {
296 stream.advance(1);296 stream.advance(1);
297 var neg_exp = false;297 var neg_exp = false;
298 if (stream.firstIs('-')) {298 if (stream.firstIs("-")) {
299 neg_exp = true;299 neg_exp = true;
300 stream.advance(1);300 stream.advance(1);
301 } else if (stream.firstIs('+')) {301 } else if (stream.firstIs("+")) {
302 stream.advance(1);302 stream.advance(1);
303 }303 }
304 var exp_num: i32 = 0;304 var exp_num: i32 = 0;
lib/std/fmt/parse_float/parse.zig+13-9
...@@ -100,6 +100,7 @@ const ParseInfo = struct {...@@ -100,6 +100,7 @@ const ParseInfo = struct {
100};100};
101101
102fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool, n: *usize, comptime info: ParseInfo) ?Number(T) {102fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool, n: *usize, comptime info: ParseInfo) ?Number(T) {
103 std.debug.assert(info.base == 10 or info.base == 16);
103 const MantissaT = common.mantissaType(T);104 const MantissaT = common.mantissaType(T);
104105
105 // parse initial digits before dot106 // parse initial digits before dot
...@@ -107,12 +108,10 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool...@@ -107,12 +108,10 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool
107 tryParseDigits(MantissaT, stream, &mantissa, info.base);108 tryParseDigits(MantissaT, stream, &mantissa, info.base);
108 const int_end = stream.offsetTrue();109 const int_end = stream.offsetTrue();
109 var n_digits = @as(isize, @intCast(stream.offsetTrue()));110 var n_digits = @as(isize, @intCast(stream.offsetTrue()));
110 // the base being 16 implies a 0x prefix, which shouldn't be included in the digit count
111 if (info.base == 16) n_digits -= 2;
112111
113 // handle dot with the following digits112 // handle dot with the following digits
114 var exponent: i64 = 0;113 var exponent: i64 = 0;
115 if (stream.firstIs('.')) {114 if (stream.firstIs(".")) {
116 stream.advance(1);115 stream.advance(1);
117 const marker = stream.offsetTrue();116 const marker = stream.offsetTrue();
118 tryParseDigits(MantissaT, stream, &mantissa, info.base);117 tryParseDigits(MantissaT, stream, &mantissa, info.base);
...@@ -132,14 +131,14 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool...@@ -132,14 +131,14 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool
132131
133 // handle scientific format132 // handle scientific format
134 var exp_number: i64 = 0;133 var exp_number: i64 = 0;
135 if (stream.firstIsLower(info.exp_char_lower)) {134 if (stream.firstIsLower(&.{info.exp_char_lower})) {
136 stream.advance(1);135 stream.advance(1);
137 exp_number = parseScientific(stream) orelse return null;136 exp_number = parseScientific(stream) orelse return null;
138 exponent += exp_number;137 exponent += exp_number;
139 }138 }
140139
141 const len = stream.offset; // length must be complete parsed length140 const len = stream.offset; // length must be complete parsed length
142 n.* = len;141 n.* += len;
143142
144 if (stream.underscore_count > 0 and !validUnderscores(stream.slice, info.base)) {143 if (stream.underscore_count > 0 and !validUnderscores(stream.slice, info.base)) {
145 return null;144 return null;
...@@ -159,7 +158,7 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool...@@ -159,7 +158,7 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool
159 n_digits -= info.max_mantissa_digits;158 n_digits -= info.max_mantissa_digits;
160 var many_digits = false;159 var many_digits = false;
161 stream.reset(); // re-parse from beginning160 stream.reset(); // re-parse from beginning
162 while (stream.firstIs3('0', '.', '_')) {161 while (stream.firstIs("0._")) {
163 // '0' = '.' + 2162 // '0' = '.' + 2
164 const next = stream.firstUnchecked();163 const next = stream.firstUnchecked();
165 if (next != '_') {164 if (next != '_') {
...@@ -193,6 +192,9 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool...@@ -193,6 +192,9 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool
193 break :blk @as(i64, @intCast(marker)) - @as(i64, @intCast(stream.offsetTrue()));192 break :blk @as(i64, @intCast(marker)) - @as(i64, @intCast(stream.offsetTrue()));
194 }193 }
195 };194 };
195 if (info.base == 16) {
196 exponent *= 4;
197 }
196 // add back the explicit part198 // add back the explicit part
197 exponent += exp_number;199 exponent += exp_number;
198 }200 }
...@@ -212,17 +214,19 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool...@@ -212,17 +214,19 @@ fn parsePartialNumberBase(comptime T: type, stream: *FloatStream, negative: bool
212/// significant digits and the decimal exponent.214/// significant digits and the decimal exponent.
213fn parsePartialNumber(comptime T: type, s: []const u8, negative: bool, n: *usize) ?Number(T) {215fn parsePartialNumber(comptime T: type, s: []const u8, negative: bool, n: *usize) ?Number(T) {
214 std.debug.assert(s.len != 0);216 std.debug.assert(s.len != 0);
215 var stream = FloatStream.init(s);
216 const MantissaT = common.mantissaType(T);217 const MantissaT = common.mantissaType(T);
218 n.* = 0;
217219
218 if (stream.hasLen(2) and stream.atUnchecked(0) == '0' and std.ascii.toLower(stream.atUnchecked(1)) == 'x') {220 if (s.len >= 2 and s[0] == '0' and std.ascii.toLower(s[1]) == 'x') {
219 stream.advance(2);221 var stream = FloatStream.init(s[2..]);
222 n.* += 2;
220 return parsePartialNumberBase(T, &stream, negative, n, .{223 return parsePartialNumberBase(T, &stream, negative, n, .{
221 .base = 16,224 .base = 16,
222 .max_mantissa_digits = if (MantissaT == u64) 16 else 32,225 .max_mantissa_digits = if (MantissaT == u64) 16 else 32,
223 .exp_char_lower = 'p',226 .exp_char_lower = 'p',
224 });227 });
225 } else {228 } else {
229 var stream = FloatStream.init(s);
226 return parsePartialNumberBase(T, &stream, negative, n, .{230 return parsePartialNumberBase(T, &stream, negative, n, .{
227 .base = 10,231 .base = 10,
228 .max_mantissa_digits = if (MantissaT == u64) 19 else 38,232 .max_mantissa_digits = if (MantissaT == u64) 19 else 38,