authorgravatar for jens.goldberg@gmail.comJens Goldberg <jens.goldberg@gmail.com> 2021-12-31 17:17:49+00:00
committergravatar for jens.goldberg@gmail.comJens Goldberg <jens.goldberg@gmail.com> 2021-12-31 17:17:49+00:00
log9a564356661d31bc7fad5b670bad8ebeecc5dad4
tree31348593f1a1365e5a41316c62dfffb5d831c8f6
parente4672c95f116eefd0a87a1f857062017e6a7fd97

tz parsing reader interface, test thicc files, and exclude tzif

4 files changed, 114 insertions(+), 97 deletions(-)

build.zig+1
...@@ -98,6 +98,7 @@ pub fn build(b: *Builder) !void {...@@ -98,6 +98,7 @@ pub fn build(b: *Builder) !void {
98 ".z.9",98 ".z.9",
99 ".gz",99 ".gz",
100 "rfc1951.txt",100 "rfc1951.txt",
101 ".tzif",
101 },102 },
102 .blank_extensions = &[_][]const u8{103 .blank_extensions = &[_][]const u8{
103 "test.zig",104 "test.zig",
lib/std/std.zig+1-1
...@@ -39,6 +39,7 @@ pub const StringArrayHashMapUnmanaged = array_hash_map.StringArrayHashMapUnmanag...@@ -39,6 +39,7 @@ pub const StringArrayHashMapUnmanaged = array_hash_map.StringArrayHashMapUnmanag
39pub const TailQueue = @import("linked_list.zig").TailQueue;39pub const TailQueue = @import("linked_list.zig").TailQueue;
40pub const Target = @import("target.zig").Target;40pub const Target = @import("target.zig").Target;
41pub const Thread = @import("Thread.zig");41pub const Thread = @import("Thread.zig");
42pub const Tz = @import("tz.zig").Tz;
4243
43pub const array_hash_map = @import("array_hash_map.zig");44pub const array_hash_map = @import("array_hash_map.zig");
44pub const atomic = @import("atomic.zig");45pub const atomic = @import("atomic.zig");
...@@ -81,7 +82,6 @@ pub const sort = @import("sort.zig");...@@ -81,7 +82,6 @@ pub const sort = @import("sort.zig");
81pub const ascii = @import("ascii.zig");82pub const ascii = @import("ascii.zig");
82pub const testing = @import("testing.zig");83pub const testing = @import("testing.zig");
83pub const time = @import("time.zig");84pub const time = @import("time.zig");
84pub const tz = @import("tz.zig");
85pub const unicode = @import("unicode.zig");85pub const unicode = @import("unicode.zig");
86pub const valgrind = @import("valgrind.zig");86pub const valgrind = @import("valgrind.zig");
87pub const wasm = @import("wasm.zig");87pub const wasm = @import("wasm.zig");
lib/std/tz.zig+112-96
...@@ -40,160 +40,163 @@ pub const Tz = struct {...@@ -40,160 +40,163 @@ pub const Tz = struct {
40 leapseconds: []const Leapsecond,40 leapseconds: []const Leapsecond,
41 footer: []const u8,41 footer: []const u8,
4242
43 pub fn parse(allocator: std.mem.Allocator, data: []const u8) !Tz {43 pub fn parse(allocator: std.mem.Allocator, reader: anytype) !Tz {
44 const header_size = 4 + 1 + 15 + 6 * 4;44 _ = allocator;
45 if (data.len < header_size) return error.BadSize;45 const Header = extern struct {
4646 magic: [4]u8,
47 const magic_l = data[0..4];47 version: u8,
48 const version_l = data[4];48 reserved: [15]u8,
49 if (!std.mem.eql(u8, magic_l, "TZif")) return error.BadHeader;49 };
50 if (version_l != '2' and version_l != '3') return error.BadVersion;50
5151 const Counts = extern struct {
52 // Parse the legacy header and skip the entire thing52 isutcnt: u32,
53 const isutcnt_l = std.mem.readIntBig(u32, data[20..24]);53 isstdcnt: u32,
54 const isstdcnt_l = std.mem.readIntBig(u32, data[24..28]);54 leapcnt: u32,
55 const leapcnt_l = std.mem.readIntBig(u32, data[28..32]);55 timecnt: u32,
56 const timecnt_l = std.mem.readIntBig(u32, data[32..36]);56 typecnt: u32,
57 const typecnt_l = std.mem.readIntBig(u32, data[36..40]);57 charcnt: u32,
58 const charcnt_l = std.mem.readIntBig(u32, data[40..44]);58 };
59 const data_block_size_legacy = timecnt_l * 5 + typecnt_l * 6 + charcnt_l + leapcnt_l * 8 + isstdcnt_l + isutcnt_l;59
60 if (data.len < header_size + data_block_size_legacy) return error.BadSize;60 // Parse and skip the legacy header and data
6161 {
62 const data2 = data[header_size + data_block_size_legacy ..];62 const header = try reader.readStruct(Header);
63 if (data2.len < header_size) return error.BadSize;63 if (!std.mem.eql(u8, &header.magic, "TZif")) return error.BadHeader;
6464 if (header.version == 0) return error.UnsupportedLegacyFormat;
65 const magic = data2[0..4];65 if (header.version != '2' and header.version != '3') return error.BadVersion;
66 const version = data2[4];66
67 if (!std.mem.eql(u8, magic, "TZif")) return error.BadHeader;67 var counts = try reader.readStruct(Counts);
68 if (version != '2' and version != '3') return error.BadVersion;68 if (builtin.target.cpu.arch.endian() != std.builtin.Endian.Big) {
6969 std.mem.bswapAllFields(Counts, &counts);
70 const isutcnt = std.mem.readIntBig(u32, data2[20..24]);70 }
71 const isstdcnt = std.mem.readIntBig(u32, data2[24..28]);71
72 const leapcnt = std.mem.readIntBig(u32, data2[28..32]);72 const skipv = counts.timecnt * 5 + counts.typecnt * 6 + counts.charcnt + counts.leapcnt * 8 + counts.isstdcnt + counts.isutcnt;
73 const timecnt = std.mem.readIntBig(u32, data2[32..36]);73 try reader.skipBytes(skipv, .{});
74 const typecnt = std.mem.readIntBig(u32, data2[36..40]);74 }
75 const charcnt = std.mem.readIntBig(u32, data2[40..44]);75
7676 const header = try reader.readStruct(Header);
77 if (isstdcnt != 0 and isstdcnt != typecnt) return error.Malformed; // rfc8536: isstdcnt [...] MUST either be zero or equal to "typecnt"77 if (!std.mem.eql(u8, &header.magic, "TZif")) return error.BadHeader;
78 if (isutcnt != 0 and isutcnt != typecnt) return error.Malformed; // rfc8536: isutcnt [...] MUST either be zero or equal to "typecnt"78 if (header.version != '2' and header.version != '3') return error.BadVersion;
79 if (typecnt == 0) return error.Malformed; // rfc8536: typecnt [...] MUST NOT be zero79
80 if (charcnt == 0) return error.Malformed; // rfc8536: charcnt [...] MUST NOT be zero80 var counts = try reader.readStruct(Counts);
8181 if (builtin.target.cpu.arch.endian() != std.builtin.Endian.Big) {
82 const data_block_size = timecnt * 9 + typecnt * 6 + charcnt + leapcnt * 12 + isstdcnt + isutcnt;82 std.mem.bswapAllFields(Counts, &counts);
83 if (data2.len < header_size + data_block_size) return error.BadSize;83 }
8484
85 var leapseconds = try allocator.alloc(Leapsecond, leapcnt);85 if (counts.isstdcnt != 0 and counts.isstdcnt != counts.typecnt) return error.Malformed; // rfc8536: isstdcnt [...] MUST either be zero or equal to "typecnt"
86 if (counts.isutcnt != 0 and counts.isutcnt != counts.typecnt) return error.Malformed; // rfc8536: isutcnt [...] MUST either be zero or equal to "typecnt"
87 if (counts.typecnt == 0) return error.Malformed; // rfc8536: typecnt [...] MUST NOT be zero
88 if (counts.charcnt == 0) return error.Malformed; // rfc8536: charcnt [...] MUST NOT be zero
89 if (counts.charcnt > 256 + 6) return error.Malformed; // Not explicitly banned by rfc8536 but nonsensical
90
91 var leapseconds = try allocator.alloc(Leapsecond, counts.leapcnt);
86 errdefer allocator.free(leapseconds);92 errdefer allocator.free(leapseconds);
87 var transitions = try allocator.alloc(Transition, timecnt);93 var transitions = try allocator.alloc(Transition, counts.timecnt);
88 errdefer allocator.free(transitions);94 errdefer allocator.free(transitions);
89 var timetypes = try allocator.alloc(Timetype, typecnt);95 var timetypes = try allocator.alloc(Timetype, counts.typecnt);
90 errdefer allocator.free(timetypes);96 errdefer allocator.free(timetypes);
9197
92 var p: usize = header_size;
93
94 // First, parse timezone designators ahead of time so that we can reject malformed files early
95 const designators = data2[header_size + timecnt * 9 + typecnt * 6 .. header_size + timecnt * 9 + typecnt * 6 + charcnt];
96 if (designators[designators.len - 1] != 0) return error.Malformed; // rfc8536: charcnt [...] includes the trailing NUL (0x00) octet
97
98 // Parse transition types98 // Parse transition types
99 var i: usize = 0;99 var i: usize = 0;
100 while (i < timecnt) : (i += 1) {100 while (i < counts.timecnt) : (i += 1) {
101 transitions[i].ts = std.mem.readIntSliceBig(i64, data2[p .. p + 8]);101 transitions[i].ts = try reader.readIntBig(i64);
102 p += 8;
103 }102 }
104103
105 i = 0;104 i = 0;
106 while (i < timecnt) : (i += 1) {105 while (i < counts.timecnt) : (i += 1) {
107 const tt = data2[p];106 const tt = try reader.readByte();
108 if (tt >= timetypes.len) return error.Malformed; // rfc8536: Each type index MUST be in the range [0, "typecnt" - 1]107 if (tt >= timetypes.len) return error.Malformed; // rfc8536: Each type index MUST be in the range [0, "typecnt" - 1]
109 transitions[i].timetype = &timetypes[tt];108 transitions[i].timetype = &timetypes[tt];
110 p += 1;
111 }109 }
112110
113 // Parse time types111 // Parse time types
114 i = 0;112 i = 0;
115 while (i < typecnt) : (i += 1) {113 while (i < counts.typecnt) : (i += 1) {
116 const offset = std.mem.readIntSliceBig(i32, data2[p .. p + 4]);114 const offset = try reader.readIntBig(i32);
117 if (offset < -2147483648) return error.Malformed; // rfc8536: utoff [...] MUST NOT be -2**31115 if (offset < -2147483648) return error.Malformed; // rfc8536: utoff [...] MUST NOT be -2**31
118 const dst = data2[p + 4];116 const dst = try reader.readByte();
119 if (dst != 0 and dst != 1) return error.Malformed; // rfc8536: (is)dst [...] The value MUST be 0 or 1.117 if (dst != 0 and dst != 1) return error.Malformed; // rfc8536: (is)dst [...] The value MUST be 0 or 1.
120 const idx = data2[p + 5];118 const idx = try reader.readByte();
121 if (idx > designators.len - 1) return error.Malformed; // rfc8536: (desig)idx [...] Each index MUST be in the range [0, "charcnt" - 1]119 if (idx > counts.charcnt - 1) return error.Malformed; // rfc8536: (desig)idx [...] Each index MUST be in the range [0, "charcnt" - 1]
122
123 const name = std.mem.sliceTo(designators[idx..], 0);
124
125 // We are mandating the "SHOULD" 6-character limit so we can pack the struct better, and to conform to POSIX.
126 if (name.len > 6) return error.Malformed; // rfc8536: Time zone designations SHOULD consist of at least three (3) and no more than six (6) ASCII characters.
127
128 timetypes[i] = .{120 timetypes[i] = .{
129 .offset = offset,121 .offset = offset,
130 .flags = dst,122 .flags = dst,
131 .name_data = undefined,123 .name_data = undefined,
132 };124 };
133125
134 std.mem.copy(u8, timetypes[i].name_data[0..], name);126 // Temporarily cache idx in name_data to be processed after we've read the designator names below
135 timetypes[i].name_data[name.len] = 0;127 timetypes[i].name_data[0] = idx;
136
137 p += 6;
138 }128 }
139129
140 // Skip the designators we got earlier130 var designators_data: [256 + 6]u8 = undefined;
141 p += charcnt;131 try reader.readNoEof(designators_data[0..counts.charcnt]);
132 const designators = designators_data[0..counts.charcnt];
133 if (designators[designators.len - 1] != 0) return error.Malformed; // rfc8536: charcnt [...] includes the trailing NUL (0x00) octet
134
135 // Iterate through the timetypes again, setting the designator names
136 for (timetypes) |*tt| {
137 const name = std.mem.sliceTo(designators[tt.name_data[0]..], 0);
138 // We are mandating the "SHOULD" 6-character limit so we can pack the struct better, and to conform to POSIX.
139 if (name.len > 6) return error.Malformed; // rfc8536: Time zone designations SHOULD consist of at least three (3) and no more than six (6) ASCII characters.
140 std.mem.copy(u8, tt.name_data[0..], name);
141 tt.name_data[name.len] = 0;
142 }
142143
143 // Parse leap seconds144 // Parse leap seconds
144 i = 0;145 i = 0;
145 while (i < leapcnt) : (i += 1) {146 while (i < counts.leapcnt) : (i += 1) {
146 const occur = std.mem.readIntSliceBig(i64, data2[p .. p + 8]);147 const occur = try reader.readIntBig(i64);
147 if (occur < 0) return error.Malformed; // rfc8536: occur [...] MUST be nonnegative148 if (occur < 0) return error.Malformed; // rfc8536: occur [...] MUST be nonnegative
148 if (i > 0 and leapseconds[i - 1].occurrence + 2419199 > occur) return error.Malformed; // rfc8536: occur [...] each later value MUST be at least 2419199 greater than the previous value149 if (i > 0 and leapseconds[i - 1].occurrence + 2419199 > occur) return error.Malformed; // rfc8536: occur [...] each later value MUST be at least 2419199 greater than the previous value
149 if (occur > std.math.maxInt(i48)) return error.Malformed; // Unreasonably far into the future150 if (occur > std.math.maxInt(i48)) return error.Malformed; // Unreasonably far into the future
150151
151 const corr = std.mem.readIntSliceBig(i32, data2[p + 8 .. p + 12]);152 const corr = try reader.readIntBig(i32);
152 if (i == 0 and corr != -1 and corr != 1) return error.Malformed; // rfc8536: The correction value in the first leap-second record, if present, MUST be either one (1) or minus one (-1)153 if (i == 0 and corr != -1 and corr != 1) return error.Malformed; // rfc8536: The correction value in the first leap-second record, if present, MUST be either one (1) or minus one (-1)
153 if (i > 0 and leapseconds[i - 1].correction != corr + 1 and leapseconds[i - 1].correction != corr - 1) return error.Malformed; // rfc8536: The correction values in adjacent leap- second records MUST differ by exactly one (1)154 if (i > 0 and leapseconds[i - 1].correction != corr + 1 and leapseconds[i - 1].correction != corr - 1) return error.Malformed; // rfc8536: The correction values in adjacent leap-second records MUST differ by exactly one (1)
154 if (corr > std.math.maxInt(i16)) return error.Malformed; // Unreasonably large correction155 if (corr > std.math.maxInt(i16)) return error.Malformed; // Unreasonably large correction
155156
156 leapseconds[i] = .{157 leapseconds[i] = .{
157 .occurrence = @intCast(i48, occur),158 .occurrence = @intCast(i48, occur),
158 .correction = @intCast(i16, corr),159 .correction = @intCast(i16, corr),
159 };160 };
160 p += 12;
161 }161 }
162162
163 // Parse standard/wall indicators163 // Parse standard/wall indicators
164 i = 0;164 i = 0;
165 while (i < isstdcnt) : (i += 1) {165 while (i < counts.isstdcnt) : (i += 1) {
166 const stdtime = data2[p];166 const stdtime = try reader.readByte();
167 if (stdtime == 1) {167 if (stdtime == 1) {
168 timetypes[i].flags |= 0x02;168 timetypes[i].flags |= 0x02;
169 }169 }
170 p += 1;
171 }170 }
172171
173 // Parse UT/local indicators172 // Parse UT/local indicators
174 i = 0;173 i = 0;
175 while (i < isutcnt) : (i += 1) {174 while (i < counts.isutcnt) : (i += 1) {
176 const ut = data2[p];175 const ut = try reader.readByte();
177 if (ut == 1) {176 if (ut == 1) {
178 timetypes[i].flags |= 0x04;177 timetypes[i].flags |= 0x04;
179 if (!timetypes[i].standardTimeIndicator()) return error.Malformed; // rfc8536: standard/wall value MUST be one (1) if the UT/local value is one (1)178 if (!timetypes[i].standardTimeIndicator()) return error.Malformed; // rfc8536: standard/wall value MUST be one (1) if the UT/local value is one (1)
180 }179 }
181 p += 1;
182 }180 }
183181
182 if ((try reader.readByte()) != '\n') return error.Malformed; // An rfc8536 footer must start with a newline
183
184 // Footer184 // Footer
185 if (data2[p..].len < 2) return error.Malformed; // rfc8536 requires at least 2 newlines185 var footerdata_buf: [128]u8 = undefined;
186 if (data2[p] != '\n') return error.Malformed; // Not a rfc8536 footer186 const footer = reader.readUntilDelimiter(&footerdata_buf, '\n') catch |err| switch (err) {
187 const footer_end = std.mem.indexOfScalar(u8, data2[p + 1 ..], '\n') orelse return error.Malformed; // No 2nd rfc8536 newline187 error.StreamTooLong => return error.OverlargeFooter, // Read more than 128 bytes, much larger than any reasonable POSIX TZ string
188 const footer = try allocator.dupe(u8, data2[p + 1 .. p + 1 + footer_end]);188 else => return err,
189 errdefer allocator.free(footer);189 };
190
191 const footer_dup = try allocator.dupe(u8, footer);
192 errdefer allocator.free(footer_dup);
190193
191 return Tz{194 return Tz{
192 .allocator = allocator,195 .allocator = allocator,
193 .transitions = transitions,196 .transitions = transitions,
194 .timetypes = timetypes,197 .timetypes = timetypes,
195 .leapseconds = leapseconds,198 .leapseconds = leapseconds,
196 .footer = footer,199 .footer = footer_dup,
197 };200 };
198 }201 }
199202
...@@ -205,14 +208,27 @@ pub const Tz = struct {...@@ -205,14 +208,27 @@ pub const Tz = struct {
205 }208 }
206};209};
207210
208test "parse" {211test "slim" {
209 // Asia/Tokyo is good for embedding, as Japan only had DST for a short while during the US occupation
210 const data = @embedFile("tz/asia_tokyo.tzif");212 const data = @embedFile("tz/asia_tokyo.tzif");
211 var tz = try Tz.parse(std.testing.allocator, data);213 var in_stream = std.io.fixedBufferStream(data);
214
215 var tz = try std.Tz.parse(std.testing.allocator, in_stream.reader());
212 defer tz.deinit();216 defer tz.deinit();
213217
214 try std.testing.expectEqual(tz.transitions.len, 9);218 try std.testing.expectEqual(tz.transitions.len, 9);
215 try std.testing.expect(std.mem.eql(u8, tz.transitions[3].timetype.name(), "JDT"));219 try std.testing.expect(std.mem.eql(u8, tz.transitions[3].timetype.name(), "JDT"));
216 try std.testing.expectEqual(tz.transitions[5].ts, -620298000); // 1950-05-06 15:00:00 (UTC)220 try std.testing.expectEqual(tz.transitions[5].ts, -620298000); // 1950-05-06 15:00:00 UTC
217 try std.testing.expectEqual(tz.leapseconds[13].occurrence, 567993613); // 1988-01-01 00:00:13 (IAT)221 try std.testing.expectEqual(tz.leapseconds[13].occurrence, 567993613); // 1988-01-01 00:00:00 UTC (+23s in TAI, and +13 in the data since it doesn't store the initial 10 second offset)
222}
223
224test "fat" {
225 const data = @embedFile("tz/antarctica_davis.tzif");
226 var in_stream = std.io.fixedBufferStream(data);
227
228 var tz = try std.Tz.parse(std.testing.allocator, in_stream.reader());
229 defer tz.deinit();
230
231 try std.testing.expectEqual(tz.transitions.len, 8);
232 try std.testing.expect(std.mem.eql(u8, tz.transitions[3].timetype.name(), "+05"));
233 try std.testing.expectEqual(tz.transitions[4].ts, 1268251224); // 2010-03-10 20:00:00 UTC
218}234}
lib/std/tz/antarctica_davis.tzif created
Binary files /dev/null and b/lib/std/tz/antarctica_davis.tzif differ