authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-14 20:34:44-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-15 10:44:35-07:00
log30b41dc51015c1ed8fa4a7c4f2c61e2a6206ff55
treeab442ad8ac96dc82a6b03ff94e099f705689abc7
parent6d7c6a0f4e4f77e10462c3d8becf4e51fe172ccf

std.compress.zstd.Decompress fixes

* std.Io.Reader: appendRemaining no longer supports alignment and has different rules about how exceeding limit. Fixed bug where it would return success instead of error.StreamTooLong like it was supposed to. * std.Io.Reader: simplify appendRemaining and appendRemainingUnlimited to be implemented based on std.Io.Writer.Allocating * std.Io.Writer: introduce unreachableRebase * std.Io.Writer: remove minimum_unused_capacity from Allocating. maybe that flexibility could have been handy, but let's see if anyone actually needs it. The field is redundant with the superlinear growth of ArrayList capacity. * std.Io.Writer: growingRebase also ensures total capacity on the preserve parameter, making it no longer necessary to do ensureTotalCapacity at the usage site of decompression streams. * std.compress.flate.Decompress: fix rebase not taking into account seek * std.compress.zstd.Decompress: split into "direct" and "indirect" usage patterns depending on whether a buffer is provided to init, matching how flate works. Remove some overzealous asserts that prevented buffer expansion from within rebase implementation. * std.zig: fix readSourceFileToAlloc returning an overaligned slice which was difficult to free correctly. fixes #24608

10 files changed, 166 insertions(+), 152 deletions(-)

lib/std/Io/Reader.zig+38-75
...@@ -8,7 +8,7 @@ const Writer = std.io.Writer;...@@ -8,7 +8,7 @@ const Writer = std.io.Writer;
8const assert = std.debug.assert;8const assert = std.debug.assert;
9const testing = std.testing;9const testing = std.testing;
10const Allocator = std.mem.Allocator;10const Allocator = std.mem.Allocator;
11const ArrayList = std.ArrayListUnmanaged;11const ArrayList = std.ArrayList;
12const Limit = std.io.Limit;12const Limit = std.io.Limit;
1313
14pub const Limited = @import("Reader/Limited.zig");14pub const Limited = @import("Reader/Limited.zig");
...@@ -290,103 +290,63 @@ pub const LimitedAllocError = Allocator.Error || ShortError || error{StreamTooLo...@@ -290,103 +290,63 @@ pub const LimitedAllocError = Allocator.Error || ShortError || error{StreamTooLo
290pub fn allocRemaining(r: *Reader, gpa: Allocator, limit: Limit) LimitedAllocError![]u8 {290pub fn allocRemaining(r: *Reader, gpa: Allocator, limit: Limit) LimitedAllocError![]u8 {
291 var buffer: ArrayList(u8) = .empty;291 var buffer: ArrayList(u8) = .empty;
292 defer buffer.deinit(gpa);292 defer buffer.deinit(gpa);
293 try appendRemaining(r, gpa, null, &buffer, limit);293 try appendRemaining(r, gpa, &buffer, limit);
294 return buffer.toOwnedSlice(gpa);294 return buffer.toOwnedSlice(gpa);
295}295}
296296
297/// Transfers all bytes from the current position to the end of the stream, up297/// Transfers all bytes from the current position to the end of the stream, up
298/// to `limit`, appending them to `list`.298/// to `limit`, appending them to `list`.
299///299///
300/// If `limit` would be exceeded, `error.StreamTooLong` is returned instead. In300/// If `limit` is reached or exceeded, `error.StreamTooLong` is returned
301/// such case, the next byte that would be read will be the first one to exceed301/// instead. In such case, the next byte that would be read will be the first
302/// `limit`, and all preceeding bytes have been appended to `list`.302/// one to exceed `limit`, and all preceeding bytes have been appended to
303///303/// `list`.
304/// If `limit` is not `Limit.unlimited`, asserts `buffer` has nonzero capacity.
305///304///
306/// See also:305/// See also:
307/// * `allocRemaining`306/// * `allocRemaining`
308pub fn appendRemaining(307pub fn appendRemaining(
309 r: *Reader,308 r: *Reader,
310 gpa: Allocator,309 gpa: Allocator,
311 comptime alignment: ?std.mem.Alignment,310 list: *ArrayList(u8),
312 list: *std.ArrayListAlignedUnmanaged(u8, alignment),
313 limit: Limit,311 limit: Limit,
314) LimitedAllocError!void {312) LimitedAllocError!void {
315 if (limit == .unlimited) return appendRemainingUnlimited(r, gpa, alignment, list, 1);313 var a: std.Io.Writer.Allocating = .initOwnedSlice(gpa, list.items);
316 assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data.314 a.writer.end = list.items.len;
317 const buffer_contents = r.buffer[r.seek..r.end];315 list.* = .empty;
318 const copy_len = limit.minInt(buffer_contents.len);316 defer {
319 try list.appendSlice(gpa, r.buffer[0..copy_len]);317 list.* = .{
320 r.seek += copy_len;318 .items = a.writer.buffer[0..a.writer.end],
321 if (buffer_contents.len - copy_len != 0) return error.StreamTooLong;319 .capacity = a.writer.buffer.len,
322 r.seek = 0;320 };
323 r.end = 0;321 }
324 var remaining = @intFromEnum(limit) - copy_len;322 var remaining = limit;
325 // From here, we leave `buffer` empty, appending directly to `list`.323 while (remaining.nonzero()) {
326 var writer: Writer = .{324 const n = stream(r, &a.writer, remaining) catch |err| switch (err) {
327 .buffer = undefined,
328 .end = undefined,
329 .vtable = &.{ .drain = Writer.fixedDrain },
330 };
331 while (true) {
332 try list.ensureUnusedCapacity(gpa, 2);
333 const cap = list.unusedCapacitySlice();
334 const dest = cap[0..@min(cap.len, remaining + 1)];
335 writer.buffer = list.allocatedSlice();
336 writer.end = list.items.len;
337 const n = r.vtable.stream(r, &writer, .limited(dest.len)) catch |err| switch (err) {
338 error.WriteFailed => unreachable, // Prevented by the limit.
339 error.EndOfStream => return,325 error.EndOfStream => return,
326 error.WriteFailed => return error.OutOfMemory,
340 error.ReadFailed => return error.ReadFailed,327 error.ReadFailed => return error.ReadFailed,
341 };328 };
342 list.items.len += n;329 remaining = remaining.subtract(n).?;
343 if (n > remaining) {
344 // Move the byte to `Reader.buffer` so it is not lost.
345 assert(n - remaining == 1);
346 assert(r.end == 0);
347 r.buffer[0] = list.items[list.items.len - 1];
348 list.items.len -= 1;
349 r.end = 1;
350 return;
351 }
352 remaining -= n;
353 }330 }
331 return error.StreamTooLong;
354}332}
355333
356pub const UnlimitedAllocError = Allocator.Error || ShortError;334pub const UnlimitedAllocError = Allocator.Error || ShortError;
357335
358pub fn appendRemainingUnlimited(336pub fn appendRemainingUnlimited(r: *Reader, gpa: Allocator, list: *ArrayList(u8)) UnlimitedAllocError!void {
359 r: *Reader,337 var a: std.Io.Writer.Allocating = .initOwnedSlice(gpa, list.items);
360 gpa: Allocator,338 a.writer.end = list.items.len;
361 comptime alignment: ?std.mem.Alignment,339 list.* = .empty;
362 list: *std.ArrayListAlignedUnmanaged(u8, alignment),340 defer {
363 bump: usize,341 list.* = .{
364) UnlimitedAllocError!void {342 .items = a.writer.buffer[0..a.writer.end],
365 const buffer_contents = r.buffer[r.seek..r.end];343 .capacity = a.writer.buffer.len,
366 try list.ensureUnusedCapacity(gpa, buffer_contents.len + bump);
367 list.appendSliceAssumeCapacity(buffer_contents);
368 // If statement protects `ending`.
369 if (r.end != 0) {
370 r.seek = 0;
371 r.end = 0;
372 }
373 // From here, we leave `buffer` empty, appending directly to `list`.
374 var writer: Writer = .{
375 .buffer = undefined,
376 .end = undefined,
377 .vtable = &.{ .drain = Writer.fixedDrain },
378 };
379 while (true) {
380 try list.ensureUnusedCapacity(gpa, bump);
381 writer.buffer = list.allocatedSlice();
382 writer.end = list.items.len;
383 const n = r.vtable.stream(r, &writer, .limited(list.unusedCapacitySlice().len)) catch |err| switch (err) {
384 error.WriteFailed => unreachable, // Prevented by the limit.
385 error.EndOfStream => return,
386 error.ReadFailed => return error.ReadFailed,
387 };344 };
388 list.items.len += n;
389 }345 }
346 _ = streamRemaining(r, &a.writer) catch |err| switch (err) {
347 error.WriteFailed => return error.OutOfMemory,
348 error.ReadFailed => return error.ReadFailed,
349 };
390}350}
391351
392/// Writes bytes from the internally tracked stream position to `data`.352/// Writes bytes from the internally tracked stream position to `data`.
...@@ -1295,7 +1255,10 @@ fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Resu...@@ -1295,7 +1255,10 @@ fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Resu
12951255
1296/// Ensures `capacity` more data can be buffered without rebasing.1256/// Ensures `capacity` more data can be buffered without rebasing.
1297pub fn rebase(r: *Reader, capacity: usize) RebaseError!void {1257pub fn rebase(r: *Reader, capacity: usize) RebaseError!void {
1298 if (r.end + capacity <= r.buffer.len) return;1258 if (r.end + capacity <= r.buffer.len) {
1259 @branchHint(.likely);
1260 return;
1261 }
1299 return r.vtable.rebase(r, capacity);1262 return r.vtable.rebase(r, capacity);
1300}1263}
13011264
lib/std/Io/Writer.zig+11-8
...@@ -329,7 +329,7 @@ pub fn rebase(w: *Writer, preserve: usize, unused_capacity_len: usize) Error!voi...@@ -329,7 +329,7 @@ pub fn rebase(w: *Writer, preserve: usize, unused_capacity_len: usize) Error!voi
329 @branchHint(.likely);329 @branchHint(.likely);
330 return;330 return;
331 }331 }
332 try w.vtable.rebase(w, preserve, unused_capacity_len);332 return w.vtable.rebase(w, preserve, unused_capacity_len);
333}333}
334334
335pub fn defaultRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void {335pub fn defaultRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void {
...@@ -2349,6 +2349,13 @@ pub fn unreachableDrain(w: *Writer, data: []const []const u8, splat: usize) Erro...@@ -2349,6 +2349,13 @@ pub fn unreachableDrain(w: *Writer, data: []const []const u8, splat: usize) Erro
2349 unreachable;2349 unreachable;
2350}2350}
23512351
2352pub fn unreachableRebase(w: *Writer, preserve: usize, capacity: usize) Error!void {
2353 _ = w;
2354 _ = preserve;
2355 _ = capacity;
2356 unreachable;
2357}
2358
2352/// Provides a `Writer` implementation based on calling `Hasher.update`, sending2359/// Provides a `Writer` implementation based on calling `Hasher.update`, sending
2353/// all data also to an underlying `Writer`.2360/// all data also to an underlying `Writer`.
2354///2361///
...@@ -2489,10 +2496,6 @@ pub fn Hashing(comptime Hasher: type) type {...@@ -2489,10 +2496,6 @@ pub fn Hashing(comptime Hasher: type) type {
2489pub const Allocating = struct {2496pub const Allocating = struct {
2490 allocator: Allocator,2497 allocator: Allocator,
2491 writer: Writer,2498 writer: Writer,
2492 /// Every call to `drain` ensures at least this amount of unused capacity
2493 /// before it returns. This prevents an infinite loop in interface logic
2494 /// that calls `drain`.
2495 minimum_unused_capacity: usize = 1,
24962499
2497 pub fn init(allocator: Allocator) Allocating {2500 pub fn init(allocator: Allocator) Allocating {
2498 return .{2501 return .{
...@@ -2604,13 +2607,12 @@ pub const Allocating = struct {...@@ -2604,13 +2607,12 @@ pub const Allocating = struct {
2604 const gpa = a.allocator;2607 const gpa = a.allocator;
2605 const pattern = data[data.len - 1];2608 const pattern = data[data.len - 1];
2606 const splat_len = pattern.len * splat;2609 const splat_len = pattern.len * splat;
2607 const bump = a.minimum_unused_capacity;
2608 var list = a.toArrayList();2610 var list = a.toArrayList();
2609 defer setArrayList(a, list);2611 defer setArrayList(a, list);
2610 const start_len = list.items.len;2612 const start_len = list.items.len;
2611 assert(data.len != 0);2613 assert(data.len != 0);
2612 for (data) |bytes| {2614 for (data) |bytes| {
2613 list.ensureUnusedCapacity(gpa, bytes.len + splat_len + bump) catch return error.WriteFailed;2615 list.ensureUnusedCapacity(gpa, bytes.len + splat_len + 1) catch return error.WriteFailed;
2614 list.appendSliceAssumeCapacity(bytes);2616 list.appendSliceAssumeCapacity(bytes);
2615 }2617 }
2616 if (splat == 0) {2618 if (splat == 0) {
...@@ -2641,11 +2643,12 @@ pub const Allocating = struct {...@@ -2641,11 +2643,12 @@ pub const Allocating = struct {
2641 }2643 }
26422644
2643 fn growingRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void {2645 fn growingRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void {
2644 _ = preserve; // This implementation always preserves the entire buffer.
2645 const a: *Allocating = @fieldParentPtr("writer", w);2646 const a: *Allocating = @fieldParentPtr("writer", w);
2646 const gpa = a.allocator;2647 const gpa = a.allocator;
2647 var list = a.toArrayList();2648 var list = a.toArrayList();
2648 defer setArrayList(a, list);2649 defer setArrayList(a, list);
2650 const total = std.math.add(usize, preserve, minimum_len) catch return error.WriteFailed;
2651 list.ensureTotalCapacity(gpa, total) catch return error.WriteFailed;
2649 list.ensureUnusedCapacity(gpa, minimum_len) catch return error.WriteFailed;2652 list.ensureUnusedCapacity(gpa, minimum_len) catch return error.WriteFailed;
2650 }2653 }
26512654
lib/std/array_list.zig+1-1
...@@ -1033,7 +1033,7 @@ pub fn Aligned(comptime T: type, comptime alignment: ?mem.Alignment) type {...@@ -1033,7 +1033,7 @@ pub fn Aligned(comptime T: type, comptime alignment: ?mem.Alignment) type {
1033 pub fn print(self: *Self, gpa: Allocator, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void {1033 pub fn print(self: *Self, gpa: Allocator, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void {
1034 comptime assert(T == u8);1034 comptime assert(T == u8);
1035 try self.ensureUnusedCapacity(gpa, fmt.len);1035 try self.ensureUnusedCapacity(gpa, fmt.len);
1036 var aw: std.io.Writer.Allocating = .fromArrayList(gpa, self);1036 var aw: std.Io.Writer.Allocating = .fromArrayList(gpa, self);
1037 defer self.* = aw.toArrayList();1037 defer self.* = aw.toArrayList();
1038 return aw.writer.print(fmt, args) catch |err| switch (err) {1038 return aw.writer.print(fmt, args) catch |err| switch (err) {
1039 error.WriteFailed => return error.OutOfMemory,1039 error.WriteFailed => return error.OutOfMemory,
lib/std/compress/flate/Decompress.zig+8-10
...@@ -62,7 +62,7 @@ pub const Error = Container.Error || error{...@@ -62,7 +62,7 @@ pub const Error = Container.Error || error{
62const direct_vtable: Reader.VTable = .{62const direct_vtable: Reader.VTable = .{
63 .stream = streamDirect,63 .stream = streamDirect,
64 .rebase = rebaseFallible,64 .rebase = rebaseFallible,
65 .discard = discard,65 .discard = discardDirect,
66 .readVec = readVec,66 .readVec = readVec,
67};67};
6868
...@@ -105,17 +105,16 @@ fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void {...@@ -105,17 +105,16 @@ fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void {
105fn rebase(r: *Reader, capacity: usize) void {105fn rebase(r: *Reader, capacity: usize) void {
106 assert(capacity <= r.buffer.len - flate.history_len);106 assert(capacity <= r.buffer.len - flate.history_len);
107 assert(r.end + capacity > r.buffer.len);107 assert(r.end + capacity > r.buffer.len);
108 const discard_n = r.end - flate.history_len;108 const discard_n = @min(r.seek, r.end - flate.history_len);
109 const keep = r.buffer[discard_n..r.end];109 const keep = r.buffer[discard_n..r.end];
110 @memmove(r.buffer[0..keep.len], keep);110 @memmove(r.buffer[0..keep.len], keep);
111 assert(keep.len != 0);
112 r.end = keep.len;111 r.end = keep.len;
113 r.seek -= discard_n;112 r.seek -= discard_n;
114}113}
115114
116/// This could be improved so that when an amount is discarded that includes an115/// This could be improved so that when an amount is discarded that includes an
117/// entire frame, skip decoding that frame.116/// entire frame, skip decoding that frame.
118fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {117fn discardDirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
119 if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);118 if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
120 var writer: Writer = .{119 var writer: Writer = .{
121 .vtable = &.{120 .vtable = &.{
...@@ -167,11 +166,14 @@ fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {...@@ -167,11 +166,14 @@ fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
167166
168fn streamIndirectInner(d: *Decompress) Reader.Error!usize {167fn streamIndirectInner(d: *Decompress) Reader.Error!usize {
169 const r = &d.reader;168 const r = &d.reader;
170 if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);169 if (r.buffer.len - r.end < flate.history_len) rebase(r, flate.history_len);
171 var writer: Writer = .{170 var writer: Writer = .{
172 .buffer = r.buffer,171 .buffer = r.buffer,
173 .end = r.end,172 .end = r.end,
174 .vtable = &.{ .drain = Writer.unreachableDrain },173 .vtable = &.{
174 .drain = Writer.unreachableDrain,
175 .rebase = Writer.unreachableRebase,
176 },
175 };177 };
176 defer r.end = writer.end;178 defer r.end = writer.end;
177 _ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) {179 _ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) {
...@@ -1251,8 +1253,6 @@ test "zlib should not overshoot" {...@@ -1251,8 +1253,6 @@ test "zlib should not overshoot" {
1251fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !void {1253fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !void {
1252 var reader: Reader = .fixed(in);1254 var reader: Reader = .fixed(in);
1253 var aw: Writer.Allocating = .init(testing.allocator);1255 var aw: Writer.Allocating = .init(testing.allocator);
1254 aw.minimum_unused_capacity = flate.history_len;
1255 try aw.ensureUnusedCapacity(flate.max_window_len);
1256 defer aw.deinit();1256 defer aw.deinit();
12571257
1258 var decompress: Decompress = .init(&reader, container, &.{});1258 var decompress: Decompress = .init(&reader, container, &.{});
...@@ -1263,8 +1263,6 @@ fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !vo...@@ -1263,8 +1263,6 @@ fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !vo
1263fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {1263fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
1264 var in: std.Io.Reader = .fixed(compressed);1264 var in: std.Io.Reader = .fixed(compressed);
1265 var aw: std.Io.Writer.Allocating = .init(testing.allocator);1265 var aw: std.Io.Writer.Allocating = .init(testing.allocator);
1266 aw.minimum_unused_capacity = flate.history_len;
1267 try aw.ensureUnusedCapacity(flate.max_window_len);
1268 defer aw.deinit();1266 defer aw.deinit();
12691267
1270 var decompress: Decompress = .init(&in, container, &.{});1268 var decompress: Decompress = .init(&in, container, &.{});
lib/std/compress/zstd.zig+9-11
...@@ -78,15 +78,14 @@ pub const table_size_max = struct {...@@ -78,15 +78,14 @@ pub const table_size_max = struct {
78};78};
7979
80fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {80fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
81 var out: std.ArrayListUnmanaged(u8) = .empty;81 var out: std.Io.Writer.Allocating = .init(gpa);
82 defer out.deinit(gpa);82 defer out.deinit();
83 try out.ensureUnusedCapacity(gpa, default_window_len);
8483
85 var in: std.io.Reader = .fixed(compressed);84 var in: std.Io.Reader = .fixed(compressed);
86 var zstd_stream: Decompress = .init(&in, &.{}, .{});85 var zstd_stream: Decompress = .init(&in, &.{}, .{});
87 try zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited);86 _ = try zstd_stream.reader.streamRemaining(&out.writer);
8887
89 return out.toOwnedSlice(gpa);88 return out.toOwnedSlice();
90}89}
9190
92fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {91fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {
...@@ -99,15 +98,14 @@ fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void...@@ -99,15 +98,14 @@ fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void
99fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {98fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
100 const gpa = std.testing.allocator;99 const gpa = std.testing.allocator;
101100
102 var out: std.ArrayListUnmanaged(u8) = .empty;101 var out: std.Io.Writer.Allocating = .init(gpa);
103 defer out.deinit(gpa);102 defer out.deinit();
104 try out.ensureUnusedCapacity(gpa, default_window_len);
105103
106 var in: std.io.Reader = .fixed(compressed);104 var in: std.Io.Reader = .fixed(compressed);
107 var zstd_stream: Decompress = .init(&in, &.{}, .{});105 var zstd_stream: Decompress = .init(&in, &.{}, .{});
108 try std.testing.expectError(106 try std.testing.expectError(
109 error.ReadFailed,107 error.ReadFailed,
110 zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited),108 zstd_stream.reader.streamRemaining(&out.writer),
111 );109 );
112 try std.testing.expectError(err, zstd_stream.err orelse {});110 try std.testing.expectError(err, zstd_stream.err orelse {});
113}111}
lib/std/compress/zstd/Decompress.zig+72-21
...@@ -73,6 +73,20 @@ pub const Error = error{...@@ -73,6 +73,20 @@ pub const Error = error{
73 WindowSizeUnknown,73 WindowSizeUnknown,
74};74};
7575
76const direct_vtable: Reader.VTable = .{
77 .stream = streamDirect,
78 .rebase = rebaseFallible,
79 .discard = discardDirect,
80 .readVec = readVec,
81};
82
83const indirect_vtable: Reader.VTable = .{
84 .stream = streamIndirect,
85 .rebase = rebaseFallible,
86 .discard = discardIndirect,
87 .readVec = readVec,
88};
89
76/// When connecting `reader` to a `Writer`, `buffer` should be empty, and90/// When connecting `reader` to a `Writer`, `buffer` should be empty, and
77/// `Writer.buffer` capacity has requirements based on `Options.window_len`.91/// `Writer.buffer` capacity has requirements based on `Options.window_len`.
78///92///
...@@ -84,12 +98,7 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {...@@ -84,12 +98,7 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
84 .verify_checksum = options.verify_checksum,98 .verify_checksum = options.verify_checksum,
85 .window_len = options.window_len,99 .window_len = options.window_len,
86 .reader = .{100 .reader = .{
87 .vtable = &.{101 .vtable = if (buffer.len == 0) &direct_vtable else &indirect_vtable,
88 .stream = stream,
89 .rebase = rebase,
90 .discard = discard,
91 .readVec = readVec,
92 },
93 .buffer = buffer,102 .buffer = buffer,
94 .seek = 0,103 .seek = 0,
95 .end = 0,104 .end = 0,
...@@ -97,11 +106,27 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {...@@ -97,11 +106,27 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
97 };106 };
98}107}
99108
100fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void {109fn streamDirect(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
110 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
111 return stream(d, w, limit);
112}
113
114fn streamIndirect(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
115 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
116 _ = limit;
117 _ = w;
118 return streamIndirectInner(d);
119}
120
121fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void {
122 rebase(r, capacity);
123}
124
125fn rebase(r: *Reader, capacity: usize) void {
101 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));126 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
102 assert(capacity <= r.buffer.len - d.window_len);127 assert(capacity <= r.buffer.len - d.window_len);
103 assert(r.end + capacity > r.buffer.len);128 assert(r.end + capacity > r.buffer.len);
104 const discard_n = r.end - d.window_len;129 const discard_n = @min(r.seek, r.end - d.window_len);
105 const keep = r.buffer[discard_n..r.end];130 const keep = r.buffer[discard_n..r.end];
106 @memmove(r.buffer[0..keep.len], keep);131 @memmove(r.buffer[0..keep.len], keep);
107 r.end = keep.len;132 r.end = keep.len;
...@@ -110,9 +135,9 @@ fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void {...@@ -110,9 +135,9 @@ fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void {
110135
111/// This could be improved so that when an amount is discarded that includes an136/// This could be improved so that when an amount is discarded that includes an
112/// entire frame, skip decoding that frame.137/// entire frame, skip decoding that frame.
113fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {138fn discardDirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
114 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));139 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
115 r.rebase(d.window_len) catch unreachable;140 rebase(r, d.window_len);
116 var writer: Writer = .{141 var writer: Writer = .{
117 .vtable = &.{142 .vtable = &.{
118 .drain = std.Io.Writer.Discarding.drain,143 .drain = std.Io.Writer.Discarding.drain,
...@@ -134,25 +159,53 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {...@@ -134,25 +159,53 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
134 return n;159 return n;
135}160}
136161
162fn discardIndirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
163 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
164 rebase(r, d.window_len);
165 var writer: Writer = .{
166 .buffer = r.buffer,
167 .end = r.end,
168 .vtable = &.{ .drain = Writer.unreachableDrain },
169 };
170 {
171 defer r.end = writer.end;
172 _ = stream(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) {
173 error.WriteFailed => unreachable,
174 else => |e| return e,
175 };
176 }
177 const n = limit.minInt(r.end - r.seek);
178 r.seek += n;
179 return n;
180}
181
137fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {182fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
138 _ = data;183 _ = data;
139 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));184 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
140 assert(r.seek == r.end);185 return streamIndirectInner(d);
141 r.rebase(d.window_len) catch unreachable;186}
187
188fn streamIndirectInner(d: *Decompress) Reader.Error!usize {
189 const r = &d.reader;
190 if (r.buffer.len - r.end < zstd.block_size_max) rebase(r, zstd.block_size_max);
191 assert(r.buffer.len - r.end >= zstd.block_size_max);
142 var writer: Writer = .{192 var writer: Writer = .{
143 .buffer = r.buffer,193 .buffer = r.buffer,
144 .end = r.end,194 .end = r.end,
145 .vtable = &.{ .drain = Writer.fixedDrain },195 .vtable = &.{
196 .drain = Writer.unreachableDrain,
197 .rebase = Writer.unreachableRebase,
198 },
146 };199 };
147 r.end += r.vtable.stream(r, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) {200 defer r.end = writer.end;
201 _ = stream(d, &writer, .limited(writer.buffer.len - writer.end)) catch |err| switch (err) {
148 error.WriteFailed => unreachable,202 error.WriteFailed => unreachable,
149 else => |e| return e,203 else => |e| return e,
150 };204 };
151 return 0;205 return 0;
152}206}
153207
154fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {208fn stream(d: *Decompress, w: *Writer, limit: Limit) Reader.StreamError!usize {
155 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
156 const in = d.input;209 const in = d.input;
157210
158 state: switch (d.state) {211 state: switch (d.state) {
...@@ -170,7 +223,7 @@ fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {...@@ -170,7 +223,7 @@ fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {
170 else => |e| return e,223 else => |e| return e,
171 };224 };
172 const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little);225 const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little);
173 initFrame(d, w.buffer.len, magic) catch |err| {226 initFrame(d, magic) catch |err| {
174 d.err = err;227 d.err = err;
175 return error.ReadFailed;228 return error.ReadFailed;
176 };229 };
...@@ -198,13 +251,13 @@ fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {...@@ -198,13 +251,13 @@ fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {
198 }251 }
199}252}
200253
201fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {254fn initFrame(d: *Decompress, magic: Frame.Magic) !void {
202 const in = d.input;255 const in = d.input;
203 switch (magic.kind() orelse return error.BadMagic) {256 switch (magic.kind() orelse return error.BadMagic) {
204 .zstandard => {257 .zstandard => {
205 const header = try Frame.Zstandard.Header.decode(in);258 const header = try Frame.Zstandard.Header.decode(in);
206 d.state = .{ .in_frame = .{259 d.state = .{ .in_frame = .{
207 .frame = try Frame.init(header, window_size_max, d.verify_checksum),260 .frame = try Frame.init(header, d.window_len, d.verify_checksum),
208 .checksum = null,261 .checksum = null,
209 .decompressed_size = 0,262 .decompressed_size = 0,
210 .decode = .init,263 .decode = .init,
...@@ -258,7 +311,6 @@ fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame)...@@ -258,7 +311,6 @@ fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame)
258 try decode.readInitialFseState(&bit_stream);311 try decode.readInitialFseState(&bit_stream);
259312
260 // Ensures the following calls to `decodeSequence` will not flush.313 // Ensures the following calls to `decodeSequence` will not flush.
261 if (window_len + frame_block_size_max > w.buffer.len) return error.OutputBufferUndersize;
262 const dest = (try w.writableSliceGreedyPreserve(window_len, frame_block_size_max))[0..frame_block_size_max];314 const dest = (try w.writableSliceGreedyPreserve(window_len, frame_block_size_max))[0..frame_block_size_max];
263 const write_pos = dest.ptr - w.buffer.ptr;315 const write_pos = dest.ptr - w.buffer.ptr;
264 for (0..sequences_header.sequence_count - 1) |_| {316 for (0..sequences_header.sequence_count - 1) |_| {
...@@ -775,7 +827,6 @@ pub const Frame = struct {...@@ -775,7 +827,6 @@ pub const Frame = struct {
775 try w.splatByteAll(d.literal_streams.one[0], len);827 try w.splatByteAll(d.literal_streams.one[0], len);
776 },828 },
777 .compressed, .treeless => {829 .compressed, .treeless => {
778 if (len > w.buffer.len) return error.OutputBufferUndersize;
779 const buf = try w.writableSlice(len);830 const buf = try w.writableSlice(len);
780 const huffman_tree = d.huffman_tree.?;831 const huffman_tree = d.huffman_tree.?;
781 const max_bit_count = huffman_tree.max_bit_count;832 const max_bit_count = huffman_tree.max_bit_count;
lib/std/debug/Dwarf.zig+1-1
...@@ -2247,7 +2247,7 @@ pub const ElfModule = struct {...@@ -2247,7 +2247,7 @@ pub const ElfModule = struct {
2247 var decompress: std.compress.flate.Decompress = .init(&section_reader, .zlib, &.{});2247 var decompress: std.compress.flate.Decompress = .init(&section_reader, .zlib, &.{});
2248 var decompressed_section: ArrayList(u8) = .empty;2248 var decompressed_section: ArrayList(u8) = .empty;
2249 defer decompressed_section.deinit(gpa);2249 defer decompressed_section.deinit(gpa);
2250 decompress.reader.appendRemainingUnlimited(gpa, null, &decompressed_section, std.compress.flate.history_len) catch {2250 decompress.reader.appendRemainingUnlimited(gpa, &decompressed_section) catch {
2251 invalidDebugInfoDetected();2251 invalidDebugInfoDetected();
2252 continue;2252 continue;
2253 };2253 };
lib/std/http/test.zig+4-7
...@@ -149,9 +149,8 @@ test "HTTP server handles a chunked transfer coding request" {...@@ -149,9 +149,8 @@ test "HTTP server handles a chunked transfer coding request" {
149 "content-type: text/plain\r\n" ++149 "content-type: text/plain\r\n" ++
150 "\r\n" ++150 "\r\n" ++
151 "message from server!\n";151 "message from server!\n";
152 var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded152 var stream_reader = stream.reader(&.{});
153 var stream_reader = stream.reader(&tiny_buffer);153 const response = try stream_reader.interface().allocRemaining(gpa, .limited(expected_response.len + 1));
154 const response = try stream_reader.interface().allocRemaining(gpa, .limited(expected_response.len));
155 defer gpa.free(response);154 defer gpa.free(response);
156 try expectEqualStrings(expected_response, response);155 try expectEqualStrings(expected_response, response);
157}156}
...@@ -293,8 +292,7 @@ test "Server.Request.respondStreaming non-chunked, unknown content-length" {...@@ -293,8 +292,7 @@ test "Server.Request.respondStreaming non-chunked, unknown content-length" {
293 var stream_writer = stream.writer(&.{});292 var stream_writer = stream.writer(&.{});
294 try stream_writer.interface.writeAll(request_bytes);293 try stream_writer.interface.writeAll(request_bytes);
295294
296 var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded295 var stream_reader = stream.reader(&.{});
297 var stream_reader = stream.reader(&tiny_buffer);
298 const response = try stream_reader.interface().allocRemaining(gpa, .unlimited);296 const response = try stream_reader.interface().allocRemaining(gpa, .unlimited);
299 defer gpa.free(response);297 defer gpa.free(response);
300298
...@@ -364,8 +362,7 @@ test "receiving arbitrary http headers from the client" {...@@ -364,8 +362,7 @@ test "receiving arbitrary http headers from the client" {
364 var stream_writer = stream.writer(&.{});362 var stream_writer = stream.writer(&.{});
365 try stream_writer.interface.writeAll(request_bytes);363 try stream_writer.interface.writeAll(request_bytes);
366364
367 var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded365 var stream_reader = stream.reader(&.{});
368 var stream_reader = stream.reader(&tiny_buffer);
369 const response = try stream_reader.interface().allocRemaining(gpa, .unlimited);366 const response = try stream_reader.interface().allocRemaining(gpa, .unlimited);
370 defer gpa.free(response);367 defer gpa.free(response);
371368
lib/std/unicode.zig+16-15
...@@ -4,6 +4,7 @@ const assert = std.debug.assert;...@@ -4,6 +4,7 @@ const assert = std.debug.assert;
4const testing = std.testing;4const testing = std.testing;
5const mem = std.mem;5const mem = std.mem;
6const native_endian = builtin.cpu.arch.endian();6const native_endian = builtin.cpu.arch.endian();
7const Allocator = std.mem.Allocator;
78
8/// Use this to replace an unknown, unrecognized, or unrepresentable character.9/// Use this to replace an unknown, unrecognized, or unrepresentable character.
9///10///
...@@ -921,7 +922,7 @@ fn utf16LeToUtf8ArrayListImpl(...@@ -921,7 +922,7 @@ fn utf16LeToUtf8ArrayListImpl(
921 comptime surrogates: Surrogates,922 comptime surrogates: Surrogates,
922) (switch (surrogates) {923) (switch (surrogates) {
923 .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,924 .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
924 .can_encode_surrogate_half => mem.Allocator.Error,925 .can_encode_surrogate_half => Allocator.Error,
925})!void {926})!void {
926 assert(result.unusedCapacitySlice().len >= utf16le.len);927 assert(result.unusedCapacitySlice().len >= utf16le.len);
927928
...@@ -965,15 +966,15 @@ fn utf16LeToUtf8ArrayListImpl(...@@ -965,15 +966,15 @@ fn utf16LeToUtf8ArrayListImpl(
965 }966 }
966}967}
967968
968pub const Utf16LeToUtf8AllocError = mem.Allocator.Error || Utf16LeToUtf8Error;969pub const Utf16LeToUtf8AllocError = Allocator.Error || Utf16LeToUtf8Error;
969970
970pub fn utf16LeToUtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {971pub fn utf16LeToUtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
971 try result.ensureUnusedCapacity(utf16le.len);972 try result.ensureUnusedCapacity(utf16le.len);
972 return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);973 return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
973}974}
974975
975/// Caller must free returned memory.976/// Caller owns returned memory.
976pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8 {977pub fn utf16LeToUtf8Alloc(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8 {
977 // optimistically guess that it will all be ascii.978 // optimistically guess that it will all be ascii.
978 var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len);979 var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len);
979 errdefer result.deinit();980 errdefer result.deinit();
...@@ -982,8 +983,8 @@ pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16L...@@ -982,8 +983,8 @@ pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16L
982 return result.toOwnedSlice();983 return result.toOwnedSlice();
983}984}
984985
985/// Caller must free returned memory.986/// Caller owns returned memory.
986pub fn utf16LeToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8 {987pub fn utf16LeToUtf8AllocZ(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8 {
987 // optimistically guess that it will all be ascii (and allocate space for the null terminator)988 // optimistically guess that it will all be ascii (and allocate space for the null terminator)
988 var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len + 1);989 var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len + 1);
989 errdefer result.deinit();990 errdefer result.deinit();
...@@ -1160,7 +1161,7 @@ pub fn utf8ToUtf16LeArrayList(result: *std.array_list.Managed(u16), utf8: []cons...@@ -1160,7 +1161,7 @@ pub fn utf8ToUtf16LeArrayList(result: *std.array_list.Managed(u16), utf8: []cons
1160 return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);1161 return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
1161}1162}
11621163
1163pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {1164pub fn utf8ToUtf16LeAlloc(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {
1164 // optimistically guess that it will not require surrogate pairs1165 // optimistically guess that it will not require surrogate pairs
1165 var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len);1166 var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len);
1166 errdefer result.deinit();1167 errdefer result.deinit();
...@@ -1169,7 +1170,7 @@ pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ Inv...@@ -1169,7 +1170,7 @@ pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ Inv
1169 return result.toOwnedSlice();1170 return result.toOwnedSlice();
1170}1171}
11711172
1172pub fn utf8ToUtf16LeAllocZ(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16 {1173pub fn utf8ToUtf16LeAllocZ(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16 {
1173 // optimistically guess that it will not require surrogate pairs1174 // optimistically guess that it will not require surrogate pairs
1174 var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len + 1);1175 var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len + 1);
1175 errdefer result.deinit();1176 errdefer result.deinit();
...@@ -1750,13 +1751,13 @@ pub const Wtf8Iterator = struct {...@@ -1750,13 +1751,13 @@ pub const Wtf8Iterator = struct {
1750 }1751 }
1751};1752};
17521753
1753pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) mem.Allocator.Error!void {1754pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Allocator.Error!void {
1754 try result.ensureUnusedCapacity(utf16le.len);1755 try result.ensureUnusedCapacity(utf16le.len);
1755 return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);1756 return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
1756}1757}
17571758
1758/// Caller must free returned memory.1759/// Caller must free returned memory.
1759pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Allocator.Error![]u8 {1760pub fn wtf16LeToWtf8Alloc(allocator: Allocator, wtf16le: []const u16) Allocator.Error![]u8 {
1760 // optimistically guess that it will all be ascii.1761 // optimistically guess that it will all be ascii.
1761 var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len);1762 var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len);
1762 errdefer result.deinit();1763 errdefer result.deinit();
...@@ -1766,7 +1767,7 @@ pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Al...@@ -1766,7 +1767,7 @@ pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Al
1766}1767}
17671768
1768/// Caller must free returned memory.1769/// Caller must free returned memory.
1769pub fn wtf16LeToWtf8AllocZ(allocator: mem.Allocator, wtf16le: []const u16) mem.Allocator.Error![:0]u8 {1770pub fn wtf16LeToWtf8AllocZ(allocator: Allocator, wtf16le: []const u16) Allocator.Error![:0]u8 {
1770 // optimistically guess that it will all be ascii (and allocate space for the null terminator)1771 // optimistically guess that it will all be ascii (and allocate space for the null terminator)
1771 var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len + 1);1772 var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len + 1);
1772 errdefer result.deinit();1773 errdefer result.deinit();
...@@ -1784,7 +1785,7 @@ pub fn wtf8ToWtf16LeArrayList(result: *std.array_list.Managed(u16), wtf8: []cons...@@ -1784,7 +1785,7 @@ pub fn wtf8ToWtf16LeArrayList(result: *std.array_list.Managed(u16), wtf8: []cons
1784 return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);1785 return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
1785}1786}
17861787
1787pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {1788pub fn wtf8ToWtf16LeAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {
1788 // optimistically guess that it will not require surrogate pairs1789 // optimistically guess that it will not require surrogate pairs
1789 var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len);1790 var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len);
1790 errdefer result.deinit();1791 errdefer result.deinit();
...@@ -1793,7 +1794,7 @@ pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ Inv...@@ -1793,7 +1794,7 @@ pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ Inv
1793 return result.toOwnedSlice();1794 return result.toOwnedSlice();
1794}1795}
17951796
1796pub fn wtf8ToWtf16LeAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16 {1797pub fn wtf8ToWtf16LeAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16 {
1797 // optimistically guess that it will not require surrogate pairs1798 // optimistically guess that it will not require surrogate pairs
1798 var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len + 1);1799 var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len + 1);
1799 errdefer result.deinit();1800 errdefer result.deinit();
...@@ -1870,7 +1871,7 @@ pub fn wtf8ToUtf8Lossy(utf8: []u8, wtf8: []const u8) error{InvalidWtf8}!void {...@@ -1870,7 +1871,7 @@ pub fn wtf8ToUtf8Lossy(utf8: []u8, wtf8: []const u8) error{InvalidWtf8}!void {
1870 }1871 }
1871}1872}
18721873
1873pub fn wtf8ToUtf8LossyAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8 {1874pub fn wtf8ToUtf8LossyAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8 {
1874 const utf8 = try allocator.alloc(u8, wtf8.len);1875 const utf8 = try allocator.alloc(u8, wtf8.len);
1875 errdefer allocator.free(utf8);1876 errdefer allocator.free(utf8);
18761877
...@@ -1879,7 +1880,7 @@ pub fn wtf8ToUtf8LossyAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ I...@@ -1879,7 +1880,7 @@ pub fn wtf8ToUtf8LossyAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ I
1879 return utf8;1880 return utf8;
1880}1881}
18811882
1882pub fn wtf8ToUtf8LossyAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8 {1883pub fn wtf8ToUtf8LossyAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8 {
1883 const utf8 = try allocator.allocSentinel(u8, wtf8.len, 0);1884 const utf8 = try allocator.allocSentinel(u8, wtf8.len, 0);
1884 errdefer allocator.free(utf8);1885 errdefer allocator.free(utf8);
18851886
lib/std/zig.zig+6-3
...@@ -554,8 +554,11 @@ test isUnderscore {...@@ -554,8 +554,11 @@ test isUnderscore {
554 try std.testing.expect(!isUnderscore("\\x5f"));554 try std.testing.expect(!isUnderscore("\\x5f"));
555}555}
556556
557/// If the source can be UTF-16LE encoded, this function asserts that `gpa`
558/// will align a byte-sized allocation to at least 2. Allocators that don't do
559/// this are rare.
557pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader) ![:0]u8 {560pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader) ![:0]u8 {
558 var buffer: std.ArrayListAlignedUnmanaged(u8, .@"2") = .empty;561 var buffer: std.ArrayList(u8) = .empty;
559 defer buffer.deinit(gpa);562 defer buffer.deinit(gpa);
560563
561 if (file_reader.getSize()) |size| {564 if (file_reader.getSize()) |size| {
...@@ -564,7 +567,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader...@@ -564,7 +567,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader
564 try buffer.ensureTotalCapacityPrecise(gpa, casted_size + 1);567 try buffer.ensureTotalCapacityPrecise(gpa, casted_size + 1);
565 } else |_| {}568 } else |_| {}
566569
567 try file_reader.interface.appendRemaining(gpa, .@"2", &buffer, .limited(max_src_size));570 try file_reader.interface.appendRemaining(gpa, &buffer, .limited(max_src_size));
568571
569 // Detect unsupported file types with their Byte Order Mark572 // Detect unsupported file types with their Byte Order Mark
570 const unsupported_boms = [_][]const u8{573 const unsupported_boms = [_][]const u8{
...@@ -581,7 +584,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader...@@ -581,7 +584,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader
581 // If the file starts with a UTF-16 little endian BOM, translate it to UTF-8584 // If the file starts with a UTF-16 little endian BOM, translate it to UTF-8
582 if (std.mem.startsWith(u8, buffer.items, "\xff\xfe")) {585 if (std.mem.startsWith(u8, buffer.items, "\xff\xfe")) {
583 if (buffer.items.len % 2 != 0) return error.InvalidEncoding;586 if (buffer.items.len % 2 != 0) return error.InvalidEncoding;
584 return std.unicode.utf16LeToUtf8AllocZ(gpa, @ptrCast(buffer.items)) catch |err| switch (err) {587 return std.unicode.utf16LeToUtf8AllocZ(gpa, @ptrCast(@alignCast(buffer.items))) catch |err| switch (err) {
585 error.DanglingSurrogateHalf => error.UnsupportedEncoding,588 error.DanglingSurrogateHalf => error.UnsupportedEncoding,
586 error.ExpectedSecondSurrogateHalf => error.UnsupportedEncoding,589 error.ExpectedSecondSurrogateHalf => error.UnsupportedEncoding,
587 error.UnexpectedSecondSurrogateHalf => error.UnsupportedEncoding,590 error.UnexpectedSecondSurrogateHalf => error.UnsupportedEncoding,