Commit 30b41dc51015c1ed8fa4a7c4f2c61e2a6206ff55 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-08-14 20:34:44-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-08-15 10:44:35-07:00
log	30b41dc51015c1ed8fa4a7c4f2c61e2a6206ff55
tree	ab442ad8ac96dc82a6b03ff94e099f705689abc7
parent	6d7c6a0f4e4f77e10462c3d8becf4e51fe172ccf

std.compress.zstd.Decompress fixes

* std.Io.Reader: appendRemaining no longer supports alignment and has
  different rules about how exceeding limit. Fixed bug where it would
  return success instead of error.StreamTooLong like it was supposed to.

* std.Io.Reader: simplify appendRemaining and appendRemainingUnlimited
  to be implemented based on std.Io.Writer.Allocating

* std.Io.Writer: introduce unreachableRebase

* std.Io.Writer: remove minimum_unused_capacity from Allocating. maybe
  that flexibility could have been handy, but let's see if anyone
  actually needs it. The field is redundant with the superlinear growth
  of ArrayList capacity.

* std.Io.Writer: growingRebase also ensures total capacity on the
  preserve parameter, making it no longer necessary to do
  ensureTotalCapacity at the usage site of decompression streams.

* std.compress.flate.Decompress: fix rebase not taking into account seek

* std.compress.zstd.Decompress: split into "direct" and "indirect" usage
  patterns depending on whether a buffer is provided to init, matching
  how flate works. Remove some overzealous asserts that prevented buffer
  expansion from within rebase implementation.

* std.zig: fix readSourceFileToAlloc returning an overaligned slice
  which was difficult to free correctly.

fixes #24608

10 files changed, 166 insertions(+), 152 deletions(-)

lib
- std
  - Io
    Reader.zig
    Writer.zig
  - array_list.zig
  - compress
    flate
    Decompress.zig
    zstd.zig
    zstd
    Decompress.zig
  - debug
    Dwarf.zig
  - http
    test.zig
  - unicode.zig
  - zig.zig

lib/std/Io/Reader.zig+38-75

...	...	@@ -8,7 +8,7 @@ const Writer = std.io.Writer;
8	8	const assert = std.debug.assert;
9	9	const testing = std.testing;
10	10	const Allocator = std.mem.Allocator;
11		const ArrayList = std.ArrayListUnmanaged;
	11	const ArrayList = std.ArrayList;
12	12	const Limit = std.io.Limit;
13	13
14	14	pub const Limited = @import("Reader/Limited.zig");
...	...	@@ -290,103 +290,63 @@ pub const LimitedAllocError = Allocator.Error \|\| ShortError \|\| error{StreamTooLo
290	290	pub fn allocRemaining(r: *Reader, gpa: Allocator, limit: Limit) LimitedAllocError![]u8 {
291	291	var buffer: ArrayList(u8) = .empty;
292	292	defer buffer.deinit(gpa);
293		try appendRemaining(r, gpa, null, &buffer, limit);
	293	try appendRemaining(r, gpa, &buffer, limit);
294	294	return buffer.toOwnedSlice(gpa);
295	295	}
296	296
297	297	/// Transfers all bytes from the current position to the end of the stream, up
298	298	/// to `limit`, appending them to `list`.
299	299	///
300		/// If `limit` would be exceeded, `error.StreamTooLong` is returned instead. In
301		/// such case, the next byte that would be read will be the first one to exceed
302		/// `limit`, and all preceeding bytes have been appended to `list`.
303		///
304		/// If `limit` is not `Limit.unlimited`, asserts `buffer` has nonzero capacity.
	300	/// If `limit` is reached or exceeded, `error.StreamTooLong` is returned
	301	/// instead. In such case, the next byte that would be read will be the first
	302	/// one to exceed `limit`, and all preceeding bytes have been appended to
	303	/// `list`.
305	304	///
306	305	/// See also:
307	306	/// * `allocRemaining`
308	307	pub fn appendRemaining(
309	308	r: *Reader,
310	309	gpa: Allocator,
311		comptime alignment: ?std.mem.Alignment,
312		list: *std.ArrayListAlignedUnmanaged(u8, alignment),
	310	list: *ArrayList(u8),
313	311	limit: Limit,
314	312	) LimitedAllocError!void {
315		if (limit == .unlimited) return appendRemainingUnlimited(r, gpa, alignment, list, 1);
316		assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data.
317		const buffer_contents = r.buffer[r.seek..r.end];
318		const copy_len = limit.minInt(buffer_contents.len);
319		try list.appendSlice(gpa, r.buffer[0..copy_len]);
320		r.seek += copy_len;
321		if (buffer_contents.len - copy_len != 0) return error.StreamTooLong;
322		r.seek = 0;
323		r.end = 0;
324		var remaining = @intFromEnum(limit) - copy_len;
325		// From here, we leave `buffer` empty, appending directly to `list`.
326		var writer: Writer = .{
327		.buffer = undefined,
328		.end = undefined,
329		.vtable = &.{ .drain = Writer.fixedDrain },
330		};
331		while (true) {
332		try list.ensureUnusedCapacity(gpa, 2);
333		const cap = list.unusedCapacitySlice();
334		const dest = cap[0..@min(cap.len, remaining + 1)];
335		writer.buffer = list.allocatedSlice();
336		writer.end = list.items.len;
337		const n = r.vtable.stream(r, &writer, .limited(dest.len)) catch \|err\| switch (err) {
338		error.WriteFailed => unreachable, // Prevented by the limit.
	313	var a: std.Io.Writer.Allocating = .initOwnedSlice(gpa, list.items);
	314	a.writer.end = list.items.len;
	315	list.* = .empty;
	316	defer {
	317	list.* = .{
	318	.items = a.writer.buffer[0..a.writer.end],
	319	.capacity = a.writer.buffer.len,
	320	};
	321	}
	322	var remaining = limit;
	323	while (remaining.nonzero()) {
	324	const n = stream(r, &a.writer, remaining) catch \|err\| switch (err) {
339	325	error.EndOfStream => return,
	326	error.WriteFailed => return error.OutOfMemory,
340	327	error.ReadFailed => return error.ReadFailed,
341	328	};
342		list.items.len += n;
343		if (n > remaining) {
344		// Move the byte to `Reader.buffer` so it is not lost.
345		assert(n - remaining == 1);
346		assert(r.end == 0);
347		r.buffer[0] = list.items[list.items.len - 1];
348		list.items.len -= 1;
349		r.end = 1;
350		return;
351		}
352		remaining -= n;
	329	remaining = remaining.subtract(n).?;
353	330	}
	331	return error.StreamTooLong;
354	332	}
355	333
356	334	pub const UnlimitedAllocError = Allocator.Error \|\| ShortError;
357	335
358		pub fn appendRemainingUnlimited(
359		r: *Reader,
360		gpa: Allocator,
361		comptime alignment: ?std.mem.Alignment,
362		list: *std.ArrayListAlignedUnmanaged(u8, alignment),
363		bump: usize,
364		) UnlimitedAllocError!void {
365		const buffer_contents = r.buffer[r.seek..r.end];
366		try list.ensureUnusedCapacity(gpa, buffer_contents.len + bump);
367		list.appendSliceAssumeCapacity(buffer_contents);
368		// If statement protects `ending`.
369		if (r.end != 0) {
370		r.seek = 0;
371		r.end = 0;
372		}
373		// From here, we leave `buffer` empty, appending directly to `list`.
374		var writer: Writer = .{
375		.buffer = undefined,
376		.end = undefined,
377		.vtable = &.{ .drain = Writer.fixedDrain },
378		};
379		while (true) {
380		try list.ensureUnusedCapacity(gpa, bump);
381		writer.buffer = list.allocatedSlice();
382		writer.end = list.items.len;
383		const n = r.vtable.stream(r, &writer, .limited(list.unusedCapacitySlice().len)) catch \|err\| switch (err) {
384		error.WriteFailed => unreachable, // Prevented by the limit.
385		error.EndOfStream => return,
386		error.ReadFailed => return error.ReadFailed,
	336	pub fn appendRemainingUnlimited(r: Reader, gpa: Allocator, list: ArrayList(u8)) UnlimitedAllocError!void {
	337	var a: std.Io.Writer.Allocating = .initOwnedSlice(gpa, list.items);
	338	a.writer.end = list.items.len;
	339	list.* = .empty;
	340	defer {
	341	list.* = .{
	342	.items = a.writer.buffer[0..a.writer.end],
	343	.capacity = a.writer.buffer.len,
387	344	};
388		list.items.len += n;
389	345	}
	346	_ = streamRemaining(r, &a.writer) catch \|err\| switch (err) {
	347	error.WriteFailed => return error.OutOfMemory,
	348	error.ReadFailed => return error.ReadFailed,
	349	};
390	350	}
391	351
392	352	/// Writes bytes from the internally tracked stream position to `data`.
...	...	@@ -1295,7 +1255,10 @@ fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Resu
1295	1255
1296	1256	/// Ensures `capacity` more data can be buffered without rebasing.
1297	1257	pub fn rebase(r: *Reader, capacity: usize) RebaseError!void {
1298		if (r.end + capacity <= r.buffer.len) return;
	1258	if (r.end + capacity <= r.buffer.len) {
	1259	@branchHint(.likely);
	1260	return;
	1261	}
1299	1262	return r.vtable.rebase(r, capacity);
1300	1263	}
1301	1264

lib/std/Io/Writer.zig+11-8

...	...	@@ -329,7 +329,7 @@ pub fn rebase(w: *Writer, preserve: usize, unused_capacity_len: usize) Error!voi
329	329	@branchHint(.likely);
330	330	return;
331	331	}
332		try w.vtable.rebase(w, preserve, unused_capacity_len);
	332	return w.vtable.rebase(w, preserve, unused_capacity_len);
333	333	}
334	334
335	335	pub fn defaultRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void {
...	...	@@ -2349,6 +2349,13 @@ pub fn unreachableDrain(w: *Writer, data: []const []const u8, splat: usize) Erro
2349	2349	unreachable;
2350	2350	}
2351	2351
	2352	pub fn unreachableRebase(w: *Writer, preserve: usize, capacity: usize) Error!void {
	2353	_ = w;
	2354	_ = preserve;
	2355	_ = capacity;
	2356	unreachable;
	2357	}
	2358
2352	2359	/// Provides a `Writer` implementation based on calling `Hasher.update`, sending
2353	2360	/// all data also to an underlying `Writer`.
2354	2361	///
...	...	@@ -2489,10 +2496,6 @@ pub fn Hashing(comptime Hasher: type) type {
2489	2496	pub const Allocating = struct {
2490	2497	allocator: Allocator,
2491	2498	writer: Writer,
2492		/// Every call to `drain` ensures at least this amount of unused capacity
2493		/// before it returns. This prevents an infinite loop in interface logic
2494		/// that calls `drain`.
2495		minimum_unused_capacity: usize = 1,
2496	2499
2497	2500	pub fn init(allocator: Allocator) Allocating {
2498	2501	return .{
...	...	@@ -2604,13 +2607,12 @@ pub const Allocating = struct {
2604	2607	const gpa = a.allocator;
2605	2608	const pattern = data[data.len - 1];
2606	2609	const splat_len = pattern.len * splat;
2607		const bump = a.minimum_unused_capacity;
2608	2610	var list = a.toArrayList();
2609	2611	defer setArrayList(a, list);
2610	2612	const start_len = list.items.len;
2611	2613	assert(data.len != 0);
2612	2614	for (data) \|bytes\| {
2613		list.ensureUnusedCapacity(gpa, bytes.len + splat_len + bump) catch return error.WriteFailed;
	2615	list.ensureUnusedCapacity(gpa, bytes.len + splat_len + 1) catch return error.WriteFailed;
2614	2616	list.appendSliceAssumeCapacity(bytes);
2615	2617	}
2616	2618	if (splat == 0) {
...	...	@@ -2641,11 +2643,12 @@ pub const Allocating = struct {
2641	2643	}
2642	2644
2643	2645	fn growingRebase(w: *Writer, preserve: usize, minimum_len: usize) Error!void {
2644		_ = preserve; // This implementation always preserves the entire buffer.
2645	2646	const a: *Allocating = @fieldParentPtr("writer", w);
2646	2647	const gpa = a.allocator;
2647	2648	var list = a.toArrayList();
2648	2649	defer setArrayList(a, list);
	2650	const total = std.math.add(usize, preserve, minimum_len) catch return error.WriteFailed;
	2651	list.ensureTotalCapacity(gpa, total) catch return error.WriteFailed;
2649	2652	list.ensureUnusedCapacity(gpa, minimum_len) catch return error.WriteFailed;
2650	2653	}
2651	2654

lib/std/array_list.zig+1-1

...	...	@@ -1033,7 +1033,7 @@ pub fn Aligned(comptime T: type, comptime alignment: ?mem.Alignment) type {
1033	1033	pub fn print(self: *Self, gpa: Allocator, comptime fmt: []const u8, args: anytype) error{OutOfMemory}!void {
1034	1034	comptime assert(T == u8);
1035	1035	try self.ensureUnusedCapacity(gpa, fmt.len);
1036		var aw: std.io.Writer.Allocating = .fromArrayList(gpa, self);
	1036	var aw: std.Io.Writer.Allocating = .fromArrayList(gpa, self);
1037	1037	defer self.* = aw.toArrayList();
1038	1038	return aw.writer.print(fmt, args) catch \|err\| switch (err) {
1039	1039	error.WriteFailed => return error.OutOfMemory,

lib/std/compress/flate/Decompress.zig+8-10

...	...	@@ -62,7 +62,7 @@ pub const Error = Container.Error \|\| error{
62	62	const direct_vtable: Reader.VTable = .{
63	63	.stream = streamDirect,
64	64	.rebase = rebaseFallible,
65		.discard = discard,
	65	.discard = discardDirect,
66	66	.readVec = readVec,
67	67	};
68	68
...	...	@@ -105,17 +105,16 @@ fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void {
105	105	fn rebase(r: *Reader, capacity: usize) void {
106	106	assert(capacity <= r.buffer.len - flate.history_len);
107	107	assert(r.end + capacity > r.buffer.len);
108		const discard_n = r.end - flate.history_len;
	108	const discard_n = @min(r.seek, r.end - flate.history_len);
109	109	const keep = r.buffer[discard_n..r.end];
110	110	@memmove(r.buffer[0..keep.len], keep);
111		assert(keep.len != 0);
112	111	r.end = keep.len;
113	112	r.seek -= discard_n;
114	113	}
115	114
116	115	/// This could be improved so that when an amount is discarded that includes an
117	116	/// entire frame, skip decoding that frame.
118		fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
	117	fn discardDirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
119	118	if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
120	119	var writer: Writer = .{
121	120	.vtable = &.{
...	...	@@ -167,11 +166,14 @@ fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
167	166
168	167	fn streamIndirectInner(d: *Decompress) Reader.Error!usize {
169	168	const r = &d.reader;
170		if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
	169	if (r.buffer.len - r.end < flate.history_len) rebase(r, flate.history_len);
171	170	var writer: Writer = .{
172	171	.buffer = r.buffer,
173	172	.end = r.end,
174		.vtable = &.{ .drain = Writer.unreachableDrain },
	173	.vtable = &.{
	174	.drain = Writer.unreachableDrain,
	175	.rebase = Writer.unreachableRebase,
	176	},
175	177	};
176	178	defer r.end = writer.end;
177	179	_ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
...	...	@@ -1251,8 +1253,6 @@ test "zlib should not overshoot" {
1251	1253	fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !void {
1252	1254	var reader: Reader = .fixed(in);
1253	1255	var aw: Writer.Allocating = .init(testing.allocator);
1254		aw.minimum_unused_capacity = flate.history_len;
1255		try aw.ensureUnusedCapacity(flate.max_window_len);
1256	1256	defer aw.deinit();
1257	1257
1258	1258	var decompress: Decompress = .init(&reader, container, &.{});
...	...	@@ -1263,8 +1263,6 @@ fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !vo
1263	1263	fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
1264	1264	var in: std.Io.Reader = .fixed(compressed);
1265	1265	var aw: std.Io.Writer.Allocating = .init(testing.allocator);
1266		aw.minimum_unused_capacity = flate.history_len;
1267		try aw.ensureUnusedCapacity(flate.max_window_len);
1268	1266	defer aw.deinit();
1269	1267
1270	1268	var decompress: Decompress = .init(&in, container, &.{});

lib/std/compress/zstd.zig+9-11

...	...	@@ -78,15 +78,14 @@ pub const table_size_max = struct {
78	78	};
79	79
80	80	fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
81		var out: std.ArrayListUnmanaged(u8) = .empty;
82		defer out.deinit(gpa);
83		try out.ensureUnusedCapacity(gpa, default_window_len);
	81	var out: std.Io.Writer.Allocating = .init(gpa);
	82	defer out.deinit();
84	83
85		var in: std.io.Reader = .fixed(compressed);
	84	var in: std.Io.Reader = .fixed(compressed);
86	85	var zstd_stream: Decompress = .init(&in, &.{}, .{});
87		try zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited);
	86	_ = try zstd_stream.reader.streamRemaining(&out.writer);
88	87
89		return out.toOwnedSlice(gpa);
	88	return out.toOwnedSlice();
90	89	}
91	90
92	91	fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {
...	...	@@ -99,15 +98,14 @@ fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void
99	98	fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
100	99	const gpa = std.testing.allocator;
101	100
102		var out: std.ArrayListUnmanaged(u8) = .empty;
103		defer out.deinit(gpa);
104		try out.ensureUnusedCapacity(gpa, default_window_len);
	101	var out: std.Io.Writer.Allocating = .init(gpa);
	102	defer out.deinit();
105	103
106		var in: std.io.Reader = .fixed(compressed);
	104	var in: std.Io.Reader = .fixed(compressed);
107	105	var zstd_stream: Decompress = .init(&in, &.{}, .{});
108	106	try std.testing.expectError(
109	107	error.ReadFailed,
110		zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited),
	108	zstd_stream.reader.streamRemaining(&out.writer),
111	109	);
112	110	try std.testing.expectError(err, zstd_stream.err orelse {});
113	111	}

lib/std/compress/zstd/Decompress.zig+72-21

...	...	@@ -73,6 +73,20 @@ pub const Error = error{
73	73	WindowSizeUnknown,
74	74	};
75	75
	76	const direct_vtable: Reader.VTable = .{
	77	.stream = streamDirect,
	78	.rebase = rebaseFallible,
	79	.discard = discardDirect,
	80	.readVec = readVec,
	81	};
	82
	83	const indirect_vtable: Reader.VTable = .{
	84	.stream = streamIndirect,
	85	.rebase = rebaseFallible,
	86	.discard = discardIndirect,
	87	.readVec = readVec,
	88	};
	89
76	90	/// When connecting `reader` to a `Writer`, `buffer` should be empty, and
77	91	/// `Writer.buffer` capacity has requirements based on `Options.window_len`.
78	92	///
...	...	@@ -84,12 +98,7 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
84	98	.verify_checksum = options.verify_checksum,
85	99	.window_len = options.window_len,
86	100	.reader = .{
87		.vtable = &.{
88		.stream = stream,
89		.rebase = rebase,
90		.discard = discard,
91		.readVec = readVec,
92		},
	101	.vtable = if (buffer.len == 0) &direct_vtable else &indirect_vtable,
93	102	.buffer = buffer,
94	103	.seek = 0,
95	104	.end = 0,
...	...	@@ -97,11 +106,27 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
97	106	};
98	107	}
99	108
100		fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void {
	109	fn streamDirect(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
	110	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	111	return stream(d, w, limit);
	112	}
	113
	114	fn streamIndirect(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
	115	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	116	_ = limit;
	117	_ = w;
	118	return streamIndirectInner(d);
	119	}
	120
	121	fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void {
	122	rebase(r, capacity);
	123	}
	124
	125	fn rebase(r: *Reader, capacity: usize) void {
101	126	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
102	127	assert(capacity <= r.buffer.len - d.window_len);
103	128	assert(r.end + capacity > r.buffer.len);
104		const discard_n = r.end - d.window_len;
	129	const discard_n = @min(r.seek, r.end - d.window_len);
105	130	const keep = r.buffer[discard_n..r.end];
106	131	@memmove(r.buffer[0..keep.len], keep);
107	132	r.end = keep.len;
...	...	@@ -110,9 +135,9 @@ fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void {
110	135
111	136	/// This could be improved so that when an amount is discarded that includes an
112	137	/// entire frame, skip decoding that frame.
113		fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
	138	fn discardDirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
114	139	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
115		r.rebase(d.window_len) catch unreachable;
	140	rebase(r, d.window_len);
116	141	var writer: Writer = .{
117	142	.vtable = &.{
118	143	.drain = std.Io.Writer.Discarding.drain,
...	...	@@ -134,25 +159,53 @@ fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
134	159	return n;
135	160	}
136	161
	162	fn discardIndirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
	163	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	164	rebase(r, d.window_len);
	165	var writer: Writer = .{
	166	.buffer = r.buffer,
	167	.end = r.end,
	168	.vtable = &.{ .drain = Writer.unreachableDrain },
	169	};
	170	{
	171	defer r.end = writer.end;
	172	_ = stream(d, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
	173	error.WriteFailed => unreachable,
	174	else => \|e\| return e,
	175	};
	176	}
	177	const n = limit.minInt(r.end - r.seek);
	178	r.seek += n;
	179	return n;
	180	}
	181
137	182	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
138	183	_ = data;
139	184	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
140		assert(r.seek == r.end);
141		r.rebase(d.window_len) catch unreachable;
	185	return streamIndirectInner(d);
	186	}
	187
	188	fn streamIndirectInner(d: *Decompress) Reader.Error!usize {
	189	const r = &d.reader;
	190	if (r.buffer.len - r.end < zstd.block_size_max) rebase(r, zstd.block_size_max);
	191	assert(r.buffer.len - r.end >= zstd.block_size_max);
142	192	var writer: Writer = .{
143	193	.buffer = r.buffer,
144	194	.end = r.end,
145		.vtable = &.{ .drain = Writer.fixedDrain },
	195	.vtable = &.{
	196	.drain = Writer.unreachableDrain,
	197	.rebase = Writer.unreachableRebase,
	198	},
146	199	};
147		r.end += r.vtable.stream(r, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
	200	defer r.end = writer.end;
	201	_ = stream(d, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
148	202	error.WriteFailed => unreachable,
149	203	else => \|e\| return e,
150	204	};
151	205	return 0;
152	206	}
153	207
154		fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {
155		const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	208	fn stream(d: Decompress, w: Writer, limit: Limit) Reader.StreamError!usize {
156	209	const in = d.input;
157	210
158	211	state: switch (d.state) {
...	...	@@ -170,7 +223,7 @@ fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {
170	223	else => \|e\| return e,
171	224	};
172	225	const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little);
173		initFrame(d, w.buffer.len, magic) catch \|err\| {
	226	initFrame(d, magic) catch \|err\| {
174	227	d.err = err;
175	228	return error.ReadFailed;
176	229	};
...	...	@@ -198,13 +251,13 @@ fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {
198	251	}
199	252	}
200	253
201		fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
	254	fn initFrame(d: *Decompress, magic: Frame.Magic) !void {
202	255	const in = d.input;
203	256	switch (magic.kind() orelse return error.BadMagic) {
204	257	.zstandard => {
205	258	const header = try Frame.Zstandard.Header.decode(in);
206	259	d.state = .{ .in_frame = .{
207		.frame = try Frame.init(header, window_size_max, d.verify_checksum),
	260	.frame = try Frame.init(header, d.window_len, d.verify_checksum),
208	261	.checksum = null,
209	262	.decompressed_size = 0,
210	263	.decode = .init,
...	...	@@ -258,7 +311,6 @@ fn readInFrame(d: Decompress, w: Writer, limit: Limit, state: *State.InFrame)
258	311	try decode.readInitialFseState(&bit_stream);
259	312
260	313	// Ensures the following calls to `decodeSequence` will not flush.
261		if (window_len + frame_block_size_max > w.buffer.len) return error.OutputBufferUndersize;
262	314	const dest = (try w.writableSliceGreedyPreserve(window_len, frame_block_size_max))[0..frame_block_size_max];
263	315	const write_pos = dest.ptr - w.buffer.ptr;
264	316	for (0..sequences_header.sequence_count - 1) \|_\| {
...	...	@@ -775,7 +827,6 @@ pub const Frame = struct {
775	827	try w.splatByteAll(d.literal_streams.one[0], len);
776	828	},
777	829	.compressed, .treeless => {
778		if (len > w.buffer.len) return error.OutputBufferUndersize;
779	830	const buf = try w.writableSlice(len);
780	831	const huffman_tree = d.huffman_tree.?;
781	832	const max_bit_count = huffman_tree.max_bit_count;

lib/std/debug/Dwarf.zig+1-1

...	...	@@ -2247,7 +2247,7 @@ pub const ElfModule = struct {
2247	2247	var decompress: std.compress.flate.Decompress = .init(&section_reader, .zlib, &.{});
2248	2248	var decompressed_section: ArrayList(u8) = .empty;
2249	2249	defer decompressed_section.deinit(gpa);
2250		decompress.reader.appendRemainingUnlimited(gpa, null, &decompressed_section, std.compress.flate.history_len) catch {
	2250	decompress.reader.appendRemainingUnlimited(gpa, &decompressed_section) catch {
2251	2251	invalidDebugInfoDetected();
2252	2252	continue;
2253	2253	};

lib/std/http/test.zig+4-7

...	...	@@ -149,9 +149,8 @@ test "HTTP server handles a chunked transfer coding request" {
149	149	"content-type: text/plain\r\n" ++
150	150	"\r\n" ++
151	151	"message from server!\n";
152		var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded
153		var stream_reader = stream.reader(&tiny_buffer);
154		const response = try stream_reader.interface().allocRemaining(gpa, .limited(expected_response.len));
	152	var stream_reader = stream.reader(&.{});
	153	const response = try stream_reader.interface().allocRemaining(gpa, .limited(expected_response.len + 1));
155	154	defer gpa.free(response);
156	155	try expectEqualStrings(expected_response, response);
157	156	}
...	...	@@ -293,8 +292,7 @@ test "Server.Request.respondStreaming non-chunked, unknown content-length" {
293	292	var stream_writer = stream.writer(&.{});
294	293	try stream_writer.interface.writeAll(request_bytes);
295	294
296		var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded
297		var stream_reader = stream.reader(&tiny_buffer);
	295	var stream_reader = stream.reader(&.{});
298	296	const response = try stream_reader.interface().allocRemaining(gpa, .unlimited);
299	297	defer gpa.free(response);
300	298
...	...	@@ -364,8 +362,7 @@ test "receiving arbitrary http headers from the client" {
364	362	var stream_writer = stream.writer(&.{});
365	363	try stream_writer.interface.writeAll(request_bytes);
366	364
367		var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded
368		var stream_reader = stream.reader(&tiny_buffer);
	365	var stream_reader = stream.reader(&.{});
369	366	const response = try stream_reader.interface().allocRemaining(gpa, .unlimited);
370	367	defer gpa.free(response);
371	368

lib/std/unicode.zig+16-15

...	...	@@ -4,6 +4,7 @@ const assert = std.debug.assert;
4	4	const testing = std.testing;
5	5	const mem = std.mem;
6	6	const native_endian = builtin.cpu.arch.endian();
	7	const Allocator = std.mem.Allocator;
7	8
8	9	/// Use this to replace an unknown, unrecognized, or unrepresentable character.
9	10	///
...	...	@@ -921,7 +922,7 @@ fn utf16LeToUtf8ArrayListImpl(
921	922	comptime surrogates: Surrogates,
922	923	) (switch (surrogates) {
923	924	.cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
924		.can_encode_surrogate_half => mem.Allocator.Error,
	925	.can_encode_surrogate_half => Allocator.Error,
925	926	})!void {
926	927	assert(result.unusedCapacitySlice().len >= utf16le.len);
927	928
...	...	@@ -965,15 +966,15 @@ fn utf16LeToUtf8ArrayListImpl(
965	966	}
966	967	}
967	968
968		pub const Utf16LeToUtf8AllocError = mem.Allocator.Error \|\| Utf16LeToUtf8Error;
	969	pub const Utf16LeToUtf8AllocError = Allocator.Error \|\| Utf16LeToUtf8Error;
969	970
970	971	pub fn utf16LeToUtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
971	972	try result.ensureUnusedCapacity(utf16le.len);
972	973	return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
973	974	}
974	975
975		/// Caller must free returned memory.
976		pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8 {
	976	/// Caller owns returned memory.
	977	pub fn utf16LeToUtf8Alloc(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8 {
977	978	// optimistically guess that it will all be ascii.
978	979	var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len);
979	980	errdefer result.deinit();
...	...	@@ -982,8 +983,8 @@ pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16L
982	983	return result.toOwnedSlice();
983	984	}
984	985
985		/// Caller must free returned memory.
986		pub fn utf16LeToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8 {
	986	/// Caller owns returned memory.
	987	pub fn utf16LeToUtf8AllocZ(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8 {
987	988	// optimistically guess that it will all be ascii (and allocate space for the null terminator)
988	989	var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len + 1);
989	990	errdefer result.deinit();
...	...	@@ -1160,7 +1161,7 @@ pub fn utf8ToUtf16LeArrayList(result: *std.array_list.Managed(u16), utf8: []cons
1160	1161	return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
1161	1162	}
1162	1163
1163		pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {
	1164	pub fn utf8ToUtf16LeAlloc(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {
1164	1165	// optimistically guess that it will not require surrogate pairs
1165	1166	var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len);
1166	1167	errdefer result.deinit();
...	...	@@ -1169,7 +1170,7 @@ pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ Inv
1169	1170	return result.toOwnedSlice();
1170	1171	}
1171	1172
1172		pub fn utf8ToUtf16LeAllocZ(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16 {
	1173	pub fn utf8ToUtf16LeAllocZ(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16 {
1173	1174	// optimistically guess that it will not require surrogate pairs
1174	1175	var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len + 1);
1175	1176	errdefer result.deinit();
...	...	@@ -1750,13 +1751,13 @@ pub const Wtf8Iterator = struct {
1750	1751	}
1751	1752	};
1752	1753
1753		pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) mem.Allocator.Error!void {
	1754	pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Allocator.Error!void {
1754	1755	try result.ensureUnusedCapacity(utf16le.len);
1755	1756	return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
1756	1757	}
1757	1758
1758	1759	/// Caller must free returned memory.
1759		pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Allocator.Error![]u8 {
	1760	pub fn wtf16LeToWtf8Alloc(allocator: Allocator, wtf16le: []const u16) Allocator.Error![]u8 {
1760	1761	// optimistically guess that it will all be ascii.
1761	1762	var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len);
1762	1763	errdefer result.deinit();
...	...	@@ -1766,7 +1767,7 @@ pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Al
1766	1767	}
1767	1768
1768	1769	/// Caller must free returned memory.
1769		pub fn wtf16LeToWtf8AllocZ(allocator: mem.Allocator, wtf16le: []const u16) mem.Allocator.Error![:0]u8 {
	1770	pub fn wtf16LeToWtf8AllocZ(allocator: Allocator, wtf16le: []const u16) Allocator.Error![:0]u8 {
1770	1771	// optimistically guess that it will all be ascii (and allocate space for the null terminator)
1771	1772	var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len + 1);
1772	1773	errdefer result.deinit();
...	...	@@ -1784,7 +1785,7 @@ pub fn wtf8ToWtf16LeArrayList(result: *std.array_list.Managed(u16), wtf8: []cons
1784	1785	return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
1785	1786	}
1786	1787
1787		pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {
	1788	pub fn wtf8ToWtf16LeAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {
1788	1789	// optimistically guess that it will not require surrogate pairs
1789	1790	var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len);
1790	1791	errdefer result.deinit();
...	...	@@ -1793,7 +1794,7 @@ pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ Inv
1793	1794	return result.toOwnedSlice();
1794	1795	}
1795	1796
1796		pub fn wtf8ToWtf16LeAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16 {
	1797	pub fn wtf8ToWtf16LeAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16 {
1797	1798	// optimistically guess that it will not require surrogate pairs
1798	1799	var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len + 1);
1799	1800	errdefer result.deinit();
...	...	@@ -1870,7 +1871,7 @@ pub fn wtf8ToUtf8Lossy(utf8: []u8, wtf8: []const u8) error{InvalidWtf8}!void {
1870	1871	}
1871	1872	}
1872	1873
1873		pub fn wtf8ToUtf8LossyAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8 {
	1874	pub fn wtf8ToUtf8LossyAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8 {
1874	1875	const utf8 = try allocator.alloc(u8, wtf8.len);
1875	1876	errdefer allocator.free(utf8);
1876	1877
...	...	@@ -1879,7 +1880,7 @@ pub fn wtf8ToUtf8LossyAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ I
1879	1880	return utf8;
1880	1881	}
1881	1882
1882		pub fn wtf8ToUtf8LossyAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8 {
	1883	pub fn wtf8ToUtf8LossyAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8 {
1883	1884	const utf8 = try allocator.allocSentinel(u8, wtf8.len, 0);
1884	1885	errdefer allocator.free(utf8);
1885	1886

lib/std/zig.zig+6-3

...	...	@@ -554,8 +554,11 @@ test isUnderscore {
554	554	try std.testing.expect(!isUnderscore("\\x5f"));
555	555	}
556	556
	557	/// If the source can be UTF-16LE encoded, this function asserts that `gpa`
	558	/// will align a byte-sized allocation to at least 2. Allocators that don't do
	559	/// this are rare.
557	560	pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader) ![:0]u8 {
558		var buffer: std.ArrayListAlignedUnmanaged(u8, .@"2") = .empty;
	561	var buffer: std.ArrayList(u8) = .empty;
559	562	defer buffer.deinit(gpa);
560	563
561	564	if (file_reader.getSize()) \|size\| {
...	...	@@ -564,7 +567,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader
564	567	try buffer.ensureTotalCapacityPrecise(gpa, casted_size + 1);
565	568	} else \|_\| {}
566	569
567		try file_reader.interface.appendRemaining(gpa, .@"2", &buffer, .limited(max_src_size));
	570	try file_reader.interface.appendRemaining(gpa, &buffer, .limited(max_src_size));
568	571
569	572	// Detect unsupported file types with their Byte Order Mark
570	573	const unsupported_boms = [_][]const u8{
...	...	@@ -581,7 +584,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader
581	584	// If the file starts with a UTF-16 little endian BOM, translate it to UTF-8
582	585	if (std.mem.startsWith(u8, buffer.items, "\xff\xfe")) {
583	586	if (buffer.items.len % 2 != 0) return error.InvalidEncoding;
584		return std.unicode.utf16LeToUtf8AllocZ(gpa, @ptrCast(buffer.items)) catch \|err\| switch (err) {
	587	return std.unicode.utf16LeToUtf8AllocZ(gpa, @ptrCast(@alignCast(buffer.items))) catch \|err\| switch (err) {
585	588	error.DanglingSurrogateHalf => error.UnsupportedEncoding,
586	589	error.ExpectedSecondSurrogateHalf => error.UnsupportedEncoding,
587	590	error.UnexpectedSecondSurrogateHalf => error.UnsupportedEncoding,