authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-06 19:19:57-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-09-06 19:19:57-04:00
logd7268cbb242d6bea6b6e4d929a8d6b99608b640a
tree73a055b7bdbad0babf52cabae6a743c9e9a534a2
parenta8a806e925ddb2362e0021bd83dd73f7389a2dbb
parent3c8e1bc25b6d122ce1295e2e33bb9f54ae801ec0
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6232 from LemonBoy/fix-readall

std: Don't trust stat() size in readAllAlloc fns

7 files changed, 63 insertions(+), 23 deletions(-)

lib/std/array_list.zig+10-2
...@@ -46,7 +46,11 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {...@@ -46,7 +46,11 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
46 /// Deinitialize with `deinit` or use `toOwnedSlice`.46 /// Deinitialize with `deinit` or use `toOwnedSlice`.
47 pub fn initCapacity(allocator: *Allocator, num: usize) !Self {47 pub fn initCapacity(allocator: *Allocator, num: usize) !Self {
48 var self = Self.init(allocator);48 var self = Self.init(allocator);
49 try self.ensureCapacity(num);49
50 const new_memory = try self.allocator.allocAdvanced(T, alignment, num, .at_least);
51 self.items.ptr = new_memory.ptr;
52 self.capacity = new_memory.len;
53
50 return self;54 return self;
51 }55 }
5256
...@@ -366,7 +370,11 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ...@@ -366,7 +370,11 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
366 /// Deinitialize with `deinit` or use `toOwnedSlice`.370 /// Deinitialize with `deinit` or use `toOwnedSlice`.
367 pub fn initCapacity(allocator: *Allocator, num: usize) !Self {371 pub fn initCapacity(allocator: *Allocator, num: usize) !Self {
368 var self = Self{};372 var self = Self{};
369 try self.ensureCapacity(allocator, num);373
374 const new_memory = try self.allocator.allocAdvanced(T, alignment, num, .at_least);
375 self.items.ptr = new_memory.ptr;
376 self.capacity = new_memory.len;
377
370 return self;378 return self;
371 }379 }
372380
lib/std/fs.zig+9-3
...@@ -1437,26 +1437,32 @@ pub const Dir = struct {...@@ -1437,26 +1437,32 @@ pub const Dir = struct {
1437 /// On success, caller owns returned buffer.1437 /// On success, caller owns returned buffer.
1438 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.1438 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
1439 pub fn readFileAlloc(self: Dir, allocator: *mem.Allocator, file_path: []const u8, max_bytes: usize) ![]u8 {1439 pub fn readFileAlloc(self: Dir, allocator: *mem.Allocator, file_path: []const u8, max_bytes: usize) ![]u8 {
1440 return self.readFileAllocOptions(allocator, file_path, max_bytes, @alignOf(u8), null);1440 return self.readFileAllocOptions(allocator, file_path, max_bytes, null, @alignOf(u8), null);
1441 }1441 }
14421442
1443 /// On success, caller owns returned buffer.1443 /// On success, caller owns returned buffer.
1444 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.1444 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
1445 /// If `size_hint` is specified the initial buffer size is calculated using
1446 /// that value, otherwise the effective file size is used instead.
1445 /// Allows specifying alignment and a sentinel value.1447 /// Allows specifying alignment and a sentinel value.
1446 pub fn readFileAllocOptions(1448 pub fn readFileAllocOptions(
1447 self: Dir,1449 self: Dir,
1448 allocator: *mem.Allocator,1450 allocator: *mem.Allocator,
1449 file_path: []const u8,1451 file_path: []const u8,
1450 max_bytes: usize,1452 max_bytes: usize,
1453 size_hint: ?usize,
1451 comptime alignment: u29,1454 comptime alignment: u29,
1452 comptime optional_sentinel: ?u8,1455 comptime optional_sentinel: ?u8,
1453 ) !(if (optional_sentinel) |s| [:s]align(alignment) u8 else []align(alignment) u8) {1456 ) !(if (optional_sentinel) |s| [:s]align(alignment) u8 else []align(alignment) u8) {
1454 var file = try self.openFile(file_path, .{});1457 var file = try self.openFile(file_path, .{});
1455 defer file.close();1458 defer file.close();
14561459
1457 const stat_size = try file.getEndPos();1460 // If the file size doesn't fit a usize it'll be certainly greater than
1461 // `max_bytes`
1462 const stat_size = size_hint orelse math.cast(usize, try file.getEndPos()) catch
1463 return error.FileTooBig;
14581464
1459 return file.readAllAllocOptions(allocator, stat_size, max_bytes, alignment, optional_sentinel);1465 return file.readToEndAllocOptions(allocator, max_bytes, stat_size, alignment, optional_sentinel);
1460 }1466 }
14611467
1462 pub const DeleteTreeError = error{1468 pub const DeleteTreeError = error{
lib/std/fs/file.zig+29-11
...@@ -363,31 +363,49 @@ pub const File = struct {...@@ -363,31 +363,49 @@ pub const File = struct {
363 try os.futimens(self.handle, &times);363 try os.futimens(self.handle, &times);
364 }364 }
365365
366 /// Reads all the bytes from the current position to the end of the file.
366 /// On success, caller owns returned buffer.367 /// On success, caller owns returned buffer.
367 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.368 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
368 pub fn readAllAlloc(self: File, allocator: *mem.Allocator, stat_size: u64, max_bytes: usize) ![]u8 {369 pub fn readToEndAlloc(self: File, allocator: *mem.Allocator, max_bytes: usize) ![]u8 {
369 return self.readAllAllocOptions(allocator, stat_size, max_bytes, @alignOf(u8), null);370 return self.readToEndAllocOptions(allocator, max_bytes, null, @alignOf(u8), null);
370 }371 }
371372
373 /// Reads all the bytes from the current position to the end of the file.
372 /// On success, caller owns returned buffer.374 /// On success, caller owns returned buffer.
373 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.375 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
376 /// If `size_hint` is specified the initial buffer size is calculated using
377 /// that value, otherwise an arbitrary value is used instead.
374 /// Allows specifying alignment and a sentinel value.378 /// Allows specifying alignment and a sentinel value.
375 pub fn readAllAllocOptions(379 pub fn readToEndAllocOptions(
376 self: File,380 self: File,
377 allocator: *mem.Allocator,381 allocator: *mem.Allocator,
378 stat_size: u64,
379 max_bytes: usize,382 max_bytes: usize,
383 size_hint: ?usize,
380 comptime alignment: u29,384 comptime alignment: u29,
381 comptime optional_sentinel: ?u8,385 comptime optional_sentinel: ?u8,
382 ) !(if (optional_sentinel) |s| [:s]align(alignment) u8 else []align(alignment) u8) {386 ) !(if (optional_sentinel) |s| [:s]align(alignment) u8 else []align(alignment) u8) {
383 const size = math.cast(usize, stat_size) catch math.maxInt(usize);387 // If no size hint is provided fall back to the size=0 code path
384 if (size > max_bytes) return error.FileTooBig;388 const size = size_hint orelse 0;
385389
386 const buf = try allocator.allocWithOptions(u8, size, alignment, optional_sentinel);390 // The file size returned by stat is used as hint to set the buffer
387 errdefer allocator.free(buf);391 // size. If the reported size is zero, as it happens on Linux for files
392 // in /proc, a small buffer is allocated instead.
393 const initial_cap = (if (size > 0) size else 1024) + @boolToInt(optional_sentinel != null);
394 var array_list = try std.ArrayListAligned(u8, alignment).initCapacity(allocator, initial_cap);
395 defer array_list.deinit();
396
397 self.reader().readAllArrayList(&array_list, max_bytes) catch |err| switch (err) {
398 error.StreamTooLong => return error.FileTooBig,
399 else => |e| return e,
400 };
388401
389 try self.reader().readNoEof(buf);402 if (optional_sentinel) |sentinel| {
390 return buf;403 try array_list.append(sentinel);
404 const buf = array_list.toOwnedSlice();
405 return buf[0 .. buf.len - 1 :sentinel];
406 } else {
407 return array_list.toOwnedSlice();
408 }
391 }409 }
392410
393 pub const ReadError = os.ReadError;411 pub const ReadError = os.ReadError;
lib/std/fs/test.zig+5-5
...@@ -188,30 +188,30 @@ test "readAllAlloc" {...@@ -188,30 +188,30 @@ test "readAllAlloc" {
188 var file = try tmp_dir.dir.createFile("test_file", .{ .read = true });188 var file = try tmp_dir.dir.createFile("test_file", .{ .read = true });
189 defer file.close();189 defer file.close();
190190
191 const buf1 = try file.readAllAlloc(testing.allocator, 0, 1024);191 const buf1 = try file.readToEndAlloc(testing.allocator, 1024);
192 defer testing.allocator.free(buf1);192 defer testing.allocator.free(buf1);
193 testing.expect(buf1.len == 0);193 testing.expect(buf1.len == 0);
194194
195 const write_buf: []const u8 = "this is a test.\nthis is a test.\nthis is a test.\nthis is a test.\n";195 const write_buf: []const u8 = "this is a test.\nthis is a test.\nthis is a test.\nthis is a test.\n";
196 try file.writeAll(write_buf);196 try file.writeAll(write_buf);
197 try file.seekTo(0);197 try file.seekTo(0);
198 const file_size = try file.getEndPos();
199198
200 // max_bytes > file_size199 // max_bytes > file_size
201 const buf2 = try file.readAllAlloc(testing.allocator, file_size, 1024);200 const buf2 = try file.readToEndAlloc(testing.allocator, 1024);
202 defer testing.allocator.free(buf2);201 defer testing.allocator.free(buf2);
203 testing.expectEqual(write_buf.len, buf2.len);202 testing.expectEqual(write_buf.len, buf2.len);
204 testing.expect(std.mem.eql(u8, write_buf, buf2));203 testing.expect(std.mem.eql(u8, write_buf, buf2));
205 try file.seekTo(0);204 try file.seekTo(0);
206205
207 // max_bytes == file_size206 // max_bytes == file_size
208 const buf3 = try file.readAllAlloc(testing.allocator, file_size, write_buf.len);207 const buf3 = try file.readToEndAlloc(testing.allocator, write_buf.len);
209 defer testing.allocator.free(buf3);208 defer testing.allocator.free(buf3);
210 testing.expectEqual(write_buf.len, buf3.len);209 testing.expectEqual(write_buf.len, buf3.len);
211 testing.expect(std.mem.eql(u8, write_buf, buf3));210 testing.expect(std.mem.eql(u8, write_buf, buf3));
211 try file.seekTo(0);
212212
213 // max_bytes < file_size213 // max_bytes < file_size
214 testing.expectError(error.FileTooBig, file.readAllAlloc(testing.allocator, file_size, write_buf.len - 1));214 testing.expectError(error.FileTooBig, file.readToEndAlloc(testing.allocator, write_buf.len - 1));
215}215}
216216
217test "directory operations on files" {217test "directory operations on files" {
src-self-hosted/Module.zig+2
...@@ -626,6 +626,7 @@ pub const Scope = struct {...@@ -626,6 +626,7 @@ pub const Scope = struct {
626 module.gpa,626 module.gpa,
627 self.sub_file_path,627 self.sub_file_path,
628 std.math.maxInt(u32),628 std.math.maxInt(u32),
629 null,
629 1,630 1,
630 0,631 0,
631 );632 );
...@@ -723,6 +724,7 @@ pub const Scope = struct {...@@ -723,6 +724,7 @@ pub const Scope = struct {
723 module.gpa,724 module.gpa,
724 self.sub_file_path,725 self.sub_file_path,
725 std.math.maxInt(u32),726 std.math.maxInt(u32),
727 null,
726 1,728 1,
727 0,729 0,
728 );730 );
src-self-hosted/main.zig+8-1
...@@ -744,6 +744,7 @@ const FmtError = error{...@@ -744,6 +744,7 @@ const FmtError = error{
744 LinkQuotaExceeded,744 LinkQuotaExceeded,
745 FileBusy,745 FileBusy,
746 EndOfStream,746 EndOfStream,
747 Unseekable,
747 NotOpenForWriting,748 NotOpenForWriting,
748} || fs.File.OpenError;749} || fs.File.OpenError;
749750
...@@ -807,7 +808,13 @@ fn fmtPathFile(...@@ -807,7 +808,13 @@ fn fmtPathFile(
807 if (stat.kind == .Directory)808 if (stat.kind == .Directory)
808 return error.IsDir;809 return error.IsDir;
809810
810 const source_code = source_file.readAllAlloc(fmt.gpa, stat.size, max_src_size) catch |err| switch (err) {811 const source_code = source_file.readToEndAllocOptions(
812 fmt.gpa,
813 max_src_size,
814 stat.size,
815 @alignOf(u8),
816 null,
817 ) catch |err| switch (err) {
811 error.ConnectionResetByPeer => unreachable,818 error.ConnectionResetByPeer => unreachable,
812 error.ConnectionTimedOut => unreachable,819 error.ConnectionTimedOut => unreachable,
813 error.NotOpenForReading => unreachable,820 error.NotOpenForReading => unreachable,
src-self-hosted/stage2.zig-1
...@@ -615,7 +615,6 @@ export fn stage2_libc_parse(stage1_libc: *Stage2LibCInstallation, libc_file_z: [...@@ -615,7 +615,6 @@ export fn stage2_libc_parse(stage1_libc: *Stage2LibCInstallation, libc_file_z: [
615 error.NotOpenForWriting => unreachable,615 error.NotOpenForWriting => unreachable,
616 error.NotOpenForReading => unreachable,616 error.NotOpenForReading => unreachable,
617 error.Unexpected => return .Unexpected,617 error.Unexpected => return .Unexpected,
618 error.EndOfStream => return .EndOfFile,
619 error.IsDir => return .IsDir,618 error.IsDir => return .IsDir,
620 error.ConnectionResetByPeer => unreachable,619 error.ConnectionResetByPeer => unreachable,
621 error.ConnectionTimedOut => unreachable,620 error.ConnectionTimedOut => unreachable,