authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-11 21:12:47-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-11 21:12:47-05:00
loged4d94a5d54bc49b3661d602301a5ec926abef61
tree13e830ea515e66e9c1a314281ba3c346d3ed9c35
parentc4e7d05ce37d60141afab997f23df33f5d4a218b

self-hosted: test all out of memory conditions

3 files changed, 121 insertions(+), 21 deletions(-)

src-self-hosted/main.zig+36-15
......@@ -8,6 +8,7 @@ const warn = std.debug.warn;
88const Tokenizer = @import("tokenizer.zig").Tokenizer;
99const Token = @import("tokenizer.zig").Token;
1010const Parser = @import("parser.zig").Parser;
11const assert = std.debug.assert;
1112
1213pub fn main() -> %void {
1314 main2() %% |err| {
......@@ -68,28 +69,48 @@ pub fn main2() -> %void {
6869
6970var fixed_buffer_mem: [100 * 1024]u8 = undefined;
7071
71fn testCanonical(source: []const u8) {
72 var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
73 const allocator = &fixed_allocator.allocator;
74
72fn testParse(source: []const u8, allocator: &mem.Allocator) -> %[]u8 {
7573 var tokenizer = Tokenizer.init(source);
7674 var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
7775 defer parser.deinit();
7876
79 const root_node = parser.parse() %% unreachable;
77 const root_node = %return parser.parse();
8078 defer parser.freeAst(root_node);
8179
82 var buffer = std.Buffer.initSize(allocator, 0) %% unreachable;
80 var buffer = %return std.Buffer.initSize(allocator, 0);
8381 var buffer_out_stream = io.BufferOutStream.init(&buffer);
84 parser.renderSource(&buffer_out_stream.stream, root_node) %% unreachable;
85
86 if (!mem.eql(u8, buffer.toSliceConst(), source)) {
87 warn("\n====== expected this output: =========\n");
88 warn("{}", source);
89 warn("\n======== instead found this: =========\n");
90 warn("{}", buffer.toSliceConst());
91 warn("\n======================================\n");
92 @panic("test failed");
82 %return parser.renderSource(&buffer_out_stream.stream, root_node);
83 return buffer.toOwnedSlice();
84}
85
86fn testCanonical(source: []const u8) {
87 const needed_alloc_count = {
88 // Try it once with unlimited memory, make sure it works
89 var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
90 var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, @maxValue(usize));
91 const result_source = testParse(source, &failing_allocator.allocator) %% @panic("test failed");
92 if (!mem.eql(u8, result_source, source)) {
93 warn("\n====== expected this output: =========\n");
94 warn("{}", source);
95 warn("\n======== instead found this: =========\n");
96 warn("{}", result_source);
97 warn("\n======================================\n");
98 @panic("test failed");
99 }
100 failing_allocator.allocator.free(result_source);
101 failing_allocator.index
102 };
103
104 var fail_index = needed_alloc_count;
105 while (fail_index != 0) {
106 fail_index -= 1;
107 var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
108 var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, fail_index);
109 if (testParse(source, &failing_allocator.allocator)) |_| {
110 @panic("non-deterministic memory usage");
111 } else |err| {
112 assert(err == error.OutOfMemory);
113 }
93114 }
94115}
95116
src-self-hosted/parser.zig+29-6
......@@ -18,6 +18,7 @@ pub const Parser = struct {
1818 put_back_tokens: [2]Token,
1919 put_back_count: usize,
2020 source_file_name: []const u8,
21 cleanup_root_node: ?&ast.NodeRoot,
2122
2223 // This memory contents are used only during a function call. It's used to repurpose memory;
2324 // specifically so that freeAst can be guaranteed to succeed.
......@@ -32,10 +33,12 @@ pub const Parser = struct {
3233 .put_back_count = 0,
3334 .source_file_name = source_file_name,
3435 .utility_bytes = []align(utility_bytes_align) u8{},
36 .cleanup_root_node = null,
3537 };
3638 }
3739
3840 pub fn deinit(self: &Parser) {
41 assert(self.cleanup_root_node == null);
3942 self.allocator.free(self.utility_bytes);
4043 }
4144
......@@ -115,13 +118,29 @@ pub const Parser = struct {
115118 }
116119
117120 pub fn parse(self: &Parser) -> %&ast.NodeRoot {
121 const result = self.parseInner() %% |err| {
122 if (self.cleanup_root_node) |root_node| {
123 self.freeAst(root_node);
124 }
125 err
126 };
127 self.cleanup_root_node = null;
128 return result;
129 }
130
131 pub fn parseInner(self: &Parser) -> %&ast.NodeRoot {
118132 var stack = self.initUtilityArrayList(State);
119133 defer self.deinitUtilityArrayList(stack);
120134
121 const root_node = %return self.createRoot();
122 %defer self.allocator.destroy(root_node);
123 %return stack.append(State.TopLevel);
124 %defer self.freeAst(root_node);
135 const root_node = {
136 const root_node = %return self.createRoot();
137 %defer self.allocator.destroy(root_node);
138 // This stack append has to succeed for freeAst to work
139 %return stack.append(State.TopLevel);
140 root_node
141 };
142 assert(self.cleanup_root_node == null);
143 self.cleanup_root_node = root_node;
125144
126145 while (true) {
127146 //{
......@@ -1063,11 +1082,15 @@ pub const Parser = struct {
10631082 const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
10641083 self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
10651084 const typed_slice = ([]T)(self.utility_bytes);
1066 return ArrayList(T).fromOwnedSlice(self.allocator, typed_slice);
1085 return ArrayList(T) {
1086 .allocator = self.allocator,
1087 .items = typed_slice,
1088 .len = 0,
1089 };
10671090 }
10681091
10691092 fn deinitUtilityArrayList(self: &Parser, list: var) {
1070 self.utility_bytes = ([]align(utility_bytes_align) u8)(list.toOwnedSlice());
1093 self.utility_bytes = ([]align(utility_bytes_align) u8)(list.items);
10711094 }
10721095
10731096};
std/debug.zig+56
......@@ -968,3 +968,59 @@ fn readILeb128(in_stream: &io.InStream) -> %i64 {
968968pub const global_allocator = &global_fixed_allocator.allocator;
969969var global_fixed_allocator = mem.FixedBufferAllocator.init(global_allocator_mem[0..]);
970970var global_allocator_mem: [100 * 1024]u8 = undefined;
971
972/// Allocator that fails after N allocations, useful for making sure out of
973/// memory conditions are handled correctly.
974pub const FailingAllocator = struct {
975 allocator: mem.Allocator,
976 index: usize,
977 fail_index: usize,
978 internal_allocator: &mem.Allocator,
979 allocated_bytes: usize,
980
981 pub fn init(allocator: &mem.Allocator, fail_index: usize) -> FailingAllocator {
982 return FailingAllocator {
983 .internal_allocator = allocator,
984 .fail_index = fail_index,
985 .index = 0,
986 .allocated_bytes = 0,
987 .allocator = mem.Allocator {
988 .allocFn = alloc,
989 .reallocFn = realloc,
990 .freeFn = free,
991 },
992 };
993 }
994
995 fn alloc(allocator: &mem.Allocator, n: usize, alignment: u29) -> %[]u8 {
996 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
997 if (self.index == self.fail_index) {
998 return error.OutOfMemory;
999 }
1000 self.index += 1;
1001 const result = %return self.internal_allocator.allocFn(self.internal_allocator, n, alignment);
1002 self.allocated_bytes += result.len;
1003 return result;
1004 }
1005
1006 fn realloc(allocator: &mem.Allocator, old_mem: []u8, new_size: usize, alignment: u29) -> %[]u8 {
1007 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
1008 if (new_size <= old_mem.len) {
1009 self.allocated_bytes -= old_mem.len - new_size;
1010 return self.internal_allocator.reallocFn(self.internal_allocator, old_mem, new_size, alignment);
1011 }
1012 if (self.index == self.fail_index) {
1013 return error.OutOfMemory;
1014 }
1015 self.index += 1;
1016 const result = %return self.internal_allocator.reallocFn(self.internal_allocator, old_mem, new_size, alignment);
1017 self.allocated_bytes += new_size - old_mem.len;
1018 return result;
1019 }
1020
1021 fn free(allocator: &mem.Allocator, bytes: []u8) {
1022 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
1023 self.allocated_bytes -= bytes.len;
1024 return self.internal_allocator.freeFn(self.internal_allocator, bytes);
1025 }
1026};