authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2021-04-02 20:59:40+02:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2021-04-08 22:47:08+02:00
log1bd5552fc1a8fd2ddcb8f0c17f35662e4eb1cbcf
treec1d7943065c0fc998ee196c3a8bd643898ca2f39
parent00b2e31589b2f4c3f67ab2bf46e140e00df3f910
signature Commit is signed but in an unrecognized format.

Calculate data length to ensure correct pointer offsets

4 files changed, 167 insertions(+), 85 deletions(-)

src/Module.zig+1-1
......@@ -3842,7 +3842,7 @@ fn allocateNewDecl(
38423842 .elf => .{ .elf = link.File.Elf.SrcFn.empty },
38433843 .macho => .{ .macho = link.File.MachO.SrcFn.empty },
38443844 .c => .{ .c = link.File.C.FnBlock.empty },
3845 .wasm => .{ .wasm = null },
3845 .wasm => .{ .wasm = .{} },
38463846 .spirv => .{ .spirv = .{} },
38473847 },
38483848 .generation = 0,
src/codegen/wasm.zig+31-11
......@@ -16,9 +16,12 @@ const Value = @import("../value.zig").Value;
1616const Compilation = @import("../Compilation.zig");
1717const AnyMCValue = @import("../codegen.zig").AnyMCValue;
1818const LazySrcLoc = Module.LazySrcLoc;
19const link = @import("../link.zig");
20const TypedValue = @import("../TypedValue.zig");
1921
2022/// Wasm Value, created when generating an instruction
2123const WValue = union(enum) {
24 /// May be referenced but is unused
2225 none: void,
2326 /// Index of the local variable
2427 local: u32,
......@@ -611,11 +614,8 @@ pub const Context = struct {
611614 }
612615
613616 /// Generates the wasm bytecode for the function declaration belonging to `Context`
614 pub fn gen(self: *Context) InnerError!Result {
615 assert(self.code.items.len == 0);
616
617 const tv = self.decl.typed_value.most_recent.typed_value;
618 switch (tv.ty.zigTypeTag()) {
617 pub fn gen(self: *Context, typed_value: TypedValue) InnerError!Result {
618 switch (typed_value.ty.zigTypeTag()) {
619619 .Fn => {
620620 try self.genFunctype();
621621
......@@ -654,21 +654,41 @@ pub const Context = struct {
654654
655655 // Fill in the size of the generated code to the reserved space at the
656656 // beginning of the buffer.
657 const size = self.code.items.len - 5 + self.decl.fn_link.wasm.?.idx_refs.items.len * 5;
657 const size = self.code.items.len - 5 + self.decl.fn_link.wasm.idx_refs.items.len * 5;
658658 leb.writeUnsignedFixed(5, self.code.items[0..5], @intCast(u32, size));
659659
660660 // codegen data has been appended to `code`
661661 return Result.appended;
662662 },
663663 .Array => {
664 if (tv.val.castTag(.bytes)) |payload| {
665 if (tv.ty.sentinel()) |sentinel| {
666 // TODO, handle sentinel correctly
664 if (typed_value.val.castTag(.bytes)) |payload| {
665 if (typed_value.ty.sentinel()) |sentinel| {
666 try self.code.appendSlice(payload.data);
667
668 switch (try self.gen(.{
669 .ty = typed_value.ty.elemType(),
670 .val = sentinel,
671 })) {
672 .appended => return Result.appended,
673 .externally_managed => |data| {
674 try self.code.appendSlice(data);
675 return Result.appended;
676 },
677 }
667678 }
668679 return Result{ .externally_managed = payload.data };
669680 } else return self.fail(.{ .node_offset = 0 }, "TODO implement gen for more kinds of arrays", .{});
670681 },
671 else => |tag| return self.fail(.{ .node_offset = 0 }, "TODO: Implement zig type codegen for type: '{s}'", .{tag}),
682 .Int => {
683 const info = typed_value.ty.intInfo(self.bin_file.base.options.target);
684 if (info.bits == 8 and info.signedness == .unsigned) {
685 const int_byte = typed_value.val.toUnsignedInt();
686 try self.code.append(@intCast(u8, int_byte));
687 return Result.appended;
688 }
689 return self.fail(self.decl.src(), "TODO: Implement codegen for int type: '{}'", .{typed_value.ty});
690 },
691 else => |tag| return self.fail(self.decl.src(), "TODO: Implement zig type codegen for type: '{s}'", .{tag}),
672692 }
673693 }
674694
......@@ -745,7 +765,7 @@ pub const Context = struct {
745765
746766 // The function index immediate argument will be filled in using this data
747767 // in link.Wasm.flush().
748 try self.decl.fn_link.wasm.?.idx_refs.append(self.gpa, .{
768 try self.decl.fn_link.wasm.idx_refs.append(self.gpa, .{
749769 .offset = @intCast(u32, self.code.items.len),
750770 .decl = target,
751771 });
src/link.zig+3-2
......@@ -147,7 +147,7 @@ pub const File = struct {
147147 coff: Coff.SrcFn,
148148 macho: MachO.SrcFn,
149149 c: C.FnBlock,
150 wasm: ?Wasm.FnData,
150 wasm: Wasm.FnData,
151151 spirv: SpirV.FnData,
152152 };
153153
......@@ -328,7 +328,8 @@ pub const File = struct {
328328 .elf => return @fieldParentPtr(Elf, "base", base).allocateDeclIndexes(decl),
329329 .macho => return @fieldParentPtr(MachO, "base", base).allocateDeclIndexes(decl),
330330 .c => return @fieldParentPtr(C, "base", base).allocateDeclIndexes(decl),
331 .wasm, .spirv => {},
331 .wasm => return @fieldParentPtr(Wasm, "base", base).allocateDeclIndexes(decl),
332 .spirv => {},
332333 }
333334 }
334335
src/link/Wasm.zig+132-71
......@@ -16,6 +16,7 @@ const link = @import("../link.zig");
1616const trace = @import("../tracy.zig").trace;
1717const build_options = @import("build_options");
1818const Cache = @import("../Cache.zig");
19const TypedValue = @import("../TypedValue.zig");
1920
2021pub const base_tag = link.File.Tag.wasm;
2122
......@@ -34,25 +35,33 @@ pub const FnData = struct {
3435/// where the offset is calculated using the previous segments and the content length
3536/// of the data
3637pub const DataSection = struct {
37 segments: std.AutoArrayHashMapUnmanaged(*const Module.Decl, []const u8) = .{},
38 segments: std.AutoArrayHashMapUnmanaged(*Module.Decl, struct { data: [*]const u8, len: u32 }) = .{},
3839
3940 /// Returns the offset into the data segment based on a given `Decl`
4041 pub fn offset(self: DataSection, decl: *const Module.Decl) u32 {
4142 var cur_offset: u32 = 0;
4243 return for (self.segments.items()) |entry| {
4344 if (entry.key == decl) break cur_offset;
44 cur_offset += @intCast(u32, entry.value.len);
45 } else cur_offset;
45 cur_offset += entry.value.len;
46 } else unreachable; // offset() called on declaration that does not live inside 'data' section
4647 }
4748
4849 /// Returns the total payload size of the data section
4950 pub fn size(self: DataSection) u32 {
5051 var total: u32 = 0;
5152 for (self.segments.items()) |entry| {
52 total += @intCast(u32, entry.value.len);
53 total += entry.value.len;
5354 }
5455 return total;
5556 }
57
58 /// Updates the data in the data segment belonging to the given decl.
59 /// It's illegal behaviour to call this before allocateDeclIndexes was called
60 /// `data` must be managed externally with a lifetime that last as long as codegen does.
61 pub fn updateData(self: DataSection, decl: *Module.Decl, data: []const u8) void {
62 const entry = self.segments.getEntry(decl).?; // called updateData before the declaration was added to data segments
63 entry.value.data = data.ptr;
64 }
5665};
5766
5867base: link.File,
......@@ -111,49 +120,92 @@ pub fn createEmpty(gpa: *Allocator, options: link.Options) !*Wasm {
111120
112121pub fn deinit(self: *Wasm) void {
113122 for (self.funcs.items) |decl| {
114 decl.fn_link.wasm.?.functype.deinit(self.base.allocator);
115 decl.fn_link.wasm.?.code.deinit(self.base.allocator);
116 decl.fn_link.wasm.?.idx_refs.deinit(self.base.allocator);
123 decl.fn_link.wasm.functype.deinit(self.base.allocator);
124 decl.fn_link.wasm.code.deinit(self.base.allocator);
125 decl.fn_link.wasm.idx_refs.deinit(self.base.allocator);
117126 }
118127 for (self.ext_funcs.items) |decl| {
119 decl.fn_link.wasm.?.functype.deinit(self.base.allocator);
120 decl.fn_link.wasm.?.code.deinit(self.base.allocator);
121 decl.fn_link.wasm.?.idx_refs.deinit(self.base.allocator);
128 decl.fn_link.wasm.functype.deinit(self.base.allocator);
129 decl.fn_link.wasm.code.deinit(self.base.allocator);
130 decl.fn_link.wasm.idx_refs.deinit(self.base.allocator);
131 }
132 for (self.data.segments.items()) |entry| {
133 // data segments only use the code section
134 entry.key.fn_link.wasm.code.deinit(self.base.allocator);
122135 }
123136 self.funcs.deinit(self.base.allocator);
124137 self.ext_funcs.deinit(self.base.allocator);
125138 self.data.segments.deinit(self.base.allocator);
126139}
127140
141pub fn allocateDeclIndexes(self: *Wasm, decl: *Module.Decl) !void {
142 std.debug.print("INIT: '{s}'\n", .{decl.name});
143 const tv = decl.typed_value.most_recent.typed_value;
144 decl.fn_link.wasm = .{};
145
146 switch (tv.ty.zigTypeTag()) {
147 .Array => {
148 // if the codegen of the given decl contributes to the data segment
149 // we must calculate its data length now so that the data offsets are available
150 // to other decls when called
151 const data_len = calcDataLen(self, tv) catch return error.AnalysisFail;
152 try self.data.segments.putNoClobber(self.base.allocator, decl, .{ .data = undefined, .len = data_len });
153 },
154 .Fn => if (self.getFuncidx(decl) == null) switch (tv.val.tag()) {
155 // dependent on function type, appends it to the correct list
156 .function => try self.funcs.append(self.base.allocator, decl),
157 .extern_fn => try self.ext_funcs.append(self.base.allocator, decl),
158 else => unreachable,
159 },
160 else => {},
161 }
162}
163
164// TODO, remove this and use the existing error mechanism
165const DataLenError = error{
166 TODO_WASM_CalcDataLenArray,
167 TODO_WASM_CalcDataLen,
168};
169/// Calculates the length of the data segment that will be occupied by the given `TypedValue`
170fn calcDataLen(bin_file: *Wasm, typed_value: TypedValue) DataLenError!u32 {
171 switch (typed_value.ty.zigTypeTag()) {
172 .Array => {
173 if (typed_value.val.castTag(.bytes)) |payload| {
174 if (typed_value.ty.sentinel()) |sentinel| {
175 return @intCast(u32, payload.data.len) + try calcDataLen(bin_file, .{
176 .ty = typed_value.ty.elemType(),
177 .val = sentinel,
178 });
179 }
180 return @intCast(u32, payload.data.len);
181 }
182 return error.TODO_WASM_CalcDataLenArray;
183 },
184 .Int => {
185 const info = typed_value.ty.intInfo(bin_file.base.options.target);
186 return info.bits / 8;
187 },
188 .Pointer => return 4,
189 else => return error.TODO_WASM_CalcDataLen,
190 }
191}
192
128193// Generate code for the Decl, storing it in memory to be later written to
129194// the file on flush().
130195pub fn updateDecl(self: *Wasm, module: *Module, decl: *Module.Decl) !void {
196 std.debug.print("Updating '{s}'\n", .{decl.name});
131197 const typed_value = decl.typed_value.most_recent.typed_value;
132198
133 if (decl.fn_link.wasm) |*fn_data| {
134 fn_data.functype.items.len = 0;
135 fn_data.code.items.len = 0;
136 fn_data.idx_refs.items.len = 0;
137 } else {
138 decl.fn_link.wasm = .{};
139 // dependent on function type, appends it to the correct list
140 switch (decl.typed_value.most_recent.typed_value.val.tag()) {
141 .function => try self.funcs.append(self.base.allocator, decl),
142 .extern_fn => try self.ext_funcs.append(self.base.allocator, decl),
143 .bytes => {},
144 else => return error.TODOImplementNonFnDeclsForWasm,
145 }
146 }
147 const fn_data = &decl.fn_link.wasm.?;
148
149 var managed_functype = fn_data.functype.toManaged(self.base.allocator);
150 var managed_code = fn_data.code.toManaged(self.base.allocator);
199 const fn_data = &decl.fn_link.wasm;
200 fn_data.functype.items.len = 0;
201 fn_data.code.items.len = 0;
202 fn_data.idx_refs.items.len = 0;
151203
152204 var context = codegen.Context{
153205 .gpa = self.base.allocator,
154206 .values = .{},
155 .code = managed_code,
156 .func_type_data = managed_functype,
207 .code = fn_data.code.toManaged(self.base.allocator),
208 .func_type_data = fn_data.functype.toManaged(self.base.allocator),
157209 .decl = decl,
158210 .err_msg = undefined,
159211 .locals = .{},
......@@ -162,7 +214,7 @@ pub fn updateDecl(self: *Wasm, module: *Module, decl: *Module.Decl) !void {
162214 defer context.deinit();
163215
164216 // generate the 'code' section for the function declaration
165 const result = context.gen() catch |err| switch (err) {
217 const result = context.gen(typed_value) catch |err| switch (err) {
166218 error.CodegenFail => {
167219 decl.analysis = .codegen_failure;
168220 try module.failed_decls.put(module.gpa, decl, context.err_msg);
......@@ -175,7 +227,7 @@ pub fn updateDecl(self: *Wasm, module: *Module, decl: *Module.Decl) !void {
175227 .Fn => {
176228 // as locals are patched afterwards, the offsets of funcidx's are off,
177229 // here we update them to correct them
178 for (decl.fn_link.wasm.?.idx_refs.items) |*func| {
230 for (decl.fn_link.wasm.idx_refs.items) |*func| {
179231 // For each local, add 6 bytes (count + type)
180232 func.offset += @intCast(u32, context.locals.items.len * 6);
181233 }
......@@ -184,8 +236,12 @@ pub fn updateDecl(self: *Wasm, module: *Module, decl: *Module.Decl) !void {
184236 fn_data.code = context.code.toUnmanaged();
185237 },
186238 .Array => switch (result) {
187 .appended => unreachable,
188 .externally_managed => |payload| try self.data.segments.put(self.base.allocator, decl, payload),
239 .appended => {
240 fn_data.functype = context.func_type_data.toUnmanaged();
241 fn_data.code = context.code.toUnmanaged();
242 self.data.updateData(decl, fn_data.code.items);
243 },
244 .externally_managed => |payload| self.data.updateData(decl, payload),
189245 },
190246 else => return error.TODO,
191247 }
......@@ -199,18 +255,18 @@ pub fn updateDeclExports(
199255) !void {}
200256
201257pub fn freeDecl(self: *Wasm, decl: *Module.Decl) void {
202 // TODO: remove this assert when non-function Decls are implemented
203 assert(decl.typed_value.most_recent.typed_value.ty.zigTypeTag() == .Fn);
204 const func_idx = self.getFuncidx(decl).?;
205 switch (decl.typed_value.most_recent.typed_value.val.tag()) {
206 .function => _ = self.funcs.swapRemove(func_idx),
207 .extern_fn => _ = self.ext_funcs.swapRemove(func_idx),
208 else => unreachable,
258 if (self.getFuncidx(decl)) |func_idx| {
259 switch (decl.typed_value.most_recent.typed_value.val.tag()) {
260 .function => _ = self.funcs.swapRemove(func_idx),
261 .extern_fn => _ = self.ext_funcs.swapRemove(func_idx),
262 else => unreachable,
263 }
209264 }
210 decl.fn_link.wasm.?.functype.deinit(self.base.allocator);
211 decl.fn_link.wasm.?.code.deinit(self.base.allocator);
212 decl.fn_link.wasm.?.idx_refs.deinit(self.base.allocator);
213 decl.fn_link.wasm = null;
265 decl.fn_link.wasm.functype.deinit(self.base.allocator);
266 decl.fn_link.wasm.code.deinit(self.base.allocator);
267 decl.fn_link.wasm.idx_refs.deinit(self.base.allocator);
268 _ = self.data.segments.orderedRemove(decl);
269 decl.fn_link.wasm = undefined;
214270}
215271
216272pub fn flush(self: *Wasm, comp: *Compilation) !void {
......@@ -238,8 +294,8 @@ pub fn flushModule(self: *Wasm, comp: *Compilation) !void {
238294
239295 // extern functions are defined in the wasm binary first through the `import`
240296 // section, so define their func types first
241 for (self.ext_funcs.items) |decl| try file.writeAll(decl.fn_link.wasm.?.functype.items);
242 for (self.funcs.items) |decl| try file.writeAll(decl.fn_link.wasm.?.functype.items);
297 for (self.ext_funcs.items) |decl| try file.writeAll(decl.fn_link.wasm.functype.items);
298 for (self.funcs.items) |decl| try file.writeAll(decl.fn_link.wasm.functype.items);
243299
244300 try writeVecSectionHeader(
245301 file,
......@@ -302,13 +358,22 @@ pub fn flushModule(self: *Wasm, comp: *Compilation) !void {
302358 const writer = file.writer();
303359
304360 try leb.writeULEB128(writer, @as(u32, 0));
305 try leb.writeULEB128(writer, @as(u32, 1));
361 // Calculate the amount of memory pages are required and write them.
362 // Wasm uses 64kB page sizes. Round up to ensure the data segments fit into the memory
363 try leb.writeULEB128(
364 writer,
365 try std.math.divCeil(
366 u32,
367 self.data.size(),
368 std.mem.page_size,
369 ),
370 );
306371 try writeVecSectionHeader(
307372 file,
308373 header_offset,
309374 .memory,
310375 @intCast(u32, (try file.getPos()) - header_offset - header_size),
311 @as(u32, 1),
376 @as(u32, 1), // wasm currently only supports 1 linear memory segment
312377 );
313378 }
314379
......@@ -360,7 +425,7 @@ pub fn flushModule(self: *Wasm, comp: *Compilation) !void {
360425 const header_offset = try reserveVecSectionHeader(file);
361426 const writer = file.writer();
362427 for (self.funcs.items) |decl| {
363 const fn_data = &decl.fn_link.wasm.?;
428 const fn_data = &decl.fn_link.wasm;
364429
365430 // Write the already generated code to the file, inserting
366431 // function indexes where required.
......@@ -387,34 +452,30 @@ pub fn flushModule(self: *Wasm, comp: *Compilation) !void {
387452 }
388453
389454 // Data section
390 {
455 if (self.data.size() != 0) {
391456 const header_offset = try reserveVecSectionHeader(file);
392457 const writer = file.writer();
393 var offset: i32 = 0;
394 for (self.data.segments.items()) |entry| {
395 // index to memory section (always 0 in current wasm version)
396 try leb.writeULEB128(writer, @as(u32, 0));
397
398 // offset into data section
399 try writer.writeByte(wasm.opcode(.i32_const));
400 try leb.writeILEB128(writer, offset);
401 try writer.writeByte(wasm.opcode(.end));
402
403 // payload size
404 const len = @intCast(u32, entry.value.len);
405 try leb.writeULEB128(writer, len);
406
407 // write payload
408 try writer.writeAll(entry.value);
409 offset += @bitCast(i32, len);
410 }
458 var len: u32 = 0;
459 // index to memory section (currently, there can only be 1 memory section in wasm)
460 try leb.writeULEB128(writer, @as(u32, 0));
461
462 // offset into data section
463 try writer.writeByte(wasm.opcode(.i32_const));
464 try leb.writeILEB128(writer, @as(i32, 0));
465 try writer.writeByte(wasm.opcode(.end));
466
467 // payload size
468 try leb.writeULEB128(writer, self.data.size());
469
470 // write payload
471 for (self.data.segments.items()) |entry| try writer.writeAll(entry.value.data[0..entry.value.len]);
411472
412473 try writeVecSectionHeader(
413474 file,
414475 header_offset,
415476 .data,
416477 @intCast(u32, (try file.getPos()) - header_offset - header_size),
417 @intCast(u32, self.data.segments.items().len),
478 @intCast(u32, 1),
418479 );
419480 }
420481}
......@@ -681,7 +742,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
681742/// Get the current index of a given Decl in the function list
682743/// This will correctly provide the index, regardless whether the function is extern or not
683744/// TODO: we could maintain a hash map to potentially make this simpler
684fn getFuncidx(self: Wasm, decl: *Module.Decl) ?u32 {
745fn getFuncidx(self: Wasm, decl: *const Module.Decl) ?u32 {
685746 var offset: u32 = 0;
686747 const slice = switch (decl.typed_value.most_recent.typed_value.val.tag()) {
687748 .function => blk: {