1prologue: []const Instruction,
2body: []const Instruction,
3epilogue: []const Instruction,
4literals: []const u32,
5nav_relocs: []const Reloc.Nav,
6uav_relocs: []const Reloc.Uav,
7lazy_relocs: []const Reloc.Lazy,
8global_relocs: []const Reloc.Global,
9literal_relocs: []const Reloc.Literal,
10
11pub const Reloc = struct {
12 label: u32,
13 addend: u64 align(@alignOf(u32)) = 0,
14
15 pub const Nav = struct {
16 nav: InternPool.Nav.Index,
17 reloc: Reloc,
18 };
19
20 pub const Uav = struct {
21 uav: InternPool.Key.Ptr.BaseAddr.Uav,
22 reloc: Reloc,
23 };
24
25 pub const Lazy = struct {
26 symbol: link.File.LazySymbol,
27 reloc: Reloc,
28 };
29
30 pub const Global = struct {
31 name: [*:0]const u8,
32 reloc: Reloc,
33 };
34
35 pub const Literal = struct {
36 label: u32,
37 };
38};
39
40pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void {
41 assert(mir.body.ptr + mir.body.len == mir.prologue.ptr);
42 assert(mir.prologue.ptr + mir.prologue.len == mir.epilogue.ptr);
43 gpa.free(mir.body.ptr[0 .. mir.body.len + mir.prologue.len + mir.epilogue.len]);
44 gpa.free(mir.literals);
45 gpa.free(mir.nav_relocs);
46 gpa.free(mir.uav_relocs);
47 gpa.free(mir.lazy_relocs);
48 gpa.free(mir.global_relocs);
49 gpa.free(mir.literal_relocs);
50 mir.* = undefined;
51}
52
53pub fn emit(
54 mir: Mir,
55 lf: *link.File,
56 pt: Zcu.PerThread,
57 func_index: InternPool.Index,
58 atom_index: link.File.AtomId,
59 w: *std.Io.Writer,
60 debug_output: link.File.DebugInfoOutput,
61) !void {
62 _ = debug_output;
63 const zcu = pt.zcu;
64 const ip = &zcu.intern_pool;
65 const func = zcu.funcInfo(func_index);
66 const nav = ip.getNav(func.owner_nav);
67 const mod = zcu.navFileScope(func.owner_nav).mod.?;
68 const target = &mod.resolved_target.result;
69 mir_log.debug("{f}:", .{nav.fqn.fmt(ip)});
70
71 const func_align = switch (nav.resolved.?.@"align") {
72 .none => switch (mod.optimize_mode) {
73 .debug, .safe, .fast => target_util.defaultFunctionAlignment(target),
74 .small => target_util.minFunctionAlignment(target),
75 },
76 else => |a| a.maxStrict(target_util.minFunctionAlignment(target)),
77 };
78 const code_len = mir.prologue.len + mir.body.len + mir.epilogue.len;
79 const literals_align_gap = -%code_len & (@divExact(
80 @as(u5, @intCast(func_align.minStrict(.@"16").toByteUnits().?)),
81 Instruction.size,
82 ) - 1);
83 try w.rebase(w.end, Instruction.size * (code_len + literals_align_gap + mir.literals.len));
84 emitInstructionsForward(w, mir.prologue) catch unreachable;
85 emitInstructionsBackward(w, mir.body) catch unreachable;
86 const body_end: u32 = @intCast(w.end);
87 emitInstructionsBackward(w, mir.epilogue) catch unreachable;
88 w.splatByteAll(0, Instruction.size * literals_align_gap) catch unreachable;
89 w.writeAll(@ptrCast(mir.literals)) catch unreachable;
90 mir_log.debug("", .{});
91
92 for (mir.nav_relocs) |nav_reloc| try emitReloc(
93 lf,
94 zcu,
95 atom_index,
96 try @import("../../codegen.zig").genNavRef(
97 lf,
98 pt,
99 nav_reloc.nav,
100 ),
101 mir.body[nav_reloc.reloc.label],
102 body_end - Instruction.size * (1 + nav_reloc.reloc.label),
103 nav_reloc.reloc.addend,
104 if (ip.getNav(nav_reloc.nav).getExtern(ip)) |_| .got_load else .direct,
105 );
106 for (mir.uav_relocs) |uav_reloc| try emitReloc(
107 lf,
108 zcu,
109 atom_index,
110 try lf.lowerUav(
111 pt,
112 uav_reloc.uav.val,
113 ZigType.fromInterned(uav_reloc.uav.orig_ty).ptrAlignment(zcu),
114 ),
115 mir.body[uav_reloc.reloc.label],
116 body_end - Instruction.size * (1 + uav_reloc.reloc.label),
117 uav_reloc.reloc.addend,
118 .direct,
119 );
120 for (mir.lazy_relocs) |lazy_reloc| try emitReloc(
121 lf,
122 zcu,
123 atom_index,
124 if (lf.cast(.elf)) |ef|
125 @fromBackingInt(@intCast(ef.zigObjectPtr().?.getOrCreateMetadataForLazySymbol(ef, pt, lazy_reloc.symbol) catch |err|
126 return zcu.codegenFail(func.owner_nav, "{s} creating lazy symbol", .{@errorName(err)})))
127 else if (lf.cast(.macho)) |mf|
128 @fromBackingInt(@intCast(mf.getZigObject().?.getOrCreateMetadataForLazySymbol(mf, pt, lazy_reloc.symbol) catch |err|
129 return zcu.codegenFail(func.owner_nav, "{s} creating lazy symbol", .{@errorName(err)})))
130 else
131 return zcu.codegenFail(func.owner_nav, "external symbols unimplemented for {t}", .{lf.tag}),
132 mir.body[lazy_reloc.reloc.label],
133 body_end - Instruction.size * (1 + lazy_reloc.reloc.label),
134 lazy_reloc.reloc.addend,
135 .direct,
136 );
137 for (mir.global_relocs) |global_reloc| try emitReloc(
138 lf,
139 zcu,
140 atom_index,
141 if (lf.cast(.elf)) |ef|
142 @fromBackingInt(@intCast(try ef.getGlobalSymbol(std.mem.span(global_reloc.name), null)))
143 else if (lf.cast(.macho)) |mf|
144 @fromBackingInt(@intCast(try mf.getGlobalSymbol(std.mem.span(global_reloc.name), null)))
145 else
146 return zcu.codegenFail(func.owner_nav, "external symbols unimplemented for {t}", .{lf.tag}),
147 mir.body[global_reloc.reloc.label],
148 body_end - Instruction.size * (1 + global_reloc.reloc.label),
149 global_reloc.reloc.addend,
150 .direct,
151 );
152 const literal_reloc_offset: i19 = @intCast(mir.epilogue.len + literals_align_gap);
153 for (mir.literal_relocs) |literal_reloc| {
154 var instruction = mir.body[literal_reloc.label];
155 instruction.load_store.register_literal.group.imm19 += literal_reloc_offset;
156 instruction.write(
157 w.buffered()[body_end - Instruction.size * (1 + literal_reloc.label) ..][0..Instruction.size],
158 );
159 }
160}
161
162fn emitInstructionsForward(w: *std.Io.Writer, instructions: []const Instruction) !void {
163 for (instructions) |instruction| try emitInstruction(w, instruction);
164}
165fn emitInstructionsBackward(w: *std.Io.Writer, instructions: []const Instruction) !void {
166 var instruction_index = instructions.len;
167 while (instruction_index > 0) {
168 instruction_index -= 1;
169 try emitInstruction(w, instructions[instruction_index]);
170 }
171}
172fn emitInstruction(w: *std.Io.Writer, instruction: Instruction) !void {
173 mir_log.debug(" {f}", .{instruction});
174 instruction.write(try w.writableArray(Instruction.size));
175}
176
177fn emitReloc(
178 lf: *link.File,
179 zcu: *Zcu,
180 atom_index: link.File.AtomId,
181 sym_index: link.File.SymbolId,
182 instruction: Instruction,
183 offset: u32,
184 addend: u64,
185 kind: enum { direct, got_load },
186) !void {
187 const gpa = zcu.gpa;
188 switch (instruction.decode()) {
189 else => unreachable,
190 .data_processing_immediate => |decoded| if (lf.cast(.elf)) |ef| {
191 const zo = ef.zigObjectPtr().?;
192 const atom = zo.symbol(@backingInt(atom_index)).atom(ef).?;
193 const r_type: std.elf.R_AARCH64 = switch (decoded.decode()) {
194 else => unreachable,
195 .pc_relative_addressing => |pc_relative_addressing| switch (pc_relative_addressing.group.op) {
196 .adr => switch (kind) {
197 .direct => .ADR_PREL_LO21,
198 .got_load => unreachable,
199 },
200 .adrp => switch (kind) {
201 .direct => .ADR_PREL_PG_HI21,
202 .got_load => .ADR_GOT_PAGE,
203 },
204 },
205 .add_subtract_immediate => |add_subtract_immediate| switch (add_subtract_immediate.group.op) {
206 .add => switch (kind) {
207 .direct => .ADD_ABS_LO12_NC,
208 .got_load => unreachable,
209 },
210 .sub => unreachable,
211 },
212 };
213 try atom.addReloc(gpa, .{
214 .r_offset = offset,
215 .r_info = @as(u64, @backingInt(sym_index)) << 32 | @backingInt(r_type),
216 .r_addend = @bitCast(addend),
217 }, zo);
218 } else if (lf.cast(.macho)) |mf| {
219 const zo = mf.getZigObject().?;
220 const atom = zo.symbols.items[@backingInt(atom_index)].getAtom(mf).?;
221 switch (decoded.decode()) {
222 else => unreachable,
223 .pc_relative_addressing => |pc_relative_addressing| switch (pc_relative_addressing.group.op) {
224 .adr => unreachable,
225 .adrp => try atom.addReloc(mf, .{
226 .tag = .@"extern",
227 .offset = offset,
228 .target = @backingInt(sym_index),
229 .addend = @bitCast(addend),
230 .type = switch (kind) {
231 .direct => .page,
232 .got_load => .got_load_page,
233 },
234 .meta = .{
235 .pcrel = true,
236 .has_subtractor = false,
237 .length = 2,
238 .symbolnum = @intCast(@backingInt(sym_index)),
239 },
240 }),
241 },
242 .add_subtract_immediate => |add_subtract_immediate| switch (add_subtract_immediate.group.op) {
243 .add => try atom.addReloc(mf, .{
244 .tag = .@"extern",
245 .offset = offset,
246 .target = @backingInt(sym_index),
247 .addend = @bitCast(addend),
248 .type = switch (kind) {
249 .direct => .pageoff,
250 .got_load => .got_load_pageoff,
251 },
252 .meta = .{
253 .pcrel = false,
254 .has_subtractor = false,
255 .length = 2,
256 .symbolnum = @intCast(@backingInt(sym_index)),
257 },
258 }),
259 .sub => unreachable,
260 },
261 }
262 },
263 .branch_exception_generating_system => |decoded| if (lf.cast(.elf)) |ef| {
264 const zo = ef.zigObjectPtr().?;
265 const atom = zo.symbol(@backingInt(atom_index)).atom(ef).?;
266 const r_type: std.elf.R_AARCH64 = switch (decoded.decode().unconditional_branch_immediate.group.op) {
267 .b => .JUMP26,
268 .bl => .CALL26,
269 };
270 try atom.addReloc(gpa, .{
271 .r_offset = offset,
272 .r_info = @as(u64, @backingInt(sym_index)) << 32 | @backingInt(r_type),
273 .r_addend = @bitCast(addend),
274 }, zo);
275 } else if (lf.cast(.macho)) |mf| {
276 const zo = mf.getZigObject().?;
277 const atom = zo.symbols.items[@backingInt(atom_index)].getAtom(mf).?;
278 try atom.addReloc(mf, .{
279 .tag = .@"extern",
280 .offset = offset,
281 .target = @backingInt(sym_index),
282 .addend = @bitCast(addend),
283 .type = .branch,
284 .meta = .{
285 .pcrel = true,
286 .has_subtractor = false,
287 .length = 2,
288 .symbolnum = @intCast(@backingInt(sym_index)),
289 },
290 });
291 },
292 .load_store => |decoded| if (lf.cast(.elf)) |ef| {
293 const zo = ef.zigObjectPtr().?;
294 const atom = zo.symbol(@backingInt(atom_index)).atom(ef).?;
295 const r_type: std.elf.R_AARCH64 = switch (decoded.decode().register_unsigned_immediate.decode()) {
296 .integer => |integer| switch (integer.decode()) {
297 .unallocated, .prfm => unreachable,
298 .strb, .ldrb, .ldrsb => switch (kind) {
299 .direct => .LDST8_ABS_LO12_NC,
300 .got_load => unreachable,
301 },
302 .strh, .ldrh, .ldrsh => switch (kind) {
303 .direct => .LDST16_ABS_LO12_NC,
304 .got_load => unreachable,
305 },
306 .ldrsw => switch (kind) {
307 .direct => .LDST32_ABS_LO12_NC,
308 .got_load => unreachable,
309 },
310 inline .str, .ldr => |encoded, mnemonic| switch (encoded.sf) {
311 .word => .LDST32_ABS_LO12_NC,
312 .doubleword => switch (kind) {
313 .direct => .LDST64_ABS_LO12_NC,
314 .got_load => switch (mnemonic) {
315 else => comptime unreachable,
316 .str => unreachable,
317 .ldr => .LD64_GOT_LO12_NC,
318 },
319 },
320 },
321 },
322 .vector => |vector| switch (kind) {
323 .direct => switch (vector.group.opc1.decode(vector.group.size)) {
324 .byte => .LDST8_ABS_LO12_NC,
325 .half => .LDST16_ABS_LO12_NC,
326 .single => .LDST32_ABS_LO12_NC,
327 .double => .LDST64_ABS_LO12_NC,
328 .quad => .LDST128_ABS_LO12_NC,
329 },
330 .got_load => unreachable,
331 },
332 };
333 try atom.addReloc(gpa, .{
334 .r_offset = offset,
335 .r_info = @as(u64, @backingInt(sym_index)) << 32 | @backingInt(r_type),
336 .r_addend = @bitCast(addend),
337 }, zo);
338 } else if (lf.cast(.macho)) |mf| {
339 const zo = mf.getZigObject().?;
340 const atom = zo.symbols.items[@backingInt(atom_index)].getAtom(mf).?;
341 try atom.addReloc(mf, .{
342 .tag = .@"extern",
343 .offset = offset,
344 .target = @backingInt(sym_index),
345 .addend = @bitCast(addend),
346 .type = switch (kind) {
347 .direct => .pageoff,
348 .got_load => .got_load_pageoff,
349 },
350 .meta = .{
351 .pcrel = false,
352 .has_subtractor = false,
353 .length = 2,
354 .symbolnum = @intCast(@backingInt(sym_index)),
355 },
356 });
357 },
358 }
359}
360
361const Air = @import("../../Air.zig");
362const assert = std.debug.assert;
363const mir_log = std.log.scoped(.mir);
364const Instruction = @import("encoding.zig").Instruction;
365const InternPool = @import("../../InternPool.zig");
366const link = @import("../../link.zig");
367const Mir = @This();
368const std = @import("std");
369const target_util = @import("../../target.zig");
370const Zcu = @import("../../Zcu.zig");
371const ZigType = @import("../../Type.zig");