| ... | @@ -273,8 +273,22 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -273,8 +273,22 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 273 | /// across each runtime branch upon joining. | 273 | /// across each runtime branch upon joining. |
| 274 | branch_stack: *std.ArrayList(Branch), | 274 | branch_stack: *std.ArrayList(Branch), |
| 275 | | 275 | |
| | 276 | /// The key must be canonical register. |
| | 277 | registers: std.AutoHashMapUnmanaged(Register, *ir.Inst) = .{}, |
| | 278 | free_registers: FreeRegInt = math.maxInt(FreeRegInt), |
| | 279 | /// Maps offset to what is stored there. |
| | 280 | stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{}, |
| | 281 | |
| | 282 | /// Offset from the stack base, representing the end of the stack frame. |
| | 283 | max_end_stack: u32 = 0, |
| | 284 | /// Represents the current end stack offset. If there is no existing slot |
| | 285 | /// to place a new stack allocation, it goes here, and then bumps `max_end_stack`. |
| | 286 | next_stack_offset: u32 = 0, |
| | 287 | |
| 276 | const MCValue = union(enum) { | 288 | const MCValue = union(enum) { |
| 277 | /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. | 289 | /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. |
| | 290 | /// TODO Look into deleting this tag and using `dead` instead, since every use |
| | 291 | /// of MCValue.none should be instead looking at the type and noticing it is 0 bits. |
| 278 | none, | 292 | none, |
| 279 | /// Control flow will not allow this value to be observed. | 293 | /// Control flow will not allow this value to be observed. |
| 280 | unreach, | 294 | unreach, |
| ... | @@ -346,71 +360,55 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -346,71 +360,55 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 346 | | 360 | |
| 347 | const Branch = struct { | 361 | const Branch = struct { |
| 348 | inst_table: std.AutoHashMapUnmanaged(*ir.Inst, MCValue) = .{}, | 362 | inst_table: std.AutoHashMapUnmanaged(*ir.Inst, MCValue) = .{}, |
| 349 | /// The key must be canonical register. | | |
| 350 | registers: std.AutoHashMapUnmanaged(Register, RegisterAllocation) = .{}, | | |
| 351 | free_registers: FreeRegInt = math.maxInt(FreeRegInt), | | |
| 352 | | | |
| 353 | /// Maps offset to what is stored there. | | |
| 354 | stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{}, | | |
| 355 | /// Offset from the stack base, representing the end of the stack frame. | | |
| 356 | max_end_stack: u32 = 0, | | |
| 357 | /// Represents the current end stack offset. If there is no existing slot | | |
| 358 | /// to place a new stack allocation, it goes here, and then bumps `max_end_stack`. | | |
| 359 | next_stack_offset: u32 = 0, | | |
| 360 | | | |
| 361 | fn markRegUsed(self: *Branch, reg: Register) void { | | |
| 362 | if (FreeRegInt == u0) return; | | |
| 363 | const index = reg.allocIndex() orelse return; | | |
| 364 | const ShiftInt = math.Log2Int(FreeRegInt); | | |
| 365 | const shift = @intCast(ShiftInt, index); | | |
| 366 | self.free_registers &= ~(@as(FreeRegInt, 1) << shift); | | |
| 367 | } | | |
| 368 | | | |
| 369 | fn markRegFree(self: *Branch, reg: Register) void { | | |
| 370 | if (FreeRegInt == u0) return; | | |
| 371 | const index = reg.allocIndex() orelse return; | | |
| 372 | const ShiftInt = math.Log2Int(FreeRegInt); | | |
| 373 | const shift = @intCast(ShiftInt, index); | | |
| 374 | self.free_registers |= @as(FreeRegInt, 1) << shift; | | |
| 375 | } | | |
| 376 | | | |
| 377 | /// Before calling, must ensureCapacity + 1 on branch.registers. | | |
| 378 | /// Returns `null` if all registers are allocated. | | |
| 379 | fn allocReg(self: *Branch, inst: *ir.Inst) ?Register { | | |
| 380 | const free_index = @ctz(FreeRegInt, self.free_registers); | | |
| 381 | if (free_index >= callee_preserved_regs.len) { | | |
| 382 | return null; | | |
| 383 | } | | |
| 384 | self.free_registers &= ~(@as(FreeRegInt, 1) << free_index); | | |
| 385 | const reg = callee_preserved_regs[free_index]; | | |
| 386 | self.registers.putAssumeCapacityNoClobber(reg, .{ .inst = inst }); | | |
| 387 | log.debug("alloc {} => {*}", .{reg, inst}); | | |
| 388 | return reg; | | |
| 389 | } | | |
| 390 | | | |
| 391 | /// Does not track the register. | | |
| 392 | fn findUnusedReg(self: *Branch) ?Register { | | |
| 393 | const free_index = @ctz(FreeRegInt, self.free_registers); | | |
| 394 | if (free_index >= callee_preserved_regs.len) { | | |
| 395 | return null; | | |
| 396 | } | | |
| 397 | return callee_preserved_regs[free_index]; | | |
| 398 | } | | |
| 399 | | 363 | |
| 400 | fn deinit(self: *Branch, gpa: *Allocator) void { | 364 | fn deinit(self: *Branch, gpa: *Allocator) void { |
| 401 | self.inst_table.deinit(gpa); | 365 | self.inst_table.deinit(gpa); |
| 402 | self.registers.deinit(gpa); | | |
| 403 | self.stack.deinit(gpa); | | |
| 404 | self.* = undefined; | 366 | self.* = undefined; |
| 405 | } | 367 | } |
| 406 | }; | 368 | }; |
| 407 | | 369 | |
| 408 | const RegisterAllocation = struct { | 370 | fn markRegUsed(self: *Self, reg: Register) void { |
| 409 | inst: *ir.Inst, | 371 | if (FreeRegInt == u0) return; |
| 410 | }; | 372 | const index = reg.allocIndex() orelse return; |
| | 373 | const ShiftInt = math.Log2Int(FreeRegInt); |
| | 374 | const shift = @intCast(ShiftInt, index); |
| | 375 | self.free_registers &= ~(@as(FreeRegInt, 1) << shift); |
| | 376 | } |
| | 377 | |
| | 378 | fn markRegFree(self: *Self, reg: Register) void { |
| | 379 | if (FreeRegInt == u0) return; |
| | 380 | const index = reg.allocIndex() orelse return; |
| | 381 | const ShiftInt = math.Log2Int(FreeRegInt); |
| | 382 | const shift = @intCast(ShiftInt, index); |
| | 383 | self.free_registers |= @as(FreeRegInt, 1) << shift; |
| | 384 | } |
| | 385 | |
| | 386 | /// Before calling, must ensureCapacity + 1 on self.registers. |
| | 387 | /// Returns `null` if all registers are allocated. |
| | 388 | fn allocReg(self: *Self, inst: *ir.Inst) ?Register { |
| | 389 | const free_index = @ctz(FreeRegInt, self.free_registers); |
| | 390 | if (free_index >= callee_preserved_regs.len) { |
| | 391 | return null; |
| | 392 | } |
| | 393 | self.free_registers &= ~(@as(FreeRegInt, 1) << free_index); |
| | 394 | const reg = callee_preserved_regs[free_index]; |
| | 395 | self.registers.putAssumeCapacityNoClobber(reg, inst); |
| | 396 | log.debug("alloc {} => {*}", .{reg, inst}); |
| | 397 | return reg; |
| | 398 | } |
| | 399 | |
| | 400 | /// Does not track the register. |
| | 401 | fn findUnusedReg(self: *Self) ?Register { |
| | 402 | const free_index = @ctz(FreeRegInt, self.free_registers); |
| | 403 | if (free_index >= callee_preserved_regs.len) { |
| | 404 | return null; |
| | 405 | } |
| | 406 | return callee_preserved_regs[free_index]; |
| | 407 | } |
| 411 | | 408 | |
| 412 | const StackAllocation = struct { | 409 | const StackAllocation = struct { |
| 413 | inst: *ir.Inst, | 410 | inst: *ir.Inst, |
| | 411 | /// TODO do we need size? should be determined by inst.ty.abiSize() |
| 414 | size: u32, | 412 | size: u32, |
| 415 | }; | 413 | }; |
| 416 | | 414 | |
| ... | @@ -435,8 +433,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -435,8 +433,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 435 | branch_stack.items[0].deinit(bin_file.allocator); | 433 | branch_stack.items[0].deinit(bin_file.allocator); |
| 436 | branch_stack.deinit(); | 434 | branch_stack.deinit(); |
| 437 | } | 435 | } |
| 438 | const branch = try branch_stack.addOne(); | 436 | try branch_stack.append(.{}); |
| 439 | branch.* = .{}; | | |
| 440 | | 437 | |
| 441 | const src_data: struct {lbrace_src: usize, rbrace_src: usize, source: []const u8} = blk: { | 438 | const src_data: struct {lbrace_src: usize, rbrace_src: usize, source: []const u8} = blk: { |
| 442 | if (module_fn.owner_decl.scope.cast(Module.Scope.File)) |scope_file| { | 439 | if (module_fn.owner_decl.scope.cast(Module.Scope.File)) |scope_file| { |
| ... | @@ -476,6 +473,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -476,6 +473,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 476 | .rbrace_src = src_data.rbrace_src, | 473 | .rbrace_src = src_data.rbrace_src, |
| 477 | .source = src_data.source, | 474 | .source = src_data.source, |
| 478 | }; | 475 | }; |
| | 476 | defer function.registers.deinit(bin_file.allocator); |
| | 477 | defer function.stack.deinit(bin_file.allocator); |
| 479 | defer function.exitlude_jump_relocs.deinit(bin_file.allocator); | 478 | defer function.exitlude_jump_relocs.deinit(bin_file.allocator); |
| 480 | | 479 | |
| 481 | var call_info = function.resolveCallingConventionValues(src, fn_type) catch |err| switch (err) { | 480 | var call_info = function.resolveCallingConventionValues(src, fn_type) catch |err| switch (err) { |
| ... | @@ -487,7 +486,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -487,7 +486,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 487 | function.args = call_info.args; | 486 | function.args = call_info.args; |
| 488 | function.ret_mcv = call_info.return_value; | 487 | function.ret_mcv = call_info.return_value; |
| 489 | function.stack_align = call_info.stack_align; | 488 | function.stack_align = call_info.stack_align; |
| 490 | branch.max_end_stack = call_info.stack_byte_count; | 489 | function.max_end_stack = call_info.stack_byte_count; |
| 491 | | 490 | |
| 492 | function.gen() catch |err| switch (err) { | 491 | function.gen() catch |err| switch (err) { |
| 493 | error.CodegenFail => return Result{ .fail = function.err_msg.? }, | 492 | error.CodegenFail => return Result{ .fail = function.err_msg.? }, |
| ... | @@ -523,7 +522,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -523,7 +522,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 523 | try self.dbgSetPrologueEnd(); | 522 | try self.dbgSetPrologueEnd(); |
| 524 | try self.genBody(self.mod_fn.analysis.success); | 523 | try self.genBody(self.mod_fn.analysis.success); |
| 525 | | 524 | |
| 526 | const stack_end = self.branch_stack.items[0].max_end_stack; | 525 | const stack_end = self.max_end_stack; |
| 527 | if (stack_end > math.maxInt(i32)) | 526 | if (stack_end > math.maxInt(i32)) |
| 528 | return self.fail(self.src, "too much stack used in call parameters", .{}); | 527 | return self.fail(self.src, "too much stack used in call parameters", .{}); |
| 529 | const aligned_stack_end = mem.alignForward(stack_end, self.stack_align); | 528 | const aligned_stack_end = mem.alignForward(stack_end, self.stack_align); |
| ... | @@ -580,13 +579,15 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -580,13 +579,15 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 580 | } | 579 | } |
| 581 | | 580 | |
| 582 | fn genBody(self: *Self, body: ir.Body) InnerError!void { | 581 | fn genBody(self: *Self, body: ir.Body) InnerError!void { |
| 583 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | | |
| 584 | const inst_table = &branch.inst_table; | | |
| 585 | for (body.instructions) |inst| { | 582 | for (body.instructions) |inst| { |
| | 583 | try self.ensureProcessDeathCapacity(@popCount(@TypeOf(inst.deaths), inst.deaths)); |
| | 584 | |
| 586 | const mcv = try self.genFuncInst(inst); | 585 | const mcv = try self.genFuncInst(inst); |
| 587 | log.debug("{*} => {}", .{inst, mcv}); | 586 | if (!inst.isUnused()) { |
| 588 | // TODO don't put void or dead things in here | 587 | log.debug("{*} => {}", .{inst, mcv}); |
| 589 | try inst_table.putNoClobber(self.gpa, inst, mcv); | 588 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| | 589 | try branch.inst_table.putNoClobber(self.gpa, inst, mcv); |
| | 590 | } |
| 590 | | 591 | |
| 591 | var i: ir.Inst.DeathsBitIndex = 0; | 592 | var i: ir.Inst.DeathsBitIndex = 0; |
| 592 | while (inst.getOperand(i)) |operand| : (i += 1) { | 593 | while (inst.getOperand(i)) |operand| : (i += 1) { |
| ... | @@ -628,21 +629,28 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -628,21 +629,28 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 628 | self.dbg_line.appendAssumeCapacity(DW.LNS_copy); | 629 | self.dbg_line.appendAssumeCapacity(DW.LNS_copy); |
| 629 | } | 630 | } |
| 630 | | 631 | |
| | 632 | /// Asserts there is already capacity to insert into top branch inst_table. |
| 631 | fn processDeath(self: *Self, inst: *ir.Inst) void { | 633 | fn processDeath(self: *Self, inst: *ir.Inst) void { |
| | 634 | if (inst.tag == .constant) return; // Constants are immortal. |
| | 635 | // When editing this function, note that the logic must synchronize with `reuseOperand`. |
| | 636 | const prev_value = self.getResolvedInstValue(inst); |
| 632 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 637 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| 633 | const entry = branch.inst_table.getEntry(inst) orelse return; | 638 | branch.inst_table.putAssumeCapacity(inst, .dead); |
| 634 | const prev_value = entry.value; | | |
| 635 | entry.value = .dead; | | |
| 636 | switch (prev_value) { | 639 | switch (prev_value) { |
| 637 | .register => |reg| { | 640 | .register => |reg| { |
| 638 | const canon_reg = toCanonicalReg(reg); | 641 | const canon_reg = toCanonicalReg(reg); |
| 639 | _ = branch.registers.remove(canon_reg); | 642 | _ = self.registers.remove(canon_reg); |
| 640 | branch.markRegFree(canon_reg); | 643 | self.markRegFree(canon_reg); |
| 641 | }, | 644 | }, |
| 642 | else => {}, // TODO process stack allocation death | 645 | else => {}, // TODO process stack allocation death |
| 643 | } | 646 | } |
| 644 | } | 647 | } |
| 645 | | 648 | |
| | 649 | fn ensureProcessDeathCapacity(self: *Self, additional_count: usize) !void { |
| | 650 | const table = &self.branch_stack.items[self.branch_stack.items.len - 1].inst_table; |
| | 651 | try table.ensureCapacity(self.gpa, table.items().len + additional_count); |
| | 652 | } |
| | 653 | |
| 646 | /// Adds a Type to the .debug_info at the current position. The bytes will be populated later, | 654 | /// Adds a Type to the .debug_info at the current position. The bytes will be populated later, |
| 647 | /// after codegen for this symbol is done. | 655 | /// after codegen for this symbol is done. |
| 648 | fn addDbgInfoTypeReloc(self: *Self, ty: Type) !void { | 656 | fn addDbgInfoTypeReloc(self: *Self, ty: Type) !void { |
| ... | @@ -705,13 +713,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -705,13 +713,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 705 | fn allocMem(self: *Self, inst: *ir.Inst, abi_size: u32, abi_align: u32) !u32 { | 713 | fn allocMem(self: *Self, inst: *ir.Inst, abi_size: u32, abi_align: u32) !u32 { |
| 706 | if (abi_align > self.stack_align) | 714 | if (abi_align > self.stack_align) |
| 707 | self.stack_align = abi_align; | 715 | self.stack_align = abi_align; |
| 708 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | | |
| 709 | // TODO find a free slot instead of always appending | 716 | // TODO find a free slot instead of always appending |
| 710 | const offset = mem.alignForwardGeneric(u32, branch.next_stack_offset, abi_align); | 717 | const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align); |
| 711 | branch.next_stack_offset = offset + abi_size; | 718 | self.next_stack_offset = offset + abi_size; |
| 712 | if (branch.next_stack_offset > branch.max_end_stack) | 719 | if (self.next_stack_offset > self.max_end_stack) |
| 713 | branch.max_end_stack = branch.next_stack_offset; | 720 | self.max_end_stack = self.next_stack_offset; |
| 714 | try branch.stack.putNoClobber(self.gpa, offset, .{ | 721 | try self.stack.putNoClobber(self.gpa, offset, .{ |
| 715 | .inst = inst, | 722 | .inst = inst, |
| 716 | .size = abi_size, | 723 | .size = abi_size, |
| 717 | }); | 724 | }); |
| ... | @@ -737,15 +744,14 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -737,15 +744,14 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 737 | const abi_align = elem_ty.abiAlignment(self.target.*); | 744 | const abi_align = elem_ty.abiAlignment(self.target.*); |
| 738 | if (abi_align > self.stack_align) | 745 | if (abi_align > self.stack_align) |
| 739 | self.stack_align = abi_align; | 746 | self.stack_align = abi_align; |
| 740 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | | |
| 741 | | 747 | |
| 742 | if (reg_ok) { | 748 | if (reg_ok) { |
| 743 | // Make sure the type can fit in a register before we try to allocate one. | 749 | // Make sure the type can fit in a register before we try to allocate one. |
| 744 | const ptr_bits = arch.ptrBitWidth(); | 750 | const ptr_bits = arch.ptrBitWidth(); |
| 745 | const ptr_bytes: u64 = @divExact(ptr_bits, 8); | 751 | const ptr_bytes: u64 = @divExact(ptr_bits, 8); |
| 746 | if (abi_size <= ptr_bytes) { | 752 | if (abi_size <= ptr_bytes) { |
| 747 | try branch.registers.ensureCapacity(self.gpa, branch.registers.items().len + 1); | 753 | try self.registers.ensureCapacity(self.gpa, self.registers.items().len + 1); |
| 748 | if (branch.allocReg(inst)) |reg| { | 754 | if (self.allocReg(inst)) |reg| { |
| 749 | return MCValue{ .register = registerAlias(reg, abi_size) }; | 755 | return MCValue{ .register = registerAlias(reg, abi_size) }; |
| 750 | } | 756 | } |
| 751 | } | 757 | } |
| ... | @@ -758,20 +764,18 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -758,20 +764,18 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 758 | /// allocated. A second call to `copyToTmpRegister` may return the same register. | 764 | /// allocated. A second call to `copyToTmpRegister` may return the same register. |
| 759 | /// This can have a side effect of spilling instructions to the stack to free up a register. | 765 | /// This can have a side effect of spilling instructions to the stack to free up a register. |
| 760 | fn copyToTmpRegister(self: *Self, src: usize, mcv: MCValue) !Register { | 766 | fn copyToTmpRegister(self: *Self, src: usize, mcv: MCValue) !Register { |
| 761 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 767 | const reg = self.findUnusedReg() orelse b: { |
| 762 | | | |
| 763 | const reg = branch.findUnusedReg() orelse b: { | | |
| 764 | // We'll take over the first register. Move the instruction that was previously | 768 | // We'll take over the first register. Move the instruction that was previously |
| 765 | // there to a stack allocation. | 769 | // there to a stack allocation. |
| 766 | const reg = callee_preserved_regs[0]; | 770 | const reg = callee_preserved_regs[0]; |
| 767 | const regs_entry = branch.registers.remove(reg).?; | 771 | const regs_entry = self.registers.remove(reg).?; |
| 768 | const spilled_inst = regs_entry.value.inst; | 772 | const spilled_inst = regs_entry.value; |
| 769 | | 773 | |
| 770 | const stack_mcv = try self.allocRegOrMem(spilled_inst, false); | 774 | const stack_mcv = try self.allocRegOrMem(spilled_inst, false); |
| 771 | const inst_entry = branch.inst_table.getEntry(spilled_inst).?; | 775 | const reg_mcv = self.getResolvedInstValue(spilled_inst); |
| 772 | const reg_mcv = inst_entry.value; | | |
| 773 | assert(reg == toCanonicalReg(reg_mcv.register)); | 776 | assert(reg == toCanonicalReg(reg_mcv.register)); |
| 774 | inst_entry.value = stack_mcv; | 777 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| | 778 | try branch.inst_table.put(self.gpa, spilled_inst, stack_mcv); |
| 775 | try self.genSetStack(src, spilled_inst.ty, stack_mcv.stack_offset, reg_mcv); | 779 | try self.genSetStack(src, spilled_inst.ty, stack_mcv.stack_offset, reg_mcv); |
| 776 | | 780 | |
| 777 | break :b reg; | 781 | break :b reg; |
| ... | @@ -784,22 +788,21 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -784,22 +788,21 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 784 | /// `reg_owner` is the instruction that gets associated with the register in the register table. | 788 | /// `reg_owner` is the instruction that gets associated with the register in the register table. |
| 785 | /// This can have a side effect of spilling instructions to the stack to free up a register. | 789 | /// This can have a side effect of spilling instructions to the stack to free up a register. |
| 786 | fn copyToNewRegister(self: *Self, reg_owner: *ir.Inst, mcv: MCValue) !MCValue { | 790 | fn copyToNewRegister(self: *Self, reg_owner: *ir.Inst, mcv: MCValue) !MCValue { |
| 787 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 791 | try self.registers.ensureCapacity(self.gpa, self.registers.items().len + 1); |
| 788 | try branch.registers.ensureCapacity(self.gpa, branch.registers.items().len + 1); | | |
| 789 | | 792 | |
| 790 | const reg = branch.allocReg(reg_owner) orelse b: { | 793 | const reg = self.allocReg(reg_owner) orelse b: { |
| 791 | // We'll take over the first register. Move the instruction that was previously | 794 | // We'll take over the first register. Move the instruction that was previously |
| 792 | // there to a stack allocation. | 795 | // there to a stack allocation. |
| 793 | const reg = callee_preserved_regs[0]; | 796 | const reg = callee_preserved_regs[0]; |
| 794 | const regs_entry = branch.registers.getEntry(reg).?; | 797 | const regs_entry = self.registers.getEntry(reg).?; |
| 795 | const spilled_inst = regs_entry.value.inst; | 798 | const spilled_inst = regs_entry.value; |
| 796 | regs_entry.value = .{ .inst = reg_owner }; | 799 | regs_entry.value = reg_owner; |
| 797 | | 800 | |
| 798 | const stack_mcv = try self.allocRegOrMem(spilled_inst, false); | 801 | const stack_mcv = try self.allocRegOrMem(spilled_inst, false); |
| 799 | const inst_entry = branch.inst_table.getEntry(spilled_inst).?; | 802 | const reg_mcv = self.getResolvedInstValue(spilled_inst); |
| 800 | const reg_mcv = inst_entry.value; | | |
| 801 | assert(reg == toCanonicalReg(reg_mcv.register)); | 803 | assert(reg == toCanonicalReg(reg_mcv.register)); |
| 802 | inst_entry.value = stack_mcv; | 804 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| | 805 | try branch.inst_table.put(self.gpa, spilled_inst, stack_mcv); |
| 803 | try self.genSetStack(reg_owner.src, spilled_inst.ty, stack_mcv.stack_offset, reg_mcv); | 806 | try self.genSetStack(reg_owner.src, spilled_inst.ty, stack_mcv.stack_offset, reg_mcv); |
| 804 | | 807 | |
| 805 | break :b reg; | 808 | break :b reg; |
| ... | @@ -934,9 +937,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -934,9 +937,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 934 | .register => |reg| { | 937 | .register => |reg| { |
| 935 | // If it's in the registers table, need to associate the register with the | 938 | // If it's in the registers table, need to associate the register with the |
| 936 | // new instruction. | 939 | // new instruction. |
| 937 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 940 | if (self.registers.getEntry(toCanonicalReg(reg))) |entry| { |
| 938 | if (branch.registers.getEntry(toCanonicalReg(reg))) |entry| { | 941 | entry.value = inst; |
| 939 | entry.value = .{ .inst = inst }; | | |
| 940 | } | 942 | } |
| 941 | log.debug("reusing {} => {*}", .{reg, inst}); | 943 | log.debug("reusing {} => {*}", .{reg, inst}); |
| 942 | }, | 944 | }, |
| ... | @@ -950,6 +952,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -950,6 +952,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 950 | // Prevent the operand deaths processing code from deallocating it. | 952 | // Prevent the operand deaths processing code from deallocating it. |
| 951 | inst.clearOperandDeath(op_index); | 953 | inst.clearOperandDeath(op_index); |
| 952 | | 954 | |
| | 955 | // That makes us responsible for doing the rest of the stuff that processDeath would have done. |
| | 956 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; |
| | 957 | branch.inst_table.putAssumeCapacity(inst.getOperand(op_index).?, .dead); |
| | 958 | |
| 953 | return true; | 959 | return true; |
| 954 | } | 960 | } |
| 955 | | 961 | |
| ... | @@ -1231,8 +1237,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -1231,8 +1237,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 1231 | if (inst.base.isUnused()) | 1237 | if (inst.base.isUnused()) |
| 1232 | return MCValue.dead; | 1238 | return MCValue.dead; |
| 1233 | | 1239 | |
| 1234 | const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; | 1240 | try self.registers.ensureCapacity(self.gpa, self.registers.items().len + 1); |
| 1235 | try branch.registers.ensureCapacity(self.gpa, branch.registers.items().len + 1); | | |
| 1236 | | 1241 | |
| 1237 | const result = self.args[self.arg_index]; | 1242 | const result = self.args[self.arg_index]; |
| 1238 | self.arg_index += 1; | 1243 | self.arg_index += 1; |
| ... | @@ -1240,8 +1245,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -1240,8 +1245,8 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 1240 | const name_with_null = inst.name[0..mem.lenZ(inst.name) + 1]; | 1245 | const name_with_null = inst.name[0..mem.lenZ(inst.name) + 1]; |
| 1241 | switch (result) { | 1246 | switch (result) { |
| 1242 | .register => |reg| { | 1247 | .register => |reg| { |
| 1243 | branch.registers.putAssumeCapacityNoClobber(toCanonicalReg(reg), .{ .inst = &inst.base }); | 1248 | self.registers.putAssumeCapacityNoClobber(toCanonicalReg(reg), &inst.base); |
| 1244 | branch.markRegUsed(reg); | 1249 | self.markRegUsed(reg); |
| 1245 | | 1250 | |
| 1246 | try self.dbg_info.ensureCapacity(self.dbg_info.items.len + 8 + name_with_null.len); | 1251 | try self.dbg_info.ensureCapacity(self.dbg_info.items.len + 8 + name_with_null.len); |
| 1247 | self.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter); | 1252 | self.dbg_info.appendAssumeCapacity(link.File.Elf.abbrev_parameter); |
| ... | @@ -1536,18 +1541,19 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -1536,18 +1541,19 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 1536 | | 1541 | |
| 1537 | fn genDbgStmt(self: *Self, inst: *ir.Inst.NoOp) !MCValue { | 1542 | fn genDbgStmt(self: *Self, inst: *ir.Inst.NoOp) !MCValue { |
| 1538 | try self.dbgAdvancePCAndLine(inst.base.src); | 1543 | try self.dbgAdvancePCAndLine(inst.base.src); |
| 1539 | return MCValue.none; | 1544 | assert(inst.base.isUnused()); |
| | 1545 | return MCValue.dead; |
| 1540 | } | 1546 | } |
| 1541 | | 1547 | |
| 1542 | fn genCondBr(self: *Self, inst: *ir.Inst.CondBr) !MCValue { | 1548 | fn genCondBr(self: *Self, inst: *ir.Inst.CondBr) !MCValue { |
| 1543 | // TODO Rework this so that the arch-independent logic isn't buried and duplicated. | 1549 | const cond = try self.resolveInst(inst.condition); |
| 1544 | switch (arch) { | 1550 | |
| 1545 | .x86_64 => { | 1551 | const reloc: Reloc = switch (arch) { |
| | 1552 | .i386, .x86_64 => reloc: { |
| 1546 | try self.code.ensureCapacity(self.code.items.len + 6); | 1553 | try self.code.ensureCapacity(self.code.items.len + 6); |
| 1547 | | 1554 | |
| 1548 | const cond = try self.resolveInst(inst.condition); | 1555 | const opcode: u8 = switch (cond) { |
| 1549 | switch (cond) { | 1556 | .compare_flags_signed => |cmp_op| blk: { |
| 1550 | .compare_flags_signed => |cmp_op| { | | |
| 1551 | // Here we map to the opposite opcode because the jump is to the false branch. | 1557 | // Here we map to the opposite opcode because the jump is to the false branch. |
| 1552 | const opcode: u8 = switch (cmp_op) { | 1558 | const opcode: u8 = switch (cmp_op) { |
| 1553 | .gte => 0x8c, | 1559 | .gte => 0x8c, |
| ... | @@ -1557,9 +1563,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -1557,9 +1563,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 1557 | .lte => 0x8f, | 1563 | .lte => 0x8f, |
| 1558 | .eq => 0x85, | 1564 | .eq => 0x85, |
| 1559 | }; | 1565 | }; |
| 1560 | return self.genX86CondBr(inst, opcode); | 1566 | break :blk opcode; |
| 1561 | }, | 1567 | }, |
| 1562 | .compare_flags_unsigned => |cmp_op| { | 1568 | .compare_flags_unsigned => |cmp_op| blk: { |
| 1563 | // Here we map to the opposite opcode because the jump is to the false branch. | 1569 | // Here we map to the opposite opcode because the jump is to the false branch. |
| 1564 | const opcode: u8 = switch (cmp_op) { | 1570 | const opcode: u8 = switch (cmp_op) { |
| 1565 | .gte => 0x82, | 1571 | .gte => 0x82, |
| ... | @@ -1569,9 +1575,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -1569,9 +1575,9 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 1569 | .lte => 0x87, | 1575 | .lte => 0x87, |
| 1570 | .eq => 0x85, | 1576 | .eq => 0x85, |
| 1571 | }; | 1577 | }; |
| 1572 | return self.genX86CondBr(inst, opcode); | 1578 | break :blk opcode; |
| 1573 | }, | 1579 | }, |
| 1574 | .register => |reg| { | 1580 | .register => |reg| blk: { |
| 1575 | // test reg, 1 | 1581 | // test reg, 1 |
| 1576 | // TODO detect al, ax, eax | 1582 | // TODO detect al, ax, eax |
| 1577 | try self.code.ensureCapacity(self.code.items.len + 4); | 1583 | try self.code.ensureCapacity(self.code.items.len + 4); |
| ... | @@ -1583,23 +1589,128 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -1583,23 +1589,128 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 1583 | @as(u8, 0xC0) | (0 << 3) | @truncate(u3, reg.id()), | 1589 | @as(u8, 0xC0) | (0 << 3) | @truncate(u3, reg.id()), |
| 1584 | 0x01, | 1590 | 0x01, |
| 1585 | }); | 1591 | }); |
| 1586 | return self.genX86CondBr(inst, 0x84); | 1592 | break :blk 0x84; |
| 1587 | }, | 1593 | }, |
| 1588 | else => return self.fail(inst.base.src, "TODO implement condbr {} when condition is {}", .{ self.target.cpu.arch, @tagName(cond) }), | 1594 | else => return self.fail(inst.base.src, "TODO implement condbr {} when condition is {}", .{ self.target.cpu.arch, @tagName(cond) }), |
| 1589 | } | 1595 | }; |
| | 1596 | self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode }); |
| | 1597 | const reloc = Reloc{ .rel32 = self.code.items.len }; |
| | 1598 | self.code.items.len += 4; |
| | 1599 | break :reloc reloc; |
| 1590 | }, | 1600 | }, |
| 1591 | else => return self.fail(inst.base.src, "TODO implement condbr for {}", .{self.target.cpu.arch}), | 1601 | else => return self.fail(inst.base.src, "TODO implement condbr {}", .{ self.target.cpu.arch }), |
| 1592 | } | 1602 | }; |
| 1593 | } | 1603 | |
| | 1604 | // Capture the state of register and stack allocation state so that we can revert to it. |
| | 1605 | const parent_next_stack_offset = self.next_stack_offset; |
| | 1606 | const parent_free_registers = self.free_registers; |
| | 1607 | var parent_stack = try self.stack.clone(self.gpa); |
| | 1608 | defer parent_stack.deinit(self.gpa); |
| | 1609 | var parent_registers = try self.registers.clone(self.gpa); |
| | 1610 | defer parent_registers.deinit(self.gpa); |
| 1594 | | 1611 | |
| 1595 | fn genX86CondBr(self: *Self, inst: *ir.Inst.CondBr, opcode: u8) !MCValue { | 1612 | try self.branch_stack.append(.{}); |
| 1596 | // TODO deal with liveness / deaths condbr's then_entry_deaths and else_entry_deaths | 1613 | |
| 1597 | self.code.appendSliceAssumeCapacity(&[_]u8{ 0x0f, opcode }); | 1614 | const then_deaths = inst.thenDeaths(); |
| 1598 | const reloc = Reloc{ .rel32 = self.code.items.len }; | 1615 | try self.ensureProcessDeathCapacity(then_deaths.len); |
| 1599 | self.code.items.len += 4; | 1616 | for (then_deaths) |operand| { |
| | 1617 | self.processDeath(operand); |
| | 1618 | } |
| 1600 | try self.genBody(inst.then_body); | 1619 | try self.genBody(inst.then_body); |
| | 1620 | |
| | 1621 | // Revert to the previous register and stack allocation state. |
| | 1622 | |
| | 1623 | var saved_then_branch = self.branch_stack.pop(); |
| | 1624 | defer saved_then_branch.deinit(self.gpa); |
| | 1625 | |
| | 1626 | self.registers.deinit(self.gpa); |
| | 1627 | self.registers = parent_registers; |
| | 1628 | parent_registers = .{}; |
| | 1629 | |
| | 1630 | self.stack.deinit(self.gpa); |
| | 1631 | self.stack = parent_stack; |
| | 1632 | parent_stack = .{}; |
| | 1633 | |
| | 1634 | self.next_stack_offset = parent_next_stack_offset; |
| | 1635 | self.free_registers = parent_free_registers; |
| | 1636 | |
| 1601 | try self.performReloc(inst.base.src, reloc); | 1637 | try self.performReloc(inst.base.src, reloc); |
| | 1638 | const else_branch = self.branch_stack.addOneAssumeCapacity(); |
| | 1639 | else_branch.* = .{}; |
| | 1640 | |
| | 1641 | const else_deaths = inst.elseDeaths(); |
| | 1642 | try self.ensureProcessDeathCapacity(else_deaths.len); |
| | 1643 | for (else_deaths) |operand| { |
| | 1644 | self.processDeath(operand); |
| | 1645 | } |
| 1602 | try self.genBody(inst.else_body); | 1646 | try self.genBody(inst.else_body); |
| | 1647 | |
| | 1648 | // At this point, each branch will possibly have conflicting values for where |
| | 1649 | // each instruction is stored. They agree, however, on which instructions are alive/dead. |
| | 1650 | // We use the first ("then") branch as canonical, and here emit |
| | 1651 | // instructions into the second ("else") branch to make it conform. |
| | 1652 | // We continue respect the data structure semantic guarantees of the else_branch so |
| | 1653 | // that we can use all the code emitting abstractions. This is why at the bottom we |
| | 1654 | // assert that parent_branch.free_registers equals the saved_then_branch.free_registers |
| | 1655 | // rather than assigning it. |
| | 1656 | const parent_branch = &self.branch_stack.items[self.branch_stack.items.len - 2]; |
| | 1657 | try parent_branch.inst_table.ensureCapacity(self.gpa, parent_branch.inst_table.items().len + |
| | 1658 | else_branch.inst_table.items().len); |
| | 1659 | for (else_branch.inst_table.items()) |else_entry| { |
| | 1660 | const canon_mcv = if (saved_then_branch.inst_table.remove(else_entry.key)) |then_entry| blk: { |
| | 1661 | // The instruction's MCValue is overridden in both branches. |
| | 1662 | parent_branch.inst_table.putAssumeCapacity(else_entry.key, then_entry.value); |
| | 1663 | if (else_entry.value == .dead) { |
| | 1664 | assert(then_entry.value == .dead); |
| | 1665 | continue; |
| | 1666 | } |
| | 1667 | break :blk then_entry.value; |
| | 1668 | } else blk: { |
| | 1669 | if (else_entry.value == .dead) |
| | 1670 | continue; |
| | 1671 | // The instruction is only overridden in the else branch. |
| | 1672 | var i: usize = self.branch_stack.items.len - 2; |
| | 1673 | while (true) { |
| | 1674 | i -= 1; // If this overflows, the question is: why wasn't the instruction marked dead? |
| | 1675 | if (self.branch_stack.items[i].inst_table.get(else_entry.key)) |mcv| { |
| | 1676 | assert(mcv != .dead); |
| | 1677 | break :blk mcv; |
| | 1678 | } |
| | 1679 | } |
| | 1680 | }; |
| | 1681 | log.debug("consolidating else_entry {*} {}=>{}", .{else_entry.key, else_entry.value, canon_mcv}); |
| | 1682 | // TODO make sure the destination stack offset / register does not already have something |
| | 1683 | // going on there. |
| | 1684 | try self.setRegOrMem(inst.base.src, else_entry.key.ty, canon_mcv, else_entry.value); |
| | 1685 | // TODO track the new register / stack allocation |
| | 1686 | } |
| | 1687 | try parent_branch.inst_table.ensureCapacity(self.gpa, parent_branch.inst_table.items().len + |
| | 1688 | saved_then_branch.inst_table.items().len); |
| | 1689 | for (saved_then_branch.inst_table.items()) |then_entry| { |
| | 1690 | // We already deleted the items from this table that matched the else_branch. |
| | 1691 | // So these are all instructions that are only overridden in the then branch. |
| | 1692 | parent_branch.inst_table.putAssumeCapacity(then_entry.key, then_entry.value); |
| | 1693 | if (then_entry.value == .dead) |
| | 1694 | continue; |
| | 1695 | const parent_mcv = blk: { |
| | 1696 | var i: usize = self.branch_stack.items.len - 2; |
| | 1697 | while (true) { |
| | 1698 | i -= 1; |
| | 1699 | if (self.branch_stack.items[i].inst_table.get(then_entry.key)) |mcv| { |
| | 1700 | assert(mcv != .dead); |
| | 1701 | break :blk mcv; |
| | 1702 | } |
| | 1703 | } |
| | 1704 | }; |
| | 1705 | log.debug("consolidating then_entry {*} {}=>{}", .{then_entry.key, parent_mcv, then_entry.value}); |
| | 1706 | // TODO make sure the destination stack offset / register does not already have something |
| | 1707 | // going on there. |
| | 1708 | try self.setRegOrMem(inst.base.src, then_entry.key.ty, parent_mcv, then_entry.value); |
| | 1709 | // TODO track the new register / stack allocation |
| | 1710 | } |
| | 1711 | |
| | 1712 | self.branch_stack.pop().deinit(self.gpa); |
| | 1713 | |
| 1603 | return MCValue.unreach; | 1714 | return MCValue.unreach; |
| 1604 | } | 1715 | } |
| 1605 | | 1716 | |
| ... | @@ -1673,11 +1784,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -1673,11 +1784,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 1673 | switch (reloc) { | 1784 | switch (reloc) { |
| 1674 | .rel32 => |pos| { | 1785 | .rel32 => |pos| { |
| 1675 | const amt = self.code.items.len - (pos + 4); | 1786 | const amt = self.code.items.len - (pos + 4); |
| 1676 | // If it wouldn't jump at all, elide it. | 1787 | // Here it would be tempting to implement testing for amt == 0 and then elide the |
| 1677 | if (amt == 0) { | 1788 | // jump. However, that will cause a problem because other jumps may assume that they |
| 1678 | self.code.items.len -= 5; | 1789 | // can jump to this code. Or maybe I didn't understand something when I was debugging. |
| 1679 | return; | 1790 | // It could be worth another look. Anyway, that's why that isn't done here. Probably the |
| 1680 | } | 1791 | // best place to elide jumps will be in semantic analysis, by inlining blocks that only |
| | 1792 | // only have 1 break instruction. |
| 1681 | const s32_amt = math.cast(i32, amt) catch | 1793 | const s32_amt = math.cast(i32, amt) catch |
| 1682 | return self.fail(src, "unable to perform relocation: jump too far", .{}); | 1794 | return self.fail(src, "unable to perform relocation: jump too far", .{}); |
| 1683 | mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt); | 1795 | mem.writeIntLittle(i32, self.code.items[pos..][0..4], s32_amt); |
| ... | @@ -2282,8 +2394,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -2282,8 +2394,12 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 2282 | } | 2394 | } |
| 2283 | | 2395 | |
| 2284 | fn resolveInst(self: *Self, inst: *ir.Inst) !MCValue { | 2396 | fn resolveInst(self: *Self, inst: *ir.Inst) !MCValue { |
| | 2397 | // If the type has no codegen bits, no need to store it. |
| | 2398 | if (!inst.ty.hasCodeGenBits()) |
| | 2399 | return MCValue.none; |
| | 2400 | |
| 2285 | // Constants have static lifetimes, so they are always memoized in the outer most table. | 2401 | // Constants have static lifetimes, so they are always memoized in the outer most table. |
| 2286 | if (inst.cast(ir.Inst.Constant)) |const_inst| { | 2402 | if (inst.castTag(.constant)) |const_inst| { |
| 2287 | const branch = &self.branch_stack.items[0]; | 2403 | const branch = &self.branch_stack.items[0]; |
| 2288 | const gop = try branch.inst_table.getOrPut(self.gpa, inst); | 2404 | const gop = try branch.inst_table.getOrPut(self.gpa, inst); |
| 2289 | if (!gop.found_existing) { | 2405 | if (!gop.found_existing) { |
| ... | @@ -2292,6 +2408,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { | ... | @@ -2292,6 +2408,10 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type { |
| 2292 | return gop.entry.value; | 2408 | return gop.entry.value; |
| 2293 | } | 2409 | } |
| 2294 | | 2410 | |
| | 2411 | return self.getResolvedInstValue(inst); |
| | 2412 | } |
| | 2413 | |
| | 2414 | fn getResolvedInstValue(self: *Self, inst: *ir.Inst) MCValue { |
| 2295 | // Treat each stack item as a "layer" on top of the previous one. | 2415 | // Treat each stack item as a "layer" on top of the previous one. |
| 2296 | var i: usize = self.branch_stack.items.len; | 2416 | var i: usize = self.branch_stack.items.len; |
| 2297 | while (true) { | 2417 | while (true) { |