| ... | ... | @@ -563,7 +563,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 563 | 563 | .is_non_err_ptr => @panic("TODO try self.airIsNonErrPtr(inst)"), |
| 564 | 564 | .is_err => try self.airIsErr(inst), |
| 565 | 565 | .is_err_ptr => @panic("TODO try self.airIsErrPtr(inst)"), |
| 566 | | .load => @panic("TODO try self.airLoad(inst)"), |
| 566 | .load => try self.airLoad(inst), |
| 567 | 567 | .loop => @panic("TODO try self.airLoop(inst)"), |
| 568 | 568 | .not => @panic("TODO try self.airNot(inst)"), |
| 569 | 569 | .ptrtoint => @panic("TODO try self.airPtrToInt(inst)"), |
| ... | ... | @@ -1242,6 +1242,36 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void { |
| 1242 | 1242 | return self.finishAir(inst, result, .{ un_op, .none, .none }); |
| 1243 | 1243 | } |
| 1244 | 1244 | |
| 1245 | fn airLoad(self: *Self, inst: Air.Inst.Index) !void { |
| 1246 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| 1247 | const elem_ty = self.air.typeOfIndex(inst); |
| 1248 | const elem_size = elem_ty.abiSize(self.target.*); |
| 1249 | const result: MCValue = result: { |
| 1250 | if (!elem_ty.hasRuntimeBits()) |
| 1251 | break :result MCValue.none; |
| 1252 | |
| 1253 | const ptr = try self.resolveInst(ty_op.operand); |
| 1254 | const is_volatile = self.air.typeOf(ty_op.operand).isVolatilePtr(); |
| 1255 | if (self.liveness.isUnused(inst) and !is_volatile) |
| 1256 | break :result MCValue.dead; |
| 1257 | |
| 1258 | const dst_mcv: MCValue = blk: { |
| 1259 | if (elem_size <= 8 and self.reuseOperand(inst, ty_op.operand, 0, ptr)) { |
| 1260 | // The MCValue that holds the pointer can be re-used as the value. |
| 1261 | break :blk switch (ptr) { |
| 1262 | .register => |r| MCValue{ .register = r }, |
| 1263 | else => ptr, |
| 1264 | }; |
| 1265 | } else { |
| 1266 | break :blk try self.allocRegOrMem(inst, true); |
| 1267 | } |
| 1268 | }; |
| 1269 | try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand)); |
| 1270 | break :result dst_mcv; |
| 1271 | }; |
| 1272 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 1273 | } |
| 1274 | |
| 1245 | 1275 | fn airRet(self: *Self, inst: Air.Inst.Index) !void { |
| 1246 | 1276 | const un_op = self.air.instructions.items(.data)[inst].un_op; |
| 1247 | 1277 | const operand = try self.resolveInst(un_op); |
| ... | ... | @@ -1263,10 +1293,15 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void { |
| 1263 | 1293 | } |
| 1264 | 1294 | |
| 1265 | 1295 | fn airStore(self: *Self, inst: Air.Inst.Index) !void { |
| 1266 | | _ = self; |
| 1267 | | _ = inst; |
| 1296 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| 1297 | const ptr = try self.resolveInst(bin_op.lhs); |
| 1298 | const value = try self.resolveInst(bin_op.rhs); |
| 1299 | const ptr_ty = self.air.typeOf(bin_op.lhs); |
| 1300 | const value_ty = self.air.typeOf(bin_op.rhs); |
| 1301 | |
| 1302 | try self.store(ptr, value, ptr_ty, value_ty); |
| 1268 | 1303 | |
| 1269 | | return self.fail("TODO implement store for {}", .{self.target.cpu.arch}); |
| 1304 | return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1270 | 1305 | } |
| 1271 | 1306 | |
| 1272 | 1307 | fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| ... | ... | @@ -1522,6 +1557,76 @@ fn genArgDbgInfo(self: *Self, inst: Air.Inst.Index, mcv: MCValue, arg_index: u32 |
| 1522 | 1557 | } |
| 1523 | 1558 | } |
| 1524 | 1559 | |
| 1560 | // TODO replace this to call to extern memcpy |
| 1561 | fn genInlineMemcpy( |
| 1562 | self: *Self, |
| 1563 | src: Register, |
| 1564 | dst: Register, |
| 1565 | len: Register, |
| 1566 | tmp: Register, |
| 1567 | ) !void { |
| 1568 | // Here we assume that len > 0. |
| 1569 | // Also we do the copy from end -> start address to save a register. |
| 1570 | |
| 1571 | // sub len, 1, len |
| 1572 | _ = try self.addInst(.{ |
| 1573 | .tag = .sub, |
| 1574 | .data = .{ .arithmetic_3op = .{ |
| 1575 | .is_imm = true, |
| 1576 | .rs1 = len, |
| 1577 | .rs2_or_imm = .{ .imm = 1 }, |
| 1578 | .rd = len, |
| 1579 | } }, |
| 1580 | }); |
| 1581 | |
| 1582 | // loop: |
| 1583 | // ldub [src + len], tmp |
| 1584 | _ = try self.addInst(.{ |
| 1585 | .tag = .ldub, |
| 1586 | .data = .{ .arithmetic_3op = .{ |
| 1587 | .is_imm = false, |
| 1588 | .rs1 = src, |
| 1589 | .rs2_or_imm = .{ .rs2 = len }, |
| 1590 | .rd = tmp, |
| 1591 | } }, |
| 1592 | }); |
| 1593 | |
| 1594 | // stb tmp, [dst + len] |
| 1595 | _ = try self.addInst(.{ |
| 1596 | .tag = .stb, |
| 1597 | .data = .{ .arithmetic_3op = .{ |
| 1598 | .is_imm = false, |
| 1599 | .rs1 = dst, |
| 1600 | .rs2_or_imm = .{ .rs2 = len }, |
| 1601 | .rd = tmp, |
| 1602 | } }, |
| 1603 | }); |
| 1604 | |
| 1605 | // brnz len, loop |
| 1606 | _ = try self.addInst(.{ |
| 1607 | .tag = .bpr, |
| 1608 | .data = .{ .branch_predict_reg = .{ |
| 1609 | .cond = .ne_zero, |
| 1610 | .rs1 = len, |
| 1611 | .inst = @intCast(u32, self.mir_instructions.len - 2), |
| 1612 | } }, |
| 1613 | }); |
| 1614 | |
| 1615 | // Delay slot: |
| 1616 | // sub len, 1, len |
| 1617 | _ = try self.addInst(.{ |
| 1618 | .tag = .sub, |
| 1619 | .data = .{ .arithmetic_3op = .{ |
| 1620 | .is_imm = true, |
| 1621 | .rs1 = len, |
| 1622 | .rs2_or_imm = .{ .imm = 1 }, |
| 1623 | .rd = len, |
| 1624 | } }, |
| 1625 | }); |
| 1626 | |
| 1627 | // end: |
| 1628 | } |
| 1629 | |
| 1525 | 1630 | fn genLoad(self: *Self, value_reg: Register, addr_reg: Register, comptime off_type: type, off: off_type, abi_size: u64) !void { |
| 1526 | 1631 | assert(off_type == Register or off_type == i13); |
| 1527 | 1632 | |
| ... | ... | @@ -1913,6 +2018,66 @@ fn iterateBigTomb(self: *Self, inst: Air.Inst.Index, operand_count: usize) !BigT |
| 1913 | 2018 | }; |
| 1914 | 2019 | } |
| 1915 | 2020 | |
| 2021 | fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void { |
| 2022 | const elem_ty = ptr_ty.elemType(); |
| 2023 | const elem_size = elem_ty.abiSize(self.target.*); |
| 2024 | |
| 2025 | switch (ptr) { |
| 2026 | .none => unreachable, |
| 2027 | .undef => unreachable, |
| 2028 | .unreach => unreachable, |
| 2029 | .dead => unreachable, |
| 2030 | .compare_flags_unsigned, |
| 2031 | .compare_flags_signed, |
| 2032 | => unreachable, // cannot hold an address |
| 2033 | .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }), |
| 2034 | .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }), |
| 2035 | .register => |addr_reg| { |
| 2036 | self.register_manager.freezeRegs(&.{addr_reg}); |
| 2037 | defer self.register_manager.unfreezeRegs(&.{addr_reg}); |
| 2038 | |
| 2039 | switch (dst_mcv) { |
| 2040 | .dead => unreachable, |
| 2041 | .undef => unreachable, |
| 2042 | .compare_flags_signed, .compare_flags_unsigned => unreachable, |
| 2043 | .register => |dst_reg| { |
| 2044 | try self.genLoad(dst_reg, addr_reg, i13, 0, elem_size); |
| 2045 | }, |
| 2046 | .stack_offset => |off| { |
| 2047 | if (elem_size <= 8) { |
| 2048 | const tmp_reg = try self.register_manager.allocReg(null); |
| 2049 | self.register_manager.freezeRegs(&.{tmp_reg}); |
| 2050 | defer self.register_manager.unfreezeRegs(&.{tmp_reg}); |
| 2051 | |
| 2052 | try self.load(.{ .register = tmp_reg }, ptr, ptr_ty); |
| 2053 | try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); |
| 2054 | } else { |
| 2055 | const regs = try self.register_manager.allocRegs(3, .{ null, null, null }); |
| 2056 | self.register_manager.freezeRegs(&regs); |
| 2057 | defer self.register_manager.unfreezeRegs(&regs); |
| 2058 | |
| 2059 | const src_reg = addr_reg; |
| 2060 | const dst_reg = regs[0]; |
| 2061 | const len_reg = regs[1]; |
| 2062 | const tmp_reg = regs[2]; |
| 2063 | |
| 2064 | try self.genSetReg(ptr_ty, dst_reg, .{ .ptr_stack_offset = off }); |
| 2065 | try self.genSetReg(Type.usize, len_reg, .{ .immediate = elem_size }); |
| 2066 | try self.genInlineMemcpy(src_reg, dst_reg, len_reg, tmp_reg); |
| 2067 | } |
| 2068 | }, |
| 2069 | else => return self.fail("TODO load from register into {}", .{dst_mcv}), |
| 2070 | } |
| 2071 | }, |
| 2072 | .memory, |
| 2073 | .stack_offset, |
| 2074 | => { |
| 2075 | const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr); |
| 2076 | try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty); |
| 2077 | }, |
| 2078 | } |
| 2079 | } |
| 2080 | |
| 1916 | 2081 | fn lowerDeclRef(self: *Self, tv: TypedValue, decl_index: Module.Decl.Index) InnerError!MCValue { |
| 1917 | 2082 | const ptr_bits = self.target.cpu.arch.ptrBitWidth(); |
| 1918 | 2083 | const ptr_bytes: u64 = @divExact(ptr_bits, 8); |
| ... | ... | @@ -2167,6 +2332,45 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void |
| 2167 | 2332 | try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv); |
| 2168 | 2333 | } |
| 2169 | 2334 | |
| 2335 | fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void { |
| 2336 | const abi_size = value_ty.abiSize(self.target.*); |
| 2337 | |
| 2338 | switch (ptr) { |
| 2339 | .none => unreachable, |
| 2340 | .undef => unreachable, |
| 2341 | .unreach => unreachable, |
| 2342 | .dead => unreachable, |
| 2343 | .compare_flags_unsigned, |
| 2344 | .compare_flags_signed, |
| 2345 | => unreachable, // cannot hold an address |
| 2346 | .immediate => |imm| { |
| 2347 | try self.setRegOrMem(value_ty, .{ .memory = imm }, value); |
| 2348 | }, |
| 2349 | .ptr_stack_offset => |off| { |
| 2350 | try self.genSetStack(value_ty, off, value); |
| 2351 | }, |
| 2352 | .register => |addr_reg| { |
| 2353 | self.register_manager.freezeRegs(&.{addr_reg}); |
| 2354 | defer self.register_manager.unfreezeRegs(&.{addr_reg}); |
| 2355 | |
| 2356 | switch (value) { |
| 2357 | .register => |value_reg| { |
| 2358 | try self.genStore(value_reg, addr_reg, i13, 0, abi_size); |
| 2359 | }, |
| 2360 | else => { |
| 2361 | return self.fail("TODO implement copying of memory", .{}); |
| 2362 | }, |
| 2363 | } |
| 2364 | }, |
| 2365 | .memory, |
| 2366 | .stack_offset, |
| 2367 | => { |
| 2368 | const addr_reg = try self.copyToTmpRegister(ptr_ty, ptr); |
| 2369 | try self.store(.{ .register = addr_reg }, value, ptr_ty, value_ty); |
| 2370 | }, |
| 2371 | } |
| 2372 | } |
| 2373 | |
| 2170 | 2374 | /// TODO support scope overrides. Also note this logic is duplicated with `Module.wantSafety`. |
| 2171 | 2375 | fn wantSafety(self: *Self) bool { |
| 2172 | 2376 | return switch (self.bin_file.options.optimize_mode) { |