| ... | @@ -1222,9 +1222,16 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1222,9 +1222,16 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void { |
| 1222 | fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { | 1222 | fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { |
| 1223 | const is_volatile = false; // TODO | 1223 | const is_volatile = false; // TODO |
| 1224 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; | 1224 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| 1225 | const result: MCValue = if (!is_volatile and self.liveness.isUnused(inst)) .dead else result: { | 1225 | |
| | 1226 | if (!is_volatile and self.liveness.isUnused(inst)) return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); |
| | 1227 | const result: MCValue = result: { |
| 1226 | const slice_mcv = try self.resolveInst(bin_op.lhs); | 1228 | const slice_mcv = try self.resolveInst(bin_op.lhs); |
| 1227 | | 1229 | |
| | 1230 | // TODO optimize for the case where the index is a constant, |
| | 1231 | // i.e. index_mcv == .immediate |
| | 1232 | const index_mcv = try self.resolveInst(bin_op.rhs); |
| | 1233 | const index_is_register = index_mcv == .register; |
| | 1234 | |
| 1228 | const slice_ty = self.air.typeOf(bin_op.lhs); | 1235 | const slice_ty = self.air.typeOf(bin_op.lhs); |
| 1229 | const elem_ty = slice_ty.childType(); | 1236 | const elem_ty = slice_ty.childType(); |
| 1230 | const elem_size = elem_ty.abiSize(self.target.*); | 1237 | const elem_size = elem_ty.abiSize(self.target.*); |
| ... | @@ -1232,12 +1239,8 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1232,12 +1239,8 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { |
| 1232 | var buf: Type.SlicePtrFieldTypeBuffer = undefined; | 1239 | var buf: Type.SlicePtrFieldTypeBuffer = undefined; |
| 1233 | const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf); | 1240 | const slice_ptr_field_type = slice_ty.slicePtrFieldType(&buf); |
| 1234 | | 1241 | |
| 1235 | // TODO optimize this for the case when elem_size is a power | 1242 | if (index_is_register) self.register_manager.freezeRegs(&.{index_mcv.register}); |
| 1236 | // of two (includes elem_size == 1) | 1243 | defer if (index_is_register) self.register_manager.unfreezeRegs(&.{index_mcv.register}); |
| 1237 | const offset_mcv = try self.genArmMulConstant(inst, bin_op.rhs, 1, @intCast(u32, elem_size)); | | |
| 1238 | assert(offset_mcv == .register); // result of multiplication should always be register | | |
| 1239 | self.register_manager.freezeRegs(&.{offset_mcv.register}); | | |
| 1240 | defer self.register_manager.unfreezeRegs(&.{offset_mcv.register}); | | |
| 1241 | | 1244 | |
| 1242 | const base_mcv: MCValue = switch (slice_mcv) { | 1245 | const base_mcv: MCValue = switch (slice_mcv) { |
| 1243 | .stack_offset => .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, slice_mcv) }, | 1246 | .stack_offset => .{ .register = try self.copyToTmpRegister(slice_ptr_field_type, slice_mcv) }, |
| ... | @@ -1246,61 +1249,67 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -1246,61 +1249,67 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { |
| 1246 | self.register_manager.freezeRegs(&.{base_mcv.register}); | 1249 | self.register_manager.freezeRegs(&.{base_mcv.register}); |
| 1247 | defer self.register_manager.unfreezeRegs(&.{base_mcv.register}); | 1250 | defer self.register_manager.unfreezeRegs(&.{base_mcv.register}); |
| 1248 | | 1251 | |
| 1249 | if (elem_size <= 4) { | 1252 | switch (elem_size) { |
| 1250 | const dst_reg = try self.register_manager.allocReg(inst); | 1253 | 1, 4 => { |
| 1251 | self.register_manager.freezeRegs(&.{dst_reg}); | 1254 | const dst_reg = try self.register_manager.allocReg(inst); |
| 1252 | defer self.register_manager.unfreezeRegs(&.{dst_reg}); | 1255 | const dst_mcv = MCValue{ .register = dst_reg }; |
| | 1256 | self.register_manager.freezeRegs(&.{dst_reg}); |
| | 1257 | defer self.register_manager.unfreezeRegs(&.{dst_reg}); |
| 1253 | | 1258 | |
| 1254 | switch (elem_size) { | 1259 | const index_reg: Register = switch (index_mcv) { |
| 1255 | 1, 4 => { | 1260 | .register => |reg| reg, |
| 1256 | const tag: Mir.Inst.Tag = switch (elem_size) { | 1261 | else => try self.copyToTmpRegister(Type.usize, index_mcv), |
| 1257 | 1 => .ldrb, | 1262 | }; |
| 1258 | 4 => .ldr, | 1263 | self.register_manager.freezeRegs(&.{index_reg}); |
| 1259 | else => unreachable, | 1264 | defer self.register_manager.unfreezeRegs(&.{index_reg}); |
| 1260 | }; | | |
| 1261 | | 1265 | |
| 1262 | _ = try self.addInst(.{ | 1266 | const tag: Mir.Inst.Tag = switch (elem_size) { |
| 1263 | .tag = tag, | 1267 | 1 => .ldrb, |
| 1264 | .data = .{ .rr_offset = .{ | 1268 | 4 => .ldr, |
| 1265 | .rt = dst_reg, | 1269 | else => unreachable, |
| 1266 | .rn = base_mcv.register, | 1270 | }; |
| 1267 | .offset = .{ .offset = Instruction.Offset.reg(offset_mcv.register, 0) }, | 1271 | const shift: u5 = switch (elem_size) { |
| 1268 | } }, | 1272 | 1 => 0, |
| 1269 | }); | 1273 | 4 => 2, |
| 1270 | }, | 1274 | else => unreachable, |
| 1271 | 2 => { | 1275 | }; |
| 1272 | _ = try self.addInst(.{ | | |
| 1273 | .tag = .ldrh, | | |
| 1274 | .data = .{ .rr_extra_offset = .{ | | |
| 1275 | .rt = dst_reg, | | |
| 1276 | .rn = base_mcv.register, | | |
| 1277 | .offset = .{ .offset = Instruction.ExtraLoadStoreOffset.reg(offset_mcv.register) }, | | |
| 1278 | } }, | | |
| 1279 | }); | | |
| 1280 | }, | | |
| 1281 | else => unreachable, | | |
| 1282 | } | | |
| 1283 | | 1276 | |
| 1284 | break :result MCValue{ .register = dst_reg }; | 1277 | _ = try self.addInst(.{ |
| 1285 | } else { | 1278 | .tag = tag, |
| 1286 | const dst_mcv = try self.allocRegOrMem(inst, false); | 1279 | .data = .{ .rr_offset = .{ |
| | 1280 | .rt = dst_reg, |
| | 1281 | .rn = base_mcv.register, |
| | 1282 | .offset = .{ .offset = Instruction.Offset.reg(index_reg, .{ .lsl = shift }) }, |
| | 1283 | } }, |
| | 1284 | }); |
| 1287 | | 1285 | |
| 1288 | const addr_reg = try self.register_manager.allocReg(null); | 1286 | break :result dst_mcv; |
| 1289 | self.register_manager.freezeRegs(&.{addr_reg}); | 1287 | }, |
| 1290 | defer self.register_manager.unfreezeRegs(&.{addr_reg}); | 1288 | else => { |
| | 1289 | const dst_mcv = try self.allocRegOrMem(inst, true); |
| | 1290 | |
| | 1291 | const offset_mcv = try self.genArmMulConstant(bin_op.rhs, @intCast(u32, elem_size)); |
| | 1292 | assert(offset_mcv == .register); // result of multiplication should always be register |
| | 1293 | self.register_manager.freezeRegs(&.{offset_mcv.register}); |
| | 1294 | defer self.register_manager.unfreezeRegs(&.{offset_mcv.register}); |
| 1291 | | 1295 | |
| 1292 | try self.genArmBinOpCode(addr_reg, base_mcv, offset_mcv, false, .add, .unsigned); | 1296 | const addr_reg = try self.register_manager.allocReg(null); |
| | 1297 | self.register_manager.freezeRegs(&.{addr_reg}); |
| | 1298 | defer self.register_manager.unfreezeRegs(&.{addr_reg}); |
| 1293 | | 1299 | |
| 1294 | // I know we will unfreeze these registers at the end of | 1300 | try self.genArmBinOpCode(addr_reg, base_mcv, offset_mcv, false, .add, .unsigned); |
| 1295 | // the scope of :result. However, at this point in time, | | |
| 1296 | // neither the base register nor the offset register | | |
| 1297 | // contains any valuable data anymore. In order to reduce | | |
| 1298 | // register pressure, unfreeze them prematurely | | |
| 1299 | self.register_manager.unfreezeRegs(&.{ base_mcv.register, offset_mcv.register }); | | |
| 1300 | | 1301 | |
| 1301 | try self.load(dst_mcv, .{ .register = addr_reg }, slice_ptr_field_type); | 1302 | // I know we will unfreeze these registers at the end of |
| | 1303 | // the scope of :result. However, at this point in time, |
| | 1304 | // neither the base register nor the offset register |
| | 1305 | // contains any valuable data anymore. In order to reduce |
| | 1306 | // register pressure, unfreeze them prematurely |
| | 1307 | self.register_manager.unfreezeRegs(&.{ base_mcv.register, offset_mcv.register }); |
| 1302 | | 1308 | |
| 1303 | break :result dst_mcv; | 1309 | try self.load(dst_mcv, .{ .register = addr_reg }, slice_ptr_field_type); |
| | 1310 | |
| | 1311 | break :result dst_mcv; |
| | 1312 | }, |
| 1304 | } | 1313 | } |
| 1305 | }; | 1314 | }; |
| 1306 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); | 1315 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| ... | @@ -1931,8 +1940,8 @@ fn genArmBinOpCode( | ... | @@ -1931,8 +1940,8 @@ fn genArmBinOpCode( |
| 1931 | .shl, .shr => { | 1940 | .shl, .shr => { |
| 1932 | assert(!swap_lhs_and_rhs); | 1941 | assert(!swap_lhs_and_rhs); |
| 1933 | const shift_amount = switch (operand) { | 1942 | const shift_amount = switch (operand) { |
| 1934 | .Register => |reg_op| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)), | 1943 | .register => |reg_op| Instruction.ShiftAmount.reg(@intToEnum(Register, reg_op.rm)), |
| 1935 | .Immediate => |imm_op| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)), | 1944 | .immediate => |imm_op| Instruction.ShiftAmount.imm(@intCast(u5, imm_op.imm)), |
| 1936 | }; | 1945 | }; |
| 1937 | | 1946 | |
| 1938 | const tag: Mir.Inst.Tag = switch (op) { | 1947 | const tag: Mir.Inst.Tag = switch (op) { |
| ... | @@ -2036,12 +2045,11 @@ fn genArmMul(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Ai | ... | @@ -2036,12 +2045,11 @@ fn genArmMul(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Ai |
| 2036 | return dst_mcv; | 2045 | return dst_mcv; |
| 2037 | } | 2046 | } |
| 2038 | | 2047 | |
| 2039 | fn genArmMulConstant(self: *Self, inst: Air.Inst.Index, op: Air.Inst.Ref, op_index: Liveness.OperandInt, imm: u32) !MCValue { | 2048 | fn genArmMulConstant(self: *Self, op: Air.Inst.Ref, imm: u32) !MCValue { |
| 2040 | const lhs = try self.resolveInst(op); | 2049 | const lhs = try self.resolveInst(op); |
| 2041 | const rhs = MCValue{ .immediate = imm }; | 2050 | const rhs = MCValue{ .immediate = imm }; |
| 2042 | | 2051 | |
| 2043 | const lhs_is_register = lhs == .register; | 2052 | const lhs_is_register = lhs == .register; |
| 2044 | const reuse_lhs = lhs_is_register and self.reuseOperand(inst, op, op_index, lhs); | | |
| 2045 | | 2053 | |
| 2046 | if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register}); | 2054 | if (lhs_is_register) self.register_manager.freezeRegs(&.{lhs.register}); |
| 2047 | defer if (lhs_is_register) self.register_manager.unfreezeRegs(&.{lhs.register}); | 2055 | defer if (lhs_is_register) self.register_manager.unfreezeRegs(&.{lhs.register}); |
| ... | @@ -2054,23 +2062,17 @@ fn genArmMulConstant(self: *Self, inst: Air.Inst.Index, op: Air.Inst.Ref, op_ind | ... | @@ -2054,23 +2062,17 @@ fn genArmMulConstant(self: *Self, inst: Air.Inst.Index, op: Air.Inst.Ref, op_ind |
| 2054 | var rhs_mcv: MCValue = rhs; | 2062 | var rhs_mcv: MCValue = rhs; |
| 2055 | | 2063 | |
| 2056 | // Allocate registers for operands and/or destination | 2064 | // Allocate registers for operands and/or destination |
| 2057 | if (reuse_lhs) { | 2065 | // Allocate 1 or 2 registers |
| 2058 | // Allocate 1 register | 2066 | if (lhs_is_register) { |
| 2059 | rhs_mcv = MCValue{ .register = try self.register_manager.allocReg(null) }; | 2067 | // Move RHS to register |
| 2060 | dst_mcv = lhs; | 2068 | dst_mcv = MCValue{ .register = try self.register_manager.allocReg(null) }; |
| | 2069 | rhs_mcv = dst_mcv; |
| 2061 | } else { | 2070 | } else { |
| 2062 | // Allocate 1 or 2 registers | 2071 | // Move LHS and RHS to register |
| 2063 | if (lhs_is_register) { | 2072 | const regs = try self.register_manager.allocRegs(2, .{ null, null }); |
| 2064 | // Move RHS to register | 2073 | lhs_mcv = MCValue{ .register = regs[0] }; |
| 2065 | dst_mcv = MCValue{ .register = try self.register_manager.allocReg(null) }; | 2074 | rhs_mcv = MCValue{ .register = regs[1] }; |
| 2066 | rhs_mcv = dst_mcv; | 2075 | dst_mcv = lhs_mcv; |
| 2067 | } else { | | |
| 2068 | // Move LHS and RHS to register | | |
| 2069 | const regs = try self.register_manager.allocRegs(2, .{ null, null }); | | |
| 2070 | lhs_mcv = MCValue{ .register = regs[0] }; | | |
| 2071 | rhs_mcv = MCValue{ .register = regs[1] }; | | |
| 2072 | dst_mcv = lhs_mcv; | | |
| 2073 | } | | |
| 2074 | } | 2076 | } |
| 2075 | | 2077 | |
| 2076 | // Move the operands to the newly allocated registers | 2078 | // Move the operands to the newly allocated registers |
| ... | @@ -2132,7 +2134,7 @@ fn genArmInlineMemcpy( | ... | @@ -2132,7 +2134,7 @@ fn genArmInlineMemcpy( |
| 2132 | .data = .{ .rr_offset = .{ | 2134 | .data = .{ .rr_offset = .{ |
| 2133 | .rt = tmp, | 2135 | .rt = tmp, |
| 2134 | .rn = src, | 2136 | .rn = src, |
| 2135 | .offset = .{ .offset = Instruction.Offset.reg(count, 0) }, | 2137 | .offset = .{ .offset = Instruction.Offset.reg(count, .none) }, |
| 2136 | } }, | 2138 | } }, |
| 2137 | }); | 2139 | }); |
| 2138 | | 2140 | |
| ... | @@ -2142,7 +2144,7 @@ fn genArmInlineMemcpy( | ... | @@ -2142,7 +2144,7 @@ fn genArmInlineMemcpy( |
| 2142 | .data = .{ .rr_offset = .{ | 2144 | .data = .{ .rr_offset = .{ |
| 2143 | .rt = tmp, | 2145 | .rt = tmp, |
| 2144 | .rn = dst, | 2146 | .rn = dst, |
| 2145 | .offset = .{ .offset = Instruction.Offset.reg(count, 0) }, | 2147 | .offset = .{ .offset = Instruction.Offset.reg(count, .none) }, |
| 2146 | } }, | 2148 | } }, |
| 2147 | }); | 2149 | }); |
| 2148 | | 2150 | |
| ... | @@ -3126,7 +3128,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro | ... | @@ -3126,7 +3128,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro |
| 3126 | 1, 4 => { | 3128 | 1, 4 => { |
| 3127 | const offset = if (math.cast(u12, adj_off)) |imm| blk: { | 3129 | const offset = if (math.cast(u12, adj_off)) |imm| blk: { |
| 3128 | break :blk Instruction.Offset.imm(imm); | 3130 | break :blk Instruction.Offset.imm(imm); |
| 3129 | } else |_| Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0); | 3131 | } else |_| Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), .none); |
| 3130 | | 3132 | |
| 3131 | const tag: Mir.Inst.Tag = switch (abi_size) { | 3133 | const tag: Mir.Inst.Tag = switch (abi_size) { |
| 3132 | 1 => .strb, | 3134 | 1 => .strb, |
| ... | @@ -3450,7 +3452,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void | ... | @@ -3450,7 +3452,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void |
| 3450 | 1, 4 => { | 3452 | 1, 4 => { |
| 3451 | const offset = if (adj_off <= math.maxInt(u12)) blk: { | 3453 | const offset = if (adj_off <= math.maxInt(u12)) blk: { |
| 3452 | break :blk Instruction.Offset.imm(@intCast(u12, adj_off)); | 3454 | break :blk Instruction.Offset.imm(@intCast(u12, adj_off)); |
| 3453 | } else Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0); | 3455 | } else Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), .none); |
| 3454 | | 3456 | |
| 3455 | const tag: Mir.Inst.Tag = switch (abi_size) { | 3457 | const tag: Mir.Inst.Tag = switch (abi_size) { |
| 3456 | 1 => .ldrb, | 3458 | 1 => .ldrb, |
| ... | @@ -3536,7 +3538,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I | ... | @@ -3536,7 +3538,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I |
| 3536 | 1, 4 => { | 3538 | 1, 4 => { |
| 3537 | const offset = if (math.cast(u12, adj_off)) |imm| blk: { | 3539 | const offset = if (math.cast(u12, adj_off)) |imm| blk: { |
| 3538 | break :blk Instruction.Offset.imm(imm); | 3540 | break :blk Instruction.Offset.imm(imm); |
| 3539 | } else |_| Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), 0); | 3541 | } else |_| Instruction.Offset.reg(try self.copyToTmpRegister(Type.initTag(.u32), MCValue{ .immediate = adj_off }), .none); |
| 3540 | | 3542 | |
| 3541 | const tag: Mir.Inst.Tag = switch (abi_size) { | 3543 | const tag: Mir.Inst.Tag = switch (abi_size) { |
| 3542 | 1 => .strb, | 3544 | 1 => .strb, |