| ... | ... | @@ -135,6 +135,8 @@ const Owner = union(enum) { |
| 135 | 135 | } |
| 136 | 136 | }; |
| 137 | 137 | |
| 138 | const MaskKind = enum { sign, all }; |
| 139 | |
| 138 | 140 | pub const MCValue = union(enum) { |
| 139 | 141 | /// No runtime bits. `void` types, empty structs, u0, enums with 1 tag, etc. |
| 140 | 142 | /// TODO Look into deleting this tag and using `dead` instead, since every use |
| ... | ... | @@ -156,10 +158,16 @@ pub const MCValue = union(enum) { |
| 156 | 158 | register: Register, |
| 157 | 159 | /// The value is split across two registers. |
| 158 | 160 | register_pair: [2]Register, |
| 161 | /// The value is split across three registers. |
| 162 | register_triple: [3]Register, |
| 163 | /// The value is split across four registers. |
| 164 | register_quadruple: [4]Register, |
| 159 | 165 | /// The value is a constant offset from the value in a register. |
| 160 | 166 | register_offset: bits.RegisterOffset, |
| 161 | 167 | /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register. |
| 162 | 168 | register_overflow: struct { reg: Register, eflags: Condition }, |
| 169 | /// The value is a bool vector stored in a vector register with a different scalar type. |
| 170 | register_mask: struct { reg: Register, kind: MaskKind, inverted: bool, scalar: Memory.Size }, |
| 163 | 171 | /// The value is in memory at a hard-coded address. |
| 164 | 172 | /// If the type is a pointer, it means the pointer address is stored at this memory location. |
| 165 | 173 | memory: u64, |
| ... | ... | @@ -195,7 +203,7 @@ pub const MCValue = union(enum) { |
| 195 | 203 | /// Payload is a frame address. |
| 196 | 204 | lea_frame: bits.FrameAddr, |
| 197 | 205 | /// Supports integer_per_element abi |
| 198 | | elementwise_regs_then_frame: packed struct { regs: u3 = 0, frame_off: i29 = 0, frame_index: FrameIndex }, |
| 206 | elementwise_regs_then_frame: packed struct { regs: u3, frame_off: i29, frame_index: FrameIndex }, |
| 199 | 207 | /// This indicates that we have already allocated a frame index for this instruction, |
| 200 | 208 | /// but it has not been spilled there yet in the current control flow. |
| 201 | 209 | /// Payload is a frame index. |
| ... | ... | @@ -210,6 +218,7 @@ pub const MCValue = union(enum) { |
| 210 | 218 | .undef, |
| 211 | 219 | .immediate, |
| 212 | 220 | .register_offset, |
| 221 | .register_mask, |
| 213 | 222 | .eflags, |
| 214 | 223 | .register_overflow, |
| 215 | 224 | .lea_symbol, |
| ... | ... | @@ -223,6 +232,8 @@ pub const MCValue = union(enum) { |
| 223 | 232 | => false, |
| 224 | 233 | .register, |
| 225 | 234 | .register_pair, |
| 235 | .register_triple, |
| 236 | .register_quadruple, |
| 226 | 237 | .memory, |
| 227 | 238 | .load_symbol, |
| 228 | 239 | .load_got, |
| ... | ... | @@ -234,13 +245,20 @@ pub const MCValue = union(enum) { |
| 234 | 245 | }; |
| 235 | 246 | } |
| 236 | 247 | |
| 237 | | fn isMemory(mcv: MCValue) bool { |
| 248 | fn isBase(mcv: MCValue) bool { |
| 238 | 249 | return switch (mcv) { |
| 239 | 250 | .memory, .indirect, .load_frame => true, |
| 240 | 251 | else => false, |
| 241 | 252 | }; |
| 242 | 253 | } |
| 243 | 254 | |
| 255 | fn isMemory(mcv: MCValue) bool { |
| 256 | return switch (mcv) { |
| 257 | .memory, .indirect, .load_frame, .load_symbol => true, |
| 258 | else => false, |
| 259 | }; |
| 260 | } |
| 261 | |
| 244 | 262 | fn isImmediate(mcv: MCValue) bool { |
| 245 | 263 | return switch (mcv) { |
| 246 | 264 | .immediate => true, |
| ... | ... | @@ -268,16 +286,23 @@ pub const MCValue = union(enum) { |
| 268 | 286 | .register => |reg| reg, |
| 269 | 287 | .register_offset, .indirect => |ro| ro.reg, |
| 270 | 288 | .register_overflow => |ro| ro.reg, |
| 289 | .register_mask => |rm| rm.reg, |
| 271 | 290 | else => null, |
| 272 | 291 | }; |
| 273 | 292 | } |
| 274 | 293 | |
| 275 | 294 | fn getRegs(mcv: *const MCValue) []const Register { |
| 276 | 295 | return switch (mcv.*) { |
| 277 | | .register => |*reg| @as(*const [1]Register, reg), |
| 278 | | .register_pair => |*regs| regs, |
| 279 | | .register_offset, .indirect => |*ro| @as(*const [1]Register, &ro.reg), |
| 280 | | .register_overflow => |*ro| @as(*const [1]Register, &ro.reg), |
| 296 | .register => |*reg| reg[0..1], |
| 297 | inline .register_pair, |
| 298 | .register_triple, |
| 299 | .register_quadruple, |
| 300 | => |*regs| regs, |
| 301 | inline .register_offset, |
| 302 | .indirect, |
| 303 | .register_overflow, |
| 304 | .register_mask, |
| 305 | => |*pl| (&pl.reg)[0..1], |
| 281 | 306 | else => &.{}, |
| 282 | 307 | }; |
| 283 | 308 | } |
| ... | ... | @@ -300,8 +325,11 @@ pub const MCValue = union(enum) { |
| 300 | 325 | .eflags, |
| 301 | 326 | .register, |
| 302 | 327 | .register_pair, |
| 328 | .register_triple, |
| 329 | .register_quadruple, |
| 303 | 330 | .register_offset, |
| 304 | 331 | .register_overflow, |
| 332 | .register_mask, |
| 305 | 333 | .lea_symbol, |
| 306 | 334 | .lea_direct, |
| 307 | 335 | .lea_got, |
| ... | ... | @@ -332,7 +360,10 @@ pub const MCValue = union(enum) { |
| 332 | 360 | .undef, |
| 333 | 361 | .eflags, |
| 334 | 362 | .register_pair, |
| 363 | .register_triple, |
| 364 | .register_quadruple, |
| 335 | 365 | .register_overflow, |
| 366 | .register_mask, |
| 336 | 367 | .memory, |
| 337 | 368 | .indirect, |
| 338 | 369 | .load_direct, |
| ... | ... | @@ -367,7 +398,10 @@ pub const MCValue = union(enum) { |
| 367 | 398 | => unreachable, // not valid |
| 368 | 399 | .eflags, |
| 369 | 400 | .register_pair, |
| 401 | .register_triple, |
| 402 | .register_quadruple, |
| 370 | 403 | .register_overflow, |
| 404 | .register_mask, |
| 371 | 405 | .memory, |
| 372 | 406 | .indirect, |
| 373 | 407 | .load_direct, |
| ... | ... | @@ -404,8 +438,11 @@ pub const MCValue = union(enum) { |
| 404 | 438 | .eflags, |
| 405 | 439 | .register, |
| 406 | 440 | .register_pair, |
| 441 | .register_triple, |
| 442 | .register_quadruple, |
| 407 | 443 | .register_offset, |
| 408 | 444 | .register_overflow, |
| 445 | .register_mask, |
| 409 | 446 | .load_direct, |
| 410 | 447 | .lea_direct, |
| 411 | 448 | .load_got, |
| ... | ... | @@ -472,10 +509,11 @@ pub const MCValue = union(enum) { |
| 472 | 509 | .memory => |pl| try writer.print("[ds:0x{x}]", .{pl}), |
| 473 | 510 | inline .eflags, .register => |pl| try writer.print("{s}", .{@tagName(pl)}), |
| 474 | 511 | .register_pair => |pl| try writer.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }), |
| 512 | .register_triple => |pl| try writer.print("{s}:{s}:{s}", .{ @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]) }), |
| 513 | .register_quadruple => |pl| try writer.print("{s}:{s}:{s}:{s}", .{ @tagName(pl[3]), @tagName(pl[2]), @tagName(pl[1]), @tagName(pl[0]) }), |
| 475 | 514 | .register_offset => |pl| try writer.print("{s} + 0x{x}", .{ @tagName(pl.reg), pl.off }), |
| 476 | | .register_overflow => |pl| try writer.print("{s}:{s}", .{ |
| 477 | | @tagName(pl.eflags), @tagName(pl.reg), |
| 478 | | }), |
| 515 | .register_overflow => |pl| try writer.print("{s}:{s}", .{ @tagName(pl.eflags), @tagName(pl.reg) }), |
| 516 | .register_mask => |pl| try writer.print("mask({s},{}):{s}", .{ @tagName(pl.kind), pl.scalar, @tagName(pl.reg) }), |
| 479 | 517 | .load_symbol => |pl| try writer.print("[sym:{} + 0x{x}]", .{ pl.sym_index, pl.off }), |
| 480 | 518 | .lea_symbol => |pl| try writer.print("sym:{} + 0x{x}", .{ pl.sym_index, pl.off }), |
| 481 | 519 | .indirect => |pl| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }), |
| ... | ... | @@ -526,8 +564,11 @@ const InstTracking = struct { |
| 526 | 564 | .eflags, |
| 527 | 565 | .register, |
| 528 | 566 | .register_pair, |
| 567 | .register_triple, |
| 568 | .register_quadruple, |
| 529 | 569 | .register_offset, |
| 530 | 570 | .register_overflow, |
| 571 | .register_mask, |
| 531 | 572 | .indirect, |
| 532 | 573 | => .none, |
| 533 | 574 | }, .short = result }; |
| ... | ... | @@ -545,17 +586,17 @@ const InstTracking = struct { |
| 545 | 586 | return self.short.getCondition(); |
| 546 | 587 | } |
| 547 | 588 | |
| 548 | | fn spill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void { |
| 589 | fn spill(self: *InstTracking, cg: *CodeGen, inst: Air.Inst.Index) !void { |
| 549 | 590 | if (std.meta.eql(self.long, self.short)) return; // Already spilled |
| 550 | 591 | // Allocate or reuse frame index |
| 551 | 592 | switch (self.long) { |
| 552 | | .none => self.long = try function.allocRegOrMem(inst, false), |
| 593 | .none => self.long = try cg.allocRegOrMem(inst, false), |
| 553 | 594 | .load_frame => {}, |
| 554 | 595 | .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } }, |
| 555 | 596 | else => unreachable, |
| 556 | 597 | } |
| 557 | 598 | tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long }); |
| 558 | | try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{}); |
| 599 | try cg.genCopy(cg.typeOfIndex(inst), self.long, self.short, .{}); |
| 559 | 600 | } |
| 560 | 601 | |
| 561 | 602 | fn reuseFrame(self: *InstTracking) void { |
| ... | ... | @@ -584,8 +625,11 @@ const InstTracking = struct { |
| 584 | 625 | .eflags, |
| 585 | 626 | .register, |
| 586 | 627 | .register_pair, |
| 628 | .register_triple, |
| 629 | .register_quadruple, |
| 587 | 630 | .register_offset, |
| 588 | 631 | .register_overflow, |
| 632 | .register_mask, |
| 589 | 633 | .indirect, |
| 590 | 634 | .elementwise_regs_then_frame, |
| 591 | 635 | .reserved_frame, |
| ... | ... | @@ -630,8 +674,11 @@ const InstTracking = struct { |
| 630 | 674 | .eflags, |
| 631 | 675 | .register, |
| 632 | 676 | .register_pair, |
| 677 | .register_triple, |
| 678 | .register_quadruple, |
| 633 | 679 | .register_offset, |
| 634 | 680 | .register_overflow, |
| 681 | .register_mask, |
| 635 | 682 | .indirect, |
| 636 | 683 | .elementwise_regs_then_frame, |
| 637 | 684 | .air_ref, |
| ... | ... | @@ -735,13 +782,13 @@ const InstTracking = struct { |
| 735 | 782 | } |
| 736 | 783 | |
| 737 | 784 | pub fn format( |
| 738 | | self: InstTracking, |
| 785 | tracking: InstTracking, |
| 739 | 786 | comptime _: []const u8, |
| 740 | 787 | _: std.fmt.FormatOptions, |
| 741 | 788 | writer: anytype, |
| 742 | 789 | ) @TypeOf(writer).Error!void { |
| 743 | | if (!std.meta.eql(self.long, self.short)) try writer.print("|{}| ", .{self.long}); |
| 744 | | try writer.print("{}", .{self.short}); |
| 790 | if (!std.meta.eql(tracking.long, tracking.short)) try writer.print("|{}| ", .{tracking.long}); |
| 791 | try writer.print("{}", .{tracking.short}); |
| 745 | 792 | } |
| 746 | 793 | }; |
| 747 | 794 | |
| ... | ... | @@ -2259,44 +2306,44 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2259 | 2306 | try self.asmPseudo(.pseudo_dbg_leave_block_none); |
| 2260 | 2307 | } |
| 2261 | 2308 | |
| 2262 | | fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2263 | | const pt = self.pt; |
| 2309 | fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2310 | const pt = cg.pt; |
| 2264 | 2311 | const zcu = pt.zcu; |
| 2265 | 2312 | const ip = &zcu.intern_pool; |
| 2266 | | const air_tags = self.air.instructions.items(.tag); |
| 2267 | | const air_datas = self.air.instructions.items(.data); |
| 2268 | | const use_old = self.target.ofmt == .coff; |
| 2313 | const air_tags = cg.air.instructions.items(.tag); |
| 2314 | const air_datas = cg.air.instructions.items(.data); |
| 2315 | const use_old = cg.target.ofmt == .coff; |
| 2269 | 2316 | |
| 2270 | | self.arg_index = 0; |
| 2317 | cg.arg_index = 0; |
| 2271 | 2318 | for (body) |inst| switch (air_tags[@intFromEnum(inst)]) { |
| 2272 | 2319 | .arg => { |
| 2273 | | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); |
| 2274 | | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); |
| 2320 | wip_mir_log.debug("{}", .{cg.fmtAir(inst)}); |
| 2321 | verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); |
| 2275 | 2322 | |
| 2276 | | self.reused_operands = .initEmpty(); |
| 2277 | | try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); |
| 2323 | cg.reused_operands = .initEmpty(); |
| 2324 | try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1); |
| 2278 | 2325 | |
| 2279 | | try self.airArg(inst); |
| 2326 | try cg.airArg(inst); |
| 2280 | 2327 | |
| 2281 | | self.resetTemps(); |
| 2282 | | self.checkInvariantsAfterAirInst(); |
| 2328 | cg.resetTemps(); |
| 2329 | cg.checkInvariantsAfterAirInst(); |
| 2283 | 2330 | }, |
| 2284 | 2331 | else => break, |
| 2285 | 2332 | }; |
| 2286 | 2333 | |
| 2287 | | if (self.arg_index == 0) try self.airDbgVarArgs(); |
| 2288 | | self.arg_index = 0; |
| 2334 | if (cg.arg_index == 0) try cg.airDbgVarArgs(); |
| 2335 | cg.arg_index = 0; |
| 2289 | 2336 | for (body) |inst| { |
| 2290 | | if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue; |
| 2291 | | wip_mir_log.debug("{}", .{self.fmtAir(inst)}); |
| 2292 | | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); |
| 2337 | if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) continue; |
| 2338 | wip_mir_log.debug("{}", .{cg.fmtAir(inst)}); |
| 2339 | verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); |
| 2293 | 2340 | |
| 2294 | | self.reused_operands = .initEmpty(); |
| 2295 | | try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); |
| 2341 | cg.reused_operands = .initEmpty(); |
| 2342 | try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1); |
| 2296 | 2343 | switch (air_tags[@intFromEnum(inst)]) { |
| 2297 | 2344 | // zig fmt: off |
| 2298 | 2345 | .not, |
| 2299 | | => |tag| try self.airUnOp(inst, tag), |
| 2346 | => |air_tag| try cg.airUnOp(inst, air_tag), |
| 2300 | 2347 | |
| 2301 | 2348 | .add, |
| 2302 | 2349 | .add_wrap, |
| ... | ... | @@ -2306,22 +2353,22 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2306 | 2353 | .bool_or, |
| 2307 | 2354 | .min, |
| 2308 | 2355 | .max, |
| 2309 | | => |tag| try self.airBinOp(inst, tag), |
| 2356 | => |air_tag| try cg.airBinOp(inst, air_tag), |
| 2310 | 2357 | |
| 2311 | | .ptr_add, .ptr_sub => |tag| try self.airPtrArithmetic(inst, tag), |
| 2358 | .ptr_add, .ptr_sub => |air_tag| try cg.airPtrArithmetic(inst, air_tag), |
| 2312 | 2359 | |
| 2313 | | .shr, .shr_exact => try self.airShlShrBinOp(inst), |
| 2314 | | .shl, .shl_exact => try self.airShlShrBinOp(inst), |
| 2360 | .shr, .shr_exact => try cg.airShlShrBinOp(inst), |
| 2361 | .shl, .shl_exact => try cg.airShlShrBinOp(inst), |
| 2315 | 2362 | |
| 2316 | | .mul => try self.airMulDivBinOp(inst), |
| 2317 | | .mul_wrap => try self.airMulDivBinOp(inst), |
| 2318 | | .rem => try self.airMulDivBinOp(inst), |
| 2319 | | .mod => try self.airMulDivBinOp(inst), |
| 2363 | .mul => try cg.airMulDivBinOp(inst), |
| 2364 | .mul_wrap => try cg.airMulDivBinOp(inst), |
| 2365 | .rem => try cg.airMulDivBinOp(inst), |
| 2366 | .mod => try cg.airMulDivBinOp(inst), |
| 2320 | 2367 | |
| 2321 | | .add_sat => try self.airAddSat(inst), |
| 2322 | | .sub_sat => try self.airSubSat(inst), |
| 2323 | | .mul_sat => try self.airMulSat(inst), |
| 2324 | | .shl_sat => try self.airShlSat(inst), |
| 2368 | .add_sat => try cg.airAddSat(inst), |
| 2369 | .sub_sat => try cg.airSubSat(inst), |
| 2370 | .mul_sat => try cg.airMulSat(inst), |
| 2371 | .shl_sat => try cg.airShlSat(inst), |
| 2325 | 2372 | |
| 2326 | 2373 | .sin, |
| 2327 | 2374 | .cos, |
| ... | ... | @@ -2332,98 +2379,97 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2332 | 2379 | .log2, |
| 2333 | 2380 | .log10, |
| 2334 | 2381 | .round, |
| 2335 | | => |tag| try self.airUnaryMath(inst, tag), |
| 2336 | | |
| 2337 | | .floor => try self.airRound(inst, .{ .mode = .down, .precision = .inexact }), |
| 2338 | | .ceil => try self.airRound(inst, .{ .mode = .up, .precision = .inexact }), |
| 2339 | | .trunc_float => try self.airRound(inst, .{ .mode = .zero, .precision = .inexact }), |
| 2340 | | .sqrt => try self.airSqrt(inst), |
| 2341 | | .neg => try self.airFloatSign(inst), |
| 2342 | | |
| 2343 | | .abs => try self.airAbs(inst), |
| 2344 | | |
| 2345 | | .add_with_overflow => try self.airAddSubWithOverflow(inst), |
| 2346 | | .sub_with_overflow => try self.airAddSubWithOverflow(inst), |
| 2347 | | .mul_with_overflow => try self.airMulWithOverflow(inst), |
| 2348 | | .shl_with_overflow => try self.airShlWithOverflow(inst), |
| 2349 | | |
| 2350 | | .div_float, .div_trunc, .div_floor, .div_exact => try self.airMulDivBinOp(inst), |
| 2351 | | |
| 2352 | | .cmp_lt => try self.airCmp(inst, .lt), |
| 2353 | | .cmp_lte => try self.airCmp(inst, .lte), |
| 2354 | | .cmp_eq => try self.airCmp(inst, .eq), |
| 2355 | | .cmp_gte => try self.airCmp(inst, .gte), |
| 2356 | | .cmp_gt => try self.airCmp(inst, .gt), |
| 2357 | | .cmp_neq => try self.airCmp(inst, .neq), |
| 2358 | | |
| 2359 | | .cmp_vector => try self.airCmpVector(inst), |
| 2360 | | .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), |
| 2361 | | |
| 2362 | | .bitcast => try self.airBitCast(inst), |
| 2363 | | .fptrunc => try self.airFptrunc(inst), |
| 2364 | | .fpext => try self.airFpext(inst), |
| 2365 | | .intcast => try self.airIntCast(inst), |
| 2366 | | .trunc => try self.airTrunc(inst), |
| 2367 | | .is_non_null => try self.airIsNonNull(inst), |
| 2368 | | .is_null => try self.airIsNull(inst), |
| 2369 | | .is_non_err => try self.airIsNonErr(inst), |
| 2370 | | .is_err => try self.airIsErr(inst), |
| 2371 | | .load => try self.airLoad(inst), |
| 2372 | | .store => try self.airStore(inst, false), |
| 2373 | | .store_safe => try self.airStore(inst, true), |
| 2374 | | .struct_field_val => try self.airStructFieldVal(inst), |
| 2375 | | .float_from_int => try self.airFloatFromInt(inst), |
| 2376 | | .int_from_float => try self.airIntFromFloat(inst), |
| 2377 | | .cmpxchg_strong => try self.airCmpxchg(inst), |
| 2378 | | .cmpxchg_weak => try self.airCmpxchg(inst), |
| 2379 | | .atomic_rmw => try self.airAtomicRmw(inst), |
| 2380 | | .atomic_load => try self.airAtomicLoad(inst), |
| 2381 | | .memcpy => try self.airMemcpy(inst), |
| 2382 | | .memset => try self.airMemset(inst, false), |
| 2383 | | .memset_safe => try self.airMemset(inst, true), |
| 2384 | | .set_union_tag => try self.airSetUnionTag(inst), |
| 2385 | | .get_union_tag => try self.airGetUnionTag(inst), |
| 2386 | | .clz => try self.airClz(inst), |
| 2387 | | .ctz => try self.airCtz(inst), |
| 2388 | | .popcount => try self.airPopCount(inst), |
| 2389 | | .byte_swap => try self.airByteSwap(inst), |
| 2390 | | .bit_reverse => try self.airBitReverse(inst), |
| 2391 | | .tag_name => try self.airTagName(inst), |
| 2392 | | .error_name => try self.airErrorName(inst), |
| 2393 | | .splat => try self.airSplat(inst), |
| 2394 | | .select => try self.airSelect(inst), |
| 2395 | | .shuffle => try self.airShuffle(inst), |
| 2396 | | .reduce => try self.airReduce(inst), |
| 2397 | | .aggregate_init => try self.airAggregateInit(inst), |
| 2398 | | .union_init => try self.airUnionInit(inst), |
| 2399 | | .prefetch => try self.airPrefetch(inst), |
| 2400 | | .mul_add => try self.airMulAdd(inst), |
| 2401 | | |
| 2402 | | .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), |
| 2403 | | .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), |
| 2404 | | .atomic_store_release => try self.airAtomicStore(inst, .release), |
| 2405 | | .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst), |
| 2406 | | |
| 2407 | | .array_elem_val => try self.airArrayElemVal(inst), |
| 2408 | | .slice_elem_val => try self.airSliceElemVal(inst), |
| 2409 | | .ptr_elem_val => try self.airPtrElemVal(inst), |
| 2410 | | |
| 2411 | | .optional_payload => try self.airOptionalPayload(inst), |
| 2412 | | .unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst), |
| 2413 | | .unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst), |
| 2414 | | .err_return_trace => try self.airErrReturnTrace(inst), |
| 2415 | | .set_err_return_trace => try self.airSetErrReturnTrace(inst), |
| 2416 | | .save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst), |
| 2417 | | |
| 2418 | | .wrap_optional => try self.airWrapOptional(inst), |
| 2419 | | .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst), |
| 2420 | | .wrap_errunion_err => try self.airWrapErrUnionErr(inst), |
| 2382 | => |air_tag| try cg.airUnaryMath(inst, air_tag), |
| 2383 | |
| 2384 | .floor => try cg.airRound(inst, .{ .mode = .down, .precision = .inexact }), |
| 2385 | .ceil => try cg.airRound(inst, .{ .mode = .up, .precision = .inexact }), |
| 2386 | .trunc_float => try cg.airRound(inst, .{ .mode = .zero, .precision = .inexact }), |
| 2387 | .sqrt => try cg.airSqrt(inst), |
| 2388 | .neg => try cg.airFloatSign(inst), |
| 2389 | |
| 2390 | .abs => try cg.airAbs(inst), |
| 2391 | |
| 2392 | .add_with_overflow => try cg.airAddSubWithOverflow(inst), |
| 2393 | .sub_with_overflow => try cg.airAddSubWithOverflow(inst), |
| 2394 | .mul_with_overflow => try cg.airMulWithOverflow(inst), |
| 2395 | .shl_with_overflow => try cg.airShlWithOverflow(inst), |
| 2396 | |
| 2397 | .div_float, .div_trunc, .div_floor, .div_exact => try cg.airMulDivBinOp(inst), |
| 2398 | |
| 2399 | .cmp_lt => try cg.airCmp(inst, .lt), |
| 2400 | .cmp_lte => try cg.airCmp(inst, .lte), |
| 2401 | .cmp_eq => try cg.airCmp(inst, .eq), |
| 2402 | .cmp_gte => try cg.airCmp(inst, .gte), |
| 2403 | .cmp_gt => try cg.airCmp(inst, .gt), |
| 2404 | .cmp_neq => try cg.airCmp(inst, .neq), |
| 2405 | |
| 2406 | .cmp_lt_errors_len => try cg.airCmpLtErrorsLen(inst), |
| 2407 | |
| 2408 | .bitcast => try cg.airBitCast(inst), |
| 2409 | .fptrunc => try cg.airFptrunc(inst), |
| 2410 | .fpext => try cg.airFpext(inst), |
| 2411 | .intcast => try cg.airIntCast(inst), |
| 2412 | .trunc => try cg.airTrunc(inst), |
| 2413 | .is_non_null => try cg.airIsNonNull(inst), |
| 2414 | .is_null => try cg.airIsNull(inst), |
| 2415 | .is_non_err => try cg.airIsNonErr(inst), |
| 2416 | .is_err => try cg.airIsErr(inst), |
| 2417 | .load => try cg.airLoad(inst), |
| 2418 | .store => try cg.airStore(inst, false), |
| 2419 | .store_safe => try cg.airStore(inst, true), |
| 2420 | .struct_field_val => try cg.airStructFieldVal(inst), |
| 2421 | .float_from_int => try cg.airFloatFromInt(inst), |
| 2422 | .int_from_float => try cg.airIntFromFloat(inst), |
| 2423 | .cmpxchg_strong => try cg.airCmpxchg(inst), |
| 2424 | .cmpxchg_weak => try cg.airCmpxchg(inst), |
| 2425 | .atomic_rmw => try cg.airAtomicRmw(inst), |
| 2426 | .atomic_load => try cg.airAtomicLoad(inst), |
| 2427 | .memcpy => try cg.airMemcpy(inst), |
| 2428 | .memset => try cg.airMemset(inst, false), |
| 2429 | .memset_safe => try cg.airMemset(inst, true), |
| 2430 | .set_union_tag => try cg.airSetUnionTag(inst), |
| 2431 | .get_union_tag => try cg.airGetUnionTag(inst), |
| 2432 | .clz => try cg.airClz(inst), |
| 2433 | .ctz => try cg.airCtz(inst), |
| 2434 | .popcount => try cg.airPopCount(inst), |
| 2435 | .byte_swap => try cg.airByteSwap(inst), |
| 2436 | .bit_reverse => try cg.airBitReverse(inst), |
| 2437 | .tag_name => try cg.airTagName(inst), |
| 2438 | .error_name => try cg.airErrorName(inst), |
| 2439 | .splat => try cg.airSplat(inst), |
| 2440 | .select => try cg.airSelect(inst), |
| 2441 | .shuffle => try cg.airShuffle(inst), |
| 2442 | .reduce => try cg.airReduce(inst), |
| 2443 | .aggregate_init => try cg.airAggregateInit(inst), |
| 2444 | .union_init => try cg.airUnionInit(inst), |
| 2445 | .prefetch => try cg.airPrefetch(inst), |
| 2446 | .mul_add => try cg.airMulAdd(inst), |
| 2447 | |
| 2448 | .atomic_store_unordered => try cg.airAtomicStore(inst, .unordered), |
| 2449 | .atomic_store_monotonic => try cg.airAtomicStore(inst, .monotonic), |
| 2450 | .atomic_store_release => try cg.airAtomicStore(inst, .release), |
| 2451 | .atomic_store_seq_cst => try cg.airAtomicStore(inst, .seq_cst), |
| 2452 | |
| 2453 | .array_elem_val => try cg.airArrayElemVal(inst), |
| 2454 | .slice_elem_val => try cg.airSliceElemVal(inst), |
| 2455 | .ptr_elem_val => try cg.airPtrElemVal(inst), |
| 2456 | |
| 2457 | .optional_payload => try cg.airOptionalPayload(inst), |
| 2458 | .unwrap_errunion_err => try cg.airUnwrapErrUnionErr(inst), |
| 2459 | .unwrap_errunion_payload => try cg.airUnwrapErrUnionPayload(inst), |
| 2460 | .err_return_trace => try cg.airErrReturnTrace(inst), |
| 2461 | .set_err_return_trace => try cg.airSetErrReturnTrace(inst), |
| 2462 | .save_err_return_trace_index=> try cg.airSaveErrReturnTraceIndex(inst), |
| 2463 | |
| 2464 | .wrap_optional => try cg.airWrapOptional(inst), |
| 2465 | .wrap_errunion_payload => try cg.airWrapErrUnionPayload(inst), |
| 2466 | .wrap_errunion_err => try cg.airWrapErrUnionErr(inst), |
| 2421 | 2467 | // zig fmt: on |
| 2422 | 2468 | |
| 2423 | 2469 | .add_safe, |
| 2424 | 2470 | .sub_safe, |
| 2425 | 2471 | .mul_safe, |
| 2426 | | => return self.fail("TODO implement safety_checked_instructions", .{}), |
| 2472 | => return cg.fail("TODO implement safety_checked_instructions", .{}), |
| 2427 | 2473 | .add_optimized, |
| 2428 | 2474 | .sub_optimized, |
| 2429 | 2475 | .mul_optimized, |
| ... | ... | @@ -2440,214 +2486,612 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2440 | 2486 | .cmp_gte_optimized, |
| 2441 | 2487 | .cmp_gt_optimized, |
| 2442 | 2488 | .cmp_neq_optimized, |
| 2443 | | .cmp_vector_optimized, |
| 2444 | 2489 | .reduce_optimized, |
| 2445 | 2490 | .int_from_float_optimized, |
| 2446 | | => return self.fail("TODO implement optimized float mode", .{}), |
| 2491 | => return cg.fail("TODO implement optimized float mode", .{}), |
| 2447 | 2492 | |
| 2448 | | .arg => try self.airDbgArg(inst), |
| 2449 | | .alloc => if (use_old) try self.airAlloc(inst) else { |
| 2450 | | var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2451 | | .index = try self.allocMemPtr(inst), |
| 2493 | .arg => try cg.airDbgArg(inst), |
| 2494 | .alloc => if (use_old) try cg.airAlloc(inst) else { |
| 2495 | var slot = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2496 | .index = try cg.allocMemPtr(inst), |
| 2452 | 2497 | } }); |
| 2453 | | try slot.moveTo(inst, self); |
| 2498 | try slot.moveTo(inst, cg); |
| 2454 | 2499 | }, |
| 2455 | 2500 | .inferred_alloc => unreachable, |
| 2456 | 2501 | .inferred_alloc_comptime => unreachable, |
| 2457 | | .ret_ptr => if (use_old) try self.airRetPtr(inst) else { |
| 2458 | | var slot = switch (self.ret_mcv.long) { |
| 2502 | .ret_ptr => if (use_old) try cg.airRetPtr(inst) else { |
| 2503 | var slot = switch (cg.ret_mcv.long) { |
| 2459 | 2504 | else => unreachable, |
| 2460 | | .none => try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2461 | | .index = try self.allocMemPtr(inst), |
| 2505 | .none => try cg.tempFromValue(cg.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2506 | .index = try cg.allocMemPtr(inst), |
| 2462 | 2507 | } }), |
| 2463 | 2508 | .load_frame => slot: { |
| 2464 | | var slot = try self.tempFromValue(self.typeOfIndex(inst), self.ret_mcv.long); |
| 2465 | | try slot.toOffset(self.ret_mcv.short.indirect.off, self); |
| 2509 | var slot = try cg.tempFromValue(cg.typeOfIndex(inst), cg.ret_mcv.long); |
| 2510 | try slot.toOffset(cg.ret_mcv.short.indirect.off, cg); |
| 2466 | 2511 | break :slot slot; |
| 2467 | 2512 | }, |
| 2468 | 2513 | }; |
| 2469 | | try slot.moveTo(inst, self); |
| 2514 | try slot.moveTo(inst, cg); |
| 2470 | 2515 | }, |
| 2471 | | .assembly => try self.airAsm(inst), |
| 2472 | | inline .bit_and, .bit_or, .xor => |tag| if (use_old) try self.airBinOp(inst, tag) else { |
| 2516 | .assembly => try cg.airAsm(inst), |
| 2517 | .bit_and, .bit_or, .xor => |air_tag| if (use_old) try cg.airBinOp(inst, air_tag) else { |
| 2473 | 2518 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| 2474 | | var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 2475 | | try self.spillEflagsIfOccupied(); |
| 2476 | | const mir_tag: Mir.Inst.Tag = switch (tag) { |
| 2477 | | else => unreachable, |
| 2478 | | .bit_and => .@"and", |
| 2479 | | .bit_or => .@"or", |
| 2480 | | .xor => .xor, |
| 2481 | | }; |
| 2519 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 2482 | 2520 | var res: [1]Temp = undefined; |
| 2483 | | try self.select(&res, &.{ &ops[0], &ops[1] }, &.{ |
| 2484 | | .{ |
| 2485 | | .required_features = &.{.avx2}, |
| 2486 | | .mir_tag = .{ .vp_, mir_tag }, |
| 2487 | | .patterns = &.{ |
| 2488 | | .{ .ops = &.{ .ymm, .ymm, .mem } }, |
| 2489 | | .{ .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| 2490 | | .{ .ops = &.{ .ymm, .ymm, .ymm } }, |
| 2491 | | }, |
| 2492 | | }, |
| 2493 | | .{ |
| 2494 | | .required_features = &.{.avx}, |
| 2495 | | .mir_tag = .{ .vp_, mir_tag }, |
| 2496 | | .patterns = &.{ |
| 2497 | | .{ .ops = &.{ .xmm, .xmm, .mem } }, |
| 2498 | | .{ .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| 2499 | | .{ .ops = &.{ .xmm, .xmm, .xmm } }, |
| 2500 | | }, |
| 2501 | | }, |
| 2502 | | .{ |
| 2503 | | .required_features = &.{.sse2}, |
| 2504 | | .mir_tag = .{ .p_, mir_tag }, |
| 2505 | | .patterns = &.{ |
| 2506 | | .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .mem } }, |
| 2507 | | .{ .ops = &.{ .xmm, .mem, .{ .implicit = 0 } } }, |
| 2508 | | .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .xmm } }, |
| 2509 | | .{ .ops = &.{ .xmm, .xmm, .{ .implicit = 0 } } }, |
| 2510 | | }, |
| 2511 | | }, |
| 2512 | | .{ |
| 2513 | | .required_features = &.{.mmx}, |
| 2514 | | .mir_tag = .{ .p_, mir_tag }, |
| 2515 | | .patterns = &.{ |
| 2516 | | .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mem } }, |
| 2517 | | .{ .ops = &.{ .mm, .mem, .{ .implicit = 0 } } }, |
| 2518 | | .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mm } }, |
| 2519 | | .{ .ops = &.{ .mm, .mm, .{ .implicit = 0 } } }, |
| 2520 | | }, |
| 2521 | | }, |
| 2522 | | .{ |
| 2523 | | .mir_tag = .{ ._, mir_tag }, |
| 2524 | | .patterns = &.{ |
| 2525 | | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .simm32 } }, |
| 2526 | | .{ .ops = &.{ .mem, .simm32, .{ .implicit = 0 } } }, |
| 2527 | | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .gpr } }, |
| 2528 | | .{ .ops = &.{ .mem, .gpr, .{ .implicit = 0 } } }, |
| 2529 | | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .simm32 } }, |
| 2530 | | .{ .ops = &.{ .gpr, .simm32, .{ .implicit = 0 } } }, |
| 2531 | | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .mem } }, |
| 2532 | | .{ .ops = &.{ .gpr, .mem, .{ .implicit = 0 } } }, |
| 2533 | | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .gpr } }, |
| 2534 | | .{ .ops = &.{ .gpr, .gpr, .{ .implicit = 0 } } }, |
| 2535 | | }, |
| 2536 | | }, |
| 2521 | try cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, pattern_sets: switch (air_tag) { |
| 2522 | else => unreachable, |
| 2523 | inline .bit_and, .bit_or, .xor => |ct_air_tag| { |
| 2524 | const mir_tag: Mir.Inst.Tag = switch (ct_air_tag) { |
| 2525 | else => unreachable, |
| 2526 | .bit_and => .@"and", |
| 2527 | .bit_or => .@"or", |
| 2528 | .xor => .xor, |
| 2529 | }; |
| 2530 | break :pattern_sets &.{ |
| 2531 | .{ |
| 2532 | .required_features = &.{.avx2}, |
| 2533 | .mir_tag = .{ .vp_, mir_tag }, |
| 2534 | .patterns = &.{ |
| 2535 | .{ .ops = &.{ .ymm, .ymm, .mem } }, |
| 2536 | .{ .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| 2537 | .{ .ops = &.{ .ymm, .ymm, .ymm } }, |
| 2538 | }, |
| 2539 | }, |
| 2540 | .{ |
| 2541 | .required_features = &.{.avx}, |
| 2542 | .mir_tag = .{ .vp_, mir_tag }, |
| 2543 | .patterns = &.{ |
| 2544 | .{ .ops = &.{ .xmm, .xmm, .mem } }, |
| 2545 | .{ .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| 2546 | .{ .ops = &.{ .xmm, .xmm, .xmm } }, |
| 2547 | }, |
| 2548 | }, |
| 2549 | .{ |
| 2550 | .required_features = &.{.sse2}, |
| 2551 | .mir_tag = .{ .p_, mir_tag }, |
| 2552 | .patterns = &.{ |
| 2553 | .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .mem } }, |
| 2554 | .{ .ops = &.{ .xmm, .mem, .{ .implicit = 0 } } }, |
| 2555 | .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .xmm } }, |
| 2556 | .{ .ops = &.{ .xmm, .xmm, .{ .implicit = 0 } } }, |
| 2557 | }, |
| 2558 | }, |
| 2559 | .{ |
| 2560 | .required_features = &.{.mmx}, |
| 2561 | .mir_tag = .{ .p_, mir_tag }, |
| 2562 | .patterns = &.{ |
| 2563 | .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mem } }, |
| 2564 | .{ .ops = &.{ .mm, .mem, .{ .implicit = 0 } } }, |
| 2565 | .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mm } }, |
| 2566 | .{ .ops = &.{ .mm, .mm, .{ .implicit = 0 } } }, |
| 2567 | }, |
| 2568 | }, |
| 2569 | .{ |
| 2570 | .clobbers = .{ .eflags = true }, |
| 2571 | .mir_tag = .{ ._, mir_tag }, |
| 2572 | .patterns = &.{ |
| 2573 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .simm32 } }, |
| 2574 | .{ .ops = &.{ .mem, .simm32, .{ .implicit = 0 } } }, |
| 2575 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .gpr } }, |
| 2576 | .{ .ops = &.{ .mem, .gpr, .{ .implicit = 0 } } }, |
| 2577 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .simm32 } }, |
| 2578 | .{ .ops = &.{ .gpr, .simm32, .{ .implicit = 0 } } }, |
| 2579 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .mem } }, |
| 2580 | .{ .ops = &.{ .gpr, .mem, .{ .implicit = 0 } } }, |
| 2581 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .gpr } }, |
| 2582 | .{ .ops = &.{ .gpr, .gpr, .{ .implicit = 0 } } }, |
| 2583 | }, |
| 2584 | }, |
| 2537 | 2585 | |
| 2538 | | .{ |
| 2539 | | .required_features = &.{.avx2}, |
| 2540 | | .loop = .bitwise, |
| 2541 | | .mir_tag = .{ .vp_, mir_tag }, |
| 2542 | | .patterns = &.{ |
| 2543 | | .{ .ops = &.{ .ymm_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2544 | | .{ .ops = &.{ .ymm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2545 | | .{ .ops = &.{ .ymm_limb, .ymm_limb, .mem_limb } }, |
| 2546 | | }, |
| 2547 | | }, |
| 2548 | | .{ |
| 2549 | | .required_features = &.{.avx}, |
| 2550 | | .loop = .bitwise, |
| 2551 | | .mir_tag = .{ .vp_, mir_tag }, |
| 2552 | | .patterns = &.{ |
| 2553 | | .{ .ops = &.{ .xmm_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2554 | | .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2555 | | .{ .ops = &.{ .xmm_limb, .xmm_limb, .mem_limb } }, |
| 2556 | | }, |
| 2557 | | }, |
| 2558 | | .{ |
| 2559 | | .required_features = &.{.sse2}, |
| 2560 | | .loop = .bitwise, |
| 2561 | | .mir_tag = .{ .p_, mir_tag }, |
| 2562 | | .patterns = &.{ |
| 2563 | | .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2564 | | .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2565 | | }, |
| 2566 | | }, |
| 2567 | | .{ |
| 2568 | | .required_features = &.{.mmx}, |
| 2569 | | .loop = .bitwise, |
| 2570 | | .mir_tag = .{ .p_, mir_tag }, |
| 2571 | | .patterns = &.{ |
| 2572 | | .{ .ops = &.{ .mm_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2573 | | .{ .ops = &.{ .mm_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2574 | | }, |
| 2575 | | }, |
| 2576 | | .{ |
| 2577 | | .loop = .bitwise, |
| 2578 | | .mir_tag = .{ ._, mir_tag }, |
| 2579 | | .patterns = &.{ |
| 2580 | | .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| 2581 | | .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } }, |
| 2582 | | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2583 | | .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2584 | | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| 2585 | | }, |
| 2586 | .{ |
| 2587 | .required_features = &.{.avx2}, |
| 2588 | .loop = .bitwise, |
| 2589 | .mir_tag = .{ .vp_, mir_tag }, |
| 2590 | .patterns = &.{ |
| 2591 | .{ .ops = &.{ .ymm_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2592 | .{ .ops = &.{ .ymm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2593 | .{ .ops = &.{ .ymm_limb, .ymm_limb, .mem_limb } }, |
| 2594 | .{ .ops = &.{ .ymm_limb, .ymm_limb, .ymm_limb } }, |
| 2595 | }, |
| 2596 | }, |
| 2597 | .{ |
| 2598 | .required_features = &.{.avx}, |
| 2599 | .loop = .bitwise, |
| 2600 | .mir_tag = .{ .vp_, mir_tag }, |
| 2601 | .patterns = &.{ |
| 2602 | .{ .ops = &.{ .xmm_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2603 | .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2604 | .{ .ops = &.{ .xmm_limb, .xmm_limb, .mem_limb } }, |
| 2605 | .{ .ops = &.{ .xmm_limb, .xmm_limb, .xmm_limb } }, |
| 2606 | }, |
| 2607 | }, |
| 2608 | .{ |
| 2609 | .required_features = &.{.sse2}, |
| 2610 | .loop = .bitwise, |
| 2611 | .mir_tag = .{ .p_, mir_tag }, |
| 2612 | .patterns = &.{ |
| 2613 | .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2614 | .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2615 | .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| 2616 | }, |
| 2617 | }, |
| 2618 | .{ |
| 2619 | .required_features = &.{.mmx}, |
| 2620 | .loop = .bitwise, |
| 2621 | .mir_tag = .{ .p_, mir_tag }, |
| 2622 | .patterns = &.{ |
| 2623 | .{ .ops = &.{ .mm_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2624 | .{ .ops = &.{ .mm_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2625 | .{ .ops = &.{ .mm_limb, .{ .implicit = 0 }, .mm_limb } }, |
| 2626 | }, |
| 2627 | }, |
| 2628 | .{ |
| 2629 | .clobbers = .{ .eflags = true }, |
| 2630 | .loop = .bitwise, |
| 2631 | .mir_tag = .{ ._, mir_tag }, |
| 2632 | .patterns = &.{ |
| 2633 | .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| 2634 | .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } }, |
| 2635 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2636 | .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2637 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| 2638 | }, |
| 2639 | }, |
| 2640 | }; |
| 2586 | 2641 | }, |
| 2587 | | }); |
| 2588 | | if (ops[0].index != res[0].index) try ops[0].die(self); |
| 2589 | | if (ops[1].index != res[0].index) try ops[1].die(self); |
| 2590 | | try res[0].moveTo(inst, self); |
| 2642 | }, .{}); |
| 2643 | if (ops[0].index != res[0].index) try ops[0].die(cg); |
| 2644 | if (ops[1].index != res[0].index) try ops[1].die(cg); |
| 2645 | try res[0].moveTo(inst, cg); |
| 2591 | 2646 | }, |
| 2592 | | .block => if (use_old) try self.airBlock(inst) else { |
| 2647 | .block => if (use_old) try cg.airBlock(inst) else { |
| 2593 | 2648 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2594 | | const extra = self.air.extraData(Air.Block, ty_pl.payload); |
| 2595 | | try self.asmPseudo(.pseudo_dbg_enter_block_none); |
| 2596 | | try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); |
| 2597 | | try self.asmPseudo(.pseudo_dbg_leave_block_none); |
| 2649 | const extra = cg.air.extraData(Air.Block, ty_pl.payload); |
| 2650 | try cg.asmPseudo(.pseudo_dbg_enter_block_none); |
| 2651 | try cg.lowerBlock(inst, @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len])); |
| 2652 | try cg.asmPseudo(.pseudo_dbg_leave_block_none); |
| 2598 | 2653 | }, |
| 2599 | | .loop => if (use_old) try self.airLoop(inst) else { |
| 2654 | .loop => if (use_old) try cg.airLoop(inst) else { |
| 2600 | 2655 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2601 | | const extra = self.air.extraData(Air.Block, ty_pl.payload); |
| 2602 | | self.scope_generation += 1; |
| 2603 | | try self.loops.putNoClobber(self.gpa, inst, .{ |
| 2604 | | .state = try self.saveState(), |
| 2605 | | .target = @intCast(self.mir_instructions.len), |
| 2656 | const extra = cg.air.extraData(Air.Block, ty_pl.payload); |
| 2657 | cg.scope_generation += 1; |
| 2658 | try cg.loops.putNoClobber(cg.gpa, inst, .{ |
| 2659 | .state = try cg.saveState(), |
| 2660 | .target = @intCast(cg.mir_instructions.len), |
| 2606 | 2661 | }); |
| 2607 | | defer assert(self.loops.remove(inst)); |
| 2608 | | try self.genBodyBlock(@ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); |
| 2662 | defer assert(cg.loops.remove(inst)); |
| 2663 | try cg.genBodyBlock(@ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len])); |
| 2609 | 2664 | }, |
| 2610 | | .repeat => if (use_old) try self.airRepeat(inst) else { |
| 2665 | .repeat => if (use_old) try cg.airRepeat(inst) else { |
| 2611 | 2666 | const repeat = air_datas[@intFromEnum(inst)].repeat; |
| 2612 | | const loop = self.loops.get(repeat.loop_inst).?; |
| 2613 | | try self.restoreState(loop.state, &.{}, .{ |
| 2667 | const loop = cg.loops.get(repeat.loop_inst).?; |
| 2668 | try cg.restoreState(loop.state, &.{}, .{ |
| 2614 | 2669 | .emit_instructions = true, |
| 2615 | 2670 | .update_tracking = false, |
| 2616 | 2671 | .resurrect = false, |
| 2617 | 2672 | .close_scope = true, |
| 2618 | 2673 | }); |
| 2619 | | _ = try self.asmJmpReloc(loop.target); |
| 2674 | _ = try cg.asmJmpReloc(loop.target); |
| 2620 | 2675 | }, |
| 2621 | | .br => try self.airBr(inst), |
| 2622 | | .trap => try self.asmOpOnly(.{ ._, .ud2 }), |
| 2623 | | .breakpoint => try self.asmOpOnly(.{ ._, .int3 }), |
| 2624 | | .ret_addr => if (use_old) try self.airRetAddr(inst) else { |
| 2625 | | var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .load_frame = .{ |
| 2676 | .br => try cg.airBr(inst), |
| 2677 | .trap => try cg.asmOpOnly(.{ ._, .ud2 }), |
| 2678 | .breakpoint => try cg.asmOpOnly(.{ ._, .int3 }), |
| 2679 | .ret_addr => if (use_old) try cg.airRetAddr(inst) else { |
| 2680 | var slot = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .load_frame = .{ |
| 2626 | 2681 | .index = .ret_addr, |
| 2627 | 2682 | } }); |
| 2628 | | while (try slot.toAnyReg(self)) {} |
| 2629 | | try slot.moveTo(inst, self); |
| 2683 | while (try slot.toAnyReg(cg)) {} |
| 2684 | try slot.moveTo(inst, cg); |
| 2630 | 2685 | }, |
| 2631 | | .frame_addr => if (use_old) try self.airFrameAddress(inst) else { |
| 2632 | | var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2686 | .frame_addr => if (use_old) try cg.airFrameAddress(inst) else { |
| 2687 | var slot = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .lea_frame = .{ |
| 2633 | 2688 | .index = .base_ptr, |
| 2634 | 2689 | } }); |
| 2635 | | try slot.moveTo(inst, self); |
| 2690 | try slot.moveTo(inst, cg); |
| 2691 | }, |
| 2692 | .call => try cg.airCall(inst, .auto), |
| 2693 | .call_always_tail => try cg.airCall(inst, .always_tail), |
| 2694 | .call_never_tail => try cg.airCall(inst, .never_tail), |
| 2695 | .call_never_inline => try cg.airCall(inst, .never_inline), |
| 2696 | |
| 2697 | .cmp_vector, .cmp_vector_optimized => if (use_old) try cg.airCmpVector(inst) else fallback: { |
| 2698 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2699 | const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data; |
| 2700 | switch (extra.compareOperator()) { |
| 2701 | .eq, .neq => {}, |
| 2702 | else => break :fallback try cg.airCmpVector(inst), |
| 2703 | } |
| 2704 | var ops = try cg.tempsFromOperands(inst, .{ extra.lhs, extra.rhs }); |
| 2705 | var res: [1]Temp = undefined; |
| 2706 | switch (extra.compareOperator()) { |
| 2707 | .lt => unreachable, |
| 2708 | .lte => unreachable, |
| 2709 | .eq, .neq => |cmp_op| try cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, &.{ |
| 2710 | .{ |
| 2711 | .required_features = &.{.avx2}, |
| 2712 | .scalar = .{ .any_int = .byte }, |
| 2713 | .mir_tag = .{ .vp_b, .cmpeq }, |
| 2714 | .patterns = &.{ |
| 2715 | .{ .ops = &.{ .ymm_mask, .ymm, .mem } }, |
| 2716 | .{ .ops = &.{ .ymm_mask, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| 2717 | .{ .ops = &.{ .ymm_mask, .ymm, .ymm } }, |
| 2718 | }, |
| 2719 | }, |
| 2720 | .{ |
| 2721 | .required_features = &.{.avx2}, |
| 2722 | .scalar = .{ .any_int = .word }, |
| 2723 | .mir_tag = .{ .vp_w, .cmpeq }, |
| 2724 | .patterns = &.{ |
| 2725 | .{ .ops = &.{ .ymm_mask, .ymm, .mem } }, |
| 2726 | .{ .ops = &.{ .ymm_mask, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| 2727 | .{ .ops = &.{ .ymm_mask, .ymm, .ymm } }, |
| 2728 | }, |
| 2729 | }, |
| 2730 | .{ |
| 2731 | .required_features = &.{.avx2}, |
| 2732 | .scalar = .{ .any_int = .dword }, |
| 2733 | .mir_tag = .{ .vp_d, .cmpeq }, |
| 2734 | .patterns = &.{ |
| 2735 | .{ .ops = &.{ .ymm_mask, .ymm, .mem } }, |
| 2736 | .{ .ops = &.{ .ymm_mask, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| 2737 | .{ .ops = &.{ .ymm_mask, .ymm, .ymm } }, |
| 2738 | }, |
| 2739 | }, |
| 2740 | .{ |
| 2741 | .required_features = &.{.avx2}, |
| 2742 | .scalar = .{ .any_int = .qword }, |
| 2743 | .mir_tag = .{ .vp_q, .cmpeq }, |
| 2744 | .patterns = &.{ |
| 2745 | .{ .ops = &.{ .ymm_mask, .ymm, .mem } }, |
| 2746 | .{ .ops = &.{ .ymm_mask, .mem, .ymm }, .commute = .{ 1, 2 } }, |
| 2747 | .{ .ops = &.{ .ymm_mask, .ymm, .ymm } }, |
| 2748 | }, |
| 2749 | }, |
| 2750 | .{ |
| 2751 | .required_features = &.{.avx}, |
| 2752 | .scalar = .{ .any_int = .byte }, |
| 2753 | .mir_tag = .{ .vp_b, .cmpeq }, |
| 2754 | .patterns = &.{ |
| 2755 | .{ .ops = &.{ .xmm_mask, .xmm, .mem } }, |
| 2756 | .{ .ops = &.{ .xmm_mask, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| 2757 | .{ .ops = &.{ .xmm_mask, .xmm, .xmm } }, |
| 2758 | }, |
| 2759 | }, |
| 2760 | .{ |
| 2761 | .required_features = &.{.avx}, |
| 2762 | .scalar = .{ .any_int = .word }, |
| 2763 | .mir_tag = .{ .vp_w, .cmpeq }, |
| 2764 | .patterns = &.{ |
| 2765 | .{ .ops = &.{ .xmm_mask, .xmm, .mem } }, |
| 2766 | .{ .ops = &.{ .xmm_mask, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| 2767 | .{ .ops = &.{ .xmm_mask, .xmm, .xmm } }, |
| 2768 | }, |
| 2769 | }, |
| 2770 | .{ |
| 2771 | .required_features = &.{.avx}, |
| 2772 | .scalar = .{ .any_int = .dword }, |
| 2773 | .mir_tag = .{ .vp_d, .cmpeq }, |
| 2774 | .patterns = &.{ |
| 2775 | .{ .ops = &.{ .xmm_mask, .xmm, .mem } }, |
| 2776 | .{ .ops = &.{ .xmm_mask, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| 2777 | .{ .ops = &.{ .xmm_mask, .xmm, .xmm } }, |
| 2778 | }, |
| 2779 | }, |
| 2780 | .{ |
| 2781 | .required_features = &.{.avx}, |
| 2782 | .scalar = .{ .any_int = .qword }, |
| 2783 | .mir_tag = .{ .vp_q, .cmpeq }, |
| 2784 | .patterns = &.{ |
| 2785 | .{ .ops = &.{ .xmm_mask, .xmm, .mem } }, |
| 2786 | .{ .ops = &.{ .xmm_mask, .mem, .xmm }, .commute = .{ 1, 2 } }, |
| 2787 | .{ .ops = &.{ .xmm_mask, .xmm, .xmm } }, |
| 2788 | }, |
| 2789 | }, |
| 2790 | .{ |
| 2791 | .required_features = &.{.sse2}, |
| 2792 | .scalar = .{ .any_int = .byte }, |
| 2793 | .mir_tag = .{ .p_b, .cmpeq }, |
| 2794 | .patterns = &.{ |
| 2795 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .mem } }, |
| 2796 | .{ .ops = &.{ .xmm_mask, .mem, .{ .implicit = 0 } } }, |
| 2797 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .xmm } }, |
| 2798 | .{ .ops = &.{ .xmm_mask, .xmm, .{ .implicit = 0 } } }, |
| 2799 | }, |
| 2800 | }, |
| 2801 | .{ |
| 2802 | .required_features = &.{.sse2}, |
| 2803 | .scalar = .{ .any_int = .word }, |
| 2804 | .mir_tag = .{ .p_w, .cmpeq }, |
| 2805 | .patterns = &.{ |
| 2806 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .mem } }, |
| 2807 | .{ .ops = &.{ .xmm_mask, .mem, .{ .implicit = 0 } } }, |
| 2808 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .xmm } }, |
| 2809 | .{ .ops = &.{ .xmm_mask, .xmm, .{ .implicit = 0 } } }, |
| 2810 | }, |
| 2811 | }, |
| 2812 | .{ |
| 2813 | .required_features = &.{.sse2}, |
| 2814 | .scalar = .{ .any_int = .dword }, |
| 2815 | .mir_tag = .{ .p_d, .cmpeq }, |
| 2816 | .patterns = &.{ |
| 2817 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .mem } }, |
| 2818 | .{ .ops = &.{ .xmm_mask, .mem, .{ .implicit = 0 } } }, |
| 2819 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .xmm } }, |
| 2820 | .{ .ops = &.{ .xmm_mask, .xmm, .{ .implicit = 0 } } }, |
| 2821 | }, |
| 2822 | }, |
| 2823 | .{ |
| 2824 | .required_features = &.{.sse4_1}, |
| 2825 | .scalar = .{ .any_int = .qword }, |
| 2826 | .mir_tag = .{ .p_q, .cmpeq }, |
| 2827 | .patterns = &.{ |
| 2828 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .mem } }, |
| 2829 | .{ .ops = &.{ .xmm_mask, .mem, .{ .implicit = 0 } } }, |
| 2830 | .{ .ops = &.{ .xmm_mask, .{ .implicit = 0 }, .xmm } }, |
| 2831 | .{ .ops = &.{ .xmm_mask, .xmm, .{ .implicit = 0 } } }, |
| 2832 | }, |
| 2833 | }, |
| 2834 | .{ |
| 2835 | .required_features = &.{.mmx}, |
| 2836 | .scalar = .{ .any_int = .byte }, |
| 2837 | .mir_tag = .{ .p_b, .cmpeq }, |
| 2838 | .patterns = &.{ |
| 2839 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mem } }, |
| 2840 | .{ .ops = &.{ .mm_mask, .mem, .{ .implicit = 0 } } }, |
| 2841 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mm } }, |
| 2842 | .{ .ops = &.{ .mm_mask, .mm, .{ .implicit = 0 } } }, |
| 2843 | }, |
| 2844 | }, |
| 2845 | .{ |
| 2846 | .required_features = &.{.mmx}, |
| 2847 | .scalar = .{ .any_int = .word }, |
| 2848 | .mir_tag = .{ .p_w, .cmpeq }, |
| 2849 | .patterns = &.{ |
| 2850 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mem } }, |
| 2851 | .{ .ops = &.{ .mm_mask, .mem, .{ .implicit = 0 } } }, |
| 2852 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mm } }, |
| 2853 | .{ .ops = &.{ .mm_mask, .mm, .{ .implicit = 0 } } }, |
| 2854 | }, |
| 2855 | }, |
| 2856 | .{ |
| 2857 | .required_features = &.{.mmx}, |
| 2858 | .scalar = .{ .any_int = .dword }, |
| 2859 | .mir_tag = .{ .p_d, .cmpeq }, |
| 2860 | .patterns = &.{ |
| 2861 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mem } }, |
| 2862 | .{ .ops = &.{ .mm_mask, .mem, .{ .implicit = 0 } } }, |
| 2863 | .{ .ops = &.{ .mm_mask, .{ .implicit = 0 }, .mm } }, |
| 2864 | .{ .ops = &.{ .mm_mask, .mm, .{ .implicit = 0 } } }, |
| 2865 | }, |
| 2866 | }, |
| 2867 | .{ |
| 2868 | .scalar = .bool, |
| 2869 | .clobbers = .{ .eflags = true }, |
| 2870 | .invert_result = true, |
| 2871 | .mir_tag = .{ ._, .xor }, |
| 2872 | .patterns = &.{ |
| 2873 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .simm32 } }, |
| 2874 | .{ .ops = &.{ .mem, .simm32, .{ .implicit = 0 } } }, |
| 2875 | .{ .ops = &.{ .mem, .{ .implicit = 0 }, .gpr } }, |
| 2876 | .{ .ops = &.{ .mem, .gpr, .{ .implicit = 0 } } }, |
| 2877 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .simm32 } }, |
| 2878 | .{ .ops = &.{ .gpr, .simm32, .{ .implicit = 0 } } }, |
| 2879 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .mem } }, |
| 2880 | .{ .ops = &.{ .gpr, .mem, .{ .implicit = 0 } } }, |
| 2881 | .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .gpr } }, |
| 2882 | .{ .ops = &.{ .gpr, .gpr, .{ .implicit = 0 } } }, |
| 2883 | }, |
| 2884 | }, |
| 2885 | |
| 2886 | .{ |
| 2887 | .required_features = &.{.avx2}, |
| 2888 | .scalar = .{ .any_int = .byte }, |
| 2889 | .loop = .elementwise, |
| 2890 | .mir_tag = .{ .vp_b, .cmpeq }, |
| 2891 | .patterns = &.{ |
| 2892 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2893 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2894 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, |
| 2895 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, |
| 2896 | }, |
| 2897 | }, |
| 2898 | .{ |
| 2899 | .required_features = &.{.avx2}, |
| 2900 | .scalar = .{ .any_int = .word }, |
| 2901 | .loop = .elementwise, |
| 2902 | .mir_tag = .{ .vp_w, .cmpeq }, |
| 2903 | .patterns = &.{ |
| 2904 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2905 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2906 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, |
| 2907 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, |
| 2908 | }, |
| 2909 | }, |
| 2910 | .{ |
| 2911 | .required_features = &.{.avx2}, |
| 2912 | .scalar = .{ .any_int = .dword }, |
| 2913 | .loop = .elementwise, |
| 2914 | .mir_tag = .{ .vp_d, .cmpeq }, |
| 2915 | .patterns = &.{ |
| 2916 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2917 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2918 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, |
| 2919 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, |
| 2920 | }, |
| 2921 | }, |
| 2922 | .{ |
| 2923 | .required_features = &.{.avx2}, |
| 2924 | .scalar = .{ .any_int = .qword }, |
| 2925 | .loop = .elementwise, |
| 2926 | .mir_tag = .{ .vp_q, .cmpeq }, |
| 2927 | .patterns = &.{ |
| 2928 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2929 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2930 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, |
| 2931 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, |
| 2932 | }, |
| 2933 | }, |
| 2934 | .{ |
| 2935 | .required_features = &.{.avx}, |
| 2936 | .scalar = .{ .any_int = .byte }, |
| 2937 | .loop = .elementwise, |
| 2938 | .mir_tag = .{ .vp_b, .cmpeq }, |
| 2939 | .patterns = &.{ |
| 2940 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2941 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2942 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, |
| 2943 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, |
| 2944 | }, |
| 2945 | }, |
| 2946 | .{ |
| 2947 | .required_features = &.{.avx}, |
| 2948 | .scalar = .{ .any_int = .word }, |
| 2949 | .loop = .elementwise, |
| 2950 | .mir_tag = .{ .vp_w, .cmpeq }, |
| 2951 | .patterns = &.{ |
| 2952 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2953 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2954 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, |
| 2955 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, |
| 2956 | }, |
| 2957 | }, |
| 2958 | .{ |
| 2959 | .required_features = &.{.avx}, |
| 2960 | .scalar = .{ .any_int = .dword }, |
| 2961 | .loop = .elementwise, |
| 2962 | .mir_tag = .{ .vp_d, .cmpeq }, |
| 2963 | .patterns = &.{ |
| 2964 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2965 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2966 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, |
| 2967 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, |
| 2968 | }, |
| 2969 | }, |
| 2970 | .{ |
| 2971 | .required_features = &.{.avx}, |
| 2972 | .scalar = .{ .any_int = .qword }, |
| 2973 | .loop = .elementwise, |
| 2974 | .mir_tag = .{ .vp_q, .cmpeq }, |
| 2975 | .patterns = &.{ |
| 2976 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, |
| 2977 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, |
| 2978 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, |
| 2979 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, |
| 2980 | }, |
| 2981 | }, |
| 2982 | .{ |
| 2983 | .required_features = &.{.sse2}, |
| 2984 | .scalar = .{ .any_int = .byte }, |
| 2985 | .loop = .elementwise, |
| 2986 | .mir_tag = .{ .p_b, .cmpeq }, |
| 2987 | .patterns = &.{ |
| 2988 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 2989 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 2990 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| 2991 | }, |
| 2992 | }, |
| 2993 | .{ |
| 2994 | .required_features = &.{.sse2}, |
| 2995 | .scalar = .{ .any_int = .word }, |
| 2996 | .loop = .elementwise, |
| 2997 | .mir_tag = .{ .p_w, .cmpeq }, |
| 2998 | .patterns = &.{ |
| 2999 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 3000 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 3001 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| 3002 | }, |
| 3003 | }, |
| 3004 | .{ |
| 3005 | .required_features = &.{.sse2}, |
| 3006 | .scalar = .{ .any_int = .dword }, |
| 3007 | .loop = .elementwise, |
| 3008 | .mir_tag = .{ .p_d, .cmpeq }, |
| 3009 | .patterns = &.{ |
| 3010 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 3011 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 3012 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| 3013 | }, |
| 3014 | }, |
| 3015 | .{ |
| 3016 | .required_features = &.{.sse4_1}, |
| 3017 | .scalar = .{ .any_int = .qword }, |
| 3018 | .loop = .elementwise, |
| 3019 | .mir_tag = .{ .p_q, .cmpeq }, |
| 3020 | .patterns = &.{ |
| 3021 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 3022 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 3023 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, |
| 3024 | }, |
| 3025 | }, |
| 3026 | .{ |
| 3027 | .required_features = &.{.mmx}, |
| 3028 | .scalar = .{ .any_int = .byte }, |
| 3029 | .loop = .elementwise, |
| 3030 | .mir_tag = .{ .p_b, .cmpeq }, |
| 3031 | .patterns = &.{ |
| 3032 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 3033 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 3034 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, |
| 3035 | }, |
| 3036 | }, |
| 3037 | .{ |
| 3038 | .required_features = &.{.mmx}, |
| 3039 | .scalar = .{ .any_int = .word }, |
| 3040 | .loop = .elementwise, |
| 3041 | .mir_tag = .{ .p_w, .cmpeq }, |
| 3042 | .patterns = &.{ |
| 3043 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 3044 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 3045 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, |
| 3046 | }, |
| 3047 | }, |
| 3048 | .{ |
| 3049 | .required_features = &.{.mmx}, |
| 3050 | .scalar = .{ .any_int = .dword }, |
| 3051 | .loop = .elementwise, |
| 3052 | .mir_tag = .{ .p_d, .cmpeq }, |
| 3053 | .patterns = &.{ |
| 3054 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 3055 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 3056 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, |
| 3057 | }, |
| 3058 | }, |
| 3059 | .{ |
| 3060 | .scalar = .bool, |
| 3061 | .clobbers = .{ .eflags = true }, |
| 3062 | .invert_result = true, |
| 3063 | .loop = .elementwise, |
| 3064 | .mir_tag = .{ ._, .xor }, |
| 3065 | .patterns = &.{ |
| 3066 | .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| 3067 | .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } }, |
| 3068 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } }, |
| 3069 | .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } }, |
| 3070 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } }, |
| 3071 | }, |
| 3072 | }, |
| 3073 | }, .{ .invert_result = switch (cmp_op) { |
| 3074 | .eq => false, |
| 3075 | .neq => true, |
| 3076 | else => unreachable, |
| 3077 | } }), |
| 3078 | .gte => unreachable, |
| 3079 | .gt => unreachable, |
| 3080 | } |
| 3081 | if (ops[0].index != res[0].index) try ops[0].die(cg); |
| 3082 | if (ops[1].index != res[0].index) try ops[1].die(cg); |
| 3083 | try res[0].moveTo(inst, cg); |
| 2636 | 3084 | }, |
| 2637 | | .call => try self.airCall(inst, .auto), |
| 2638 | | .call_always_tail => try self.airCall(inst, .always_tail), |
| 2639 | | .call_never_tail => try self.airCall(inst, .never_tail), |
| 2640 | | .call_never_inline => try self.airCall(inst, .never_inline), |
| 2641 | | |
| 2642 | | .cond_br => try self.airCondBr(inst), |
| 2643 | | .switch_br => try self.airSwitchBr(inst), |
| 2644 | | .loop_switch_br => try self.airLoopSwitchBr(inst), |
| 2645 | | .switch_dispatch => try self.airSwitchDispatch(inst), |
| 2646 | | .@"try", .try_cold => try self.airTry(inst), |
| 2647 | | .try_ptr, .try_ptr_cold => try self.airTryPtr(inst), |
| 2648 | | .dbg_stmt => if (use_old) try self.airDbgStmt(inst) else { |
| 3085 | |
| 3086 | .cond_br => try cg.airCondBr(inst), |
| 3087 | .switch_br => try cg.airSwitchBr(inst), |
| 3088 | .loop_switch_br => try cg.airLoopSwitchBr(inst), |
| 3089 | .switch_dispatch => try cg.airSwitchDispatch(inst), |
| 3090 | .@"try", .try_cold => try cg.airTry(inst), |
| 3091 | .try_ptr, .try_ptr_cold => try cg.airTryPtr(inst), |
| 3092 | .dbg_stmt => if (use_old) try cg.airDbgStmt(inst) else { |
| 2649 | 3093 | const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt; |
| 2650 | | _ = try self.addInst(.{ |
| 3094 | _ = try cg.addInst(.{ |
| 2651 | 3095 | .tag = .pseudo, |
| 2652 | 3096 | .ops = .pseudo_dbg_line_line_column, |
| 2653 | 3097 | .data = .{ .line_column = .{ |
| ... | ... | @@ -2656,50 +3100,50 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2656 | 3100 | } }, |
| 2657 | 3101 | }); |
| 2658 | 3102 | }, |
| 2659 | | .dbg_empty_stmt => if (use_old) try self.airDbgEmptyStmt() else { |
| 2660 | | if (self.mir_instructions.len > 0) { |
| 2661 | | const prev_mir_op = &self.mir_instructions.items(.ops)[self.mir_instructions.len - 1]; |
| 3103 | .dbg_empty_stmt => if (use_old) try cg.airDbgEmptyStmt() else { |
| 3104 | if (cg.mir_instructions.len > 0) { |
| 3105 | const prev_mir_op = &cg.mir_instructions.items(.ops)[cg.mir_instructions.len - 1]; |
| 2662 | 3106 | if (prev_mir_op.* == .pseudo_dbg_line_stmt_line_column) |
| 2663 | 3107 | prev_mir_op.* = .pseudo_dbg_line_line_column; |
| 2664 | 3108 | } |
| 2665 | | try self.asmOpOnly(.{ ._, .nop }); |
| 3109 | try cg.asmOpOnly(.{ ._, .nop }); |
| 2666 | 3110 | }, |
| 2667 | | .dbg_inline_block => if (use_old) try self.airDbgInlineBlock(inst) else { |
| 3111 | .dbg_inline_block => if (use_old) try cg.airDbgInlineBlock(inst) else { |
| 2668 | 3112 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2669 | | const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload); |
| 2670 | | const old_inline_func = self.inline_func; |
| 2671 | | defer self.inline_func = old_inline_func; |
| 2672 | | self.inline_func = extra.data.func; |
| 2673 | | _ = try self.addInst(.{ |
| 3113 | const extra = cg.air.extraData(Air.DbgInlineBlock, ty_pl.payload); |
| 3114 | const old_inline_func = cg.inline_func; |
| 3115 | defer cg.inline_func = old_inline_func; |
| 3116 | cg.inline_func = extra.data.func; |
| 3117 | _ = try cg.addInst(.{ |
| 2674 | 3118 | .tag = .pseudo, |
| 2675 | 3119 | .ops = .pseudo_dbg_enter_inline_func, |
| 2676 | 3120 | .data = .{ .func = extra.data.func }, |
| 2677 | 3121 | }); |
| 2678 | | try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); |
| 2679 | | _ = try self.addInst(.{ |
| 3122 | try cg.lowerBlock(inst, @ptrCast(cg.air.extra[extra.end..][0..extra.data.body_len])); |
| 3123 | _ = try cg.addInst(.{ |
| 2680 | 3124 | .tag = .pseudo, |
| 2681 | 3125 | .ops = .pseudo_dbg_leave_inline_func, |
| 2682 | 3126 | .data = .{ .func = old_inline_func }, |
| 2683 | 3127 | }); |
| 2684 | 3128 | }, |
| 2685 | | .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try self.airDbgVar(inst) else { |
| 3129 | .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try cg.airDbgVar(inst) else { |
| 2686 | 3130 | const pl_op = air_datas[@intFromEnum(inst)].pl_op; |
| 2687 | | var ops = try self.tempsFromOperands(inst, .{pl_op.operand}); |
| 2688 | | try self.genLocalDebugInfo(inst, ops[0].tracking(self).short); |
| 2689 | | try ops[0].die(self); |
| 3131 | var ops = try cg.tempsFromOperands(inst, .{pl_op.operand}); |
| 3132 | try cg.genLocalDebugInfo(inst, ops[0].tracking(cg).short); |
| 3133 | try ops[0].die(cg); |
| 2690 | 3134 | }, |
| 2691 | | .is_null_ptr => if (use_old) try self.airIsNullPtr(inst) else { |
| 3135 | .is_null_ptr => if (use_old) try cg.airIsNullPtr(inst) else { |
| 2692 | 3136 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2693 | | const opt_ty = self.typeOf(un_op).childType(zcu); |
| 3137 | const opt_ty = cg.typeOf(un_op).childType(zcu); |
| 2694 | 3138 | const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); |
| 2695 | 3139 | const opt_child_ty = opt_ty.optionalChild(zcu); |
| 2696 | 3140 | const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); |
| 2697 | | var ops = try self.tempsFromOperands(inst, .{un_op}); |
| 2698 | | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self); |
| 2699 | | while (try ops[0].toLea(self)) {} |
| 2700 | | try self.asmMemoryImmediate( |
| 3141 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 3142 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); |
| 3143 | while (try ops[0].toLea(cg)) {} |
| 3144 | try cg.asmMemoryImmediate( |
| 2701 | 3145 | .{ ._, .cmp }, |
| 2702 | | try ops[0].tracking(self).short.deref().mem(self, .{ .size = if (!opt_repr_is_pl) |
| 3146 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) |
| 2703 | 3147 | .byte |
| 2704 | 3148 | else if (opt_child_ty.isSlice(zcu)) |
| 2705 | 3149 | .qword |
| ... | ... | @@ -2707,22 +3151,22 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2707 | 3151 | .fromSize(opt_child_abi_size) }), |
| 2708 | 3152 | .u(0), |
| 2709 | 3153 | ); |
| 2710 | | var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e }); |
| 2711 | | try ops[0].die(self); |
| 2712 | | try is_null.moveTo(inst, self); |
| 3154 | var is_null = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .eflags = .e }); |
| 3155 | try ops[0].die(cg); |
| 3156 | try is_null.moveTo(inst, cg); |
| 2713 | 3157 | }, |
| 2714 | | .is_non_null_ptr => if (use_old) try self.airIsNonNullPtr(inst) else { |
| 3158 | .is_non_null_ptr => if (use_old) try cg.airIsNonNullPtr(inst) else { |
| 2715 | 3159 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2716 | | const opt_ty = self.typeOf(un_op).childType(zcu); |
| 3160 | const opt_ty = cg.typeOf(un_op).childType(zcu); |
| 2717 | 3161 | const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); |
| 2718 | 3162 | const opt_child_ty = opt_ty.optionalChild(zcu); |
| 2719 | 3163 | const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); |
| 2720 | | var ops = try self.tempsFromOperands(inst, .{un_op}); |
| 2721 | | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self); |
| 2722 | | while (try ops[0].toLea(self)) {} |
| 2723 | | try self.asmMemoryImmediate( |
| 3164 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 3165 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); |
| 3166 | while (try ops[0].toLea(cg)) {} |
| 3167 | try cg.asmMemoryImmediate( |
| 2724 | 3168 | .{ ._, .cmp }, |
| 2725 | | try ops[0].tracking(self).short.deref().mem(self, .{ .size = if (!opt_repr_is_pl) |
| 3169 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) |
| 2726 | 3170 | .byte |
| 2727 | 3171 | else if (opt_child_ty.isSlice(zcu)) |
| 2728 | 3172 | .qword |
| ... | ... | @@ -2730,244 +3174,244 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2730 | 3174 | .fromSize(opt_child_abi_size) }), |
| 2731 | 3175 | .u(0), |
| 2732 | 3176 | ); |
| 2733 | | var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne }); |
| 2734 | | try ops[0].die(self); |
| 2735 | | try is_non_null.moveTo(inst, self); |
| 3177 | var is_non_null = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .eflags = .ne }); |
| 3178 | try ops[0].die(cg); |
| 3179 | try is_non_null.moveTo(inst, cg); |
| 2736 | 3180 | }, |
| 2737 | | .is_err_ptr => if (use_old) try self.airIsErrPtr(inst) else { |
| 3181 | .is_err_ptr => if (use_old) try cg.airIsErrPtr(inst) else { |
| 2738 | 3182 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2739 | | const eu_ty = self.typeOf(un_op).childType(zcu); |
| 3183 | const eu_ty = cg.typeOf(un_op).childType(zcu); |
| 2740 | 3184 | const eu_err_ty = eu_ty.errorUnionSet(zcu); |
| 2741 | 3185 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2742 | 3186 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 2743 | | var ops = try self.tempsFromOperands(inst, .{un_op}); |
| 2744 | | try ops[0].toOffset(eu_err_off, self); |
| 2745 | | while (try ops[0].toLea(self)) {} |
| 2746 | | try self.asmMemoryImmediate( |
| 3187 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 3188 | try ops[0].toOffset(eu_err_off, cg); |
| 3189 | while (try ops[0].toLea(cg)) {} |
| 3190 | try cg.asmMemoryImmediate( |
| 2747 | 3191 | .{ ._, .cmp }, |
| 2748 | | try ops[0].tracking(self).short.deref().mem(self, .{ .size = self.memSize(eu_err_ty) }), |
| 3192 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), |
| 2749 | 3193 | .u(0), |
| 2750 | 3194 | ); |
| 2751 | | var is_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne }); |
| 2752 | | try ops[0].die(self); |
| 2753 | | try is_err.moveTo(inst, self); |
| 3195 | var is_err = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .eflags = .ne }); |
| 3196 | try ops[0].die(cg); |
| 3197 | try is_err.moveTo(inst, cg); |
| 2754 | 3198 | }, |
| 2755 | | .is_non_err_ptr => if (use_old) try self.airIsNonErrPtr(inst) else { |
| 3199 | .is_non_err_ptr => if (use_old) try cg.airIsNonErrPtr(inst) else { |
| 2756 | 3200 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2757 | | const eu_ty = self.typeOf(un_op).childType(zcu); |
| 3201 | const eu_ty = cg.typeOf(un_op).childType(zcu); |
| 2758 | 3202 | const eu_err_ty = eu_ty.errorUnionSet(zcu); |
| 2759 | 3203 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2760 | 3204 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 2761 | | var ops = try self.tempsFromOperands(inst, .{un_op}); |
| 2762 | | try ops[0].toOffset(eu_err_off, self); |
| 2763 | | while (try ops[0].toLea(self)) {} |
| 2764 | | try self.asmMemoryImmediate( |
| 3205 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 3206 | try ops[0].toOffset(eu_err_off, cg); |
| 3207 | while (try ops[0].toLea(cg)) {} |
| 3208 | try cg.asmMemoryImmediate( |
| 2765 | 3209 | .{ ._, .cmp }, |
| 2766 | | try ops[0].tracking(self).short.deref().mem(self, .{ .size = self.memSize(eu_err_ty) }), |
| 3210 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(eu_err_ty) }), |
| 2767 | 3211 | .u(0), |
| 2768 | 3212 | ); |
| 2769 | | var is_non_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e }); |
| 2770 | | try ops[0].die(self); |
| 2771 | | try is_non_err.moveTo(inst, self); |
| 3213 | var is_non_err = try cg.tempFromValue(cg.typeOfIndex(inst), .{ .eflags = .e }); |
| 3214 | try ops[0].die(cg); |
| 3215 | try is_non_err.moveTo(inst, cg); |
| 2772 | 3216 | }, |
| 2773 | | .int_from_ptr => if (use_old) try self.airIntFromPtr(inst) else { |
| 3217 | .int_from_ptr => if (use_old) try cg.airIntFromPtr(inst) else { |
| 2774 | 3218 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2775 | | var ops = try self.tempsFromOperands(inst, .{un_op}); |
| 2776 | | try ops[0].toLimb(0, self); |
| 2777 | | try ops[0].moveTo(inst, self); |
| 3219 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 3220 | try ops[0].toLimb(0, cg); |
| 3221 | try ops[0].moveTo(inst, cg); |
| 2778 | 3222 | }, |
| 2779 | | .int_from_bool => if (use_old) try self.airIntFromBool(inst) else { |
| 3223 | .int_from_bool => if (use_old) try cg.airIntFromBool(inst) else { |
| 2780 | 3224 | const un_op = air_datas[@intFromEnum(inst)].un_op; |
| 2781 | | var ops = try self.tempsFromOperands(inst, .{un_op}); |
| 2782 | | try ops[0].moveTo(inst, self); |
| 3225 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 3226 | try ops[0].moveTo(inst, cg); |
| 2783 | 3227 | }, |
| 2784 | | .ret => try self.airRet(inst, false), |
| 2785 | | .ret_safe => try self.airRet(inst, true), |
| 2786 | | .ret_load => try self.airRetLoad(inst), |
| 3228 | .ret => try cg.airRet(inst, false), |
| 3229 | .ret_safe => try cg.airRet(inst, true), |
| 3230 | .ret_load => try cg.airRetLoad(inst), |
| 2787 | 3231 | .unreach => {}, |
| 2788 | | .optional_payload_ptr => if (use_old) try self.airOptionalPayloadPtr(inst) else { |
| 3232 | .optional_payload_ptr => if (use_old) try cg.airOptionalPayloadPtr(inst) else { |
| 2789 | 3233 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2790 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2791 | | try ops[0].moveTo(inst, self); |
| 3234 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3235 | try ops[0].moveTo(inst, cg); |
| 2792 | 3236 | }, |
| 2793 | | .optional_payload_ptr_set => if (use_old) try self.airOptionalPayloadPtrSet(inst) else { |
| 3237 | .optional_payload_ptr_set => if (use_old) try cg.airOptionalPayloadPtrSet(inst) else { |
| 2794 | 3238 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2795 | | const opt_ty = self.typeOf(ty_op.operand).childType(zcu); |
| 2796 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 3239 | const opt_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 3240 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 2797 | 3241 | if (!opt_ty.optionalReprIsPayload(zcu)) { |
| 2798 | 3242 | const opt_child_ty = opt_ty.optionalChild(zcu); |
| 2799 | 3243 | const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu)); |
| 2800 | | try ops[0].toOffset(opt_child_abi_size, self); |
| 2801 | | var has_value = try self.tempFromValue(Type.bool, .{ .immediate = 1 }); |
| 2802 | | try ops[0].store(&has_value, self); |
| 2803 | | try has_value.die(self); |
| 2804 | | try ops[0].toOffset(-opt_child_abi_size, self); |
| 3244 | try ops[0].toOffset(opt_child_abi_size, cg); |
| 3245 | var has_value = try cg.tempFromValue(Type.bool, .{ .immediate = 1 }); |
| 3246 | try ops[0].store(&has_value, cg); |
| 3247 | try has_value.die(cg); |
| 3248 | try ops[0].toOffset(-opt_child_abi_size, cg); |
| 2805 | 3249 | } |
| 2806 | | try ops[0].moveTo(inst, self); |
| 3250 | try ops[0].moveTo(inst, cg); |
| 2807 | 3251 | }, |
| 2808 | | .unwrap_errunion_payload_ptr => if (use_old) try self.airUnwrapErrUnionPayloadPtr(inst) else { |
| 3252 | .unwrap_errunion_payload_ptr => if (use_old) try cg.airUnwrapErrUnionPayloadPtr(inst) else { |
| 2809 | 3253 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2810 | | const eu_ty = self.typeOf(ty_op.operand).childType(zcu); |
| 3254 | const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 2811 | 3255 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2812 | 3256 | const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); |
| 2813 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2814 | | try ops[0].toOffset(eu_pl_off, self); |
| 2815 | | try ops[0].moveTo(inst, self); |
| 3257 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3258 | try ops[0].toOffset(eu_pl_off, cg); |
| 3259 | try ops[0].moveTo(inst, cg); |
| 2816 | 3260 | }, |
| 2817 | | .unwrap_errunion_err_ptr => if (use_old) try self.airUnwrapErrUnionErrPtr(inst) else { |
| 3261 | .unwrap_errunion_err_ptr => if (use_old) try cg.airUnwrapErrUnionErrPtr(inst) else { |
| 2818 | 3262 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2819 | | const eu_ty = self.typeOf(ty_op.operand).childType(zcu); |
| 3263 | const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 2820 | 3264 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2821 | 3265 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 2822 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2823 | | try ops[0].toOffset(eu_err_off, self); |
| 2824 | | var err = try ops[0].load(eu_ty.errorUnionSet(zcu), self); |
| 2825 | | try ops[0].die(self); |
| 2826 | | try err.moveTo(inst, self); |
| 3266 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3267 | try ops[0].toOffset(eu_err_off, cg); |
| 3268 | var err = try ops[0].load(eu_ty.errorUnionSet(zcu), cg); |
| 3269 | try ops[0].die(cg); |
| 3270 | try err.moveTo(inst, cg); |
| 2827 | 3271 | }, |
| 2828 | | .errunion_payload_ptr_set => if (use_old) try self.airErrUnionPayloadPtrSet(inst) else { |
| 3272 | .errunion_payload_ptr_set => if (use_old) try cg.airErrUnionPayloadPtrSet(inst) else { |
| 2829 | 3273 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2830 | | const eu_ty = self.typeOf(ty_op.operand).childType(zcu); |
| 3274 | const eu_ty = cg.typeOf(ty_op.operand).childType(zcu); |
| 2831 | 3275 | const eu_err_ty = eu_ty.errorUnionSet(zcu); |
| 2832 | 3276 | const eu_pl_ty = eu_ty.errorUnionPayload(zcu); |
| 2833 | 3277 | const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); |
| 2834 | 3278 | const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); |
| 2835 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2836 | | try ops[0].toOffset(eu_err_off, self); |
| 2837 | | var no_err = try self.tempFromValue(eu_err_ty, .{ .immediate = 0 }); |
| 2838 | | try ops[0].store(&no_err, self); |
| 2839 | | try no_err.die(self); |
| 2840 | | try ops[0].toOffset(eu_pl_off - eu_err_off, self); |
| 2841 | | try ops[0].moveTo(inst, self); |
| 3279 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3280 | try ops[0].toOffset(eu_err_off, cg); |
| 3281 | var no_err = try cg.tempFromValue(eu_err_ty, .{ .immediate = 0 }); |
| 3282 | try ops[0].store(&no_err, cg); |
| 3283 | try no_err.die(cg); |
| 3284 | try ops[0].toOffset(eu_pl_off - eu_err_off, cg); |
| 3285 | try ops[0].moveTo(inst, cg); |
| 2842 | 3286 | }, |
| 2843 | | .struct_field_ptr => if (use_old) try self.airStructFieldPtr(inst) else { |
| 3287 | .struct_field_ptr => if (use_old) try cg.airStructFieldPtr(inst) else { |
| 2844 | 3288 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2845 | | const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; |
| 2846 | | var ops = try self.tempsFromOperands(inst, .{extra.struct_operand}); |
| 2847 | | try ops[0].toOffset(self.fieldOffset( |
| 2848 | | self.typeOf(extra.struct_operand), |
| 2849 | | self.typeOfIndex(inst), |
| 3289 | const extra = cg.air.extraData(Air.StructField, ty_pl.payload).data; |
| 3290 | var ops = try cg.tempsFromOperands(inst, .{extra.struct_operand}); |
| 3291 | try ops[0].toOffset(cg.fieldOffset( |
| 3292 | cg.typeOf(extra.struct_operand), |
| 3293 | cg.typeOfIndex(inst), |
| 2850 | 3294 | extra.field_index, |
| 2851 | | ), self); |
| 2852 | | try ops[0].moveTo(inst, self); |
| 3295 | ), cg); |
| 3296 | try ops[0].moveTo(inst, cg); |
| 2853 | 3297 | }, |
| 2854 | | .struct_field_ptr_index_0 => if (use_old) try self.airStructFieldPtrIndex(inst, 0) else { |
| 3298 | .struct_field_ptr_index_0 => if (use_old) try cg.airStructFieldPtrIndex(inst, 0) else { |
| 2855 | 3299 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2856 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2857 | | try ops[0].toOffset(self.fieldOffset( |
| 2858 | | self.typeOf(ty_op.operand), |
| 2859 | | self.typeOfIndex(inst), |
| 3300 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3301 | try ops[0].toOffset(cg.fieldOffset( |
| 3302 | cg.typeOf(ty_op.operand), |
| 3303 | cg.typeOfIndex(inst), |
| 2860 | 3304 | 0, |
| 2861 | | ), self); |
| 2862 | | try ops[0].moveTo(inst, self); |
| 3305 | ), cg); |
| 3306 | try ops[0].moveTo(inst, cg); |
| 2863 | 3307 | }, |
| 2864 | | .struct_field_ptr_index_1 => if (use_old) try self.airStructFieldPtrIndex(inst, 1) else { |
| 3308 | .struct_field_ptr_index_1 => if (use_old) try cg.airStructFieldPtrIndex(inst, 1) else { |
| 2865 | 3309 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2866 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2867 | | try ops[0].toOffset(self.fieldOffset( |
| 2868 | | self.typeOf(ty_op.operand), |
| 2869 | | self.typeOfIndex(inst), |
| 3310 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3311 | try ops[0].toOffset(cg.fieldOffset( |
| 3312 | cg.typeOf(ty_op.operand), |
| 3313 | cg.typeOfIndex(inst), |
| 2870 | 3314 | 1, |
| 2871 | | ), self); |
| 2872 | | try ops[0].moveTo(inst, self); |
| 3315 | ), cg); |
| 3316 | try ops[0].moveTo(inst, cg); |
| 2873 | 3317 | }, |
| 2874 | | .struct_field_ptr_index_2 => if (use_old) try self.airStructFieldPtrIndex(inst, 2) else { |
| 3318 | .struct_field_ptr_index_2 => if (use_old) try cg.airStructFieldPtrIndex(inst, 2) else { |
| 2875 | 3319 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2876 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2877 | | try ops[0].toOffset(self.fieldOffset( |
| 2878 | | self.typeOf(ty_op.operand), |
| 2879 | | self.typeOfIndex(inst), |
| 3320 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3321 | try ops[0].toOffset(cg.fieldOffset( |
| 3322 | cg.typeOf(ty_op.operand), |
| 3323 | cg.typeOfIndex(inst), |
| 2880 | 3324 | 2, |
| 2881 | | ), self); |
| 2882 | | try ops[0].moveTo(inst, self); |
| 3325 | ), cg); |
| 3326 | try ops[0].moveTo(inst, cg); |
| 2883 | 3327 | }, |
| 2884 | | .struct_field_ptr_index_3 => if (use_old) try self.airStructFieldPtrIndex(inst, 3) else { |
| 3328 | .struct_field_ptr_index_3 => if (use_old) try cg.airStructFieldPtrIndex(inst, 3) else { |
| 2885 | 3329 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2886 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2887 | | try ops[0].toOffset(self.fieldOffset( |
| 2888 | | self.typeOf(ty_op.operand), |
| 2889 | | self.typeOfIndex(inst), |
| 3330 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3331 | try ops[0].toOffset(cg.fieldOffset( |
| 3332 | cg.typeOf(ty_op.operand), |
| 3333 | cg.typeOfIndex(inst), |
| 2890 | 3334 | 3, |
| 2891 | | ), self); |
| 2892 | | try ops[0].moveTo(inst, self); |
| 3335 | ), cg); |
| 3336 | try ops[0].moveTo(inst, cg); |
| 2893 | 3337 | }, |
| 2894 | | .slice => if (use_old) try self.airSlice(inst) else { |
| 3338 | .slice => if (use_old) try cg.airSlice(inst) else { |
| 2895 | 3339 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2896 | | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 2897 | | var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 2898 | | try ops[0].toPair(&ops[1], self); |
| 2899 | | try ops[0].moveTo(inst, self); |
| 3340 | const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; |
| 3341 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 3342 | try ops[0].toPair(&ops[1], cg); |
| 3343 | try ops[0].moveTo(inst, cg); |
| 2900 | 3344 | }, |
| 2901 | | .slice_len => if (use_old) try self.airSliceLen(inst) else { |
| 3345 | .slice_len => if (use_old) try cg.airSliceLen(inst) else { |
| 2902 | 3346 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2903 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2904 | | try ops[0].toLimb(1, self); |
| 2905 | | try ops[0].moveTo(inst, self); |
| 3347 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3348 | try ops[0].toLimb(1, cg); |
| 3349 | try ops[0].moveTo(inst, cg); |
| 2906 | 3350 | }, |
| 2907 | | .slice_ptr => if (use_old) try self.airSlicePtr(inst) else { |
| 3351 | .slice_ptr => if (use_old) try cg.airSlicePtr(inst) else { |
| 2908 | 3352 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2909 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2910 | | try ops[0].toLimb(0, self); |
| 2911 | | try ops[0].moveTo(inst, self); |
| 3353 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3354 | try ops[0].toLimb(0, cg); |
| 3355 | try ops[0].moveTo(inst, cg); |
| 2912 | 3356 | }, |
| 2913 | | .ptr_slice_len_ptr => if (use_old) try self.airPtrSliceLenPtr(inst) else { |
| 3357 | .ptr_slice_len_ptr => if (use_old) try cg.airPtrSliceLenPtr(inst) else { |
| 2914 | 3358 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2915 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2916 | | try ops[0].toOffset(8, self); |
| 2917 | | try ops[0].moveTo(inst, self); |
| 3359 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3360 | try ops[0].toOffset(8, cg); |
| 3361 | try ops[0].moveTo(inst, cg); |
| 2918 | 3362 | }, |
| 2919 | | .ptr_slice_ptr_ptr => if (use_old) try self.airPtrSlicePtrPtr(inst) else { |
| 3363 | .ptr_slice_ptr_ptr => if (use_old) try cg.airPtrSlicePtrPtr(inst) else { |
| 2920 | 3364 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2921 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2922 | | try ops[0].toOffset(0, self); |
| 2923 | | try ops[0].moveTo(inst, self); |
| 3365 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3366 | try ops[0].toOffset(0, cg); |
| 3367 | try ops[0].moveTo(inst, cg); |
| 2924 | 3368 | }, |
| 2925 | 3369 | .slice_elem_ptr, .ptr_elem_ptr => |tag| if (use_old) switch (tag) { |
| 2926 | 3370 | else => unreachable, |
| 2927 | | .slice_elem_ptr => try self.airSliceElemPtr(inst), |
| 2928 | | .ptr_elem_ptr => try self.airPtrElemPtr(inst), |
| 3371 | .slice_elem_ptr => try cg.airSliceElemPtr(inst), |
| 3372 | .ptr_elem_ptr => try cg.airPtrElemPtr(inst), |
| 2929 | 3373 | } else { |
| 2930 | 3374 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2931 | | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 2932 | | var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 3375 | const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; |
| 3376 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 2933 | 3377 | switch (tag) { |
| 2934 | 3378 | else => unreachable, |
| 2935 | | .slice_elem_ptr => try ops[0].toLimb(0, self), |
| 3379 | .slice_elem_ptr => try ops[0].toLimb(0, cg), |
| 2936 | 3380 | .ptr_elem_ptr => {}, |
| 2937 | 3381 | } |
| 2938 | | const dst_ty = self.typeOfIndex(inst); |
| 3382 | const dst_ty = cg.typeOfIndex(inst); |
| 2939 | 3383 | if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: { |
| 2940 | 3384 | const elem_size = dst_ty.childType(zcu).abiSize(zcu); |
| 2941 | 3385 | if (elem_size == 0) break :zero_offset; |
| 2942 | 3386 | while (true) for (&ops) |*op| { |
| 2943 | | if (try op.toAnyReg(self)) break; |
| 3387 | if (try op.toAnyReg(cg)) break; |
| 2944 | 3388 | } else break; |
| 2945 | | const lhs_reg = ops[0].unwrap(self).temp.tracking(self).short.register.to64(); |
| 2946 | | const rhs_reg = ops[1].unwrap(self).temp.tracking(self).short.register.to64(); |
| 3389 | const lhs_reg = ops[0].unwrap(cg).temp.tracking(cg).short.register.to64(); |
| 3390 | const rhs_reg = ops[1].unwrap(cg).temp.tracking(cg).short.register.to64(); |
| 2947 | 3391 | if (!std.math.isPowerOfTwo(elem_size)) { |
| 2948 | | try self.spillEflagsIfOccupied(); |
| 2949 | | try self.asmRegisterRegisterImmediate( |
| 3392 | try cg.spillEflagsIfOccupied(); |
| 3393 | try cg.asmRegisterRegisterImmediate( |
| 2950 | 3394 | .{ .i_, .mul }, |
| 2951 | 3395 | rhs_reg, |
| 2952 | 3396 | rhs_reg, |
| 2953 | 3397 | .u(elem_size), |
| 2954 | 3398 | ); |
| 2955 | | try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 3399 | try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 2956 | 3400 | .base = .{ .reg = lhs_reg }, |
| 2957 | 3401 | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, |
| 2958 | 3402 | }); |
| 2959 | 3403 | } else if (elem_size > 8) { |
| 2960 | | try self.spillEflagsIfOccupied(); |
| 2961 | | try self.asmRegisterImmediate( |
| 3404 | try cg.spillEflagsIfOccupied(); |
| 3405 | try cg.asmRegisterImmediate( |
| 2962 | 3406 | .{ ._l, .sh }, |
| 2963 | 3407 | rhs_reg, |
| 2964 | 3408 | .u(std.math.log2_int(u64, elem_size)), |
| 2965 | 3409 | ); |
| 2966 | | try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 3410 | try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 2967 | 3411 | .base = .{ .reg = lhs_reg }, |
| 2968 | 3412 | .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, |
| 2969 | 3413 | }); |
| 2970 | | } else try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 3414 | } else try cg.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ |
| 2971 | 3415 | .base = .{ .reg = lhs_reg }, |
| 2972 | 3416 | .mod = .{ .rm = .{ |
| 2973 | 3417 | .size = .qword, |
| ... | ... | @@ -2976,57 +3420,57 @@ fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2976 | 3420 | } }, |
| 2977 | 3421 | }); |
| 2978 | 3422 | } |
| 2979 | | try ops[1].die(self); |
| 2980 | | try ops[0].moveTo(inst, self); |
| 3423 | try ops[1].die(cg); |
| 3424 | try ops[0].moveTo(inst, cg); |
| 2981 | 3425 | }, |
| 2982 | | .array_to_slice => if (use_old) try self.airArrayToSlice(inst) else { |
| 3426 | .array_to_slice => if (use_old) try cg.airArrayToSlice(inst) else { |
| 2983 | 3427 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 2984 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 2985 | | var len = try self.tempFromValue(Type.usize, .{ |
| 2986 | | .immediate = self.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), |
| 3428 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3429 | var len = try cg.tempFromValue(Type.usize, .{ |
| 3430 | .immediate = cg.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), |
| 2987 | 3431 | }); |
| 2988 | | try ops[0].toPair(&len, self); |
| 2989 | | try ops[0].moveTo(inst, self); |
| 3432 | try ops[0].toPair(&len, cg); |
| 3433 | try ops[0].moveTo(inst, cg); |
| 2990 | 3434 | }, |
| 2991 | | .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}), |
| 2992 | | .field_parent_ptr => if (use_old) try self.airFieldParentPtr(inst) else { |
| 3435 | .error_set_has_value => return cg.fail("TODO implement error_set_has_value", .{}), |
| 3436 | .field_parent_ptr => if (use_old) try cg.airFieldParentPtr(inst) else { |
| 2993 | 3437 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2994 | | const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; |
| 2995 | | var ops = try self.tempsFromOperands(inst, .{extra.field_ptr}); |
| 2996 | | try ops[0].toOffset(-self.fieldOffset( |
| 2997 | | self.typeOfIndex(inst), |
| 2998 | | self.typeOf(extra.field_ptr), |
| 3438 | const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; |
| 3439 | var ops = try cg.tempsFromOperands(inst, .{extra.field_ptr}); |
| 3440 | try ops[0].toOffset(-cg.fieldOffset( |
| 3441 | cg.typeOfIndex(inst), |
| 3442 | cg.typeOf(extra.field_ptr), |
| 2999 | 3443 | extra.field_index, |
| 3000 | | ), self); |
| 3001 | | try ops[0].moveTo(inst, self); |
| 3444 | ), cg); |
| 3445 | try ops[0].moveTo(inst, cg); |
| 3002 | 3446 | }, |
| 3003 | 3447 | |
| 3004 | | .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}), |
| 3448 | .is_named_enum_value => return cg.fail("TODO implement is_named_enum_value", .{}), |
| 3005 | 3449 | |
| 3006 | 3450 | .wasm_memory_size => unreachable, |
| 3007 | 3451 | .wasm_memory_grow => unreachable, |
| 3008 | 3452 | |
| 3009 | 3453 | .addrspace_cast => { |
| 3010 | 3454 | const ty_op = air_datas[@intFromEnum(inst)].ty_op; |
| 3011 | | var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); |
| 3012 | | try ops[0].moveTo(inst, self); |
| 3455 | var ops = try cg.tempsFromOperands(inst, .{ty_op.operand}); |
| 3456 | try ops[0].moveTo(inst, cg); |
| 3013 | 3457 | }, |
| 3014 | 3458 | |
| 3015 | | .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}), |
| 3459 | .vector_store_elem => return cg.fail("TODO implement vector_store_elem", .{}), |
| 3016 | 3460 | |
| 3017 | | .c_va_arg => try self.airVaArg(inst), |
| 3018 | | .c_va_copy => try self.airVaCopy(inst), |
| 3019 | | .c_va_end => try self.airVaEnd(inst), |
| 3020 | | .c_va_start => try self.airVaStart(inst), |
| 3461 | .c_va_arg => try cg.airVaArg(inst), |
| 3462 | .c_va_copy => try cg.airVaCopy(inst), |
| 3463 | .c_va_end => try cg.airVaEnd(inst), |
| 3464 | .c_va_start => try cg.airVaStart(inst), |
| 3021 | 3465 | |
| 3022 | 3466 | .work_item_id => unreachable, |
| 3023 | 3467 | .work_group_size => unreachable, |
| 3024 | 3468 | .work_group_id => unreachable, |
| 3025 | 3469 | } |
| 3026 | | self.resetTemps(); |
| 3027 | | self.checkInvariantsAfterAirInst(); |
| 3470 | cg.resetTemps(); |
| 3471 | cg.checkInvariantsAfterAirInst(); |
| 3028 | 3472 | } |
| 3029 | | verbose_tracking_log.debug("{}", .{self.fmtTracking()}); |
| 3473 | verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); |
| 3030 | 3474 | } |
| 3031 | 3475 | |
| 3032 | 3476 | fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { |
| ... | ... | @@ -3117,12 +3561,16 @@ fn freeReg(self: *CodeGen, reg: Register) !void { |
| 3117 | 3561 | fn freeValue(self: *CodeGen, value: MCValue) !void { |
| 3118 | 3562 | switch (value) { |
| 3119 | 3563 | .register => |reg| try self.freeReg(reg), |
| 3120 | | .register_pair => |regs| for (regs) |reg| try self.freeReg(reg), |
| 3564 | inline .register_pair, |
| 3565 | .register_triple, |
| 3566 | .register_quadruple, |
| 3567 | => |regs| for (regs) |reg| try self.freeReg(reg), |
| 3121 | 3568 | .register_offset, .indirect => |reg_off| try self.freeReg(reg_off.reg), |
| 3122 | 3569 | .register_overflow => |reg_ov| { |
| 3123 | 3570 | try self.freeReg(reg_ov.reg); |
| 3124 | 3571 | self.eflags_inst = null; |
| 3125 | 3572 | }, |
| 3573 | .register_mask => |reg_mask| try self.freeReg(reg_mask.reg), |
| 3126 | 3574 | .eflags => self.eflags_inst = null, |
| 3127 | 3575 | else => {}, // TODO process stack allocation death |
| 3128 | 3576 | } |
| ... | ... | @@ -3323,11 +3771,11 @@ fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok |
| 3323 | 3771 | }, |
| 3324 | 3772 | .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { |
| 3325 | 3773 | .float => switch (ty.childType(zcu).floatBits(self.target.*)) { |
| 3326 | | 16, 32, 64, 128 => if (self.hasFeature(.avx)) 32 else 16, |
| 3774 | 16, 32, 64, 128 => self.vectorSize(.float), |
| 3327 | 3775 | 80 => break :need_mem, |
| 3328 | 3776 | else => unreachable, |
| 3329 | 3777 | }, |
| 3330 | | else => if (self.hasFeature(.avx)) 32 else 16, |
| 3778 | else => self.vectorSize(.int), |
| 3331 | 3779 | }, |
| 3332 | 3780 | else => 8, |
| 3333 | 3781 | })) { |
| ... | ... | @@ -3374,6 +3822,28 @@ fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet { |
| 3374 | 3822 | return regSetForRegClass(self.regClassForType(ty)); |
| 3375 | 3823 | } |
| 3376 | 3824 | |
| 3825 | fn vectorSize(cg: *CodeGen, kind: enum { int, float }) u6 { |
| 3826 | return if (cg.hasFeature(switch (kind) { |
| 3827 | .int => .avx2, |
| 3828 | .float => .avx, |
| 3829 | })) 32 else if (cg.hasFeature(.sse)) 16 else 8; |
| 3830 | } |
| 3831 | |
| 3832 | fn limbType(cg: *CodeGen, ty: Type) Type { |
| 3833 | const pt = cg.pt; |
| 3834 | const zcu = pt.zcu; |
| 3835 | const vector_size = cg.vectorSize(if (ty.isRuntimeFloat()) .float else .int); |
| 3836 | const scalar_ty, const scalar_size = scalar: { |
| 3837 | const scalar_ty = ty.scalarType(zcu); |
| 3838 | const scalar_size = scalar_ty.abiSize(zcu); |
| 3839 | if (scalar_size <= vector_size) break :scalar .{ scalar_ty, scalar_size }; |
| 3840 | }; |
| 3841 | pt.vectorType(.{ |
| 3842 | .len = @divExact(vector_size, scalar_size), |
| 3843 | .child = scalar_ty.toIntern(), |
| 3844 | }); |
| 3845 | } |
| 3846 | |
| 3377 | 3847 | const State = struct { |
| 3378 | 3848 | registers: RegisterManager.TrackedRegisters, |
| 3379 | 3849 | reg_tracking: [RegisterManager.RegisterBitSet.bit_length]InstTracking, |
| ... | ... | @@ -3639,7 +4109,7 @@ fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3639 | 4109 | } |
| 3640 | 4110 | } else { |
| 3641 | 4111 | assert(src_bits == 64 and dst_bits == 32); |
| 3642 | | if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| 4112 | if (self.hasFeature(.avx)) if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 3643 | 4113 | .{ .v_ss, .cvtsd2 }, |
| 3644 | 4114 | dst_reg, |
| 3645 | 4115 | dst_reg, |
| ... | ... | @@ -3652,7 +4122,7 @@ fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3652 | 4122 | src_mcv.getReg().? |
| 3653 | 4123 | else |
| 3654 | 4124 | try self.copyToTmpRegister(src_ty, src_mcv)).to128(), |
| 3655 | | ) else if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 4125 | ) else if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 3656 | 4126 | .{ ._ss, .cvtsd2 }, |
| 3657 | 4127 | dst_reg, |
| 3658 | 4128 | try src_mcv.mem(self, .{ .size = .qword }), |
| ... | ... | @@ -3751,7 +4221,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3751 | 4221 | } else { |
| 3752 | 4222 | assert(src_bits == 32 and dst_bits == 64); |
| 3753 | 4223 | if (self.hasFeature(.avx)) switch (vec_len) { |
| 3754 | | 1 => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| 4224 | 1 => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 3755 | 4225 | .{ .v_sd, .cvtss2 }, |
| 3756 | 4226 | dst_alias, |
| 3757 | 4227 | dst_alias, |
| ... | ... | @@ -3765,7 +4235,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3765 | 4235 | else |
| 3766 | 4236 | try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), |
| 3767 | 4237 | ), |
| 3768 | | 2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 4238 | 2...4 => if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 3769 | 4239 | .{ .v_pd, .cvtps2 }, |
| 3770 | 4240 | dst_alias, |
| 3771 | 4241 | try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), |
| ... | ... | @@ -3778,7 +4248,7 @@ fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3778 | 4248 | try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size), |
| 3779 | 4249 | ), |
| 3780 | 4250 | else => break :result null, |
| 3781 | | } else if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 4251 | } else if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 3782 | 4252 | switch (vec_len) { |
| 3783 | 4253 | 1 => .{ ._sd, .cvtss2 }, |
| 3784 | 4254 | 2 => .{ ._pd, .cvtps2 }, |
| ... | ... | @@ -3827,7 +4297,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3827 | 4297 | if (dst_ty.isVector(zcu)) { |
| 3828 | 4298 | const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu)); |
| 3829 | 4299 | const max_abi_size = @max(dst_abi_size, src_abi_size); |
| 3830 | | if (max_abi_size > @as(u32, if (self.hasFeature(.avx2)) 32 else 16)) break :result null; |
| 4300 | if (max_abi_size > self.vectorSize(.int)) break :result null; |
| 3831 | 4301 | const has_avx = self.hasFeature(.avx); |
| 3832 | 4302 | |
| 3833 | 4303 | const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu); |
| ... | ... | @@ -3919,7 +4389,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 3919 | 4389 | const dst_reg = dst_mcv.getReg().?; |
| 3920 | 4390 | const dst_alias = registerAlias(dst_reg, dst_abi_size); |
| 3921 | 4391 | |
| 3922 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 4392 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 3923 | 4393 | mir_tag, |
| 3924 | 4394 | dst_alias, |
| 3925 | 4395 | try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }), |
| ... | ... | @@ -4017,7 +4487,7 @@ fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4017 | 4487 | const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable; |
| 4018 | 4488 | const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable; |
| 4019 | 4489 | |
| 4020 | | const high_mcv: MCValue = if (dst_mcv.isMemory()) |
| 4490 | const high_mcv: MCValue = if (dst_mcv.isBase()) |
| 4021 | 4491 | dst_mcv.address().offset((src_limbs_len - 1) * 8).deref() |
| 4022 | 4492 | else |
| 4023 | 4493 | .{ .register = dst_mcv.register_pair[1] }; |
| ... | ... | @@ -4392,7 +4862,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4392 | 4862 | else => null, |
| 4393 | 4863 | }; |
| 4394 | 4864 | defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4395 | | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 4865 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 4396 | 4866 | .{ ._, .mov }, |
| 4397 | 4867 | tmp_reg, |
| 4398 | 4868 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | ... | @@ -4416,7 +4886,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4416 | 4886 | else => null, |
| 4417 | 4887 | }; |
| 4418 | 4888 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4419 | | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 4889 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 4420 | 4890 | .{ ._, .xor }, |
| 4421 | 4891 | tmp_reg, |
| 4422 | 4892 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | ... | @@ -4518,7 +4988,7 @@ fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4518 | 4988 | |
| 4519 | 4989 | for (tmp_regs, dst_regs) |tmp_reg, dst_reg| |
| 4520 | 4990 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg); |
| 4521 | | if (mat_rhs_mcv.isMemory()) { |
| 4991 | if (mat_rhs_mcv.isBase()) { |
| 4522 | 4992 | try self.asmRegisterMemory( |
| 4523 | 4993 | .{ ._, .add }, |
| 4524 | 4994 | tmp_regs[0], |
| ... | ... | @@ -4793,7 +5263,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4793 | 5263 | else => null, |
| 4794 | 5264 | }; |
| 4795 | 5265 | defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4796 | | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 5266 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 4797 | 5267 | .{ ._, .mov }, |
| 4798 | 5268 | tmp_reg, |
| 4799 | 5269 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | ... | @@ -4817,7 +5287,7 @@ fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 4817 | 5287 | else => null, |
| 4818 | 5288 | }; |
| 4819 | 5289 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 4820 | | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 5290 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 4821 | 5291 | .{ ._, .xor }, |
| 4822 | 5292 | tmp_reg, |
| 4823 | 5293 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | ... | @@ -5348,7 +5818,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 5348 | 5818 | }; |
| 5349 | 5819 | defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 5350 | 5820 | |
| 5351 | | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 5821 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 5352 | 5822 | .{ ._, .mov }, |
| 5353 | 5823 | .rax, |
| 5354 | 5824 | try mat_lhs_mcv.mem(self, .{ .size = .qword }), |
| ... | ... | @@ -5357,7 +5827,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 5357 | 5827 | .rax, |
| 5358 | 5828 | mat_lhs_mcv.register_pair[0], |
| 5359 | 5829 | ); |
| 5360 | | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 5830 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 5361 | 5831 | .{ ._, .mov }, |
| 5362 | 5832 | tmp_regs[0], |
| 5363 | 5833 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | ... | @@ -5370,14 +5840,14 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 5370 | 5840 | try self.asmSetccRegister(.nz, tmp_regs[1].to8()); |
| 5371 | 5841 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax); |
| 5372 | 5842 | try self.asmSetccRegister(.o, tmp_regs[2].to8()); |
| 5373 | | if (mat_rhs_mcv.isMemory()) |
| 5843 | if (mat_rhs_mcv.isBase()) |
| 5374 | 5844 | try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword })) |
| 5375 | 5845 | else |
| 5376 | 5846 | try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]); |
| 5377 | 5847 | try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]); |
| 5378 | 5848 | try self.asmSetccRegister(.c, tmp_regs[3].to8()); |
| 5379 | 5849 | try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8()); |
| 5380 | | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 5850 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 5381 | 5851 | .{ ._, .mov }, |
| 5382 | 5852 | tmp_regs[0], |
| 5383 | 5853 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| ... | ... | @@ -5394,7 +5864,7 @@ fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 5394 | 5864 | tmp_regs[3].to8(), |
| 5395 | 5865 | ); |
| 5396 | 5866 | try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8()); |
| 5397 | | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 5867 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 5398 | 5868 | .{ .i_, .mul }, |
| 5399 | 5869 | tmp_regs[0], |
| 5400 | 5870 | try mat_rhs_mcv.mem(self, .{ .size = .qword }), |
| ... | ... | @@ -6512,7 +6982,7 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6512 | 6982 | |
| 6513 | 6983 | const index_ty = self.typeOf(bin_op.rhs); |
| 6514 | 6984 | const index_mcv = try self.resolveInst(bin_op.rhs); |
| 6515 | | const index_lock: ?RegisterLock = switch (index_mcv) { |
| 6985 | const index_lock = switch (index_mcv) { |
| 6516 | 6986 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 6517 | 6987 | else => null, |
| 6518 | 6988 | }; |
| ... | ... | @@ -6520,48 +6990,102 @@ fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6520 | 6990 | |
| 6521 | 6991 | try self.spillEflagsIfOccupied(); |
| 6522 | 6992 | if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) { |
| 6523 | | const index_reg = switch (index_mcv) { |
| 6524 | | .register => |reg| reg, |
| 6525 | | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| 6993 | const array_mat_mcv: MCValue = switch (array_mcv) { |
| 6994 | else => array_mcv, |
| 6995 | .register_mask => .{ .register = try self.copyToTmpRegister(array_ty, array_mcv) }, |
| 6526 | 6996 | }; |
| 6527 | | switch (array_mcv) { |
| 6997 | const array_mat_lock = switch (array_mat_mcv) { |
| 6998 | .register => |reg| self.register_manager.lockReg(reg), |
| 6999 | else => null, |
| 7000 | }; |
| 7001 | defer if (array_mat_lock) |lock| self.register_manager.unlockReg(lock); |
| 7002 | |
| 7003 | switch (array_mat_mcv) { |
| 6528 | 7004 | .register => |array_reg| switch (array_reg.class()) { |
| 6529 | | .general_purpose => try self.asmRegisterRegister( |
| 6530 | | .{ ._, .bt }, |
| 6531 | | array_reg.to64(), |
| 6532 | | index_reg.to64(), |
| 6533 | | ), |
| 7005 | .general_purpose => switch (index_mcv) { |
| 7006 | .immediate => |index_imm| try self.asmRegisterImmediate( |
| 7007 | .{ ._, .bt }, |
| 7008 | array_reg.to64(), |
| 7009 | .u(index_imm), |
| 7010 | ), |
| 7011 | else => try self.asmRegisterRegister( |
| 7012 | .{ ._, .bt }, |
| 7013 | array_reg.to64(), |
| 7014 | switch (index_mcv) { |
| 7015 | .register => |index_reg| index_reg, |
| 7016 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| 7017 | }.to64(), |
| 7018 | ), |
| 7019 | }, |
| 6534 | 7020 | .sse => { |
| 6535 | 7021 | const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu)); |
| 6536 | | try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{}); |
| 6537 | | try self.asmMemoryRegister( |
| 6538 | | .{ ._, .bt }, |
| 6539 | | .{ |
| 6540 | | .base = .{ .frame = frame_index }, |
| 6541 | | .mod = .{ .rm = .{ .size = .qword } }, |
| 6542 | | }, |
| 6543 | | index_reg.to64(), |
| 6544 | | ); |
| 7022 | try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mat_mcv, .{}); |
| 7023 | switch (index_mcv) { |
| 7024 | .immediate => |index_imm| try self.asmMemoryImmediate( |
| 7025 | .{ ._, .bt }, |
| 7026 | .{ |
| 7027 | .base = .{ .frame = frame_index }, |
| 7028 | .mod = .{ .rm = .{ .size = .qword } }, |
| 7029 | }, |
| 7030 | .u(index_imm), |
| 7031 | ), |
| 7032 | else => try self.asmMemoryRegister( |
| 7033 | .{ ._, .bt }, |
| 7034 | .{ |
| 7035 | .base = .{ .frame = frame_index }, |
| 7036 | .mod = .{ .rm = .{ .size = .qword } }, |
| 7037 | }, |
| 7038 | switch (index_mcv) { |
| 7039 | .register => |index_reg| index_reg, |
| 7040 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| 7041 | }.to64(), |
| 7042 | ), |
| 7043 | } |
| 6545 | 7044 | }, |
| 6546 | 7045 | else => unreachable, |
| 6547 | 7046 | }, |
| 6548 | | .load_frame => try self.asmMemoryRegister( |
| 6549 | | .{ ._, .bt }, |
| 6550 | | try array_mcv.mem(self, .{ .size = .qword }), |
| 6551 | | index_reg.to64(), |
| 6552 | | ), |
| 6553 | | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister( |
| 6554 | | .{ ._, .bt }, |
| 6555 | | .{ |
| 6556 | | .base = .{ |
| 6557 | | .reg = try self.copyToTmpRegister(Type.usize, array_mcv.address()), |
| 7047 | .load_frame => switch (index_mcv) { |
| 7048 | .immediate => |index_imm| try self.asmMemoryImmediate( |
| 7049 | .{ ._, .bt }, |
| 7050 | try array_mat_mcv.mem(self, .{ .size = .qword }), |
| 7051 | .u(index_imm), |
| 7052 | ), |
| 7053 | else => try self.asmMemoryRegister( |
| 7054 | .{ ._, .bt }, |
| 7055 | try array_mat_mcv.mem(self, .{ .size = .qword }), |
| 7056 | switch (index_mcv) { |
| 7057 | .register => |index_reg| index_reg, |
| 7058 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| 7059 | }.to64(), |
| 7060 | ), |
| 7061 | }, |
| 7062 | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => switch (index_mcv) { |
| 7063 | .immediate => |index_imm| try self.asmMemoryImmediate( |
| 7064 | .{ ._, .bt }, |
| 7065 | .{ |
| 7066 | .base = .{ |
| 7067 | .reg = try self.copyToTmpRegister(Type.usize, array_mat_mcv.address()), |
| 7068 | }, |
| 7069 | .mod = .{ .rm = .{ .size = .qword } }, |
| 6558 | 7070 | }, |
| 6559 | | .mod = .{ .rm = .{ .size = .qword } }, |
| 6560 | | }, |
| 6561 | | index_reg.to64(), |
| 6562 | | ), |
| 7071 | .u(index_imm), |
| 7072 | ), |
| 7073 | else => try self.asmMemoryRegister( |
| 7074 | .{ ._, .bt }, |
| 7075 | .{ |
| 7076 | .base = .{ |
| 7077 | .reg = try self.copyToTmpRegister(Type.usize, array_mat_mcv.address()), |
| 7078 | }, |
| 7079 | .mod = .{ .rm = .{ .size = .qword } }, |
| 7080 | }, |
| 7081 | switch (index_mcv) { |
| 7082 | .register => |index_reg| index_reg, |
| 7083 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| 7084 | }.to64(), |
| 7085 | ), |
| 7086 | }, |
| 6563 | 7087 | else => return self.fail("TODO airArrayElemVal for {s} of {}", .{ |
| 6564 | | @tagName(array_mcv), array_ty.fmt(pt), |
| 7088 | @tagName(array_mat_mcv), array_ty.fmt(pt), |
| 6565 | 7089 | }), |
| 6566 | 7090 | } |
| 6567 | 7091 | |
| ... | ... | @@ -6856,6 +7380,15 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6856 | 7380 | const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); |
| 6857 | 7381 | const has_lzcnt = self.hasFeature(.lzcnt); |
| 6858 | 7382 | if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) { |
| 7383 | const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) { |
| 7384 | .load_frame => |src_frame_addr| src_frame_addr, |
| 7385 | else => { |
| 7386 | const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu)); |
| 7387 | try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{}); |
| 7388 | break :src_frame_addr .{ .index = src_frame_addr }; |
| 7389 | }, |
| 7390 | }; |
| 7391 | |
| 6859 | 7392 | const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; |
| 6860 | 7393 | const extra_bits = abi_size * 8 - src_bits; |
| 6861 | 7394 | |
| ... | ... | @@ -6881,22 +7414,22 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6881 | 7414 | try self.asmRegister(.{ ._, .dec }, index_reg.to32()); |
| 6882 | 7415 | } |
| 6883 | 7416 | try self.asmMemoryImmediate(.{ ._, .cmp }, .{ |
| 6884 | | .base = .{ .frame = src_mcv.load_frame.index }, |
| 7417 | .base = .{ .frame = src_frame_addr.index }, |
| 6885 | 7418 | .mod = .{ .rm = .{ |
| 6886 | 7419 | .size = .qword, |
| 6887 | 7420 | .index = index_reg.to64(), |
| 6888 | 7421 | .scale = .@"8", |
| 6889 | | .disp = src_mcv.load_frame.off, |
| 7422 | .disp = src_frame_addr.off, |
| 6890 | 7423 | } }, |
| 6891 | 7424 | }, .u(0)); |
| 6892 | 7425 | _ = try self.asmJccReloc(.e, loop); |
| 6893 | 7426 | try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{ |
| 6894 | | .base = .{ .frame = src_mcv.load_frame.index }, |
| 7427 | .base = .{ .frame = src_frame_addr.index }, |
| 6895 | 7428 | .mod = .{ .rm = .{ |
| 6896 | 7429 | .size = .qword, |
| 6897 | 7430 | .index = index_reg.to64(), |
| 6898 | 7431 | .scale = .@"8", |
| 6899 | | .disp = src_mcv.load_frame.off, |
| 7432 | .disp = src_frame_addr.off, |
| 6900 | 7433 | } }, |
| 6901 | 7434 | }); |
| 6902 | 7435 | self.performReloc(zero); |
| ... | ... | @@ -6935,7 +7468,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6935 | 7468 | .{ ._, .lzcnt }, |
| 6936 | 7469 | Type.u64, |
| 6937 | 7470 | dst_mcv, |
| 6938 | | if (mat_src_mcv.isMemory()) |
| 7471 | if (mat_src_mcv.isBase()) |
| 6939 | 7472 | mat_src_mcv |
| 6940 | 7473 | else |
| 6941 | 7474 | .{ .register = mat_src_mcv.register_pair[0] }, |
| ... | ... | @@ -6945,7 +7478,7 @@ fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 6945 | 7478 | .{ ._, .lzcnt }, |
| 6946 | 7479 | Type.u64, |
| 6947 | 7480 | tmp_mcv, |
| 6948 | | if (mat_src_mcv.isMemory()) |
| 7481 | if (mat_src_mcv.isBase()) |
| 6949 | 7482 | mat_src_mcv.address().offset(8).deref() |
| 6950 | 7483 | else |
| 6951 | 7484 | .{ .register = mat_src_mcv.register_pair[1] }, |
| ... | ... | @@ -7053,6 +7586,15 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 7053 | 7586 | const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); |
| 7054 | 7587 | const has_bmi = self.hasFeature(.bmi); |
| 7055 | 7588 | if (src_bits > @as(u32, if (has_bmi) 128 else 64)) { |
| 7589 | const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) { |
| 7590 | .load_frame => |src_frame_addr| src_frame_addr, |
| 7591 | else => { |
| 7592 | const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu)); |
| 7593 | try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{}); |
| 7594 | break :src_frame_addr .{ .index = src_frame_addr }; |
| 7595 | }, |
| 7596 | }; |
| 7597 | |
| 7056 | 7598 | const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; |
| 7057 | 7599 | const extra_bits = abi_size * 8 - src_bits; |
| 7058 | 7600 | |
| ... | ... | @@ -7079,22 +7621,22 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 7079 | 7621 | try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len)); |
| 7080 | 7622 | const zero = try self.asmJccReloc(.nb, undefined); |
| 7081 | 7623 | try self.asmMemoryImmediate(.{ ._, .cmp }, .{ |
| 7082 | | .base = .{ .frame = src_mcv.load_frame.index }, |
| 7624 | .base = .{ .frame = src_frame_addr.index }, |
| 7083 | 7625 | .mod = .{ .rm = .{ |
| 7084 | 7626 | .size = .qword, |
| 7085 | 7627 | .index = index_reg.to64(), |
| 7086 | 7628 | .scale = .@"8", |
| 7087 | | .disp = src_mcv.load_frame.off, |
| 7629 | .disp = src_frame_addr.off, |
| 7088 | 7630 | } }, |
| 7089 | 7631 | }, .u(0)); |
| 7090 | 7632 | _ = try self.asmJccReloc(.e, loop); |
| 7091 | 7633 | try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{ |
| 7092 | | .base = .{ .frame = src_mcv.load_frame.index }, |
| 7634 | .base = .{ .frame = src_frame_addr.index }, |
| 7093 | 7635 | .mod = .{ .rm = .{ |
| 7094 | 7636 | .size = .qword, |
| 7095 | 7637 | .index = index_reg.to64(), |
| 7096 | 7638 | .scale = .@"8", |
| 7097 | | .disp = src_mcv.load_frame.off, |
| 7639 | .disp = src_frame_addr.off, |
| 7098 | 7640 | } }, |
| 7099 | 7641 | }); |
| 7100 | 7642 | self.performReloc(zero); |
| ... | ... | @@ -7131,11 +7673,11 @@ fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 7131 | 7673 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7132 | 7674 | defer self.register_manager.unlockReg(tmp_lock); |
| 7133 | 7675 | |
| 7134 | | const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isMemory()) |
| 7676 | const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isBase()) |
| 7135 | 7677 | mat_src_mcv |
| 7136 | 7678 | else |
| 7137 | 7679 | .{ .register = mat_src_mcv.register_pair[0] }; |
| 7138 | | const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isMemory()) |
| 7680 | const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isBase()) |
| 7139 | 7681 | mat_src_mcv.address().offset(8).deref() |
| 7140 | 7682 | else |
| 7141 | 7683 | .{ .register = mat_src_mcv.register_pair[1] }; |
| ... | ... | @@ -7224,13 +7766,13 @@ fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 7224 | 7766 | const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs); |
| 7225 | 7767 | defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock); |
| 7226 | 7768 | |
| 7227 | | try self.genPopCount(tmp_regs[0], Type.usize, if (mat_src_mcv.isMemory()) |
| 7769 | try self.genPopCount(tmp_regs[0], Type.usize, if (mat_src_mcv.isBase()) |
| 7228 | 7770 | mat_src_mcv |
| 7229 | 7771 | else |
| 7230 | 7772 | .{ .register = mat_src_mcv.register_pair[0] }, false); |
| 7231 | 7773 | const src_info = src_ty.intInfo(zcu); |
| 7232 | 7774 | const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1); |
| 7233 | | try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isMemory()) |
| 7775 | try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isBase()) |
| 7234 | 7776 | mat_src_mcv.address().offset(8).deref() |
| 7235 | 7777 | else |
| 7236 | 7778 | .{ .register = mat_src_mcv.register_pair[1] }, false); |
| ... | ... | @@ -7388,7 +7930,7 @@ fn genByteSwap( |
| 7388 | 7930 | defer for (dst_locks) |lock| self.register_manager.unlockReg(lock); |
| 7389 | 7931 | |
| 7390 | 7932 | for (dst_regs, 0..) |dst_reg, limb_index| { |
| 7391 | | if (src_mcv.isMemory()) { |
| 7933 | if (src_mcv.isBase()) { |
| 7392 | 7934 | try self.asmRegisterMemory( |
| 7393 | 7935 | .{ ._, if (has_movbe) .movbe else .mov }, |
| 7394 | 7936 | dst_reg.to64(), |
| ... | ... | @@ -7706,7 +8248,7 @@ fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Ty |
| 7706 | 8248 | .abs => try vec_ty.maxInt(pt, vec_ty), |
| 7707 | 8249 | else => unreachable, |
| 7708 | 8250 | }); |
| 7709 | | const sign_mem: Memory = if (sign_mcv.isMemory()) |
| 8251 | const sign_mem: Memory = if (sign_mcv.isBase()) |
| 7710 | 8252 | try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) }) |
| 7711 | 8253 | else |
| 7712 | 8254 | .{ |
| ... | ... | @@ -7888,7 +8430,7 @@ fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: |
| 7888 | 8430 | const abi_size: u32 = @intCast(ty.abiSize(pt.zcu)); |
| 7889 | 8431 | const dst_alias = registerAlias(dst_reg, abi_size); |
| 7890 | 8432 | switch (mir_tag[0]) { |
| 7891 | | .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 8433 | .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 7892 | 8434 | mir_tag, |
| 7893 | 8435 | dst_alias, |
| 7894 | 8436 | dst_alias, |
| ... | ... | @@ -7904,7 +8446,7 @@ fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: |
| 7904 | 8446 | try self.copyToTmpRegister(ty, src_mcv), abi_size), |
| 7905 | 8447 | .u(@as(u5, @bitCast(mode))), |
| 7906 | 8448 | ), |
| 7907 | | else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 8449 | else => if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 7908 | 8450 | mir_tag, |
| 7909 | 8451 | dst_alias, |
| 7910 | 8452 | try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), |
| ... | ... | @@ -8105,7 +8647,7 @@ fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8105 | 8647 | else |
| 8106 | 8648 | try self.register_manager.allocReg(inst, self.regSetForType(ty)); |
| 8107 | 8649 | const dst_alias = registerAlias(dst_reg, abi_size); |
| 8108 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 8650 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 8109 | 8651 | mir_tag, |
| 8110 | 8652 | dst_alias, |
| 8111 | 8653 | try src_mcv.mem(self, .{ .size = self.memSize(ty) }), |
| ... | ... | @@ -8212,7 +8754,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8212 | 8754 | }, |
| 8213 | 8755 | 2...8 => { |
| 8214 | 8756 | const wide_reg = registerAlias(dst_reg, abi_size * 2); |
| 8215 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 8757 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 8216 | 8758 | .{ .v_ps, .cvtph2 }, |
| 8217 | 8759 | wide_reg, |
| 8218 | 8760 | try src_mcv.mem(self, .{ .size = .fromSize( |
| ... | ... | @@ -8257,7 +8799,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8257 | 8799 | else => unreachable, |
| 8258 | 8800 | }) orelse return self.fail("TODO implement airSqrt for {}", .{ty.fmt(pt)}); |
| 8259 | 8801 | switch (mir_tag[0]) { |
| 8260 | | .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| 8802 | .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 8261 | 8803 | mir_tag, |
| 8262 | 8804 | dst_reg, |
| 8263 | 8805 | dst_reg, |
| ... | ... | @@ -8271,7 +8813,7 @@ fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8271 | 8813 | else |
| 8272 | 8814 | try self.copyToTmpRegister(ty, src_mcv), abi_size), |
| 8273 | 8815 | ), |
| 8274 | | else => if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 8816 | else => if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 8275 | 8817 | mir_tag, |
| 8276 | 8818 | dst_reg, |
| 8277 | 8819 | try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }), |
| ... | ... | @@ -8339,7 +8881,7 @@ fn reuseOperandAdvanced( |
| 8339 | 8881 | return false; |
| 8340 | 8882 | |
| 8341 | 8883 | switch (mcv) { |
| 8342 | | .register, .register_pair, .register_overflow => for (mcv.getRegs()) |reg| { |
| 8884 | .register, .register_pair, .register_overflow, .register_mask => for (mcv.getRegs()) |reg| { |
| 8343 | 8885 | // If it's in the registers table, need to associate the register(s) with the |
| 8344 | 8886 | // new instruction. |
| 8345 | 8887 | if (maybe_tracked_inst) |tracked_inst| { |
| ... | ... | @@ -8486,7 +9028,10 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE |
| 8486 | 9028 | .undef, |
| 8487 | 9029 | .eflags, |
| 8488 | 9030 | .register_pair, |
| 9031 | .register_triple, |
| 9032 | .register_quadruple, |
| 8489 | 9033 | .register_overflow, |
| 9034 | .register_mask, |
| 8490 | 9035 | .elementwise_regs_then_frame, |
| 8491 | 9036 | .reserved_frame, |
| 8492 | 9037 | => unreachable, // not a valid pointer |
| ... | ... | @@ -8694,7 +9239,10 @@ fn store( |
| 8694 | 9239 | .undef, |
| 8695 | 9240 | .eflags, |
| 8696 | 9241 | .register_pair, |
| 9242 | .register_triple, |
| 9243 | .register_quadruple, |
| 8697 | 9244 | .register_overflow, |
| 9245 | .register_mask, |
| 8698 | 9246 | .elementwise_regs_then_frame, |
| 8699 | 9247 | .reserved_frame, |
| 8700 | 9248 | => unreachable, // not a valid pointer |
| ... | ... | @@ -8986,7 +9534,7 @@ fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 8986 | 9534 | try self.genCopy(field_ty, dst_mcv, off_mcv, .{}); |
| 8987 | 9535 | break :dst dst_mcv; |
| 8988 | 9536 | }; |
| 8989 | | if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) { |
| 9537 | if (field_abi_size * 8 > field_bit_size and dst_mcv.isBase()) { |
| 8990 | 9538 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 8991 | 9539 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 8992 | 9540 | defer self.register_manager.unlockReg(tmp_lock); |
| ... | ... | @@ -9194,6 +9742,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: |
| 9194 | 9742 | .register_offset, |
| 9195 | 9743 | .eflags, |
| 9196 | 9744 | .register_overflow, |
| 9745 | .register_mask, |
| 9197 | 9746 | .lea_symbol, |
| 9198 | 9747 | .lea_direct, |
| 9199 | 9748 | .lea_got, |
| ... | ... | @@ -9204,7 +9753,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: |
| 9204 | 9753 | .air_ref, |
| 9205 | 9754 | => unreachable, // unmodifiable destination |
| 9206 | 9755 | .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)), |
| 9207 | | .register_pair => unreachable, // unimplemented |
| 9756 | .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented |
| 9208 | 9757 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => { |
| 9209 | 9758 | const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 9210 | 9759 | const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); |
| ... | ... | @@ -9974,27 +10523,27 @@ fn genMulDivBinOp( |
| 9974 | 10523 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 9975 | 10524 | defer self.register_manager.unlockReg(tmp_lock); |
| 9976 | 10525 | |
| 9977 | | if (mat_lhs_mcv.isMemory()) |
| 10526 | if (mat_lhs_mcv.isBase()) |
| 9978 | 10527 | try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .{ .size = .qword })) |
| 9979 | 10528 | else |
| 9980 | 10529 | try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]); |
| 9981 | | if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 10530 | if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 9982 | 10531 | .{ ._, .mov }, |
| 9983 | 10532 | tmp_reg, |
| 9984 | 10533 | try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| 9985 | 10534 | ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]); |
| 9986 | 10535 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax); |
| 9987 | | if (mat_rhs_mcv.isMemory()) |
| 10536 | if (mat_rhs_mcv.isBase()) |
| 9988 | 10537 | try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword })) |
| 9989 | 10538 | else |
| 9990 | 10539 | try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]); |
| 9991 | 10540 | try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg); |
| 9992 | | if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 10541 | if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 9993 | 10542 | .{ ._, .mov }, |
| 9994 | 10543 | tmp_reg, |
| 9995 | 10544 | try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }), |
| 9996 | 10545 | ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]); |
| 9997 | | if (mat_rhs_mcv.isMemory()) |
| 10546 | if (mat_rhs_mcv.isBase()) |
| 9998 | 10547 | try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .{ .size = .qword })) |
| 9999 | 10548 | else |
| 10000 | 10549 | try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]); |
| ... | ... | @@ -10414,7 +10963,7 @@ fn genBinOp( |
| 10414 | 10963 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 10415 | 10964 | defer self.register_manager.unlockReg(tmp_lock); |
| 10416 | 10965 | |
| 10417 | | if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 10966 | if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 10418 | 10967 | .{ .vp_w, .insr }, |
| 10419 | 10968 | dst_reg, |
| 10420 | 10969 | dst_reg, |
| ... | ... | @@ -10439,7 +10988,7 @@ fn genBinOp( |
| 10439 | 10988 | 64 => .{ .v_sd, .add }, |
| 10440 | 10989 | else => unreachable, |
| 10441 | 10990 | }; |
| 10442 | | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| 10991 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 10443 | 10992 | mir_tag, |
| 10444 | 10993 | dst_reg, |
| 10445 | 10994 | dst_reg, |
| ... | ... | @@ -10459,7 +11008,7 @@ fn genBinOp( |
| 10459 | 11008 | 64 => .{ ._sd, .add }, |
| 10460 | 11009 | else => unreachable, |
| 10461 | 11010 | }; |
| 10462 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 11011 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 10463 | 11012 | mir_tag, |
| 10464 | 11013 | dst_reg, |
| 10465 | 11014 | try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }), |
| ... | ... | @@ -10514,7 +11063,7 @@ fn genBinOp( |
| 10514 | 11063 | }; |
| 10515 | 11064 | if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and |
| 10516 | 11065 | lhs_ty.scalarType(zcu).floatBits(self.target.*) == 80) or |
| 10517 | | lhs_ty.abiSize(zcu) > @as(u6, if (self.hasFeature(.avx)) 32 else 16))) |
| 11066 | lhs_ty.abiSize(zcu) > self.vectorSize(.float))) |
| 10518 | 11067 | return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(pt) }); |
| 10519 | 11068 | |
| 10520 | 11069 | const maybe_mask_reg = switch (air_tag) { |
| ... | ... | @@ -10731,7 +11280,7 @@ fn genBinOp( |
| 10731 | 11280 | }; |
| 10732 | 11281 | |
| 10733 | 11282 | try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); |
| 10734 | | if (src_mcv.isMemory()) { |
| 11283 | if (src_mcv.isBase()) { |
| 10735 | 11284 | try self.asmRegisterMemory( |
| 10736 | 11285 | .{ ._, .cmp }, |
| 10737 | 11286 | dst_regs[0], |
| ... | ... | @@ -10828,8 +11377,11 @@ fn genBinOp( |
| 10828 | 11377 | .immediate, |
| 10829 | 11378 | .eflags, |
| 10830 | 11379 | .register_pair, |
| 11380 | .register_triple, |
| 11381 | .register_quadruple, |
| 10831 | 11382 | .register_offset, |
| 10832 | 11383 | .register_overflow, |
| 11384 | .register_mask, |
| 10833 | 11385 | .load_symbol, |
| 10834 | 11386 | .lea_symbol, |
| 10835 | 11387 | .load_direct, |
| ... | ... | @@ -10909,7 +11461,7 @@ fn genBinOp( |
| 10909 | 11461 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 10910 | 11462 | defer self.register_manager.unlockReg(tmp_lock); |
| 10911 | 11463 | |
| 10912 | | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 11464 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 10913 | 11465 | .{ .vp_w, .insr }, |
| 10914 | 11466 | dst_reg, |
| 10915 | 11467 | dst_reg, |
| ... | ... | @@ -11355,7 +11907,7 @@ fn genBinOp( |
| 11355 | 11907 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 11356 | 11908 | defer self.register_manager.unlockReg(tmp_lock); |
| 11357 | 11909 | |
| 11358 | | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 11910 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 11359 | 11911 | .{ .vp_w, .insr }, |
| 11360 | 11912 | dst_reg, |
| 11361 | 11913 | dst_reg, |
| ... | ... | @@ -11402,7 +11954,7 @@ fn genBinOp( |
| 11402 | 11954 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 11403 | 11955 | defer self.register_manager.unlockReg(tmp_lock); |
| 11404 | 11956 | |
| 11405 | | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 11957 | if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 11406 | 11958 | .{ .vp_d, .insr }, |
| 11407 | 11959 | dst_reg, |
| 11408 | 11960 | try src_mcv.mem(self, .{ .size = .dword }), |
| ... | ... | @@ -11454,7 +12006,7 @@ fn genBinOp( |
| 11454 | 12006 | defer self.register_manager.unlockReg(tmp_lock); |
| 11455 | 12007 | |
| 11456 | 12008 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); |
| 11457 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 12009 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 11458 | 12010 | .{ .v_ps, .cvtph2 }, |
| 11459 | 12011 | tmp_reg, |
| 11460 | 12012 | try src_mcv.mem(self, .{ .size = .qword }), |
| ... | ... | @@ -11497,7 +12049,7 @@ fn genBinOp( |
| 11497 | 12049 | defer self.register_manager.unlockReg(tmp_lock); |
| 11498 | 12050 | |
| 11499 | 12051 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), dst_reg); |
| 11500 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 12052 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 11501 | 12053 | .{ .v_ps, .cvtph2 }, |
| 11502 | 12054 | tmp_reg, |
| 11503 | 12055 | try src_mcv.mem(self, .{ .size = .xword }), |
| ... | ... | @@ -11659,7 +12211,7 @@ fn genBinOp( |
| 11659 | 12211 | else => if (self.hasFeature(.avx)) { |
| 11660 | 12212 | const lhs_reg = |
| 11661 | 12213 | if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); |
| 11662 | | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| 12214 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 11663 | 12215 | mir_tag, |
| 11664 | 12216 | dst_reg, |
| 11665 | 12217 | lhs_reg, |
| ... | ... | @@ -11678,7 +12230,7 @@ fn genBinOp( |
| 11678 | 12230 | ); |
| 11679 | 12231 | } else { |
| 11680 | 12232 | assert(copied_to_dst); |
| 11681 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 12233 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 11682 | 12234 | mir_tag, |
| 11683 | 12235 | dst_reg, |
| 11684 | 12236 | try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { |
| ... | ... | @@ -11705,7 +12257,7 @@ fn genBinOp( |
| 11705 | 12257 | if (self.hasFeature(.avx)) { |
| 11706 | 12258 | const lhs_reg = |
| 11707 | 12259 | if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size); |
| 11708 | | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 12260 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 11709 | 12261 | mir_tag, |
| 11710 | 12262 | dst_reg, |
| 11711 | 12263 | lhs_reg, |
| ... | ... | @@ -11726,7 +12278,7 @@ fn genBinOp( |
| 11726 | 12278 | ); |
| 11727 | 12279 | } else { |
| 11728 | 12280 | assert(copied_to_dst); |
| 11729 | | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 12281 | if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 11730 | 12282 | mir_tag, |
| 11731 | 12283 | dst_reg, |
| 11732 | 12284 | try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) { |
| ... | ... | @@ -11937,7 +12489,7 @@ fn genBinOp( |
| 11937 | 12489 | => { |
| 11938 | 12490 | const unsigned_ty = try lhs_ty.toUnsigned(pt); |
| 11939 | 12491 | const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty)); |
| 11940 | | const not_mem: Memory = if (not_mcv.isMemory()) |
| 12492 | const not_mem: Memory = if (not_mcv.isBase()) |
| 11941 | 12493 | try not_mcv.mem(self, .{ .size = .fromSize(abi_size) }) |
| 11942 | 12494 | else |
| 11943 | 12495 | .{ .base = .{ |
| ... | ... | @@ -12017,6 +12569,7 @@ fn genBinOpMir( |
| 12017 | 12569 | .immediate, |
| 12018 | 12570 | .eflags, |
| 12019 | 12571 | .register_overflow, |
| 12572 | .register_mask, |
| 12020 | 12573 | .lea_direct, |
| 12021 | 12574 | .lea_got, |
| 12022 | 12575 | .lea_tlv, |
| ... | ... | @@ -12026,9 +12579,9 @@ fn genBinOpMir( |
| 12026 | 12579 | .reserved_frame, |
| 12027 | 12580 | .air_ref, |
| 12028 | 12581 | => unreachable, // unmodifiable destination |
| 12029 | | .register, .register_pair, .register_offset => { |
| 12582 | .register, .register_pair, .register_triple, .register_quadruple, .register_offset => { |
| 12030 | 12583 | switch (dst_mcv) { |
| 12031 | | .register, .register_pair => {}, |
| 12584 | .register, .register_pair, .register_triple, .register_quadruple => {}, |
| 12032 | 12585 | .register_offset => |ro| assert(ro.off == 0), |
| 12033 | 12586 | else => unreachable, |
| 12034 | 12587 | } |
| ... | ... | @@ -12057,10 +12610,15 @@ fn genBinOpMir( |
| 12057 | 12610 | .dead, |
| 12058 | 12611 | .undef, |
| 12059 | 12612 | .register_overflow, |
| 12613 | .register_mask, |
| 12060 | 12614 | .elementwise_regs_then_frame, |
| 12061 | 12615 | .reserved_frame, |
| 12062 | 12616 | => unreachable, |
| 12063 | | .register, .register_pair => try self.asmRegisterRegister( |
| 12617 | .register, |
| 12618 | .register_pair, |
| 12619 | .register_triple, |
| 12620 | .register_quadruple, |
| 12621 | => try self.asmRegisterRegister( |
| 12064 | 12622 | mir_limb_tag, |
| 12065 | 12623 | dst_alias, |
| 12066 | 12624 | registerAlias(src_mcv.getRegs()[dst_reg_i], limb_abi_size), |
| ... | ... | @@ -12216,6 +12774,7 @@ fn genBinOpMir( |
| 12216 | 12774 | .dead, |
| 12217 | 12775 | .undef, |
| 12218 | 12776 | .register_overflow, |
| 12777 | .register_mask, |
| 12219 | 12778 | .elementwise_regs_then_frame, |
| 12220 | 12779 | .reserved_frame, |
| 12221 | 12780 | .air_ref, |
| ... | ... | @@ -12224,6 +12783,8 @@ fn genBinOpMir( |
| 12224 | 12783 | .eflags, |
| 12225 | 12784 | .register, |
| 12226 | 12785 | .register_pair, |
| 12786 | .register_triple, |
| 12787 | .register_quadruple, |
| 12227 | 12788 | .register_offset, |
| 12228 | 12789 | .indirect, |
| 12229 | 12790 | .lea_direct, |
| ... | ... | @@ -12311,6 +12872,7 @@ fn genBinOpMir( |
| 12311 | 12872 | .dead, |
| 12312 | 12873 | .undef, |
| 12313 | 12874 | .register_overflow, |
| 12875 | .register_mask, |
| 12314 | 12876 | .elementwise_regs_then_frame, |
| 12315 | 12877 | .reserved_frame, |
| 12316 | 12878 | .air_ref, |
| ... | ... | @@ -12364,6 +12926,8 @@ fn genBinOpMir( |
| 12364 | 12926 | }, |
| 12365 | 12927 | .register, |
| 12366 | 12928 | .register_pair, |
| 12929 | .register_triple, |
| 12930 | .register_quadruple, |
| 12367 | 12931 | .register_offset, |
| 12368 | 12932 | .eflags, |
| 12369 | 12933 | .memory, |
| ... | ... | @@ -12382,7 +12946,7 @@ fn genBinOpMir( |
| 12382 | 12946 | const src_limb_mcv: MCValue = if (src_info) |info| .{ |
| 12383 | 12947 | .indirect = .{ .reg = info.addr_reg, .off = off }, |
| 12384 | 12948 | } else switch (resolved_src_mcv) { |
| 12385 | | .register, .register_pair => .{ |
| 12949 | .register, .register_pair, .register_triple, .register_quadruple => .{ |
| 12386 | 12950 | .register = resolved_src_mcv.getRegs()[limb_i], |
| 12387 | 12951 | }, |
| 12388 | 12952 | .eflags, |
| ... | ... | @@ -12438,6 +13002,7 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv |
| 12438 | 13002 | .eflags, |
| 12439 | 13003 | .register_offset, |
| 12440 | 13004 | .register_overflow, |
| 13005 | .register_mask, |
| 12441 | 13006 | .lea_symbol, |
| 12442 | 13007 | .lea_direct, |
| 12443 | 13008 | .lea_got, |
| ... | ... | @@ -12462,7 +13027,10 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv |
| 12462 | 13027 | .dead, |
| 12463 | 13028 | .undef, |
| 12464 | 13029 | .register_pair, |
| 13030 | .register_triple, |
| 13031 | .register_quadruple, |
| 12465 | 13032 | .register_overflow, |
| 13033 | .register_mask, |
| 12466 | 13034 | .elementwise_regs_then_frame, |
| 12467 | 13035 | .reserved_frame, |
| 12468 | 13036 | .air_ref, |
| ... | ... | @@ -12539,7 +13107,7 @@ fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv |
| 12539 | 13107 | ), |
| 12540 | 13108 | } |
| 12541 | 13109 | }, |
| 12542 | | .register_pair => unreachable, // unimplemented |
| 13110 | .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented |
| 12543 | 13111 | .memory, .indirect, .load_symbol, .load_direct, .load_got, .load_tlv, .load_frame => { |
| 12544 | 13112 | const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); |
| 12545 | 13113 | const tmp_mcv = MCValue{ .register = tmp_reg }; |
| ... | ... | @@ -12892,7 +13460,7 @@ fn genCall(self: *CodeGen, info: union(enum) { |
| 12892 | 13460 | const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); |
| 12893 | 13461 | defer self.register_manager.unlockReg(index_lock); |
| 12894 | 13462 | |
| 12895 | | const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .{ .size = .dword }) else .{ |
| 13463 | const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{ |
| 12896 | 13464 | .base = .{ .reg = try self.copyToTmpRegister( |
| 12897 | 13465 | Type.usize, |
| 12898 | 13466 | switch (src_arg) { |
| ... | ... | @@ -12984,7 +13552,7 @@ fn genCall(self: *CodeGen, info: union(enum) { |
| 12984 | 13552 | .lea_frame = .{ .index = frame_index, .off = -reg_off.off }, |
| 12985 | 13553 | }, .{}), |
| 12986 | 13554 | .elementwise_regs_then_frame => |regs_frame_addr| { |
| 12987 | | const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .{ .size = .dword }) else .{ |
| 13555 | const src_mem: Memory = if (src_arg.isBase()) try src_arg.mem(self, .{ .size = .dword }) else .{ |
| 12988 | 13556 | .base = .{ .reg = try self.copyToTmpRegister( |
| 12989 | 13557 | Type.usize, |
| 12990 | 13558 | switch (src_arg) { |
| ... | ... | @@ -13100,6 +13668,8 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void { |
| 13100 | 13668 | .none => {}, |
| 13101 | 13669 | .register, |
| 13102 | 13670 | .register_pair, |
| 13671 | .register_triple, |
| 13672 | .register_quadruple, |
| 13103 | 13673 | => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety }), |
| 13104 | 13674 | .indirect => |reg_off| { |
| 13105 | 13675 | try self.register_manager.getReg(reg_off.reg, null); |
| ... | ... | @@ -13226,7 +13796,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13226 | 13796 | const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg); |
| 13227 | 13797 | defer self.register_manager.unlockReg(temp_lhs_lock); |
| 13228 | 13798 | |
| 13229 | | if (lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 13799 | if (lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 13230 | 13800 | .{ ._, .mov }, |
| 13231 | 13801 | temp_lhs_reg.to8(), |
| 13232 | 13802 | try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }), |
| ... | ... | @@ -13240,7 +13810,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13240 | 13810 | } |
| 13241 | 13811 | |
| 13242 | 13812 | const payload_compare = payload_compare: { |
| 13243 | | if (rhs_mcv.isMemory()) { |
| 13813 | if (rhs_mcv.isBase()) { |
| 13244 | 13814 | const rhs_mem = |
| 13245 | 13815 | try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }); |
| 13246 | 13816 | try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8()); |
| ... | ... | @@ -13291,13 +13861,13 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13291 | 13861 | } else .may_flip; |
| 13292 | 13862 | |
| 13293 | 13863 | const flipped = switch (may_flip) { |
| 13294 | | .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isMemory(), |
| 13864 | .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isBase(), |
| 13295 | 13865 | .must_flip => true, |
| 13296 | 13866 | .must_not_flip => false, |
| 13297 | 13867 | }; |
| 13298 | 13868 | const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv; |
| 13299 | 13869 | const dst_mcv = if (unmat_dst_mcv.isRegister() or |
| 13300 | | (abi_size <= 8 and unmat_dst_mcv.isMemory())) unmat_dst_mcv else dst: { |
| 13870 | (abi_size <= 8 and unmat_dst_mcv.isBase())) unmat_dst_mcv else dst: { |
| 13301 | 13871 | const dst_mcv = try self.allocTempRegOrMem(ty, true); |
| 13302 | 13872 | try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{}); |
| 13303 | 13873 | break :dst dst_mcv; |
| ... | ... | @@ -13335,6 +13905,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13335 | 13905 | .register, |
| 13336 | 13906 | .register_offset, |
| 13337 | 13907 | .register_overflow, |
| 13908 | .register_mask, |
| 13338 | 13909 | .indirect, |
| 13339 | 13910 | .lea_direct, |
| 13340 | 13911 | .lea_got, |
| ... | ... | @@ -13345,7 +13916,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13345 | 13916 | .reserved_frame, |
| 13346 | 13917 | .air_ref, |
| 13347 | 13918 | => unreachable, |
| 13348 | | .register_pair, .load_frame => null, |
| 13919 | .register_pair, .register_triple, .register_quadruple, .load_frame => null, |
| 13349 | 13920 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: { |
| 13350 | 13921 | switch (resolved_dst_mcv) { |
| 13351 | 13922 | .memory => |addr| if (std.math.cast( |
| ... | ... | @@ -13396,6 +13967,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13396 | 13967 | .register, |
| 13397 | 13968 | .register_offset, |
| 13398 | 13969 | .register_overflow, |
| 13970 | .register_mask, |
| 13399 | 13971 | .indirect, |
| 13400 | 13972 | .lea_symbol, |
| 13401 | 13973 | .lea_direct, |
| ... | ... | @@ -13406,7 +13978,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13406 | 13978 | .reserved_frame, |
| 13407 | 13979 | .air_ref, |
| 13408 | 13980 | => unreachable, |
| 13409 | | .register_pair, .load_frame => null, |
| 13981 | .register_pair, .register_triple, .register_quadruple, .load_frame => null, |
| 13410 | 13982 | .memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: { |
| 13411 | 13983 | switch (resolved_src_mcv) { |
| 13412 | 13984 | .memory => |addr| if (std.math.cast( |
| ... | ... | @@ -13457,7 +14029,10 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13457 | 14029 | try self.genSetReg(tmp_reg, Type.usize, if (dst_info) |info| .{ |
| 13458 | 14030 | .indirect = .{ .reg = info.addr_reg, .off = off }, |
| 13459 | 14031 | } else switch (resolved_dst_mcv) { |
| 13460 | | .register_pair => |dst_regs| .{ .register = dst_regs[limb_i] }, |
| 14032 | inline .register_pair, |
| 14033 | .register_triple, |
| 14034 | .register_quadruple, |
| 14035 | => |dst_regs| .{ .register = dst_regs[limb_i] }, |
| 13461 | 14036 | .memory => |dst_addr| .{ |
| 13462 | 14037 | .memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off), |
| 13463 | 14038 | }, |
| ... | ... | @@ -13479,9 +14054,10 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13479 | 14054 | if (src_info) |info| .{ |
| 13480 | 14055 | .indirect = .{ .reg = info.addr_reg, .off = off }, |
| 13481 | 14056 | } else switch (resolved_src_mcv) { |
| 13482 | | .register_pair => |src_regs| .{ |
| 13483 | | .register = src_regs[limb_i], |
| 13484 | | }, |
| 14057 | inline .register_pair, |
| 14058 | .register_triple, |
| 14059 | .register_quadruple, |
| 14060 | => |src_regs| .{ .register = src_regs[limb_i] }, |
| 13485 | 14061 | .memory => |src_addr| .{ |
| 13486 | 14062 | .memory = @bitCast(@as(i64, @bitCast(src_addr)) + off), |
| 13487 | 14063 | }, |
| ... | ... | @@ -13539,7 +14115,7 @@ fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !v |
| 13539 | 14115 | const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg); |
| 13540 | 14116 | defer self.register_manager.unlockReg(tmp2_lock); |
| 13541 | 14117 | |
| 13542 | | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 14118 | if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 13543 | 14119 | .{ .vp_w, .insr }, |
| 13544 | 14120 | tmp1_reg, |
| 13545 | 14121 | dst_reg.to128(), |
| ... | ... | @@ -13840,8 +14416,11 @@ fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) |
| 13840 | 14416 | .undef, |
| 13841 | 14417 | .immediate, |
| 13842 | 14418 | .eflags, |
| 14419 | .register_triple, |
| 14420 | .register_quadruple, |
| 13843 | 14421 | .register_offset, |
| 13844 | 14422 | .register_overflow, |
| 14423 | .register_mask, |
| 13845 | 14424 | .lea_direct, |
| 13846 | 14425 | .lea_got, |
| 13847 | 14426 | .lea_tlv, |
| ... | ... | @@ -15481,6 +16060,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 15481 | 16060 | .immediate, |
| 15482 | 16061 | .eflags, |
| 15483 | 16062 | .register_overflow, |
| 16063 | .register_mask, |
| 15484 | 16064 | .lea_direct, |
| 15485 | 16065 | .lea_got, |
| 15486 | 16066 | .lea_tlv, |
| ... | ... | @@ -15510,7 +16090,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 15510 | 16090 | .off = -dst_reg_off.off, |
| 15511 | 16091 | } }, |
| 15512 | 16092 | }, opts), |
| 15513 | | .register_pair => |dst_regs| { |
| 16093 | inline .register_pair, .register_triple, .register_quadruple => |dst_regs| { |
| 15514 | 16094 | const src_info: ?struct { addr_reg: Register, addr_lock: RegisterLock } = switch (src_mcv) { |
| 15515 | 16095 | .register_pair, .memory, .indirect, .load_frame => null, |
| 15516 | 16096 | .load_symbol, .load_direct, .load_got, .load_tlv => src: { |
| ... | ... | @@ -15535,9 +16115,12 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C |
| 15535 | 16115 | defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock); |
| 15536 | 16116 | |
| 15537 | 16117 | var part_disp: i32 = 0; |
| 15538 | | for (dst_regs, try self.splitType(ty), 0..) |dst_reg, dst_ty, part_i| { |
| 16118 | for (dst_regs, try self.splitType(dst_regs.len, ty), 0..) |dst_reg, dst_ty, part_i| { |
| 15539 | 16119 | try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) { |
| 15540 | | .register_pair => |src_regs| .{ .register = src_regs[part_i] }, |
| 16120 | inline .register_pair, |
| 16121 | .register_triple, |
| 16122 | .register_quadruple, |
| 16123 | => |src_regs| .{ .register = src_regs[part_i] }, |
| 15541 | 16124 | .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(), |
| 15542 | 16125 | .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{ |
| 15543 | 16126 | .reg = src_info.?.addr_reg, |
| ... | ... | @@ -15733,7 +16316,10 @@ fn genSetReg( |
| 15733 | 16316 | }, |
| 15734 | 16317 | .ip => unreachable, |
| 15735 | 16318 | }, |
| 15736 | | .register_pair => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts), |
| 16319 | inline .register_pair, |
| 16320 | .register_triple, |
| 16321 | .register_quadruple, |
| 16322 | => |src_regs| try self.genSetReg(dst_reg, ty, .{ .register = src_regs[0] }, opts), |
| 15737 | 16323 | .register_offset, |
| 15738 | 16324 | .indirect, |
| 15739 | 16325 | .load_frame, |
| ... | ... | @@ -15770,6 +16356,47 @@ fn genSetReg( |
| 15770 | 16356 | }, |
| 15771 | 16357 | else => unreachable, |
| 15772 | 16358 | }), |
| 16359 | .register_mask => |src_reg_mask| { |
| 16360 | assert(src_reg_mask.reg.class() == .sse); |
| 16361 | const has_avx = self.hasFeature(.avx); |
| 16362 | const bits_reg = switch (dst_reg.class()) { |
| 16363 | .general_purpose => dst_reg, |
| 16364 | else => try self.register_manager.allocReg(null, abi.RegisterClass.gp), |
| 16365 | }; |
| 16366 | const bits_lock = self.register_manager.lockReg(bits_reg); |
| 16367 | defer if (bits_lock) |lock| self.register_manager.unlockReg(lock); |
| 16368 | |
| 16369 | const pack_reg = switch (src_reg_mask.scalar) { |
| 16370 | else => src_reg_mask.reg, |
| 16371 | .word => try self.register_manager.allocReg(null, abi.RegisterClass.sse), |
| 16372 | }; |
| 16373 | const pack_lock = self.register_manager.lockReg(pack_reg); |
| 16374 | defer if (pack_lock) |lock| self.register_manager.unlockReg(lock); |
| 16375 | |
| 16376 | var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.scalar.bitSize(), 8)); |
| 16377 | switch (src_reg_mask.scalar) { |
| 16378 | else => {}, |
| 16379 | .word => { |
| 16380 | const src_alias = registerAlias(src_reg_mask.reg, mask_size); |
| 16381 | const pack_alias = registerAlias(pack_reg, mask_size); |
| 16382 | if (has_avx) { |
| 16383 | try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias); |
| 16384 | } else { |
| 16385 | try self.asmRegisterRegister(.{ ._, .movdqa }, pack_alias, src_alias); |
| 16386 | try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias); |
| 16387 | } |
| 16388 | mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable; |
| 16389 | }, |
| 16390 | } |
| 16391 | try self.asmRegisterRegister(.{ switch (src_reg_mask.scalar) { |
| 16392 | .byte, .word => if (has_avx) .vp_b else .p_b, |
| 16393 | .dword => if (has_avx) .v_ps else ._ps, |
| 16394 | .qword => if (has_avx) .v_pd else ._pd, |
| 16395 | else => unreachable, |
| 16396 | }, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size)); |
| 16397 | if (src_reg_mask.inverted) try self.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size)); |
| 16398 | try self.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{}); |
| 16399 | }, |
| 15773 | 16400 | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => { |
| 15774 | 16401 | switch (src_mcv) { |
| 15775 | 16402 | .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| |
| ... | ... | @@ -15998,9 +16625,9 @@ fn genSetMem( |
| 15998 | 16625 | src_alias, |
| 15999 | 16626 | ); |
| 16000 | 16627 | }, |
| 16001 | | .register_pair => |src_regs| { |
| 16628 | inline .register_pair, .register_triple, .register_quadruple => |src_regs| { |
| 16002 | 16629 | var part_disp: i32 = disp; |
| 16003 | | for (try self.splitType(ty), src_regs) |src_ty, src_reg| { |
| 16630 | for (try self.splitType(src_regs.len, ty), src_regs) |src_ty, src_reg| { |
| 16004 | 16631 | try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg }, opts); |
| 16005 | 16632 | part_disp += @intCast(src_ty.abiSize(zcu)); |
| 16006 | 16633 | } |
| ... | ... | @@ -16065,6 +16692,13 @@ fn genSetMem( |
| 16065 | 16692 | |
| 16066 | 16693 | try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); |
| 16067 | 16694 | }, |
| 16695 | .register_mask => { |
| 16696 | const src_reg = try self.copyToTmpRegister(ty, src_mcv); |
| 16697 | const src_lock = self.register_manager.lockRegAssumeUnused(src_reg); |
| 16698 | defer self.register_manager.unlockReg(src_lock); |
| 16699 | |
| 16700 | try self.genSetMem(base, disp, ty, .{ .register = src_reg }, opts); |
| 16701 | }, |
| 16068 | 16702 | .memory, |
| 16069 | 16703 | .indirect, |
| 16070 | 16704 | .load_direct, |
| ... | ... | @@ -16283,7 +16917,7 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 16283 | 16917 | const dst_mcv = try self.allocRegOrMem(inst, true); |
| 16284 | 16918 | try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { |
| 16285 | 16919 | .lt => dst_ty, |
| 16286 | | .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty, |
| 16920 | .eq => if (!dst_mcv.isBase() or src_mcv.isBase()) dst_ty else src_ty, |
| 16287 | 16921 | .gt => src_ty, |
| 16288 | 16922 | }, dst_mcv, src_mcv, .{}); |
| 16289 | 16923 | break :dst dst_mcv; |
| ... | ... | @@ -16729,7 +17363,7 @@ fn atomicOp( |
| 16729 | 17363 | }); |
| 16730 | 17364 | try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }, .{}); |
| 16731 | 17365 | switch (mir_tag[0]) { |
| 16732 | | .v_ss, .v_sd => if (val_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| 17366 | .v_ss, .v_sd => if (val_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 16733 | 17367 | mir_tag, |
| 16734 | 17368 | sse_reg.to128(), |
| 16735 | 17369 | sse_reg.to128(), |
| ... | ... | @@ -16743,7 +17377,7 @@ fn atomicOp( |
| 16743 | 17377 | else |
| 16744 | 17378 | try self.copyToTmpRegister(val_ty, val_mcv)).to128(), |
| 16745 | 17379 | ), |
| 16746 | | ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory( |
| 17380 | ._ss, ._sd => if (val_mcv.isBase()) try self.asmRegisterMemory( |
| 16747 | 17381 | mir_tag, |
| 16748 | 17382 | sse_reg.to128(), |
| 16749 | 17383 | try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }), |
| ... | ... | @@ -17443,7 +18077,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17443 | 18077 | defer self.register_manager.unlockReg(dst_lock); |
| 17444 | 18078 | |
| 17445 | 18079 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17446 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 18080 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17447 | 18081 | mir_tag, |
| 17448 | 18082 | registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))), |
| 17449 | 18083 | try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }), |
| ... | ... | @@ -17519,7 +18153,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17519 | 18153 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17520 | 18154 | if (self.hasFeature(.avx)) { |
| 17521 | 18155 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17522 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 18156 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17523 | 18157 | .{ .v_ss, .broadcast }, |
| 17524 | 18158 | dst_reg.to128(), |
| 17525 | 18159 | try src_mcv.mem(self, .{ .size = .dword }), |
| ... | ... | @@ -17556,7 +18190,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17556 | 18190 | 5...8 => if (self.hasFeature(.avx)) { |
| 17557 | 18191 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17558 | 18192 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17559 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 18193 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17560 | 18194 | .{ .v_ss, .broadcast }, |
| 17561 | 18195 | dst_reg.to256(), |
| 17562 | 18196 | try src_mcv.mem(self, .{ .size = .dword }), |
| ... | ... | @@ -17602,7 +18236,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17602 | 18236 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17603 | 18237 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17604 | 18238 | if (self.hasFeature(.sse3)) { |
| 17605 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 18239 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17606 | 18240 | if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup }, |
| 17607 | 18241 | dst_reg.to128(), |
| 17608 | 18242 | try src_mcv.mem(self, .{ .size = .qword }), |
| ... | ... | @@ -17627,7 +18261,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17627 | 18261 | 3...4 => if (self.hasFeature(.avx)) { |
| 17628 | 18262 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17629 | 18263 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17630 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 18264 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17631 | 18265 | .{ .v_sd, .broadcast }, |
| 17632 | 18266 | dst_reg.to256(), |
| 17633 | 18267 | try src_mcv.mem(self, .{ .size = .qword }), |
| ... | ... | @@ -17670,7 +18304,7 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17670 | 18304 | 2 => if (self.hasFeature(.avx)) { |
| 17671 | 18305 | const src_mcv = try self.resolveInst(ty_op.operand); |
| 17672 | 18306 | const dst_reg = try self.register_manager.allocReg(inst, dst_rc); |
| 17673 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 18307 | if (src_mcv.isBase()) try self.asmRegisterMemory( |
| 17674 | 18308 | .{ .v_f128, .broadcast }, |
| 17675 | 18309 | dst_reg.to256(), |
| 17676 | 18310 | try src_mcv.mem(self, .{ .size = .xword }), |
| ... | ... | @@ -17779,7 +18413,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17779 | 18413 | else => unreachable, |
| 17780 | 18414 | }, .broadcast }, |
| 17781 | 18415 | mask_alias, |
| 17782 | | if (pred_mcv.isMemory()) try pred_mcv.mem(self, .{ .size = .byte }) else .{ |
| 18416 | if (pred_mcv.isBase()) try pred_mcv.mem(self, .{ .size = .byte }) else .{ |
| 17783 | 18417 | .base = .{ .reg = (try self.copyToTmpRegister( |
| 17784 | 18418 | Type.usize, |
| 17785 | 18419 | pred_mcv.address(), |
| ... | ... | @@ -17973,7 +18607,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17973 | 18607 | try self.genSetReg(dst_reg, ty, rhs_mcv, .{}); |
| 17974 | 18608 | break :rhs dst_alias; |
| 17975 | 18609 | }; |
| 17976 | | if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister( |
| 18610 | if (lhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( |
| 17977 | 18611 | mir_tag, |
| 17978 | 18612 | dst_alias, |
| 17979 | 18613 | rhs_alias, |
| ... | ... | @@ -17989,7 +18623,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 17989 | 18623 | try self.copyToTmpRegister(ty, lhs_mcv), abi_size), |
| 17990 | 18624 | mask_alias, |
| 17991 | 18625 | ); |
| 17992 | | } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister( |
| 18626 | } else if (has_blend) if (lhs_mcv.isBase()) try self.asmRegisterMemoryRegister( |
| 17993 | 18627 | mir_tag, |
| 17994 | 18628 | dst_alias, |
| 17995 | 18629 | try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }), |
| ... | ... | @@ -18014,7 +18648,7 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18014 | 18648 | }, |
| 18015 | 18649 | }) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(pt)}); |
| 18016 | 18650 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); |
| 18017 | | if (rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 18651 | if (rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 18018 | 18652 | .{ mir_fixes, .andn }, |
| 18019 | 18653 | mask_alias, |
| 18020 | 18654 | try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }), |
| ... | ... | @@ -18093,10 +18727,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18093 | 18727 | |
| 18094 | 18728 | for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: { |
| 18095 | 18729 | if (elem_abi_size > 8) break :unpck; |
| 18096 | | if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4) |
| 18097 | | has_avx |
| 18098 | | else |
| 18099 | | self.hasFeature(.avx2)) 32 else 16)) break :unpck; |
| 18730 | if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck; |
| 18100 | 18731 | |
| 18101 | 18732 | var sources: [2]?u1 = @splat(null); |
| 18102 | 18733 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { |
| ... | ... | @@ -18154,7 +18785,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18154 | 18785 | }, |
| 18155 | 18786 | else => unreachable, |
| 18156 | 18787 | } }; |
| 18157 | | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| 18788 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 18158 | 18789 | mir_tag, |
| 18159 | 18790 | dst_alias, |
| 18160 | 18791 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| ... | ... | @@ -18167,7 +18798,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18167 | 18798 | rhs_mcv.getReg().? |
| 18168 | 18799 | else |
| 18169 | 18800 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| 18170 | | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 18801 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemory( |
| 18171 | 18802 | mir_tag, |
| 18172 | 18803 | dst_alias, |
| 18173 | 18804 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), |
| ... | ... | @@ -18184,7 +18815,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18184 | 18815 | |
| 18185 | 18816 | pshufd: { |
| 18186 | 18817 | if (elem_abi_size != 4) break :pshufd; |
| 18187 | | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd; |
| 18818 | if (max_abi_size > self.vectorSize(.float)) break :pshufd; |
| 18188 | 18819 | |
| 18189 | 18820 | var control: u8 = 0b00_00_00_00; |
| 18190 | 18821 | var sources: [1]?u1 = @splat(null); |
| ... | ... | @@ -18216,7 +18847,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18216 | 18847 | try self.register_manager.allocReg(inst, abi.RegisterClass.sse); |
| 18217 | 18848 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| 18218 | 18849 | |
| 18219 | | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 18850 | if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18220 | 18851 | .{ if (has_avx) .vp_d else .p_d, .shuf }, |
| 18221 | 18852 | dst_alias, |
| 18222 | 18853 | try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), |
| ... | ... | @@ -18235,7 +18866,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18235 | 18866 | |
| 18236 | 18867 | shufps: { |
| 18237 | 18868 | if (elem_abi_size != 4) break :shufps; |
| 18238 | | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps; |
| 18869 | if (max_abi_size > self.vectorSize(.float)) break :shufps; |
| 18239 | 18870 | |
| 18240 | 18871 | var control: u8 = 0b00_00_00_00; |
| 18241 | 18872 | var sources: [2]?u1 = @splat(null); |
| ... | ... | @@ -18272,7 +18903,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18272 | 18903 | const dst_reg = dst_mcv.getReg().?; |
| 18273 | 18904 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| 18274 | 18905 | |
| 18275 | | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 18906 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18276 | 18907 | .{ .v_ps, .shuf }, |
| 18277 | 18908 | dst_alias, |
| 18278 | 18909 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| ... | ... | @@ -18287,7 +18918,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18287 | 18918 | else |
| 18288 | 18919 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| 18289 | 18920 | .u(control), |
| 18290 | | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 18921 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18291 | 18922 | .{ ._ps, .shuf }, |
| 18292 | 18923 | dst_alias, |
| 18293 | 18924 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), |
| ... | ... | @@ -18306,7 +18937,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18306 | 18937 | |
| 18307 | 18938 | shufpd: { |
| 18308 | 18939 | if (elem_abi_size != 8) break :shufpd; |
| 18309 | | if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd; |
| 18940 | if (max_abi_size > self.vectorSize(.float)) break :shufpd; |
| 18310 | 18941 | |
| 18311 | 18942 | var control: u4 = 0b0_0_0_0; |
| 18312 | 18943 | var sources: [2]?u1 = @splat(null); |
| ... | ... | @@ -18339,7 +18970,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18339 | 18970 | const dst_reg = dst_mcv.getReg().?; |
| 18340 | 18971 | const dst_alias = registerAlias(dst_reg, max_abi_size); |
| 18341 | 18972 | |
| 18342 | | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 18973 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18343 | 18974 | .{ .v_pd, .shuf }, |
| 18344 | 18975 | dst_alias, |
| 18345 | 18976 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), |
| ... | ... | @@ -18354,7 +18985,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18354 | 18985 | else |
| 18355 | 18986 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), |
| 18356 | 18987 | .u(control), |
| 18357 | | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 18988 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18358 | 18989 | .{ ._pd, .shuf }, |
| 18359 | 18990 | dst_alias, |
| 18360 | 18991 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), |
| ... | ... | @@ -18373,7 +19004,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18373 | 19004 | |
| 18374 | 19005 | blend: { |
| 18375 | 19006 | if (elem_abi_size < 2) break :blend; |
| 18376 | | if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend; |
| 19007 | if (dst_abi_size > self.vectorSize(.float)) break :blend; |
| 18377 | 19008 | if (!self.hasFeature(.sse4_1)) break :blend; |
| 18378 | 19009 | |
| 18379 | 19010 | var control: u8 = 0b0_0_0_0_0_0_0_0; |
| ... | ... | @@ -18409,7 +19040,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18409 | 19040 | |
| 18410 | 19041 | const rhs_mcv = try self.resolveInst(extra.b); |
| 18411 | 19042 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); |
| 18412 | | if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 19043 | if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18413 | 19044 | .{ .vp_d, .blend }, |
| 18414 | 19045 | registerAlias(dst_reg, dst_abi_size), |
| 18415 | 19046 | registerAlias(lhs_reg, dst_abi_size), |
| ... | ... | @@ -18461,7 +19092,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18461 | 19092 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); |
| 18462 | 19093 | const dst_reg = dst_mcv.getReg().?; |
| 18463 | 19094 | |
| 18464 | | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 19095 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18465 | 19096 | .{ .vp_w, .blend }, |
| 18466 | 19097 | registerAlias(dst_reg, dst_abi_size), |
| 18467 | 19098 | registerAlias(if (lhs_mcv.isRegister()) |
| ... | ... | @@ -18482,7 +19113,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18482 | 19113 | else |
| 18483 | 19114 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| 18484 | 19115 | .u(expanded_control), |
| 18485 | | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 19116 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18486 | 19117 | .{ .p_w, .blend }, |
| 18487 | 19118 | registerAlias(dst_reg, dst_abi_size), |
| 18488 | 19119 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), |
| ... | ... | @@ -18518,7 +19149,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18518 | 19149 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); |
| 18519 | 19150 | const dst_reg = dst_mcv.getReg().?; |
| 18520 | 19151 | |
| 18521 | | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 19152 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( |
| 18522 | 19153 | switch (elem_abi_size) { |
| 18523 | 19154 | 4 => .{ .v_ps, .blend }, |
| 18524 | 19155 | 8, 16 => .{ .v_pd, .blend }, |
| ... | ... | @@ -18547,7 +19178,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18547 | 19178 | else |
| 18548 | 19179 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| 18549 | 19180 | .u(expanded_control), |
| 18550 | | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 19181 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( |
| 18551 | 19182 | switch (elem_abi_size) { |
| 18552 | 19183 | 4 => .{ ._ps, .blend }, |
| 18553 | 19184 | 8, 16 => .{ ._pd, .blend }, |
| ... | ... | @@ -18573,10 +19204,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18573 | 19204 | } |
| 18574 | 19205 | |
| 18575 | 19206 | blendv: { |
| 18576 | | if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4) |
| 18577 | | has_avx |
| 18578 | | else |
| 18579 | | self.hasFeature(.avx2)) 32 else 16)) break :blendv; |
| 19207 | if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv; |
| 18580 | 19208 | |
| 18581 | 19209 | const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8); |
| 18582 | 19210 | const select_mask_ty = try pt.vectorType(.{ |
| ... | ... | @@ -18637,7 +19265,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18637 | 19265 | const dst_reg = dst_mcv.getReg().?; |
| 18638 | 19266 | const dst_alias = registerAlias(dst_reg, dst_abi_size); |
| 18639 | 19267 | |
| 18640 | | if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister( |
| 19268 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( |
| 18641 | 19269 | mir_tag, |
| 18642 | 19270 | dst_alias, |
| 18643 | 19271 | if (lhs_mcv.isRegister()) |
| ... | ... | @@ -18658,7 +19286,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18658 | 19286 | else |
| 18659 | 19287 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), |
| 18660 | 19288 | select_mask_alias, |
| 18661 | | ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister( |
| 19289 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister( |
| 18662 | 19290 | mir_tag, |
| 18663 | 19291 | dst_alias, |
| 18664 | 19292 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), |
| ... | ... | @@ -18701,7 +19329,7 @@ fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18701 | 19329 | else |
| 18702 | 19330 | .p_; |
| 18703 | 19331 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); |
| 18704 | | if (lhs_mcv.isMemory()) try self.asmRegisterMemory( |
| 19332 | if (lhs_mcv.isBase()) try self.asmRegisterMemory( |
| 18705 | 19333 | .{ mir_fixes, .andn }, |
| 18706 | 19334 | mask_alias, |
| 18707 | 19335 | try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), |
| ... | ... | @@ -18851,22 +19479,61 @@ fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18851 | 19479 | if (operand_ty.isVector(zcu) and operand_ty.childType(zcu).toIntern() == .bool_type) { |
| 18852 | 19480 | try self.spillEflagsIfOccupied(); |
| 18853 | 19481 | |
| 18854 | | const operand_mcv = try self.resolveInst(reduce.operand); |
| 18855 | | const mask_len = (std.math.cast(u6, operand_ty.vectorLen(zcu)) orelse |
| 18856 | | return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)})); |
| 18857 | | const mask = (@as(u64, 1) << mask_len) - 1; |
| 18858 | 19482 | const abi_size: u32 = @intCast(operand_ty.abiSize(zcu)); |
| 19483 | const operand_mcv = try self.resolveInst(reduce.operand); |
| 19484 | const mask_len = operand_ty.vectorLen(zcu); |
| 19485 | const mask_len_minus_one = (std.math.cast(u6, mask_len - 1) orelse { |
| 19486 | const acc_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 19487 | const acc_lock = self.register_manager.lockRegAssumeUnused(acc_reg); |
| 19488 | defer self.register_manager.unlockReg(acc_lock); |
| 19489 | var limb_offset: i31 = 0; |
| 19490 | while (limb_offset < abi_size) : (limb_offset += 8) { |
| 19491 | try self.asmRegisterMemory( |
| 19492 | .{ ._, if (limb_offset == 0) .mov else switch (reduce.operation) { |
| 19493 | .Or => .@"or", |
| 19494 | .And => .@"and", |
| 19495 | else => return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)}), |
| 19496 | } }, |
| 19497 | acc_reg.to64(), |
| 19498 | try operand_mcv.mem(self, .{ |
| 19499 | .size = .qword, |
| 19500 | .disp = limb_offset, |
| 19501 | }), |
| 19502 | ); |
| 19503 | } |
| 19504 | switch (reduce.operation) { |
| 19505 | .Or => { |
| 19506 | try self.asmRegisterRegister(.{ ._, .@"test" }, acc_reg.to64(), acc_reg.to64()); |
| 19507 | break :result .{ .eflags = .nz }; |
| 19508 | }, |
| 19509 | .And => { |
| 19510 | try self.asmRegisterImmediate(.{ ._, .cmp }, acc_reg.to64(), .s(-1)); |
| 19511 | break :result .{ .eflags = .z }; |
| 19512 | }, |
| 19513 | else => unreachable, |
| 19514 | } |
| 19515 | }); |
| 19516 | const mask = @as(u64, std.math.maxInt(u64)) >> ~mask_len_minus_one; |
| 18859 | 19517 | switch (reduce.operation) { |
| 18860 | 19518 | .Or => { |
| 18861 | | if (operand_mcv.isMemory()) try self.asmMemoryImmediate( |
| 19519 | if (operand_mcv.isBase()) try self.asmMemoryImmediate( |
| 18862 | 19520 | .{ ._, .@"test" }, |
| 18863 | 19521 | try operand_mcv.mem(self, .{ .size = .fromSize(abi_size) }), |
| 18864 | | .u(mask), |
| 18865 | | ) else { |
| 18866 | | const operand_reg = registerAlias(if (operand_mcv.isRegister()) |
| 18867 | | operand_mcv.getReg().? |
| 19522 | if (mask_len < abi_size * 8) |
| 19523 | .u(mask) |
| 18868 | 19524 | else |
| 18869 | | try self.copyToTmpRegister(operand_ty, operand_mcv), abi_size); |
| 19525 | .s(-1), |
| 19526 | ) else { |
| 19527 | const operand_reg = registerAlias(operand_reg: { |
| 19528 | if (operand_mcv.isRegister()) { |
| 19529 | const operand_reg = operand_mcv.getReg().?; |
| 19530 | if (operand_reg.class() == .general_purpose) break :operand_reg operand_reg; |
| 19531 | } |
| 19532 | break :operand_reg try self.copyToTmpRegister(operand_ty, operand_mcv); |
| 19533 | }, abi_size); |
| 19534 | const operand_lock = self.register_manager.lockReg(operand_reg); |
| 19535 | defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); |
| 19536 | |
| 18870 | 19537 | if (mask_len < abi_size * 8) try self.asmRegisterImmediate( |
| 18871 | 19538 | .{ ._, .@"test" }, |
| 18872 | 19539 | operand_reg, |
| ... | ... | @@ -18880,7 +19547,10 @@ fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 18880 | 19547 | break :result .{ .eflags = .nz }; |
| 18881 | 19548 | }, |
| 18882 | 19549 | .And => { |
| 18883 | | const tmp_reg = try self.copyToTmpRegister(operand_ty, operand_mcv); |
| 19550 | const tmp_reg = registerAlias( |
| 19551 | try self.copyToTmpRegister(operand_ty, operand_mcv), |
| 19552 | abi_size, |
| 19553 | ); |
| 18884 | 19554 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 18885 | 19555 | defer self.register_manager.unlockReg(tmp_lock); |
| 18886 | 19556 | |
| ... | ... | @@ -19184,7 +19854,7 @@ fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 19184 | 19854 | if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) { |
| 19185 | 19855 | order[op_index] = 1; |
| 19186 | 19856 | unused.unset(0); |
| 19187 | | } else if (unused.isSet(2) and mcv.isMemory()) { |
| 19857 | } else if (unused.isSet(2) and mcv.isBase()) { |
| 19188 | 19858 | order[op_index] = 3; |
| 19189 | 19859 | unused.unset(2); |
| 19190 | 19860 | } |
| ... | ... | @@ -19531,7 +20201,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 19531 | 20201 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 19532 | 20202 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 19533 | 20203 | |
| 19534 | | if (self.hasFeature(.avx)) if (promote_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| 20204 | if (self.hasFeature(.avx)) if (promote_mcv.isBase()) try self.asmRegisterRegisterMemory( |
| 19535 | 20205 | .{ .v_ss, .cvtsd2 }, |
| 19536 | 20206 | dst_reg, |
| 19537 | 20207 | dst_reg, |
| ... | ... | @@ -19544,7 +20214,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 19544 | 20214 | promote_mcv.getReg().? |
| 19545 | 20215 | else |
| 19546 | 20216 | try self.copyToTmpRegister(promote_ty, promote_mcv)).to128(), |
| 19547 | | ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory( |
| 20217 | ) else if (promote_mcv.isBase()) try self.asmRegisterMemory( |
| 19548 | 20218 | .{ ._ss, .cvtsd2 }, |
| 19549 | 20219 | dst_reg, |
| 19550 | 20220 | try promote_mcv.mem(self, .{ .size = .qword }), |
| ... | ... | @@ -19751,7 +20421,7 @@ fn resolveCallingConventionValues( |
| 19751 | 20421 | // TODO: is this even possible for C calling convention? |
| 19752 | 20422 | result.return_value = .init(.none); |
| 19753 | 20423 | } else { |
| 19754 | | var ret_tracking: [2]InstTracking = undefined; |
| 20424 | var ret_tracking: [4]InstTracking = undefined; |
| 19755 | 20425 | var ret_tracking_i: usize = 0; |
| 19756 | 20426 | |
| 19757 | 20427 | const classes = switch (resolved_cc) { |
| ... | ... | @@ -19771,14 +20441,17 @@ fn resolveCallingConventionValues( |
| 19771 | 20441 | ret_tracking_i += 1; |
| 19772 | 20442 | }, |
| 19773 | 20443 | .sse, .float, .float_combine, .win_i128 => { |
| 19774 | | const ret_sse_reg = registerAlias( |
| 19775 | | abi.getCAbiSseReturnRegs(resolved_cc)[ret_sse_reg_i], |
| 19776 | | @intCast(ret_ty.abiSize(zcu)), |
| 19777 | | ); |
| 19778 | | ret_sse_reg_i += 1; |
| 19779 | | |
| 19780 | | ret_tracking[ret_tracking_i] = .init(.{ .register = ret_sse_reg }); |
| 19781 | | ret_tracking_i += 1; |
| 20444 | const ret_sse_regs = abi.getCAbiSseReturnRegs(resolved_cc); |
| 20445 | const abi_size: u32 = @intCast(ret_ty.abiSize(zcu)); |
| 20446 | const reg_size = @min(abi_size, self.vectorSize(.float)); |
| 20447 | var byte_offset: u32 = 0; |
| 20448 | while (byte_offset < abi_size) : (byte_offset += reg_size) { |
| 20449 | const ret_sse_reg = registerAlias(ret_sse_regs[ret_sse_reg_i], reg_size); |
| 20450 | ret_sse_reg_i += 1; |
| 20451 | |
| 20452 | ret_tracking[ret_tracking_i] = .init(.{ .register = ret_sse_reg }); |
| 20453 | ret_tracking_i += 1; |
| 20454 | } |
| 19782 | 20455 | }, |
| 19783 | 20456 | .sseup => assert(ret_tracking[ret_tracking_i - 1].short.register.class() == .sse), |
| 19784 | 20457 | .x87 => { |
| ... | ... | @@ -19811,6 +20484,17 @@ fn resolveCallingConventionValues( |
| 19811 | 20484 | ret_tracking[0].short.register, |
| 19812 | 20485 | ret_tracking[1].short.register, |
| 19813 | 20486 | } }), |
| 20487 | 3 => .init(.{ .register_triple = .{ |
| 20488 | ret_tracking[0].short.register, |
| 20489 | ret_tracking[1].short.register, |
| 20490 | ret_tracking[2].short.register, |
| 20491 | } }), |
| 20492 | 4 => .init(.{ .register_quadruple = .{ |
| 20493 | ret_tracking[0].short.register, |
| 20494 | ret_tracking[1].short.register, |
| 20495 | ret_tracking[2].short.register, |
| 20496 | ret_tracking[3].short.register, |
| 20497 | } }), |
| 19814 | 20498 | }; |
| 19815 | 20499 | } |
| 19816 | 20500 | |
| ... | ... | @@ -19826,7 +20510,7 @@ fn resolveCallingConventionValues( |
| 19826 | 20510 | else => unreachable, |
| 19827 | 20511 | } |
| 19828 | 20512 | |
| 19829 | | var arg_mcv: [2]MCValue = undefined; |
| 20513 | var arg_mcv: [4]MCValue = undefined; |
| 19830 | 20514 | var arg_mcv_i: usize = 0; |
| 19831 | 20515 | |
| 19832 | 20516 | const classes = switch (resolved_cc) { |
| ... | ... | @@ -19834,15 +20518,13 @@ fn resolveCallingConventionValues( |
| 19834 | 20518 | .x86_64_win => &.{abi.classifyWindows(ty, zcu)}, |
| 19835 | 20519 | else => unreachable, |
| 19836 | 20520 | }; |
| 19837 | | for (classes) |class| switch (class) { |
| 20521 | classes: for (classes) |class| switch (class) { |
| 19838 | 20522 | .integer => { |
| 19839 | 20523 | const param_int_regs = abi.getCAbiIntParamRegs(resolved_cc); |
| 19840 | 20524 | if (param_int_reg_i >= param_int_regs.len) break; |
| 19841 | 20525 | |
| 19842 | | const param_int_reg = registerAlias( |
| 19843 | | abi.getCAbiIntParamRegs(resolved_cc)[param_int_reg_i], |
| 19844 | | @intCast(@min(ty.abiSize(zcu), 8)), |
| 19845 | | ); |
| 20526 | const param_int_reg = |
| 20527 | registerAlias(param_int_regs[param_int_reg_i], @intCast(@min(ty.abiSize(zcu), 8))); |
| 19846 | 20528 | param_int_reg_i += 1; |
| 19847 | 20529 | |
| 19848 | 20530 | arg_mcv[arg_mcv_i] = .{ .register = param_int_reg }; |
| ... | ... | @@ -19850,16 +20532,18 @@ fn resolveCallingConventionValues( |
| 19850 | 20532 | }, |
| 19851 | 20533 | .sse, .float, .float_combine => { |
| 19852 | 20534 | const param_sse_regs = abi.getCAbiSseParamRegs(resolved_cc); |
| 19853 | | if (param_sse_reg_i >= param_sse_regs.len) break; |
| 20535 | const abi_size: u32 = @intCast(ty.abiSize(zcu)); |
| 20536 | const reg_size = @min(abi_size, self.vectorSize(.float)); |
| 20537 | var byte_offset: u32 = 0; |
| 20538 | while (byte_offset < abi_size) : (byte_offset += reg_size) { |
| 20539 | if (param_sse_reg_i >= param_sse_regs.len) break :classes; |
| 19854 | 20540 | |
| 19855 | | const param_sse_reg = registerAlias( |
| 19856 | | abi.getCAbiSseParamRegs(resolved_cc)[param_sse_reg_i], |
| 19857 | | @intCast(ty.abiSize(zcu)), |
| 19858 | | ); |
| 19859 | | param_sse_reg_i += 1; |
| 20541 | const param_sse_reg = registerAlias(param_sse_regs[param_sse_reg_i], reg_size); |
| 20542 | param_sse_reg_i += 1; |
| 19860 | 20543 | |
| 19861 | | arg_mcv[arg_mcv_i] = .{ .register = param_sse_reg }; |
| 19862 | | arg_mcv_i += 1; |
| 20544 | arg_mcv[arg_mcv_i] = .{ .register = param_sse_reg }; |
| 20545 | arg_mcv_i += 1; |
| 20546 | } |
| 19863 | 20547 | }, |
| 19864 | 20548 | .sseup => assert(arg_mcv[arg_mcv_i - 1].register.class() == .sse), |
| 19865 | 20549 | .x87, .x87up, .complex_x87, .memory, .win_i128 => switch (resolved_cc) { |
| ... | ... | @@ -19908,7 +20592,21 @@ fn resolveCallingConventionValues( |
| 19908 | 20592 | arg.* = switch (arg_mcv_i) { |
| 19909 | 20593 | else => unreachable, |
| 19910 | 20594 | 1 => arg_mcv[0], |
| 19911 | | 2 => .{ .register_pair = .{ arg_mcv[0].register, arg_mcv[1].register } }, |
| 20595 | 2 => .{ .register_pair = .{ |
| 20596 | arg_mcv[0].register, |
| 20597 | arg_mcv[1].register, |
| 20598 | } }, |
| 20599 | 3 => .{ .register_triple = .{ |
| 20600 | arg_mcv[0].register, |
| 20601 | arg_mcv[1].register, |
| 20602 | arg_mcv[2].register, |
| 20603 | } }, |
| 20604 | 4 => .{ .register_quadruple = .{ |
| 20605 | arg_mcv[0].register, |
| 20606 | arg_mcv[1].register, |
| 20607 | arg_mcv[2].register, |
| 20608 | arg_mcv[3].register, |
| 20609 | } }, |
| 19912 | 20610 | }; |
| 19913 | 20611 | continue; |
| 19914 | 20612 | } |
| ... | ... | @@ -20052,32 +20750,40 @@ fn memSize(self: *CodeGen, ty: Type) Memory.Size { |
| 20052 | 20750 | }; |
| 20053 | 20751 | } |
| 20054 | 20752 | |
| 20055 | | fn splitType(self: *CodeGen, ty: Type) ![2]Type { |
| 20753 | fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Type { |
| 20056 | 20754 | const pt = self.pt; |
| 20057 | 20755 | const zcu = pt.zcu; |
| 20756 | var parts: [parts_len]Type = undefined; |
| 20757 | if (ty.isVector(zcu)) if (std.math.divExact(u32, ty.vectorLen(zcu), parts_len)) |vec_len| return .{ |
| 20758 | try pt.vectorType(.{ .len = vec_len, .child = ty.scalarType(zcu).toIntern() }), |
| 20759 | } ** parts_len else |err| switch (err) { |
| 20760 | error.DivisionByZero => unreachable, |
| 20761 | error.UnexpectedRemainder => {}, |
| 20762 | }; |
| 20058 | 20763 | const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none); |
| 20059 | | var parts: [2]Type = undefined; |
| 20060 | | if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| { |
| 20764 | if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| { |
| 20061 | 20765 | part.* = switch (class) { |
| 20062 | | .integer => switch (part_i) { |
| 20063 | | 0 => Type.u64, |
| 20064 | | 1 => part: { |
| 20065 | | const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?; |
| 20066 | | const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8)); |
| 20067 | | break :part switch (@divExact(ty.abiSize(zcu) - 8, elem_size)) { |
| 20068 | | 1 => elem_ty, |
| 20069 | | else => |len| try pt.arrayType(.{ .len = len, .child = elem_ty.toIntern() }), |
| 20070 | | }; |
| 20071 | | }, |
| 20072 | | else => unreachable, |
| 20766 | .integer => if (part_i < parts_len - 1) |
| 20767 | Type.u64 |
| 20768 | else part: { |
| 20769 | const elem_size = ty.abiAlignment(zcu).minStrict(.@"8").toByteUnits().?; |
| 20770 | const elem_ty = try pt.intType(.unsigned, @intCast(elem_size * 8)); |
| 20771 | break :part switch (@divExact(ty.abiSize(zcu) - part_i * 8, elem_size)) { |
| 20772 | 1 => elem_ty, |
| 20773 | else => |array_len| try pt.arrayType(.{ .len = array_len, .child = elem_ty.toIntern() }), |
| 20774 | }; |
| 20073 | 20775 | }, |
| 20074 | 20776 | .float => Type.f32, |
| 20075 | 20777 | .float_combine => try pt.arrayType(.{ .len = 2, .child = .f32_type }), |
| 20076 | 20778 | .sse => Type.f64, |
| 20077 | 20779 | else => break, |
| 20078 | 20780 | }; |
| 20079 | | } else if (parts[0].abiSize(zcu) + parts[1].abiSize(zcu) == ty.abiSize(zcu)) return parts; |
| 20080 | | return self.fail("TODO implement splitType for {}", .{ty.fmt(pt)}); |
| 20781 | } else { |
| 20782 | var part_sizes: u64 = 0; |
| 20783 | for (parts) |part| part_sizes += part.abiSize(zcu); |
| 20784 | if (part_sizes == ty.abiSize(zcu)) return parts; |
| 20785 | }; |
| 20786 | return self.fail("TODO implement splitType({d}, {})", .{ parts_len, ty.fmt(pt) }); |
| 20081 | 20787 | } |
| 20082 | 20788 | |
| 20083 | 20789 | /// Truncates the value in the register in place. |
| ... | ... | @@ -20297,6 +21003,7 @@ const Temp = struct { |
| 20297 | 21003 | .immediate, |
| 20298 | 21004 | .eflags, |
| 20299 | 21005 | .register_offset, |
| 21006 | .register_mask, |
| 20300 | 21007 | .memory, |
| 20301 | 21008 | .load_symbol, |
| 20302 | 21009 | .lea_symbol, |
| ... | ... | @@ -20314,6 +21021,8 @@ const Temp = struct { |
| 20314 | 21021 | => false, |
| 20315 | 21022 | .register, |
| 20316 | 21023 | .register_pair, |
| 21024 | .register_triple, |
| 21025 | .register_quadruple, |
| 20317 | 21026 | .register_overflow, |
| 20318 | 21027 | => true, |
| 20319 | 21028 | .load_frame => |frame_addr| !frame_addr.index.isNamed(), |
| ... | ... | @@ -20330,7 +21039,7 @@ const Temp = struct { |
| 20330 | 21039 | cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize; |
| 20331 | 21040 | cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); |
| 20332 | 21041 | switch (temp.tracking(cg).short) { |
| 20333 | | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), |
| 21042 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20334 | 21043 | .register => |reg| { |
| 20335 | 21044 | const new_reg = |
| 20336 | 21045 | try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); |
| ... | ... | @@ -20434,7 +21143,7 @@ const Temp = struct { |
| 20434 | 21143 | const new_temp_index = cg.next_temp_index; |
| 20435 | 21144 | cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize; |
| 20436 | 21145 | switch (temp.tracking(cg).short) { |
| 20437 | | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), |
| 21146 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20438 | 21147 | .immediate => |imm| { |
| 20439 | 21148 | assert(limb_index == 0); |
| 20440 | 21149 | new_temp_index.tracking(cg).* = .init(.{ .immediate = imm }); |
| ... | ... | @@ -20635,6 +21344,17 @@ const Temp = struct { |
| 20635 | 21344 | first_temp.* = result_temp; |
| 20636 | 21345 | } |
| 20637 | 21346 | |
| 21347 | fn asMask(temp: Temp, kind: MaskKind, inverted: bool, scalar: Memory.Size, cg: *CodeGen) void { |
| 21348 | assert(scalar != .none); |
| 21349 | const mcv = &temp.unwrap(cg).temp.tracking(cg).short; |
| 21350 | mcv.* = .{ .register_mask = .{ |
| 21351 | .reg = mcv.register, |
| 21352 | .kind = kind, |
| 21353 | .inverted = inverted, |
| 21354 | .scalar = scalar, |
| 21355 | } }; |
| 21356 | } |
| 21357 | |
| 20638 | 21358 | fn toLea(temp: *Temp, cg: *CodeGen) !bool { |
| 20639 | 21359 | switch (temp.tracking(cg).short) { |
| 20640 | 21360 | .none, |
| ... | ... | @@ -20643,7 +21363,10 @@ const Temp = struct { |
| 20643 | 21363 | .undef, |
| 20644 | 21364 | .eflags, |
| 20645 | 21365 | .register_pair, |
| 21366 | .register_triple, |
| 21367 | .register_quadruple, |
| 20646 | 21368 | .register_overflow, |
| 21369 | .register_mask, |
| 20647 | 21370 | .elementwise_regs_then_frame, |
| 20648 | 21371 | .reserved_frame, |
| 20649 | 21372 | .air_ref, |
| ... | ... | @@ -20677,10 +21400,7 @@ const Temp = struct { |
| 20677 | 21400 | |
| 20678 | 21401 | fn toBase(temp: *Temp, cg: *CodeGen) !bool { |
| 20679 | 21402 | const temp_tracking = temp.tracking(cg); |
| 20680 | | switch (temp_tracking.short) { |
| 20681 | | else => {}, |
| 20682 | | .indirect, .load_frame => return false, |
| 20683 | | } |
| 21403 | if (temp_tracking.short.isBase()) return false; |
| 20684 | 21404 | const new_temp_index = cg.next_temp_index; |
| 20685 | 21405 | cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg); |
| 20686 | 21406 | const new_reg = |
| ... | ... | @@ -20697,7 +21417,7 @@ const Temp = struct { |
| 20697 | 21417 | const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu)); |
| 20698 | 21418 | const val = try cg.tempAlloc(val_ty); |
| 20699 | 21419 | switch (val.tracking(cg).short) { |
| 20700 | | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), |
| 21420 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20701 | 21421 | .register => |val_reg| { |
| 20702 | 21422 | while (try ptr.toLea(cg)) {} |
| 20703 | 21423 | switch (val_reg.class()) { |
| ... | ... | @@ -20706,7 +21426,7 @@ const Temp = struct { |
| 20706 | 21426 | registerAlias(val_reg, val_abi_size), |
| 20707 | 21427 | try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }), |
| 20708 | 21428 | ), |
| 20709 | | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), |
| 21429 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20710 | 21430 | } |
| 20711 | 21431 | }, |
| 20712 | 21432 | .load_frame => |val_frame_addr| { |
| ... | ... | @@ -20724,7 +21444,7 @@ const Temp = struct { |
| 20724 | 21444 | const val_ty = val.typeOf(cg); |
| 20725 | 21445 | const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu)); |
| 20726 | 21446 | val: switch (val.tracking(cg).short) { |
| 20727 | | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), |
| 21447 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20728 | 21448 | .immediate => |imm| if (std.math.cast(i32, imm)) |s| { |
| 20729 | 21449 | while (try ptr.toLea(cg)) {} |
| 20730 | 21450 | try cg.asmMemoryImmediate( |
| ... | ... | @@ -20742,7 +21462,7 @@ const Temp = struct { |
| 20742 | 21462 | try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }), |
| 20743 | 21463 | registerAlias(val_reg, val_abi_size), |
| 20744 | 21464 | ), |
| 20745 | | else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), |
| 21465 | else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), |
| 20746 | 21466 | } |
| 20747 | 21467 | }, |
| 20748 | 21468 | } |
| ... | ... | @@ -20876,6 +21596,7 @@ fn reuseTemp( |
| 20876 | 21596 | .register_pair, |
| 20877 | 21597 | .register_offset, |
| 20878 | 21598 | .register_overflow, |
| 21599 | .register_mask, |
| 20879 | 21600 | .indirect, |
| 20880 | 21601 | => for (tracking.short.getRegs()) |tracked_reg| { |
| 20881 | 21602 | if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| { |
| ... | ... | @@ -20913,6 +21634,26 @@ fn tempAllocReg(cg: *CodeGen, ty: Type, rc: RegisterManager.RegisterBitSet) !Tem |
| 20913 | 21634 | return .{ .index = temp_index.toIndex() }; |
| 20914 | 21635 | } |
| 20915 | 21636 | |
| 21637 | fn tempAllocRegPair(cg: *CodeGen, ty: Type, rc: RegisterManager.RegisterBitSet) !Temp { |
| 21638 | const temp_index = cg.next_temp_index; |
| 21639 | temp_index.tracking(cg).* = .init( |
| 21640 | .{ .register_pair = try cg.register_manager.allocRegs(2, temp_index.toIndex(), rc) }, |
| 21641 | ); |
| 21642 | cg.temp_type[@intFromEnum(temp_index)] = ty; |
| 21643 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| 21644 | return .{ .index = temp_index.toIndex() }; |
| 21645 | } |
| 21646 | |
| 21647 | fn tempAllocMem(cg: *CodeGen, ty: Type) !Temp { |
| 21648 | const temp_index = cg.next_temp_index; |
| 21649 | temp_index.tracking(cg).* = .init( |
| 21650 | try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false), |
| 21651 | ); |
| 21652 | cg.temp_type[@intFromEnum(temp_index)] = ty; |
| 21653 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| 21654 | return .{ .index = temp_index.toIndex() }; |
| 21655 | } |
| 21656 | |
| 20916 | 21657 | fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp { |
| 20917 | 21658 | const temp_index = cg.next_temp_index; |
| 20918 | 21659 | temp_index.tracking(cg).* = .init(value); |
| ... | ... | @@ -20993,30 +21734,23 @@ const Operand = union(enum) { |
| 20993 | 21734 | inst: Mir.Inst.Index, |
| 20994 | 21735 | }; |
| 20995 | 21736 | |
| 20996 | | const SelectLoop = struct { |
| 20997 | | element_reloc: Mir.Inst.Index, |
| 20998 | | element_offset: union(enum) { |
| 20999 | | unused, |
| 21000 | | known: u31, |
| 21001 | | temp: Temp, |
| 21002 | | }, |
| 21003 | | element_size: ?u13, |
| 21004 | | limb_reloc: Mir.Inst.Index, |
| 21005 | | limb_offset: union(enum) { |
| 21006 | | unused, |
| 21007 | | known: u31, |
| 21008 | | temp: Temp, |
| 21009 | | }, |
| 21010 | | limb_size: ?u8, |
| 21011 | | remaining_size: ?u64, |
| 21012 | | }; |
| 21013 | | |
| 21014 | 21737 | const Pattern = struct { |
| 21015 | 21738 | ops: []const Op, |
| 21016 | 21739 | commute: struct { u8, u8 } = .{ 0, 0 }, |
| 21017 | 21740 | |
| 21018 | 21741 | const Set = struct { |
| 21019 | 21742 | required_features: []const std.Target.x86.Feature = &.{}, |
| 21743 | scalar: union(enum) { |
| 21744 | any, |
| 21745 | bool, |
| 21746 | float: Memory.Size, |
| 21747 | any_int: Memory.Size, |
| 21748 | signed_int: Memory.Size, |
| 21749 | unsigned_int: Memory.Size, |
| 21750 | any_float_or_int: Memory.Size, |
| 21751 | } = .any, |
| 21752 | clobbers: struct { eflags: bool = false } = .{}, |
| 21753 | invert_result: bool = false, |
| 21020 | 21754 | loop: enum { |
| 21021 | 21755 | /// only execute the instruction once |
| 21022 | 21756 | once, |
| ... | ... | @@ -21050,6 +21784,18 @@ const Pattern = struct { |
| 21050 | 21784 | xmm, |
| 21051 | 21785 | /// any 256-bit sse register |
| 21052 | 21786 | ymm, |
| 21787 | /// a 64-bit mmx register mask |
| 21788 | mm_mask, |
| 21789 | /// a 128-bit sse register mask |
| 21790 | xmm_mask, |
| 21791 | /// a 256-bit sse register mask |
| 21792 | ymm_mask, |
| 21793 | /// a 64-bit mmx register sign mask |
| 21794 | mm_sign_mask, |
| 21795 | /// a 128-bit sse register sign mask |
| 21796 | xmm_sign_mask, |
| 21797 | /// a 256-bit sse register sign mask |
| 21798 | ymm_sign_mask, |
| 21053 | 21799 | /// any memory |
| 21054 | 21800 | mem, |
| 21055 | 21801 | /// a limb stored in a gpr |
| ... | ... | @@ -21062,41 +21808,82 @@ const Pattern = struct { |
| 21062 | 21808 | ymm_limb, |
| 21063 | 21809 | /// a limb stored in memory |
| 21064 | 21810 | mem_limb, |
| 21811 | /// a limb stored in a 64-bit mmx register mask |
| 21812 | mm_mask_limb, |
| 21813 | /// a limb stored in a 128-bit sse register masuk |
| 21814 | xmm_mask_limb, |
| 21815 | /// a limb stored in a 256-bit sse register masuk |
| 21816 | ymm_mask_limb, |
| 21065 | 21817 | /// specific immediate |
| 21066 | 21818 | imm: i8, |
| 21067 | 21819 | /// any immediate signed extended from 32 bits |
| 21068 | 21820 | simm32, |
| 21821 | /// a temp general purpose register containing all ones |
| 21822 | umax_gpr, |
| 21823 | /// a temp 64-bit mmx register containing all ones |
| 21824 | umax_mm, |
| 21825 | /// a temp 128-bit sse register containing all ones |
| 21826 | umax_xmm, |
| 21827 | /// a temp 256-bit sse register containing all ones |
| 21828 | umax_ymm, |
| 21069 | 21829 | |
| 21070 | 21830 | fn matches(op: Op, is_mut: bool, temp: Temp, cg: *CodeGen) bool { |
| 21071 | | const abi_size = temp.typeOf(cg).abiSize(cg.pt.zcu); |
| 21072 | | return switch (op) { |
| 21831 | switch (op) { |
| 21073 | 21832 | .implicit, .explicit => unreachable, |
| 21833 | else => {}, |
| 21834 | // temp is undefined |
| 21835 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => return true, |
| 21836 | } |
| 21837 | const temp_ty = temp.typeOf(cg); |
| 21838 | const abi_size = temp_ty.abiSize(cg.pt.zcu); |
| 21839 | return switch (op) { |
| 21840 | .implicit, .explicit, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, |
| 21074 | 21841 | .gpr => abi_size <= 8 and switch (temp.tracking(cg).short) { |
| 21075 | 21842 | .register => |reg| reg.class() == .general_purpose, |
| 21076 | 21843 | .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and |
| 21077 | 21844 | reg_off.off == 0, |
| 21078 | | else => cg.regClassForType(temp.typeOf(cg)) == .general_purpose, |
| 21845 | else => cg.regClassForType(temp_ty) == .general_purpose, |
| 21079 | 21846 | }, |
| 21080 | | .mm => abi_size <= 8 and switch (temp.tracking(cg).short) { |
| 21847 | .mm, .mm_mask, .mm_sign_mask => abi_size <= 8 and switch (temp.tracking(cg).short) { |
| 21081 | 21848 | .register => |reg| reg.class() == .mmx, |
| 21082 | 21849 | .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0, |
| 21083 | | else => cg.regClassForType(temp.typeOf(cg)) == .mmx, |
| 21850 | else => cg.regClassForType(temp_ty) == .mmx, |
| 21084 | 21851 | }, |
| 21085 | | .xmm => abi_size > 8 and abi_size <= 16 and switch (temp.tracking(cg).short) { |
| 21852 | .xmm, .xmm_mask, .xmm_sign_mask => abi_size > 8 and abi_size <= 16 and switch (temp.tracking(cg).short) { |
| 21086 | 21853 | .register => |reg| reg.class() == .sse, |
| 21087 | 21854 | .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0, |
| 21088 | | else => cg.regClassForType(temp.typeOf(cg)) == .sse, |
| 21855 | else => cg.regClassForType(temp_ty) == .sse, |
| 21089 | 21856 | }, |
| 21090 | | .ymm => abi_size > 16 and abi_size <= 32 and switch (temp.tracking(cg).short) { |
| 21857 | .ymm, .ymm_mask, .ymm_sign_mask => abi_size > 16 and abi_size <= 32 and switch (temp.tracking(cg).short) { |
| 21091 | 21858 | .register => |reg| reg.class() == .sse, |
| 21092 | 21859 | .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0, |
| 21093 | | else => cg.regClassForType(temp.typeOf(cg)) == .sse, |
| 21860 | else => cg.regClassForType(temp_ty) == .sse, |
| 21094 | 21861 | }, |
| 21095 | 21862 | .mem, .mem_limb => (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(), |
| 21096 | | .gpr_limb => abi_size > 8, |
| 21097 | | .mm_limb => abi_size > 8 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory() and cg.regClassForType(temp.typeOf(cg)) == .mmx, |
| 21098 | | .xmm_limb => abi_size > 16 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(), |
| 21099 | | .ymm_limb => abi_size > 32 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(), |
| 21863 | .gpr_limb => abi_size > 8 and switch (temp.tracking(cg).short) { |
| 21864 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { |
| 21865 | if (reg.class() != .general_purpose) break false; |
| 21866 | } else true, |
| 21867 | else => |mcv| mcv.isMemory(), |
| 21868 | }, |
| 21869 | .mm_limb, .mm_mask_limb => abi_size > 8 and switch (temp.tracking(cg).short) { |
| 21870 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { |
| 21871 | if (reg.class() != .mmx) break false; |
| 21872 | } else true, |
| 21873 | else => |mcv| mcv.isMemory() and cg.regClassForType(temp_ty) == .mmx, |
| 21874 | }, |
| 21875 | .xmm_limb, .xmm_mask_limb => abi_size > 16 and switch (temp.tracking(cg).short) { |
| 21876 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { |
| 21877 | if (reg.class() != .sse) break false; |
| 21878 | } else true, |
| 21879 | else => |mcv| mcv.isMemory(), |
| 21880 | }, |
| 21881 | .ymm_limb, .ymm_mask_limb => abi_size > 32 and switch (temp.tracking(cg).short) { |
| 21882 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { |
| 21883 | if (reg.class() != .sse) break false; |
| 21884 | } else true, |
| 21885 | else => |mcv| mcv.isMemory(), |
| 21886 | }, |
| 21100 | 21887 | .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(cg).short) { |
| 21101 | 21888 | .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm, |
| 21102 | 21889 | else => false, |
| ... | ... | @@ -21109,78 +21896,176 @@ const Pattern = struct { |
| 21109 | 21896 | } |
| 21110 | 21897 | }; |
| 21111 | 21898 | }; |
| 21112 | | fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_sets: []const Pattern.Set) !void { |
| 21113 | | var loop: SelectLoop = .{ |
| 21899 | const SelectOptions = struct { |
| 21900 | invert_result: bool = false, |
| 21901 | }; |
| 21902 | fn select( |
| 21903 | cg: *CodeGen, |
| 21904 | dst_temps: []Temp, |
| 21905 | dst_tys: []const Type, |
| 21906 | src_temps: []Temp, |
| 21907 | pattern_sets: []const Pattern.Set, |
| 21908 | opts: SelectOptions, |
| 21909 | ) !void { |
| 21910 | var loop: struct { |
| 21911 | element_reloc: Mir.Inst.Index, |
| 21912 | element_offset: Offset, |
| 21913 | element_size: ?u13, |
| 21914 | limb_reloc: Mir.Inst.Index, |
| 21915 | limb_offset: Offset, |
| 21916 | limb_size: ?u8, |
| 21917 | mask_limb_temp: ?Temp, |
| 21918 | mask_limb_offset: Offset, |
| 21919 | mask_limb_offset_lock: ?RegisterLock, |
| 21920 | mask_limb_bit_size: ?u7, |
| 21921 | mask_store_temp: ?Temp, |
| 21922 | mask_store_reg: ?Register, |
| 21923 | mask_store_bit_size: ?u7, |
| 21924 | remaining_size: ?u64, |
| 21925 | |
| 21926 | const Offset = union(enum) { |
| 21927 | unused, |
| 21928 | known: u31, |
| 21929 | temp: Temp, |
| 21930 | }; |
| 21931 | } = .{ |
| 21114 | 21932 | .element_reloc = undefined, |
| 21115 | 21933 | .element_offset = .unused, |
| 21116 | 21934 | .element_size = null, |
| 21117 | 21935 | .limb_reloc = undefined, |
| 21118 | 21936 | .limb_offset = .unused, |
| 21119 | 21937 | .limb_size = null, |
| 21938 | .mask_limb_temp = null, |
| 21939 | .mask_limb_offset = .unused, |
| 21940 | .mask_limb_offset_lock = null, |
| 21941 | .mask_limb_bit_size = null, |
| 21942 | .mask_store_temp = null, |
| 21943 | .mask_store_reg = null, |
| 21944 | .mask_store_bit_size = null, |
| 21120 | 21945 | .remaining_size = null, |
| 21121 | 21946 | }; |
| 21122 | 21947 | var extra_temps: [4]?Temp = @splat(null); |
| 21123 | 21948 | pattern_sets: for (pattern_sets) |pattern_set| { |
| 21124 | 21949 | for (pattern_set.required_features) |required_feature| if (!cg.hasFeature(required_feature)) continue :pattern_sets; |
| 21950 | for (src_temps) |src_temp| switch (pattern_set.scalar) { |
| 21951 | .any => {}, |
| 21952 | .bool => if (src_temp.typeOf(cg).scalarType(cg.pt.zcu).toIntern() != .bool_type) continue :pattern_sets, |
| 21953 | .float => |size| { |
| 21954 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| 21955 | if (!scalar_ty.isRuntimeFloat()) continue :pattern_sets; |
| 21956 | if (scalar_ty.floatBits(cg.target.*) != size.bitSize()) continue :pattern_sets; |
| 21957 | }, |
| 21958 | .any_int => |size| { |
| 21959 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| 21960 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; |
| 21961 | if (scalar_ty.intInfo(cg.pt.zcu).bits > size.bitSize()) continue :pattern_sets; |
| 21962 | }, |
| 21963 | .signed_int => |size| { |
| 21964 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| 21965 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; |
| 21966 | const scalar_info = scalar_ty.intInfo(cg.pt.zcu); |
| 21967 | if (scalar_info.signedness != .signed) continue :pattern_sets; |
| 21968 | if (scalar_info.bits > size.bitSize()) continue :pattern_sets; |
| 21969 | }, |
| 21970 | .unsigned_int => |size| { |
| 21971 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| 21972 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; |
| 21973 | const scalar_info = scalar_ty.intInfo(cg.pt.zcu); |
| 21974 | if (scalar_info.signedness != .unsigned) continue :pattern_sets; |
| 21975 | if (scalar_info.bits > size.bitSize()) continue :pattern_sets; |
| 21976 | }, |
| 21977 | .any_float_or_int => |size| { |
| 21978 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); |
| 21979 | if (scalar_ty.isRuntimeFloat()) { |
| 21980 | if (scalar_ty.floatBits(cg.target.*) != size.bitSize()) continue :pattern_sets; |
| 21981 | } else if (scalar_ty.isAbiInt(cg.pt.zcu)) { |
| 21982 | if (scalar_ty.intInfo(cg.pt.zcu).bits > size.bitSize()) continue :pattern_sets; |
| 21983 | } else continue :pattern_sets; |
| 21984 | }, |
| 21985 | }; |
| 21125 | 21986 | patterns: for (pattern_set.patterns) |pattern| { |
| 21126 | 21987 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { |
| 21127 | 21988 | const ref_src_op, const is_mut = switch (src_op) { |
| 21128 | | .implicit, .explicit => |op_index| .{ pattern.ops[op_index], true }, |
| 21989 | .implicit, .explicit => |linked_index| .{ pattern.ops[linked_index], true }, |
| 21129 | 21990 | else => .{ src_op, false }, |
| 21130 | 21991 | }; |
| 21131 | | if (!ref_src_op.matches(is_mut, src_temp.*, cg)) continue :patterns; |
| 21992 | if (!ref_src_op.matches(is_mut, src_temp, cg)) continue :patterns; |
| 21132 | 21993 | } |
| 21133 | | while (true) for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { |
| 21134 | | if (switch (switch (src_op) { |
| 21135 | | .implicit, .explicit => |op_index| pattern.ops[op_index], |
| 21994 | |
| 21995 | while (true) for (src_temps, pattern.ops[dst_temps.len..]) |*src_temp, src_op| { |
| 21996 | if (changed: switch (switch (src_op) { |
| 21997 | .implicit, .explicit => |linked_index| pattern.ops[linked_index], |
| 21136 | 21998 | else => src_op, |
| 21137 | 21999 | }) { |
| 21138 | 22000 | .implicit, .explicit => unreachable, |
| 21139 | 22001 | .gpr => try src_temp.toRegClass(.general_purpose, cg), |
| 21140 | | .mm => try src_temp.toRegClass(.mmx, cg), |
| 21141 | | .xmm, .ymm => try src_temp.toRegClass(.sse, cg), |
| 21142 | | .mem, .imm, .simm32 => false, |
| 21143 | | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb, .mem_limb => switch (src_temp.tracking(cg).short) { |
| 21144 | | .register_pair => false, |
| 22002 | .mm, .mm_mask, .mm_sign_mask => try src_temp.toRegClass(.mmx, cg), |
| 22003 | .xmm, |
| 22004 | .ymm, |
| 22005 | .xmm_mask, |
| 22006 | .ymm_mask, |
| 22007 | .xmm_sign_mask, |
| 22008 | .ymm_sign_mask, |
| 22009 | => try src_temp.toRegClass(.sse, cg), |
| 22010 | .mem => try src_temp.toBase(cg), |
| 22011 | .imm, .simm32 => false, |
| 22012 | .gpr_limb, |
| 22013 | .mm_limb, |
| 22014 | .xmm_limb, |
| 22015 | .ymm_limb, |
| 22016 | .mem_limb, |
| 22017 | => switch (src_temp.tracking(cg).short) { |
| 22018 | .register_pair, .register_triple, .register_quadruple => false, |
| 21145 | 22019 | else => try src_temp.toBase(cg), |
| 21146 | 22020 | }, |
| 22021 | .mm_mask_limb, |
| 22022 | .xmm_mask_limb, |
| 22023 | .ymm_mask_limb, |
| 22024 | => if (!cg.hasFeature(.bmi2) and !cg.register_manager.isKnownRegFree(.rcx)) { |
| 22025 | try cg.register_manager.getKnownReg(.rcx, null); |
| 22026 | loop.mask_limb_offset_lock = cg.register_manager.lockKnownRegAssumeUnused(.rcx); |
| 22027 | break :changed true; |
| 22028 | } else false, |
| 22029 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => false, |
| 21147 | 22030 | }) break; |
| 21148 | 22031 | } else break; |
| 22032 | |
| 22033 | var dst_is_linked: std.StaticBitSet(4) = .initEmpty(); |
| 21149 | 22034 | var mir_ops_len = dst_temps.len; |
| 21150 | 22035 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { |
| 21151 | | const ref_src_op, const extra_temp = op: switch (src_op) { |
| 21152 | | .implicit => |op_index| { |
| 21153 | | dst_temps[op_index] = if (src_temp.isMut(cg)) |
| 21154 | | src_temp.* |
| 21155 | | else |
| 21156 | | try cg.tempAlloc(src_temp.typeOf(cg)); |
| 21157 | | break :op .{ pattern.ops[op_index], &extra_temps[op_index] }; |
| 21158 | | }, |
| 21159 | | .explicit => |op_index| { |
| 21160 | | dst_temps[op_index] = if (src_temp.isMut(cg)) |
| 21161 | | src_temp.* |
| 21162 | | else |
| 21163 | | try cg.tempAlloc(src_temp.typeOf(cg)); |
| 21164 | | defer mir_ops_len += 1; |
| 21165 | | break :op .{ pattern.ops[op_index], &extra_temps[mir_ops_len] }; |
| 21166 | | }, |
| 21167 | | else => { |
| 21168 | | defer mir_ops_len += 1; |
| 21169 | | break :op .{ src_op, &extra_temps[mir_ops_len] }; |
| 22036 | defer mir_ops_len += @intFromBool(src_op != .implicit); |
| 22037 | const linked_src_op, const extra_temp = op: switch (src_op) { |
| 22038 | .implicit, .explicit => |linked_index| { |
| 22039 | if (src_temp.isMut(cg)) { |
| 22040 | dst_temps[linked_index] = src_temp; |
| 22041 | dst_is_linked.set(linked_index); |
| 22042 | } |
| 22043 | break :op .{ pattern.ops[linked_index], &extra_temps[linked_index] }; |
| 21170 | 22044 | }, |
| 22045 | else => .{ src_op, &extra_temps[mir_ops_len] }, |
| 21171 | 22046 | }; |
| 21172 | | const limb_size: u8, const rc = switch (ref_src_op) { |
| 22047 | const limb_size: u8, const rc = switch (linked_src_op) { |
| 21173 | 22048 | else => continue, |
| 21174 | 22049 | .gpr_limb => .{ 8, abi.RegisterClass.gp }, |
| 21175 | | .mm_limb => .{ 8, @panic("TODO") }, |
| 21176 | | .xmm_limb => .{ 16, abi.RegisterClass.sse }, |
| 21177 | | .ymm_limb => .{ 32, abi.RegisterClass.sse }, |
| 22050 | .mm_limb, .mm_mask_limb => .{ 8, @panic("TODO") }, |
| 22051 | .xmm_limb, .xmm_mask_limb => .{ 16, abi.RegisterClass.sse }, |
| 22052 | .ymm_limb, .ymm_mask_limb => .{ 32, abi.RegisterClass.sse }, |
| 22053 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => { |
| 22054 | assert(extra_temp.* == null); |
| 22055 | extra_temp.* = try cg.tempAllocReg(Type.usize, switch (linked_src_op) { |
| 22056 | else => unreachable, |
| 22057 | .umax_gpr => abi.RegisterClass.gp, |
| 22058 | .umax_mm => @panic("TODO"), |
| 22059 | .umax_xmm, .umax_ymm => abi.RegisterClass.sse, |
| 22060 | }); |
| 22061 | continue; |
| 22062 | }, |
| 21178 | 22063 | }; |
| 21179 | 22064 | assert(loop.limb_size == null or loop.limb_size == limb_size); |
| 21180 | 22065 | loop.limb_size = limb_size; |
| 21181 | 22066 | loop.remaining_size = loop.remaining_size orelse src_temp.typeOf(cg).abiSize(cg.pt.zcu); |
| 21182 | 22067 | switch (src_temp.tracking(cg).short) { |
| 21183 | | .register_pair => switch (loop.limb_offset) { |
| 22068 | .register_pair, .register_triple, .register_quadruple => switch (loop.limb_offset) { |
| 21184 | 22069 | .unused, .temp => loop.limb_offset = .{ .known = 0 }, |
| 21185 | 22070 | .known => {}, |
| 21186 | 22071 | }, |
| ... | ... | @@ -21189,25 +22074,130 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21189 | 22074 | .unused => loop.limb_offset = .{ .temp = undefined }, |
| 21190 | 22075 | .known, .temp => {}, |
| 21191 | 22076 | } |
| 22077 | assert(extra_temp.* == null); |
| 21192 | 22078 | extra_temp.* = try cg.tempAllocReg(Type.usize, rc); |
| 21193 | 22079 | }, |
| 21194 | 22080 | } |
| 21195 | 22081 | } |
| 22082 | for ( |
| 22083 | 0.., |
| 22084 | dst_temps, |
| 22085 | pattern.ops[0..dst_temps.len], |
| 22086 | dst_tys, |
| 22087 | extra_temps[0..dst_temps.len], |
| 22088 | ) |dst_index, *dst_temp, dst_op, dst_ty, *extra_temp| switch (dst_op) { |
| 22089 | else => if (!dst_is_linked.isSet(dst_index)) { |
| 22090 | dst_temp.* = dst_temp: switch (dst_op) { |
| 22091 | .implicit => unreachable, |
| 22092 | .explicit => |linked_index| dst_temps[linked_index], |
| 22093 | .gpr => try cg.tempAllocReg(dst_ty, abi.RegisterClass.gp), |
| 22094 | .mm, .mm_mask, .mm_sign_mask => @panic("TODO"), |
| 22095 | .xmm, .xmm_mask, .xmm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse), |
| 22096 | .ymm, .ymm_mask, .ymm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse), |
| 22097 | .mem => @panic("TODO"), |
| 22098 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => { |
| 22099 | if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(Type.usize, switch (dst_op) { |
| 22100 | else => unreachable, |
| 22101 | .gpr_limb => abi.RegisterClass.gp, |
| 22102 | .mm_limb => @panic("TODO"), |
| 22103 | .xmm_limb, .ymm_limb => abi.RegisterClass.sse, |
| 22104 | }); |
| 22105 | break :dst_temp try cg.tempAlloc(dst_ty); |
| 22106 | }, |
| 22107 | .mem_limb => try cg.tempAlloc(dst_ty), |
| 22108 | .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => unreachable, // already checked |
| 22109 | .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination |
| 22110 | }; |
| 22111 | }, |
| 22112 | .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => { |
| 22113 | const scalar_size = @divExact(switch (pattern_set.scalar) { |
| 22114 | .any, .bool => unreachable, |
| 22115 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, |
| 22116 | }.bitSize(), 8); |
| 22117 | const mask_bit_size = @divExact(loop.remaining_size.?, scalar_size); |
| 22118 | const mask_limb_bit_size: u7 = @intCast(@divExact(loop.limb_size.?, scalar_size)); |
| 22119 | assert(loop.mask_limb_bit_size == null or loop.mask_limb_bit_size == mask_limb_bit_size); |
| 22120 | loop.mask_limb_bit_size = mask_limb_bit_size; |
| 22121 | const mask_store_bit_size = mask_store_bit_size: { |
| 22122 | // Try to match limb size so that no shifting will be needed. |
| 22123 | if (mask_limb_bit_size % 8 == 0) break :mask_store_bit_size mask_limb_bit_size; |
| 22124 | // If abi size <= 8 the entire value can be stored at once, |
| 22125 | // enabling store forwarding and minimizing store buffer usage. |
| 22126 | // Otherwise, we will be performing shifts that need to wrap at |
| 22127 | // store size, which for x86 requires 32 or 64, so just pick 64 |
| 22128 | // for the same reasons as above. |
| 22129 | break :mask_store_bit_size @min(mask_bit_size, 64); |
| 22130 | }; |
| 22131 | assert(loop.mask_store_bit_size == null or loop.mask_store_bit_size == mask_store_bit_size); |
| 22132 | loop.mask_store_bit_size = mask_store_bit_size; |
| 22133 | loop.mask_limb_offset = loop.limb_offset; |
| 22134 | if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(Type.usize, switch (dst_op) { |
| 22135 | else => unreachable, |
| 22136 | .mm_mask_limb => @panic("TODO"), |
| 22137 | .xmm_mask_limb, .ymm_mask_limb => abi.RegisterClass.sse, |
| 22138 | }); |
| 22139 | if (loop.mask_limb_temp == null) loop.mask_limb_temp = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| 22140 | if (mask_limb_bit_size < mask_store_bit_size and loop.mask_store_reg == null) { |
| 22141 | loop.mask_store_temp = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| 22142 | loop.mask_store_reg = loop.mask_store_temp.?.tracking(cg).short.register; |
| 22143 | } |
| 22144 | dst_temp.* = if (mask_store_bit_size < mask_bit_size) |
| 22145 | try cg.tempAllocMem(dst_ty) |
| 22146 | else if (loop.mask_store_temp) |mask_store_temp| dst_temp: { |
| 22147 | loop.mask_store_temp = null; |
| 22148 | break :dst_temp mask_store_temp; |
| 22149 | } else try cg.tempAlloc(dst_ty); |
| 22150 | }, |
| 22151 | }; |
| 22152 | switch (loop.mask_limb_offset) { |
| 22153 | .unused, .known => {}, |
| 22154 | .temp => |*mask_limb_offset| { |
| 22155 | if (cg.hasFeature(.bmi2)) { |
| 22156 | assert(loop.mask_limb_offset_lock == null); |
| 22157 | mask_limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| 22158 | } else { |
| 22159 | if (loop.mask_limb_offset_lock) |lock| cg.register_manager.unlockReg(lock); |
| 22160 | loop.mask_limb_offset_lock = null; |
| 22161 | mask_limb_offset.* = try cg.tempFromValue(Type.usize, .{ .register = .rcx }); |
| 22162 | } |
| 22163 | if (loop.mask_store_reg) |mask_store_reg| { |
| 22164 | const mask_store_alias = registerAlias( |
| 22165 | mask_store_reg, |
| 22166 | @min(std.math.divCeil(u7, loop.mask_store_bit_size.?, 8) catch unreachable, 4), |
| 22167 | ); |
| 22168 | try cg.spillEflagsIfOccupied(); |
| 22169 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); |
| 22170 | } |
| 22171 | }, |
| 22172 | } |
| 21196 | 22173 | switch (loop.element_offset) { |
| 21197 | 22174 | .unused, .known => {}, |
| 21198 | 22175 | .temp => |*element_offset| { |
| 21199 | 22176 | element_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| 21200 | 22177 | const element_offset_reg = element_offset.tracking(cg).short.register; |
| 22178 | try cg.spillEflagsIfOccupied(); |
| 21201 | 22179 | try cg.asmRegisterRegister(.{ ._, .xor }, element_offset_reg.to32(), element_offset_reg.to32()); |
| 21202 | 22180 | loop.element_reloc = @intCast(cg.mir_instructions.len); |
| 21203 | 22181 | }, |
| 21204 | 22182 | } |
| 22183 | switch (loop.limb_offset) { |
| 22184 | .unused, .known => {}, |
| 22185 | .temp => |*limb_offset| limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp), |
| 22186 | } |
| 21205 | 22187 | while (true) { |
| 22188 | switch (loop.mask_limb_offset) { |
| 22189 | .unused, .known => {}, |
| 22190 | .temp => |mask_limb_offset| { |
| 22191 | const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32(); |
| 22192 | try cg.spillEflagsIfOccupied(); |
| 22193 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_limb_offset_reg, mask_limb_offset_reg); |
| 22194 | }, |
| 22195 | } |
| 21206 | 22196 | switch (loop.limb_offset) { |
| 21207 | 22197 | .unused, .known => {}, |
| 21208 | | .temp => |*limb_offset| { |
| 21209 | | limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| 21210 | | const limb_offset_reg = limb_offset.tracking(cg).short.register; |
| 22198 | .temp => |limb_offset| { |
| 22199 | const limb_offset_reg = limb_offset.tracking(cg).short.register.to32(); |
| 22200 | try cg.spillEflagsIfOccupied(); |
| 21211 | 22201 | try cg.asmRegisterRegister(.{ ._, .xor }, limb_offset_reg.to32(), limb_offset_reg.to32()); |
| 21212 | 22202 | loop.limb_reloc = @intCast(cg.mir_instructions.len); |
| 21213 | 22203 | }, |
| ... | ... | @@ -21216,56 +22206,67 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21216 | 22206 | var mir_ops: [4]Operand = @splat(.none); |
| 21217 | 22207 | mir_ops_len = dst_temps.len; |
| 21218 | 22208 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { |
| 21219 | | const mir_op, const ref_src_op, const extra_temp = op: switch (src_op) { |
| 21220 | | .implicit => |op_index| .{ &mir_ops[op_index], pattern.ops[op_index], &extra_temps[op_index] }, |
| 21221 | | .explicit => |op_index| { |
| 21222 | | defer mir_ops_len += 1; |
| 21223 | | break :op .{ &mir_ops[mir_ops_len], pattern.ops[op_index], &extra_temps[mir_ops_len] }; |
| 21224 | | }, |
| 21225 | | else => { |
| 21226 | | defer mir_ops_len += 1; |
| 21227 | | break :op .{ &mir_ops[mir_ops_len], src_op, &extra_temps[mir_ops_len] }; |
| 21228 | | }, |
| 22209 | defer mir_ops_len += @intFromBool(src_op != .implicit); |
| 22210 | const mir_op, const linked_src_op, const extra_temp = switch (src_op) { |
| 22211 | .implicit => |linked_index| .{ &mir_ops[linked_index], pattern.ops[linked_index], extra_temps[linked_index] }, |
| 22212 | .explicit => |linked_index| .{ &mir_ops[mir_ops_len], pattern.ops[linked_index], extra_temps[linked_index] }, |
| 22213 | else => .{ &mir_ops[mir_ops_len], src_op, extra_temps[mir_ops_len] }, |
| 21229 | 22214 | }; |
| 21230 | | const src_mcv = src_temp.tracking(cg).short; |
| 21231 | | switch (ref_src_op) { |
| 21232 | | else => {}, |
| 21233 | | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (src_mcv) { |
| 21234 | | .register_pair => {}, |
| 21235 | | else => try cg.asmRegisterMemory( |
| 21236 | | switch (ref_src_op) { |
| 21237 | | else => unreachable, |
| 21238 | | .gpr_limb => .{ ._, .mov }, |
| 21239 | | .mm_limb => .{ ._q, .mov }, |
| 21240 | | .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, |
| 22215 | const src_mcv = switch (linked_src_op) { |
| 22216 | else => src_temp, |
| 22217 | // src_temp is undefined |
| 22218 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => extra_temp.?, |
| 22219 | }.tracking(cg).short; |
| 22220 | copy_limb: switch (src_mcv) { |
| 22221 | .register_pair, .register_triple, .register_quadruple => {}, |
| 22222 | else => try cg.asmRegisterMemory( |
| 22223 | switch (linked_src_op) { |
| 22224 | else => break :copy_limb, |
| 22225 | .gpr_limb => .{ ._, .mov }, |
| 22226 | .mm_limb, .mm_mask_limb => .{ ._q, .mov }, |
| 22227 | .xmm_limb, |
| 22228 | .ymm_limb, |
| 22229 | .xmm_mask_limb, |
| 22230 | .ymm_mask_limb, |
| 22231 | => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, |
| 22232 | }, |
| 22233 | registerAlias(extra_temp.?.tracking(cg).short.register, loop.limb_size.?), |
| 22234 | try src_mcv.mem(cg, switch (loop.limb_offset) { |
| 22235 | .unused => unreachable, |
| 22236 | .known => |limb_offset| .{ |
| 22237 | .size = .fromSize(loop.limb_size.?), |
| 22238 | .disp = limb_offset, |
| 21241 | 22239 | }, |
| 21242 | | registerAlias(extra_temp.*.?.tracking(cg).short.register, loop.limb_size.?), |
| 21243 | | try src_mcv.mem(cg, switch (loop.limb_offset) { |
| 21244 | | .unused => unreachable, |
| 21245 | | .known => |limb_offset| .{ |
| 21246 | | .size = .fromSize(loop.limb_size.?), |
| 21247 | | .disp = limb_offset, |
| 21248 | | }, |
| 21249 | | .temp => |limb_offset| .{ |
| 21250 | | .size = .fromSize(loop.limb_size.?), |
| 21251 | | .index = limb_offset.tracking(cg).short.register.to64(), |
| 21252 | | }, |
| 21253 | | }), |
| 21254 | | ), |
| 21255 | | }, |
| 22240 | .temp => |limb_offset| .{ |
| 22241 | .size = .fromSize(loop.limb_size.?), |
| 22242 | .index = limb_offset.tracking(cg).short.register.to64(), |
| 22243 | }, |
| 22244 | }), |
| 22245 | ), |
| 21256 | 22246 | } |
| 21257 | | mir_op.* = switch (ref_src_op) { |
| 22247 | mir_op.* = switch (linked_src_op) { |
| 21258 | 22248 | .implicit, .explicit => unreachable, |
| 21259 | 22249 | .gpr => .{ .reg = registerAlias( |
| 21260 | 22250 | src_mcv.register, |
| 21261 | 22251 | @intCast(src_temp.typeOf(cg).abiSize(cg.pt.zcu)), |
| 21262 | 22252 | ) }, |
| 21263 | | .mm => .{ .reg = src_mcv.register }, |
| 21264 | | .xmm => .{ .reg = src_mcv.register.to128() }, |
| 21265 | | .ymm => .{ .reg = src_mcv.register.to256() }, |
| 22253 | .umax_gpr => .{ .reg = src_mcv.register.to64() }, // TODO: use other op size? |
| 22254 | .mm, .mm_mask, .mm_sign_mask, .umax_mm => .{ .reg = src_mcv.register }, |
| 22255 | .xmm, .xmm_mask, .xmm_sign_mask, .umax_xmm => .{ .reg = src_mcv.register.to128() }, |
| 22256 | .ymm, .ymm_mask, .ymm_sign_mask, .umax_ymm => .{ .reg = src_mcv.register.to256() }, |
| 21266 | 22257 | .mem => .{ .mem = try src_mcv.mem(cg, .{ .size = cg.memSize(src_temp.typeOf(cg)) }) }, |
| 21267 | | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (src_mcv) { |
| 21268 | | .register_pair => |src_regs| switch (loop.limb_offset) { |
| 22258 | .gpr_limb, |
| 22259 | .mm_limb, |
| 22260 | .xmm_limb, |
| 22261 | .ymm_limb, |
| 22262 | .mm_mask_limb, |
| 22263 | .xmm_mask_limb, |
| 22264 | .ymm_mask_limb, |
| 22265 | => switch (src_mcv) { |
| 22266 | inline .register_pair, |
| 22267 | .register_triple, |
| 22268 | .register_quadruple, |
| 22269 | => |src_regs| switch (loop.limb_offset) { |
| 21269 | 22270 | .unused => unreachable, |
| 21270 | 22271 | .known => |limb_offset| .{ .reg = registerAlias( |
| 21271 | 22272 | src_regs[@divExact(limb_offset, loop.limb_size.?)], |
| ... | ... | @@ -21274,12 +22275,12 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21274 | 22275 | .temp => unreachable, |
| 21275 | 22276 | }, |
| 21276 | 22277 | else => .{ .reg = registerAlias( |
| 21277 | | extra_temp.*.?.tracking(cg).short.register, |
| 22278 | extra_temp.?.tracking(cg).short.register, |
| 21278 | 22279 | loop.limb_size.?, |
| 21279 | 22280 | ) }, |
| 21280 | 22281 | }, |
| 21281 | 22282 | .mem_limb => .{ .mem = switch (src_mcv) { |
| 21282 | | .register_pair => unreachable, |
| 22283 | .register_pair, .register_triple, .register_quadruple => unreachable, |
| 21283 | 22284 | else => switch (loop.limb_offset) { |
| 21284 | 22285 | .unused => unreachable, |
| 21285 | 22286 | .known => |limb_offset| try src_mcv.mem(cg, .{ |
| ... | ... | @@ -21309,76 +22310,53 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21309 | 22310 | .u(@as(u32, @intCast(src_mcv.immediate))) }, |
| 21310 | 22311 | }, |
| 21311 | 22312 | }; |
| 22313 | switch (src_op) { |
| 22314 | else => {}, |
| 22315 | .explicit => |linked_index| mir_ops[linked_index] = mir_op.*, |
| 22316 | } |
| 21312 | 22317 | } |
| 21313 | 22318 | for ( |
| 22319 | mir_ops[0..dst_temps.len], |
| 21314 | 22320 | pattern.ops[0..dst_temps.len], |
| 21315 | 22321 | dst_temps, |
| 21316 | | mir_ops[0..dst_temps.len], |
| 22322 | dst_tys, |
| 21317 | 22323 | extra_temps[0..dst_temps.len], |
| 21318 | | ) |dst_op, *dst_temp, *mir_op, *extra_temp| { |
| 22324 | ) |*mir_op, dst_op, dst_temp, dst_ty, extra_temp| { |
| 21319 | 22325 | if (mir_op.* != .none) continue; |
| 21320 | | const ty = src_temps[0].typeOf(cg); |
| 21321 | | switch (dst_op) { |
| 22326 | mir_op.* = switch (dst_op) { |
| 21322 | 22327 | .implicit => unreachable, |
| 21323 | | .explicit => |op_index| { |
| 21324 | | dst_temp.* = dst_temps[op_index]; |
| 21325 | | mir_op.* = mir_ops[op_index]; |
| 21326 | | }, |
| 21327 | | .gpr => { |
| 21328 | | dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.gp); |
| 21329 | | mir_op.* = .{ .reg = registerAlias( |
| 21330 | | dst_temp.tracking(cg).short.register, |
| 21331 | | @intCast(ty.abiSize(cg.pt.zcu)), |
| 21332 | | ) }; |
| 21333 | | }, |
| 21334 | | .mm => @panic("TODO"), |
| 21335 | | .xmm => { |
| 21336 | | dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.sse); |
| 21337 | | mir_op.* = .{ .reg = dst_temp.tracking(cg).short.register.to128() }; |
| 21338 | | }, |
| 21339 | | .ymm => { |
| 21340 | | dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.sse); |
| 21341 | | mir_op.* = .{ .reg = dst_temp.tracking(cg).short.register.to256() }; |
| 21342 | | }, |
| 22328 | .explicit => |linked_index| mir_ops[linked_index], |
| 22329 | .gpr => .{ .reg = registerAlias( |
| 22330 | dst_temp.tracking(cg).short.register, |
| 22331 | @intCast(dst_ty.abiSize(cg.pt.zcu)), |
| 22332 | ) }, |
| 22333 | .mm, .mm_mask, .mm_sign_mask => @panic("TODO"), |
| 22334 | .xmm, .xmm_mask, .xmm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to128() }, |
| 22335 | .ymm, .ymm_mask, .ymm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to256() }, |
| 21343 | 22336 | .mem => @panic("TODO"), |
| 21344 | | .gpr_limb => { |
| 21345 | | dst_temp.* = try cg.tempAlloc(ty); |
| 21346 | | extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp); |
| 21347 | | mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to64() }; |
| 21348 | | }, |
| 21349 | | .mm_limb => { |
| 21350 | | dst_temp.* = try cg.tempAlloc(ty); |
| 21351 | | extra_temp.* = try cg.tempAllocReg(Type.usize, @panic("TODO")); |
| 21352 | | mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register }; |
| 21353 | | }, |
| 21354 | | .xmm_limb => { |
| 21355 | | dst_temp.* = try cg.tempAlloc(ty); |
| 21356 | | extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.sse); |
| 21357 | | mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to128() }; |
| 21358 | | }, |
| 21359 | | .ymm_limb => { |
| 21360 | | dst_temp.* = try cg.tempAlloc(ty); |
| 21361 | | extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.sse); |
| 21362 | | mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to256() }; |
| 21363 | | }, |
| 21364 | | .mem_limb => { |
| 21365 | | dst_temp.* = try cg.tempAlloc(ty); |
| 21366 | | mir_op.* = .{ .mem = try dst_temp.tracking(cg).short.mem(cg, switch (loop.limb_offset) { |
| 21367 | | .unused => unreachable, |
| 21368 | | .known => |limb_offset| .{ |
| 21369 | | .size = .fromSize(loop.limb_size.?), |
| 21370 | | .disp = limb_offset, |
| 21371 | | }, |
| 21372 | | .temp => |limb_offset| .{ |
| 21373 | | .size = .fromSize(loop.limb_size.?), |
| 21374 | | .index = limb_offset.tracking(cg).short.register.to64(), |
| 21375 | | }, |
| 21376 | | }) }; |
| 21377 | | }, |
| 21378 | | .imm, .simm32 => unreachable, // unmodifiable destination |
| 21379 | | } |
| 22337 | .gpr_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to64() }, |
| 22338 | .mm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register }, |
| 22339 | .xmm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() }, |
| 22340 | .ymm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() }, |
| 22341 | .mem_limb => .{ .mem = try dst_temp.tracking(cg).short.mem(cg, switch (loop.limb_offset) { |
| 22342 | .unused => unreachable, |
| 22343 | .known => |limb_offset| .{ |
| 22344 | .size = .fromSize(loop.limb_size.?), |
| 22345 | .disp = limb_offset, |
| 22346 | }, |
| 22347 | .temp => |limb_offset| .{ |
| 22348 | .size = .fromSize(loop.limb_size.?), |
| 22349 | .index = limb_offset.tracking(cg).short.register.to64(), |
| 22350 | }, |
| 22351 | }) }, |
| 22352 | .mm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register }, |
| 22353 | .xmm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() }, |
| 22354 | .ymm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() }, |
| 22355 | .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination |
| 22356 | }; |
| 21380 | 22357 | } |
| 21381 | 22358 | std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]); |
| 22359 | if (pattern_set.clobbers.eflags) try cg.spillEflagsIfOccupied(); |
| 21382 | 22360 | cg.asmOps(pattern_set.mir_tag, mir_ops) catch |err| switch (err) { |
| 21383 | 22361 | error.InvalidInstruction => { |
| 21384 | 22362 | const fixes = @tagName(pattern_set.mir_tag[0]); |
| ... | ... | @@ -21398,42 +22376,223 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21398 | 22376 | }, |
| 21399 | 22377 | else => |e| return e, |
| 21400 | 22378 | }; |
| 22379 | const invert_result = opts.invert_result != pattern_set.invert_result; |
| 21401 | 22380 | for ( |
| 21402 | 22381 | extra_temps[0..dst_temps.len], |
| 21403 | 22382 | pattern.ops[0..dst_temps.len], |
| 22383 | mir_ops[0..dst_temps.len], |
| 21404 | 22384 | dst_temps, |
| 21405 | | ) |maybe_extra_temp, dst_op, dst_temp| if (maybe_extra_temp) |extra_temp| switch (dst_op) { |
| 21406 | | else => {}, |
| 21407 | | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (dst_temp.tracking(cg).short) { |
| 21408 | | .register_pair => |dst_regs| switch (loop.limb_offset) { |
| 21409 | | .unused => unreachable, |
| 21410 | | .known => |limb_offset| try cg.asmRegisterRegister( |
| 21411 | | .{ ._, .mov }, |
| 21412 | | dst_regs[@divExact(limb_offset, loop.limb_size.?)].to64(), |
| 21413 | | extra_temp.tracking(cg).short.register.to64(), |
| 22385 | ) |extra_temp, dst_op, mir_op, dst_temp| switch (dst_op) { |
| 22386 | else => if (invert_result) { |
| 22387 | try cg.spillEflagsIfOccupied(); |
| 22388 | cg.asmOps( |
| 22389 | .{ ._, .not }, |
| 22390 | .{ mir_op, .none, .none, .none }, |
| 22391 | ) catch |err| switch (err) { |
| 22392 | error.InvalidInstruction => return cg.fail( |
| 22393 | "invalid instruction: 'not {s} none none none'", |
| 22394 | .{@tagName(mir_op)}, |
| 21414 | 22395 | ), |
| 21415 | | .temp => unreachable, |
| 21416 | | }, |
| 21417 | | else => |dst_mcv| try cg.asmMemoryRegister( |
| 21418 | | switch (dst_op) { |
| 21419 | | else => unreachable, |
| 21420 | | .gpr_limb => .{ ._, .mov }, |
| 21421 | | .mm_limb => .{ ._q, .mov }, |
| 21422 | | .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, |
| 22396 | else => |e| return e, |
| 22397 | }; |
| 22398 | }, |
| 22399 | .mm_mask, |
| 22400 | .xmm_mask, |
| 22401 | .ymm_mask, |
| 22402 | .mm_sign_mask, |
| 22403 | .xmm_sign_mask, |
| 22404 | .ymm_sign_mask, |
| 22405 | => dst_temp.asMask(switch (dst_op) { |
| 22406 | else => unreachable, |
| 22407 | .mm_mask, .xmm_mask, .ymm_mask => .all, |
| 22408 | .mm_sign_mask, .xmm_sign_mask, .ymm_sign_mask => .sign, |
| 22409 | }, invert_result, switch (pattern_set.scalar) { |
| 22410 | .any, .bool => unreachable, |
| 22411 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, |
| 22412 | }, cg), |
| 22413 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => if (extra_temp) |limb_temp| |
| 22414 | switch (dst_temp.tracking(cg).short) { |
| 22415 | inline .register_pair, |
| 22416 | .register_triple, |
| 22417 | .register_quadruple, |
| 22418 | => |dst_regs| switch (loop.limb_offset) { |
| 22419 | .unused => unreachable, |
| 22420 | .known => |limb_offset| try cg.asmRegisterRegister( |
| 22421 | .{ ._, .mov }, |
| 22422 | dst_regs[@divExact(limb_offset, loop.limb_size.?)].to64(), |
| 22423 | limb_temp.tracking(cg).short.register.to64(), |
| 22424 | ), |
| 22425 | .temp => unreachable, |
| 21423 | 22426 | }, |
| 21424 | | try dst_mcv.mem(cg, switch (loop.limb_offset) { |
| 22427 | else => |dst_mcv| try cg.asmMemoryRegister( |
| 22428 | switch (dst_op) { |
| 22429 | else => unreachable, |
| 22430 | .gpr_limb => .{ ._, .mov }, |
| 22431 | .mm_limb => .{ ._q, .mov }, |
| 22432 | .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, |
| 22433 | }, |
| 22434 | try dst_mcv.mem(cg, switch (loop.limb_offset) { |
| 22435 | .unused => unreachable, |
| 22436 | .known => |limb_offset| .{ |
| 22437 | .size = .fromSize(loop.limb_size.?), |
| 22438 | .disp = limb_offset, |
| 22439 | }, |
| 22440 | .temp => |limb_offset| .{ |
| 22441 | .size = .fromSize(loop.limb_size.?), |
| 22442 | .index = limb_offset.tracking(cg).short.register.to64(), |
| 22443 | }, |
| 22444 | }), |
| 22445 | registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?), |
| 22446 | ), |
| 22447 | }, |
| 22448 | .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => { |
| 22449 | const scalar_size = switch (pattern_set.scalar) { |
| 22450 | .any, .bool => unreachable, |
| 22451 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, |
| 22452 | }; |
| 22453 | switch (scalar_size) { |
| 22454 | else => {}, |
| 22455 | .word => if (cg.hasFeature(.avx)) try cg.asmRegisterRegisterRegister( |
| 22456 | .{ .vp_b, .ackssw }, |
| 22457 | mir_op.reg, |
| 22458 | mir_op.reg, |
| 22459 | mir_op.reg, |
| 22460 | ) else try cg.asmRegisterRegister( |
| 22461 | .{ .p_b, .ackssw }, |
| 22462 | mir_op.reg, |
| 22463 | mir_op.reg, |
| 22464 | ), |
| 22465 | } |
| 22466 | const mask_store_size: u4 = |
| 22467 | @intCast(std.math.divCeil(u7, loop.mask_store_bit_size.?, 8) catch unreachable); |
| 22468 | const mask_limb_reg = registerAlias( |
| 22469 | loop.mask_limb_temp.?.tracking(cg).short.register, |
| 22470 | mask_store_size, |
| 22471 | ); |
| 22472 | try cg.asmRegisterRegister(switch (scalar_size) { |
| 22473 | else => unreachable, |
| 22474 | .byte, .word => .{ if (cg.hasFeature(.avx)) .vp_b else .p_b, .movmsk }, |
| 22475 | .dword => .{ if (cg.hasFeature(.avx)) .v_ps else ._ps, .movmsk }, |
| 22476 | .qword => .{ if (cg.hasFeature(.avx)) .v_pd else ._pd, .movmsk }, |
| 22477 | }, mask_limb_reg.to32(), mir_op.reg); |
| 22478 | if (invert_result) if (loop.mask_store_reg) |_| { |
| 22479 | try cg.spillEflagsIfOccupied(); |
| 22480 | try cg.asmRegisterImmediate( |
| 22481 | .{ ._, .xor }, |
| 22482 | registerAlias(mask_limb_reg, @min(mask_store_size, 4)), |
| 22483 | .u((@as(u32, 1) << @intCast(loop.mask_limb_bit_size.?)) - 1), |
| 22484 | ); |
| 22485 | } else try cg.asmRegister(.{ ._, .not }, mask_limb_reg); |
| 22486 | if (loop.mask_store_reg) |mask_store_reg| { |
| 22487 | const mask_store_alias = registerAlias(mask_store_reg, mask_store_size); |
| 22488 | switch (loop.mask_limb_offset) { |
| 21425 | 22489 | .unused => unreachable, |
| 21426 | | .known => |limb_offset| .{ |
| 21427 | | .size = .fromSize(loop.limb_size.?), |
| 21428 | | .disp = limb_offset, |
| 22490 | .known => |mask_limb_offset| switch (mask_limb_offset & (loop.mask_store_bit_size.? - 1)) { |
| 22491 | 0 => try cg.asmRegisterRegister(.{ ._, .mov }, mask_store_alias, mask_limb_reg), |
| 22492 | else => |shl_count| { |
| 22493 | try cg.spillEflagsIfOccupied(); |
| 22494 | try cg.asmRegisterImmediate(.{ ._l, .sh }, mask_limb_reg, .u(shl_count)); |
| 22495 | try cg.spillEflagsIfOccupied(); |
| 22496 | try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg); |
| 22497 | }, |
| 21429 | 22498 | }, |
| 21430 | | .temp => |limb_offset| .{ |
| 21431 | | .size = .fromSize(loop.limb_size.?), |
| 21432 | | .index = limb_offset.tracking(cg).short.register.to64(), |
| 22499 | .temp => |mask_limb_offset| { |
| 22500 | if (cg.hasFeature(.bmi2)) { |
| 22501 | const shlx_size = @max(mask_store_size, 4); |
| 22502 | const shlx_mask_limb_reg = registerAlias(mask_limb_reg, shlx_size); |
| 22503 | try cg.asmRegisterRegisterRegister( |
| 22504 | .{ ._lx, .sh }, |
| 22505 | shlx_mask_limb_reg, |
| 22506 | shlx_mask_limb_reg, |
| 22507 | registerAlias(mask_limb_offset.tracking(cg).short.register, shlx_size), |
| 22508 | ); |
| 22509 | } else { |
| 22510 | try cg.spillEflagsIfOccupied(); |
| 22511 | try cg.asmRegisterRegister( |
| 22512 | .{ ._l, .sh }, |
| 22513 | mask_limb_reg, |
| 22514 | mask_limb_offset.tracking(cg).short.register.to8(), |
| 22515 | ); |
| 22516 | } |
| 22517 | try cg.spillEflagsIfOccupied(); |
| 22518 | try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg); |
| 21433 | 22519 | }, |
| 21434 | | }), |
| 21435 | | registerAlias(extra_temp.tracking(cg).short.register, loop.limb_size.?), |
| 21436 | | ), |
| 22520 | } |
| 22521 | } |
| 22522 | const dst_mcv = dst_temp.tracking(cg).short; |
| 22523 | switch (loop.mask_limb_offset) { |
| 22524 | .unused => unreachable, |
| 22525 | .known => |*mask_limb_offset| { |
| 22526 | mask_limb_offset.* += loop.mask_limb_bit_size.?; |
| 22527 | if (mask_limb_offset.* & (loop.mask_store_bit_size.? - 1) == 0) { |
| 22528 | switch (dst_mcv) { |
| 22529 | .register => {}, |
| 22530 | else => try cg.asmMemoryRegister( |
| 22531 | .{ ._, .mov }, |
| 22532 | try dst_mcv.mem(cg, .{ |
| 22533 | .size = .fromSize(mask_store_size), |
| 22534 | .disp = @divExact(mask_limb_offset.*, 8) - mask_store_size, |
| 22535 | }), |
| 22536 | registerAlias(loop.mask_store_reg orelse mask_limb_reg, mask_store_size), |
| 22537 | ), |
| 22538 | } |
| 22539 | if (loop.mask_store_reg) |mask_store_reg| { |
| 22540 | const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4)); |
| 22541 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); |
| 22542 | } |
| 22543 | } |
| 22544 | }, |
| 22545 | .temp => |mask_limb_offset| { |
| 22546 | const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32(); |
| 22547 | if (loop.mask_store_reg) |mask_store_reg| { |
| 22548 | try cg.asmRegisterMemory(.{ ._, .lea }, mask_limb_offset_reg, .{ |
| 22549 | .base = .{ .reg = mask_limb_offset_reg.to64() }, |
| 22550 | .mod = .{ .rm = .{ |
| 22551 | .size = .qword, |
| 22552 | .disp = loop.mask_limb_bit_size.?, |
| 22553 | } }, |
| 22554 | }); |
| 22555 | switch (dst_mcv) { |
| 22556 | .register => {}, |
| 22557 | else => { |
| 22558 | try cg.spillEflagsIfOccupied(); |
| 22559 | try cg.asmRegisterImmediate( |
| 22560 | .{ ._, .@"test" }, |
| 22561 | mask_limb_offset_reg, |
| 22562 | .u(loop.mask_store_bit_size.? - 1), |
| 22563 | ); |
| 22564 | const skip_store_reloc = try cg.asmJccReloc(.nz, undefined); |
| 22565 | const mask_store_offset_reg = mask_limb_reg.to32(); |
| 22566 | try cg.asmRegisterRegister(.{ ._, .mov }, mask_store_offset_reg, mask_limb_offset_reg); |
| 22567 | try cg.asmRegisterImmediate(.{ ._r, .sh }, mask_store_offset_reg, .u(3)); |
| 22568 | try cg.asmMemoryRegister(.{ ._, .mov }, try dst_mcv.mem(cg, .{ |
| 22569 | .size = .fromSize(mask_store_size), |
| 22570 | .index = mask_store_offset_reg.to64(), |
| 22571 | .disp = -@as(i8, mask_store_size), |
| 22572 | }), registerAlias(mask_store_reg, mask_store_size)); |
| 22573 | const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4)); |
| 22574 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); |
| 22575 | cg.performReloc(skip_store_reloc); |
| 22576 | }, |
| 22577 | } |
| 22578 | } else { |
| 22579 | switch (dst_mcv) { |
| 22580 | .register => {}, |
| 22581 | else => try cg.asmMemoryRegister(.{ ._, .mov }, try dst_mcv.mem(cg, .{ |
| 22582 | .size = .fromSize(mask_store_size), |
| 22583 | .index = mask_limb_offset_reg.to64(), |
| 22584 | }), mask_limb_reg), |
| 22585 | } |
| 22586 | try cg.asmRegisterMemory(.{ ._, .lea }, mask_limb_offset_reg, .{ |
| 22587 | .base = .{ .reg = mask_limb_offset_reg.to64() }, |
| 22588 | .mod = .{ .rm = .{ |
| 22589 | .size = .qword, |
| 22590 | .disp = mask_store_size, |
| 22591 | } }, |
| 22592 | }); |
| 22593 | } |
| 22594 | }, |
| 22595 | } |
| 21437 | 22596 | }, |
| 21438 | 22597 | }; |
| 21439 | 22598 | switch (pattern_set.loop) { |
| ... | ... | @@ -21442,7 +22601,7 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21442 | 22601 | .limbwise_carry => @panic("TODO"), |
| 21443 | 22602 | .limbwise_pairs_forward => @panic("TODO"), |
| 21444 | 22603 | .limbwise_pairs_reverse => @panic("TODO"), |
| 21445 | | .elementwise => @panic("TODO"), |
| 22604 | .elementwise => {}, |
| 21446 | 22605 | } |
| 21447 | 22606 | switch (loop.limb_offset) { |
| 21448 | 22607 | .unused => break, |
| ... | ... | @@ -21452,6 +22611,11 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21452 | 22611 | if (loop.remaining_size.? < loop.limb_size.? or |
| 21453 | 22612 | (loop.element_size != null and limb_offset.* >= loop.element_size.?)) |
| 21454 | 22613 | { |
| 22614 | switch (loop.mask_limb_offset) { |
| 22615 | .unused => {}, |
| 22616 | .known => |*mask_limb_offset| mask_limb_offset.* = 0, |
| 22617 | .temp => unreachable, |
| 22618 | } |
| 21455 | 22619 | limb_offset.* = 0; |
| 21456 | 22620 | break; |
| 21457 | 22621 | } |
| ... | ... | @@ -21465,6 +22629,7 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21465 | 22629 | .disp = loop.limb_size.?, |
| 21466 | 22630 | } }, |
| 21467 | 22631 | }); |
| 22632 | try cg.spillEflagsIfOccupied(); |
| 21468 | 22633 | try cg.asmRegisterImmediate( |
| 21469 | 22634 | .{ ._, .cmp }, |
| 21470 | 22635 | limb_offset_reg.to32(), |
| ... | ... | @@ -21476,6 +22641,12 @@ fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_set |
| 21476 | 22641 | }, |
| 21477 | 22642 | } |
| 21478 | 22643 | } |
| 22644 | if (loop.mask_limb_temp) |mask_limb_temp| try mask_limb_temp.die(cg); |
| 22645 | if (loop.mask_store_temp) |mask_store_temp| try mask_store_temp.die(cg); |
| 22646 | switch (loop.mask_limb_offset) { |
| 22647 | .unused, .known => {}, |
| 22648 | .temp => |mask_limb_offset| try mask_limb_offset.die(cg), |
| 22649 | } |
| 21479 | 22650 | switch (loop.element_offset) { |
| 21480 | 22651 | .unused => break :pattern_sets, |
| 21481 | 22652 | .known => |*element_offset| { |
Jacob Young <jacobly@ziglang.org> 2024-12-01 23:53:22-05:00