authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-11-09 06:09:42-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-01-16 20:42:08-05:00
log7c713251cadf831fb2009ae8908b099f218cf42c
tree3dd3dc0c5a7990218dc382c17db0dcd870563498
parentbeadf702b8d0421cc84c47ebc9644ce07e22c306

x86_64: looped instructions

5 files changed, 1274 insertions(+), 866 deletions(-)

lib/std/crypto/chacha20.zig+1-3
...@@ -499,11 +499,9 @@ fn ChaChaNonVecImpl(comptime rounds_nb: usize) type {...@@ -499,11 +499,9 @@ fn ChaChaNonVecImpl(comptime rounds_nb: usize) type {
499fn ChaChaImpl(comptime rounds_nb: usize) type {499fn ChaChaImpl(comptime rounds_nb: usize) type {
500 switch (builtin.cpu.arch) {500 switch (builtin.cpu.arch) {
501 .x86_64 => {501 .x86_64 => {
502 if (builtin.zig_backend == .stage2_x86_64) return ChaChaNonVecImpl(rounds_nb);
503
504 const has_avx2 = std.Target.x86.featureSetHas(builtin.cpu.features, .avx2);502 const has_avx2 = std.Target.x86.featureSetHas(builtin.cpu.features, .avx2);
505 const has_avx512f = std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);503 const has_avx512f = std.Target.x86.featureSetHas(builtin.cpu.features, .avx512f);
506 if (has_avx512f) return ChaChaVecImpl(rounds_nb, 4);504 if (builtin.zig_backend != .stage2_x86_64 and has_avx512f) return ChaChaVecImpl(rounds_nb, 4);
507 if (has_avx2) return ChaChaVecImpl(rounds_nb, 2);505 if (has_avx2) return ChaChaVecImpl(rounds_nb, 2);
508 return ChaChaVecImpl(rounds_nb, 1);506 return ChaChaVecImpl(rounds_nb, 1);
509 },507 },
lib/std/crypto/tls/Client.zig+7-56
...@@ -356,14 +356,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client...@@ -356,14 +356,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
356 if (ciphertext.len > cleartext_fragment_buf.len) return error.TlsRecordOverflow;356 if (ciphertext.len > cleartext_fragment_buf.len) return error.TlsRecordOverflow;
357 const cleartext = cleartext_fragment_buf[0..ciphertext.len];357 const cleartext = cleartext_fragment_buf[0..ciphertext.len];
358 const auth_tag = record_decoder.array(P.AEAD.tag_length).*;358 const auth_tag = record_decoder.array(P.AEAD.tag_length).*;
359 const nonce = if (builtin.zig_backend == .stage2_x86_64 and359 const nonce = nonce: {
360 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
361 nonce: {
362 var nonce = pv.server_handshake_iv;
363 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
364 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ read_seq, .big);
365 break :nonce nonce;
366 } else nonce: {
367 const V = @Vector(P.AEAD.nonce_length, u8);360 const V = @Vector(P.AEAD.nonce_length, u8);
368 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);361 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
369 const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));362 const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));
...@@ -400,14 +393,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client...@@ -400,14 +393,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
400 const record_iv = record_decoder.array(P.record_iv_length).*;393 const record_iv = record_decoder.array(P.record_iv_length).*;
401 const masked_read_seq = read_seq &394 const masked_read_seq = read_seq &
402 comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);395 comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);
403 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and396 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
404 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
405 nonce: {
406 var nonce = pv.app_cipher.server_write_IV ++ record_iv;
407 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
408 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ masked_read_seq, .big);
409 break :nonce nonce;
410 } else nonce: {
411 const V = @Vector(P.AEAD.nonce_length, u8);397 const V = @Vector(P.AEAD.nonce_length, u8);
412 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);398 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
413 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));399 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
...@@ -750,14 +736,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client...@@ -750,14 +736,7 @@ pub fn init(stream: anytype, options: Options) InitError(@TypeOf(stream))!Client
750 .app_cipher = std.mem.bytesToValue(P.Tls_1_2, &key_block),736 .app_cipher = std.mem.bytesToValue(P.Tls_1_2, &key_block),
751 } };737 } };
752 const pv = &p.version.tls_1_2;738 const pv = &p.version.tls_1_2;
753 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and739 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
754 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
755 nonce: {
756 var nonce = pv.app_cipher.client_write_IV ++ pv.app_cipher.client_salt;
757 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
758 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ write_seq, .big);
759 break :nonce nonce;
760 } else nonce: {
761 const V = @Vector(P.AEAD.nonce_length, u8);740 const V = @Vector(P.AEAD.nonce_length, u8);
762 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);741 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
763 const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));742 const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));
...@@ -1043,14 +1022,7 @@ fn prepareCiphertextRecord(...@@ -1043,14 +1022,7 @@ fn prepareCiphertextRecord(
1043 ciphertext_end += ciphertext_len;1022 ciphertext_end += ciphertext_len;
1044 const auth_tag = ciphertext_buf[ciphertext_end..][0..P.AEAD.tag_length];1023 const auth_tag = ciphertext_buf[ciphertext_end..][0..P.AEAD.tag_length];
1045 ciphertext_end += auth_tag.len;1024 ciphertext_end += auth_tag.len;
1046 const nonce = if (builtin.zig_backend == .stage2_x86_64 and1025 const nonce = nonce: {
1047 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1048 nonce: {
1049 var nonce = pv.client_iv;
1050 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1051 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.write_seq, .big);
1052 break :nonce nonce;
1053 } else nonce: {
1054 const V = @Vector(P.AEAD.nonce_length, u8);1026 const V = @Vector(P.AEAD.nonce_length, u8);
1055 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);1027 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
1056 const operand: V = pad ++ std.mem.toBytes(big(c.write_seq));1028 const operand: V = pad ++ std.mem.toBytes(big(c.write_seq));
...@@ -1098,14 +1070,7 @@ fn prepareCiphertextRecord(...@@ -1098,14 +1070,7 @@ fn prepareCiphertextRecord(
1098 const ad = std.mem.toBytes(big(c.write_seq)) ++ record_header[0 .. 1 + 2] ++ int(u16, message_len);1070 const ad = std.mem.toBytes(big(c.write_seq)) ++ record_header[0 .. 1 + 2] ++ int(u16, message_len);
1099 const record_iv = ciphertext_buf[ciphertext_end..][0..P.record_iv_length];1071 const record_iv = ciphertext_buf[ciphertext_end..][0..P.record_iv_length];
1100 ciphertext_end += P.record_iv_length;1072 ciphertext_end += P.record_iv_length;
1101 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and1073 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
1102 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1103 nonce: {
1104 var nonce = pv.client_write_IV ++ pv.client_salt;
1105 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1106 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.write_seq, .big);
1107 break :nonce nonce;
1108 } else nonce: {
1109 const V = @Vector(P.AEAD.nonce_length, u8);1074 const V = @Vector(P.AEAD.nonce_length, u8);
1110 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);1075 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
1111 const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));1076 const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));
...@@ -1374,14 +1339,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove...@@ -1374,14 +1339,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
1374 const ciphertext = frag[in..][0..ciphertext_len];1339 const ciphertext = frag[in..][0..ciphertext_len];
1375 in += ciphertext_len;1340 in += ciphertext_len;
1376 const auth_tag = frag[in..][0..P.AEAD.tag_length].*;1341 const auth_tag = frag[in..][0..P.AEAD.tag_length].*;
1377 const nonce = if (builtin.zig_backend == .stage2_x86_64 and1342 const nonce = nonce: {
1378 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1379 nonce: {
1380 var nonce = pv.server_iv;
1381 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1382 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ c.read_seq, .big);
1383 break :nonce nonce;
1384 } else nonce: {
1385 const V = @Vector(P.AEAD.nonce_length, u8);1343 const V = @Vector(P.AEAD.nonce_length, u8);
1386 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);1344 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
1387 const operand: V = pad ++ std.mem.toBytes(big(c.read_seq));1345 const operand: V = pad ++ std.mem.toBytes(big(c.read_seq));
...@@ -1409,14 +1367,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove...@@ -1409,14 +1367,7 @@ pub fn readvAdvanced(c: *Client, stream: anytype, iovecs: []const std.posix.iove
1409 in += P.record_iv_length;1367 in += P.record_iv_length;
1410 const masked_read_seq = c.read_seq &1368 const masked_read_seq = c.read_seq &
1411 comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);1369 comptime std.math.shl(u64, std.math.maxInt(u64), 8 * P.record_iv_length);
1412 const nonce: [P.AEAD.nonce_length]u8 = if (builtin.zig_backend == .stage2_x86_64 and1370 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
1413 P.AEAD.nonce_length > comptime std.simd.suggestVectorLength(u8) orelse 1)
1414 nonce: {
1415 var nonce = pv.server_write_IV ++ record_iv;
1416 const operand = std.mem.readInt(u64, nonce[nonce.len - 8 ..], .big);
1417 std.mem.writeInt(u64, nonce[nonce.len - 8 ..], operand ^ masked_read_seq, .big);
1418 break :nonce nonce;
1419 } else nonce: {
1420 const V = @Vector(P.AEAD.nonce_length, u8);1371 const V = @Vector(P.AEAD.nonce_length, u8);
1421 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);1372 const pad = [1]u8{0} ** (P.AEAD.nonce_length - 8);
1422 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));1373 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
lib/std/http/protocol.zig-1
...@@ -4,7 +4,6 @@ const testing = std.testing;...@@ -4,7 +4,6 @@ const testing = std.testing;
4const mem = std.mem;4const mem = std.mem;
55
6const assert = std.debug.assert;6const assert = std.debug.assert;
7const use_vectors = builtin.zig_backend != .stage2_x86_64;
87
9pub const State = enum {8pub const State = enum {
10 invalid,9 invalid,
src/arch/x86_64/CodeGen.zig+1261-803
...@@ -104,7 +104,7 @@ const Owner = union(enum) {...@@ -104,7 +104,7 @@ const Owner = union(enum) {
104 nav_index: InternPool.Nav.Index,104 nav_index: InternPool.Nav.Index,
105 lazy_sym: link.File.LazySymbol,105 lazy_sym: link.File.LazySymbol,
106106
107 fn getSymbolIndex(owner: Owner, ctx: *Self) !u32 {107 fn getSymbolIndex(owner: Owner, ctx: *CodeGen) !u32 {
108 const pt = ctx.pt;108 const pt = ctx.pt;
109 switch (owner) {109 switch (owner) {
110 .nav_index => |nav_index| if (ctx.bin_file.cast(.elf)) |elf_file| {110 .nav_index => |nav_index| if (ctx.bin_file.cast(.elf)) |elf_file| {
...@@ -394,7 +394,7 @@ pub const MCValue = union(enum) {...@@ -394,7 +394,7 @@ pub const MCValue = union(enum) {
394 };394 };
395 }395 }
396396
397 fn mem(mcv: MCValue, function: *Self, size: Memory.Size) !Memory {397 fn mem(mcv: MCValue, function: *CodeGen, mod_rm: Memory.Mod.Rm) !Memory {
398 return switch (mcv) {398 return switch (mcv) {
399 .none,399 .none,
400 .unreach,400 .unreach,
...@@ -420,22 +420,28 @@ pub const MCValue = union(enum) {...@@ -420,22 +420,28 @@ pub const MCValue = union(enum) {
420 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{420 .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{
421 .base = .{ .reg = .ds },421 .base = .{ .reg = .ds },
422 .mod = .{ .rm = .{422 .mod = .{ .rm = .{
423 .size = size,423 .size = mod_rm.size,
424 .disp = small_addr,424 .index = mod_rm.index,
425 .scale = mod_rm.scale,
426 .disp = small_addr + mod_rm.disp,
425 } },427 } },
426 } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },428 } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },
427 .indirect => |reg_off| .{429 .indirect => |reg_off| .{
428 .base = .{ .reg = reg_off.reg },430 .base = .{ .reg = reg_off.reg },
429 .mod = .{ .rm = .{431 .mod = .{ .rm = .{
430 .size = size,432 .size = mod_rm.size,
431 .disp = reg_off.off,433 .index = mod_rm.index,
434 .scale = mod_rm.scale,
435 .disp = reg_off.off + mod_rm.disp,
432 } },436 } },
433 },437 },
434 .load_frame => |frame_addr| .{438 .load_frame => |frame_addr| .{
435 .base = .{ .frame = frame_addr.index },439 .base = .{ .frame = frame_addr.index },
436 .mod = .{ .rm = .{440 .mod = .{ .rm = .{
437 .size = size,441 .size = mod_rm.size,
438 .disp = frame_addr.off,442 .index = mod_rm.index,
443 .scale = mod_rm.scale,
444 .disp = frame_addr.off + mod_rm.disp,
439 } },445 } },
440 },446 },
441 .load_symbol => |sym_off| {447 .load_symbol => |sym_off| {
...@@ -443,12 +449,14 @@ pub const MCValue = union(enum) {...@@ -443,12 +449,14 @@ pub const MCValue = union(enum) {
443 return .{449 return .{
444 .base = .{ .reloc = sym_off.sym_index },450 .base = .{ .reloc = sym_off.sym_index },
445 .mod = .{ .rm = .{451 .mod = .{ .rm = .{
446 .size = size,452 .size = mod_rm.size,
447 .disp = sym_off.off,453 .index = mod_rm.index,
454 .scale = mod_rm.scale,
455 .disp = sym_off.off + mod_rm.disp,
448 } },456 } },
449 };457 };
450 },458 },
451 .air_ref => |ref| (try function.resolveInst(ref)).mem(function, size),459 .air_ref => |ref| (try function.resolveInst(ref)).mem(function, mod_rm),
452 };460 };
453 }461 }
454462
...@@ -537,7 +545,7 @@ const InstTracking = struct {...@@ -537,7 +545,7 @@ const InstTracking = struct {
537 return self.short.getCondition();545 return self.short.getCondition();
538 }546 }
539547
540 fn spill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {548 fn spill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
541 if (std.meta.eql(self.long, self.short)) return; // Already spilled549 if (std.meta.eql(self.long, self.short)) return; // Already spilled
542 // Allocate or reuse frame index550 // Allocate or reuse frame index
543 switch (self.long) {551 switch (self.long) {
...@@ -586,7 +594,7 @@ const InstTracking = struct {...@@ -586,7 +594,7 @@ const InstTracking = struct {
586 };594 };
587 }595 }
588596
589 fn trackSpill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {597 fn trackSpill(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
590 try function.freeValue(self.short);598 try function.freeValue(self.short);
591 self.reuseFrame();599 self.reuseFrame();
592 tracking_log.debug("{} => {} (spilled)", .{ inst, self.* });600 tracking_log.debug("{} => {} (spilled)", .{ inst, self.* });
...@@ -633,7 +641,7 @@ const InstTracking = struct {...@@ -633,7 +641,7 @@ const InstTracking = struct {
633641
634 fn materialize(642 fn materialize(
635 self: *InstTracking,643 self: *InstTracking,
636 function: *Self,644 function: *CodeGen,
637 inst: Air.Inst.Index,645 inst: Air.Inst.Index,
638 target: InstTracking,646 target: InstTracking,
639 ) !void {647 ) !void {
...@@ -643,7 +651,7 @@ const InstTracking = struct {...@@ -643,7 +651,7 @@ const InstTracking = struct {
643651
644 fn materializeUnsafe(652 fn materializeUnsafe(
645 self: InstTracking,653 self: InstTracking,
646 function: *Self,654 function: *CodeGen,
647 inst: Air.Inst.Index,655 inst: Air.Inst.Index,
648 target: InstTracking,656 target: InstTracking,
649 ) !void {657 ) !void {
...@@ -675,7 +683,7 @@ const InstTracking = struct {...@@ -675,7 +683,7 @@ const InstTracking = struct {
675 }683 }
676 }684 }
677685
678 fn die(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void {686 fn die(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) !void {
679 if (self.short == .dead) return;687 if (self.short == .dead) return;
680 try function.freeValue(self.short);688 try function.freeValue(self.short);
681 self.short = .{ .dead = function.scope_generation };689 self.short = .{ .dead = function.scope_generation };
...@@ -684,7 +692,7 @@ const InstTracking = struct {...@@ -684,7 +692,7 @@ const InstTracking = struct {
684692
685 fn reuse(693 fn reuse(
686 self: *InstTracking,694 self: *InstTracking,
687 function: *Self,695 function: *CodeGen,
688 new_inst: ?Air.Inst.Index,696 new_inst: ?Air.Inst.Index,
689 old_inst: Air.Inst.Index,697 old_inst: Air.Inst.Index,
690 ) void {698 ) void {
...@@ -692,7 +700,7 @@ const InstTracking = struct {...@@ -692,7 +700,7 @@ const InstTracking = struct {
692 tracking_log.debug("{?} => {} (reuse {})", .{ new_inst, self.*, old_inst });700 tracking_log.debug("{?} => {} (reuse {})", .{ new_inst, self.*, old_inst });
693 }701 }
694702
695 fn liveOut(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void {703 fn liveOut(self: *InstTracking, function: *CodeGen, inst: Air.Inst.Index) void {
696 for (self.getRegs()) |reg| {704 for (self.getRegs()) |reg| {
697 if (function.register_manager.isRegFree(reg)) {705 if (function.register_manager.isRegFree(reg)) {
698 tracking_log.debug("{} => {} (live-out)", .{ inst, self.* });706 tracking_log.debug("{} => {} (live-out)", .{ inst, self.* });
...@@ -789,7 +797,7 @@ const BlockData = struct {...@@ -789,7 +797,7 @@ const BlockData = struct {
789 }797 }
790};798};
791799
792const Self = @This();800const CodeGen = @This();
793801
794pub fn generate(802pub fn generate(
795 bin_file: *link.File,803 bin_file: *link.File,
...@@ -809,7 +817,7 @@ pub fn generate(...@@ -809,7 +817,7 @@ pub fn generate(
809 const fn_type: Type = .fromInterned(func.ty);817 const fn_type: Type = .fromInterned(func.ty);
810 const mod = zcu.navFileScope(func.owner_nav).mod;818 const mod = zcu.navFileScope(func.owner_nav).mod;
811819
812 var function: Self = .{820 var function: CodeGen = .{
813 .gpa = gpa,821 .gpa = gpa,
814 .pt = pt,822 .pt = pt,
815 .air = air,823 .air = air,
...@@ -962,7 +970,7 @@ pub fn generateLazy(...@@ -962,7 +970,7 @@ pub fn generateLazy(
962 const gpa = comp.gpa;970 const gpa = comp.gpa;
963 // This function is for generating global code, so we use the root module.971 // This function is for generating global code, so we use the root module.
964 const mod = comp.root_mod;972 const mod = comp.root_mod;
965 var function: Self = .{973 var function: CodeGen = .{
966 .gpa = gpa,974 .gpa = gpa,
967 .pt = pt,975 .pt = pt,
968 .air = undefined,976 .air = undefined,
...@@ -1050,7 +1058,7 @@ fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Format...@@ -1050,7 +1058,7 @@ fn fmtNav(nav_index: InternPool.Nav.Index, ip: *const InternPool) std.fmt.Format
1050}1058}
10511059
1052const FormatAirData = struct {1060const FormatAirData = struct {
1053 self: *Self,1061 self: *CodeGen,
1054 inst: Air.Inst.Index,1062 inst: Air.Inst.Index,
1055};1063};
1056fn formatAir(1064fn formatAir(
...@@ -1066,12 +1074,12 @@ fn formatAir(...@@ -1066,12 +1074,12 @@ fn formatAir(
1066 data.self.liveness,1074 data.self.liveness,
1067 );1075 );
1068}1076}
1069fn fmtAir(self: *Self, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {1077fn fmtAir(self: *CodeGen, inst: Air.Inst.Index) std.fmt.Formatter(formatAir) {
1070 return .{ .data = .{ .self = self, .inst = inst } };1078 return .{ .data = .{ .self = self, .inst = inst } };
1071}1079}
10721080
1073const FormatWipMirData = struct {1081const FormatWipMirData = struct {
1074 self: *Self,1082 self: *CodeGen,
1075 inst: Mir.Inst.Index,1083 inst: Mir.Inst.Index,
1076};1084};
1077fn formatWipMir(1085fn formatWipMir(
...@@ -1192,12 +1200,12 @@ fn formatWipMir(...@@ -1192,12 +1200,12 @@ fn formatWipMir(
1192 }1200 }
1193 }1201 }
1194}1202}
1195fn fmtWipMir(self: *Self, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {1203fn fmtWipMir(self: *CodeGen, inst: Mir.Inst.Index) std.fmt.Formatter(formatWipMir) {
1196 return .{ .data = .{ .self = self, .inst = inst } };1204 return .{ .data = .{ .self = self, .inst = inst } };
1197}1205}
11981206
1199const FormatTrackingData = struct {1207const FormatTrackingData = struct {
1200 self: *Self,1208 self: *CodeGen,
1201};1209};
1202fn formatTracking(1210fn formatTracking(
1203 data: FormatTrackingData,1211 data: FormatTrackingData,
...@@ -1208,26 +1216,26 @@ fn formatTracking(...@@ -1208,26 +1216,26 @@ fn formatTracking(
1208 var it = data.self.inst_tracking.iterator();1216 var it = data.self.inst_tracking.iterator();
1209 while (it.next()) |entry| try writer.print("\n{} = {}", .{ entry.key_ptr.*, entry.value_ptr.* });1217 while (it.next()) |entry| try writer.print("\n{} = {}", .{ entry.key_ptr.*, entry.value_ptr.* });
1210}1218}
1211fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) {1219fn fmtTracking(self: *CodeGen) std.fmt.Formatter(formatTracking) {
1212 return .{ .data = .{ .self = self } };1220 return .{ .data = .{ .self = self } };
1213}1221}
12141222
1215fn addInst(self: *Self, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {1223fn addInst(self: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!Mir.Inst.Index {
1216 const gpa = self.gpa;1224 const gpa = self.gpa;
1217 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);1225 try self.mir_instructions.ensureUnusedCapacity(gpa, 1);
1218 const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);1226 const result_index: Mir.Inst.Index = @intCast(self.mir_instructions.len);
1219 self.mir_instructions.appendAssumeCapacity(inst);1227 self.mir_instructions.appendAssumeCapacity(inst);
1220 wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});1228 if (inst.ops != .pseudo_dead_none) wip_mir_log.debug("{}", .{self.fmtWipMir(result_index)});
1221 return result_index;1229 return result_index;
1222}1230}
12231231
1224fn addExtra(self: *Self, extra: anytype) Allocator.Error!u32 {1232fn addExtra(self: *CodeGen, extra: anytype) Allocator.Error!u32 {
1225 const fields = std.meta.fields(@TypeOf(extra));1233 const fields = std.meta.fields(@TypeOf(extra));
1226 try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);1234 try self.mir_extra.ensureUnusedCapacity(self.gpa, fields.len);
1227 return self.addExtraAssumeCapacity(extra);1235 return self.addExtraAssumeCapacity(extra);
1228}1236}
12291237
1230fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {1238fn addExtraAssumeCapacity(self: *CodeGen, extra: anytype) u32 {
1231 const fields = std.meta.fields(@TypeOf(extra));1239 const fields = std.meta.fields(@TypeOf(extra));
1232 const result: u32 = @intCast(self.mir_extra.items.len);1240 const result: u32 = @intCast(self.mir_extra.items.len);
1233 inline for (fields) |field| {1241 inline for (fields) |field| {
...@@ -1241,7 +1249,7 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {...@@ -1241,7 +1249,7 @@ fn addExtraAssumeCapacity(self: *Self, extra: anytype) u32 {
1241 return result;1249 return result;
1242}1250}
12431251
1244fn asmOps(self: *Self, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {1252fn asmOps(self: *CodeGen, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
1245 return switch (ops[0]) {1253 return switch (ops[0]) {
1246 .none => self.asmOpOnly(tag),1254 .none => self.asmOpOnly(tag),
1247 .reg => |reg0| switch (ops[1]) {1255 .reg => |reg0| switch (ops[1]) {
...@@ -1316,7 +1324,7 @@ fn asmOps(self: *Self, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {...@@ -1316,7 +1324,7 @@ fn asmOps(self: *Self, tag: Mir.Inst.FixedTag, ops: [4]Operand) !void {
1316}1324}
13171325
1318/// A `cc` of `.z_and_np` clobbers `reg2`!1326/// A `cc` of `.z_and_np` clobbers `reg2`!
1319fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: Register) !void {1327fn asmCmovccRegisterRegister(self: *CodeGen, cc: Condition, reg1: Register, reg2: Register) !void {
1320 _ = try self.addInst(.{1328 _ = try self.addInst(.{
1321 .tag = switch (cc) {1329 .tag = switch (cc) {
1322 else => .cmov,1330 else => .cmov,
...@@ -1339,7 +1347,7 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R...@@ -1339,7 +1347,7 @@ fn asmCmovccRegisterRegister(self: *Self, cc: Condition, reg1: Register, reg2: R
1339}1347}
13401348
1341/// A `cc` of `.z_and_np` is not supported by this encoding!1349/// A `cc` of `.z_and_np` is not supported by this encoding!
1342fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory) !void {1350fn asmCmovccRegisterMemory(self: *CodeGen, cc: Condition, reg: Register, m: Memory) !void {
1343 _ = try self.addInst(.{1351 _ = try self.addInst(.{
1344 .tag = switch (cc) {1352 .tag = switch (cc) {
1345 else => .cmov,1353 else => .cmov,
...@@ -1363,7 +1371,7 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)...@@ -1363,7 +1371,7 @@ fn asmCmovccRegisterMemory(self: *Self, cc: Condition, reg: Register, m: Memory)
1363 });1371 });
1364}1372}
13651373
1366fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {1374fn asmSetccRegister(self: *CodeGen, cc: Condition, reg: Register) !void {
1367 _ = try self.addInst(.{1375 _ = try self.addInst(.{
1368 .tag = switch (cc) {1376 .tag = switch (cc) {
1369 else => .set,1377 else => .set,
...@@ -1387,7 +1395,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {...@@ -1387,7 +1395,7 @@ fn asmSetccRegister(self: *Self, cc: Condition, reg: Register) !void {
1387 });1395 });
1388}1396}
13891397
1390fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {1398fn asmSetccMemory(self: *CodeGen, cc: Condition, m: Memory) !void {
1391 const payload = try self.addExtra(Mir.Memory.encode(m));1399 const payload = try self.addExtra(Mir.Memory.encode(m));
1392 _ = try self.addInst(.{1400 _ = try self.addInst(.{
1393 .tag = switch (cc) {1401 .tag = switch (cc) {
...@@ -1412,7 +1420,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {...@@ -1412,7 +1420,7 @@ fn asmSetccMemory(self: *Self, cc: Condition, m: Memory) !void {
1412 });1420 });
1413}1421}
14141422
1415fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {1423fn asmJmpReloc(self: *CodeGen, target: Mir.Inst.Index) !Mir.Inst.Index {
1416 return self.addInst(.{1424 return self.addInst(.{
1417 .tag = .jmp,1425 .tag = .jmp,
1418 .ops = .inst,1426 .ops = .inst,
...@@ -1422,7 +1430,7 @@ fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {...@@ -1422,7 +1430,7 @@ fn asmJmpReloc(self: *Self, target: Mir.Inst.Index) !Mir.Inst.Index {
1422 });1430 });
1423}1431}
14241432
1425fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index {1433fn asmJccReloc(self: *CodeGen, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Index {
1426 return self.addInst(.{1434 return self.addInst(.{
1427 .tag = switch (cc) {1435 .tag = switch (cc) {
1428 else => .j,1436 else => .j,
...@@ -1443,7 +1451,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind...@@ -1443,7 +1451,7 @@ fn asmJccReloc(self: *Self, cc: Condition, target: Mir.Inst.Index) !Mir.Inst.Ind
1443 });1451 });
1444}1452}
14451453
1446fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {1454fn asmReloc(self: *CodeGen, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
1447 _ = try self.addInst(.{1455 _ = try self.addInst(.{
1448 .tag = tag[1],1456 .tag = tag[1],
1449 .ops = .inst,1457 .ops = .inst,
...@@ -1454,7 +1462,7 @@ fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {...@@ -1454,7 +1462,7 @@ fn asmReloc(self: *Self, tag: Mir.Inst.FixedTag, target: Mir.Inst.Index) !void {
1454 });1462 });
1455}1463}
14561464
1457fn asmPlaceholder(self: *Self) !Mir.Inst.Index {1465fn asmPlaceholder(self: *CodeGen) !Mir.Inst.Index {
1458 return self.addInst(.{1466 return self.addInst(.{
1459 .tag = .pseudo,1467 .tag = .pseudo,
1460 .ops = .pseudo_dead_none,1468 .ops = .pseudo_dead_none,
...@@ -1464,7 +1472,7 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {...@@ -1464,7 +1472,7 @@ fn asmPlaceholder(self: *Self) !Mir.Inst.Index {
14641472
1465const MirTagAir = enum { dbg_local };1473const MirTagAir = enum { dbg_local };
14661474
1467fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {1475fn asmAir(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index) !void {
1468 _ = try self.addInst(.{1476 _ = try self.addInst(.{
1469 .tag = .pseudo,1477 .tag = .pseudo,
1470 .ops = switch (tag) {1478 .ops = switch (tag) {
...@@ -1474,7 +1482,7 @@ fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {...@@ -1474,7 +1482,7 @@ fn asmAir(self: *Self, tag: MirTagAir, inst: Air.Inst.Index) !void {
1474 });1482 });
1475}1483}
14761484
1477fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immediate) !void {1485fn asmAirImmediate(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index, imm: Immediate) !void {
1478 switch (imm) {1486 switch (imm) {
1479 .signed => |s| _ = try self.addInst(.{1487 .signed => |s| _ = try self.addInst(.{
1480 .tag = .pseudo,1488 .tag = .pseudo,
...@@ -1528,7 +1536,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed...@@ -1528,7 +1536,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed
1528}1536}
15291537
1530fn asmAirRegisterImmediate(1538fn asmAirRegisterImmediate(
1531 self: *Self,1539 self: *CodeGen,
1532 tag: MirTagAir,1540 tag: MirTagAir,
1533 inst: Air.Inst.Index,1541 inst: Air.Inst.Index,
1534 reg: Register,1542 reg: Register,
...@@ -1550,7 +1558,7 @@ fn asmAirRegisterImmediate(...@@ -1550,7 +1558,7 @@ fn asmAirRegisterImmediate(
1550}1558}
15511559
1552fn asmAirFrameAddress(1560fn asmAirFrameAddress(
1553 self: *Self,1561 self: *CodeGen,
1554 tag: MirTagAir,1562 tag: MirTagAir,
1555 inst: Air.Inst.Index,1563 inst: Air.Inst.Index,
1556 frame_addr: bits.FrameAddr,1564 frame_addr: bits.FrameAddr,
...@@ -1567,7 +1575,7 @@ fn asmAirFrameAddress(...@@ -1567,7 +1575,7 @@ fn asmAirFrameAddress(
1567 });1575 });
1568}1576}
15691577
1570fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !void {1578fn asmAirMemory(self: *CodeGen, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !void {
1571 _ = try self.addInst(.{1579 _ = try self.addInst(.{
1572 .tag = .pseudo,1580 .tag = .pseudo,
1573 .ops = switch (tag) {1581 .ops = switch (tag) {
...@@ -1580,7 +1588,7 @@ fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !v...@@ -1580,7 +1588,7 @@ fn asmAirMemory(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, m: Memory) !v
1580 });1588 });
1581}1589}
15821590
1583fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {1591fn asmOpOnly(self: *CodeGen, tag: Mir.Inst.FixedTag) !void {
1584 _ = try self.addInst(.{1592 _ = try self.addInst(.{
1585 .tag = tag[1],1593 .tag = tag[1],
1586 .ops = .none,1594 .ops = .none,
...@@ -1590,7 +1598,7 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {...@@ -1590,7 +1598,7 @@ fn asmOpOnly(self: *Self, tag: Mir.Inst.FixedTag) !void {
1590 });1598 });
1591}1599}
15921600
1593fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {1601fn asmPseudo(self: *CodeGen, ops: Mir.Inst.Ops) !void {
1594 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and1602 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1595 std.mem.endsWith(u8, @tagName(ops), "_none"));1603 std.mem.endsWith(u8, @tagName(ops), "_none"));
1596 _ = try self.addInst(.{1604 _ = try self.addInst(.{
...@@ -1600,7 +1608,7 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {...@@ -1600,7 +1608,7 @@ fn asmPseudo(self: *Self, ops: Mir.Inst.Ops) !void {
1600 });1608 });
1601}1609}
16021610
1603fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {1611fn asmPseudoRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register) !void {
1604 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and1612 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1605 std.mem.endsWith(u8, @tagName(ops), "_r"));1613 std.mem.endsWith(u8, @tagName(ops), "_r"));
1606 _ = try self.addInst(.{1614 _ = try self.addInst(.{
...@@ -1610,7 +1618,7 @@ fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {...@@ -1610,7 +1618,7 @@ fn asmPseudoRegister(self: *Self, ops: Mir.Inst.Ops, reg: Register) !void {
1610 });1618 });
1611}1619}
16121620
1613fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {1621fn asmPseudoImmediate(self: *CodeGen, ops: Mir.Inst.Ops, imm: Immediate) !void {
1614 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and1622 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1615 std.mem.endsWith(u8, @tagName(ops), "_i_s"));1623 std.mem.endsWith(u8, @tagName(ops), "_i_s"));
1616 _ = try self.addInst(.{1624 _ = try self.addInst(.{
...@@ -1620,7 +1628,7 @@ fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {...@@ -1620,7 +1628,7 @@ fn asmPseudoImmediate(self: *Self, ops: Mir.Inst.Ops, imm: Immediate) !void {
1620 });1628 });
1621}1629}
16221630
1623fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void {1631fn asmPseudoRegisterRegister(self: *CodeGen, ops: Mir.Inst.Ops, reg1: Register, reg2: Register) !void {
1624 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and1632 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1625 std.mem.endsWith(u8, @tagName(ops), "_rr"));1633 std.mem.endsWith(u8, @tagName(ops), "_rr"));
1626 _ = try self.addInst(.{1634 _ = try self.addInst(.{
...@@ -1630,7 +1638,7 @@ fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg...@@ -1630,7 +1638,7 @@ fn asmPseudoRegisterRegister(self: *Self, ops: Mir.Inst.Ops, reg1: Register, reg
1630 });1638 });
1631}1639}
16321640
1633fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void {1641fn asmPseudoRegisterImmediate(self: *CodeGen, ops: Mir.Inst.Ops, reg: Register, imm: Immediate) !void {
1634 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and1642 assert(std.mem.startsWith(u8, @tagName(ops), "pseudo_") and
1635 std.mem.endsWith(u8, @tagName(ops), "_ri_s"));1643 std.mem.endsWith(u8, @tagName(ops), "_ri_s"));
1636 _ = try self.addInst(.{1644 _ = try self.addInst(.{
...@@ -1640,7 +1648,7 @@ fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm...@@ -1640,7 +1648,7 @@ fn asmPseudoRegisterImmediate(self: *Self, ops: Mir.Inst.Ops, reg: Register, imm
1640 });1648 });
1641}1649}
16421650
1643fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {1651fn asmRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register) !void {
1644 _ = try self.addInst(.{1652 _ = try self.addInst(.{
1645 .tag = tag[1],1653 .tag = tag[1],
1646 .ops = .r,1654 .ops = .r,
...@@ -1651,7 +1659,7 @@ fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {...@@ -1651,7 +1659,7 @@ fn asmRegister(self: *Self, tag: Mir.Inst.FixedTag, reg: Register) !void {
1651 });1659 });
1652}1660}
16531661
1654fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {1662fn asmImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1655 _ = try self.addInst(.{1663 _ = try self.addInst(.{
1656 .tag = tag[1],1664 .tag = tag[1],
1657 .ops = switch (imm) {1665 .ops = switch (imm) {
...@@ -1676,7 +1684,7 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {...@@ -1676,7 +1684,7 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void {
1676 });1684 });
1677}1685}
16781686
1679fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {1687fn asmRegisterRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, reg1: Register, reg2: Register) !void {
1680 _ = try self.addInst(.{1688 _ = try self.addInst(.{
1681 .tag = tag[1],1689 .tag = tag[1],
1682 .ops = .rr,1690 .ops = .rr,
...@@ -1688,7 +1696,7 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2...@@ -1688,7 +1696,7 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2
1688 });1696 });
1689}1697}
16901698
1691fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {1699fn asmRegisterImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void {
1692 const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {1700 const ops: Mir.Inst.Ops, const i: u32 = switch (imm) {
1693 .signed => |s| .{ .ri_s, @bitCast(s) },1701 .signed => |s| .{ .ri_s, @bitCast(s) },
1694 .unsigned => |u| if (std.math.cast(u32, u)) |small|1702 .unsigned => |u| if (std.math.cast(u32, u)) |small|
...@@ -1709,7 +1717,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm:...@@ -1709,7 +1717,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm:
1709}1717}
17101718
1711fn asmRegisterRegisterRegister(1719fn asmRegisterRegisterRegister(
1712 self: *Self,1720 self: *CodeGen,
1713 tag: Mir.Inst.FixedTag,1721 tag: Mir.Inst.FixedTag,
1714 reg1: Register,1722 reg1: Register,
1715 reg2: Register,1723 reg2: Register,
...@@ -1728,7 +1736,7 @@ fn asmRegisterRegisterRegister(...@@ -1728,7 +1736,7 @@ fn asmRegisterRegisterRegister(
1728}1736}
17291737
1730fn asmRegisterRegisterRegisterRegister(1738fn asmRegisterRegisterRegisterRegister(
1731 self: *Self,1739 self: *CodeGen,
1732 tag: Mir.Inst.FixedTag,1740 tag: Mir.Inst.FixedTag,
1733 reg1: Register,1741 reg1: Register,
1734 reg2: Register,1742 reg2: Register,
...@@ -1749,7 +1757,7 @@ fn asmRegisterRegisterRegisterRegister(...@@ -1749,7 +1757,7 @@ fn asmRegisterRegisterRegisterRegister(
1749}1757}
17501758
1751fn asmRegisterRegisterRegisterImmediate(1759fn asmRegisterRegisterRegisterImmediate(
1752 self: *Self,1760 self: *CodeGen,
1753 tag: Mir.Inst.FixedTag,1761 tag: Mir.Inst.FixedTag,
1754 reg1: Register,1762 reg1: Register,
1755 reg2: Register,1763 reg2: Register,
...@@ -1774,7 +1782,7 @@ fn asmRegisterRegisterRegisterImmediate(...@@ -1774,7 +1782,7 @@ fn asmRegisterRegisterRegisterImmediate(
1774}1782}
17751783
1776fn asmRegisterRegisterImmediate(1784fn asmRegisterRegisterImmediate(
1777 self: *Self,1785 self: *CodeGen,
1778 tag: Mir.Inst.FixedTag,1786 tag: Mir.Inst.FixedTag,
1779 reg1: Register,1787 reg1: Register,
1780 reg2: Register,1788 reg2: Register,
...@@ -1801,7 +1809,7 @@ fn asmRegisterRegisterImmediate(...@@ -1801,7 +1809,7 @@ fn asmRegisterRegisterImmediate(
1801}1809}
18021810
1803fn asmRegisterRegisterMemory(1811fn asmRegisterRegisterMemory(
1804 self: *Self,1812 self: *CodeGen,
1805 tag: Mir.Inst.FixedTag,1813 tag: Mir.Inst.FixedTag,
1806 reg1: Register,1814 reg1: Register,
1807 reg2: Register,1815 reg2: Register,
...@@ -1820,7 +1828,7 @@ fn asmRegisterRegisterMemory(...@@ -1820,7 +1828,7 @@ fn asmRegisterRegisterMemory(
1820}1828}
18211829
1822fn asmRegisterRegisterMemoryRegister(1830fn asmRegisterRegisterMemoryRegister(
1823 self: *Self,1831 self: *CodeGen,
1824 tag: Mir.Inst.FixedTag,1832 tag: Mir.Inst.FixedTag,
1825 reg1: Register,1833 reg1: Register,
1826 reg2: Register,1834 reg2: Register,
...@@ -1840,7 +1848,7 @@ fn asmRegisterRegisterMemoryRegister(...@@ -1840,7 +1848,7 @@ fn asmRegisterRegisterMemoryRegister(
1840 });1848 });
1841}1849}
18421850
1843fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {1851fn asmMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory) !void {
1844 _ = try self.addInst(.{1852 _ = try self.addInst(.{
1845 .tag = tag[1],1853 .tag = tag[1],
1846 .ops = .m,1854 .ops = .m,
...@@ -1851,7 +1859,7 @@ fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {...@@ -1851,7 +1859,7 @@ fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1851 });1859 });
1852}1860}
18531861
1854fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {1862fn asmRegisterMemory(self: *CodeGen, tag: Mir.Inst.FixedTag, reg: Register, m: Memory) !void {
1855 _ = try self.addInst(.{1863 _ = try self.addInst(.{
1856 .tag = tag[1],1864 .tag = tag[1],
1857 .ops = .rm,1865 .ops = .rm,
...@@ -1864,7 +1872,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo...@@ -1864,7 +1872,7 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo
1864}1872}
18651873
1866fn asmRegisterMemoryRegister(1874fn asmRegisterMemoryRegister(
1867 self: *Self,1875 self: *CodeGen,
1868 tag: Mir.Inst.FixedTag,1876 tag: Mir.Inst.FixedTag,
1869 reg1: Register,1877 reg1: Register,
1870 m: Memory,1878 m: Memory,
...@@ -1883,7 +1891,7 @@ fn asmRegisterMemoryRegister(...@@ -1883,7 +1891,7 @@ fn asmRegisterMemoryRegister(
1883}1891}
18841892
1885fn asmRegisterMemoryImmediate(1893fn asmRegisterMemoryImmediate(
1886 self: *Self,1894 self: *CodeGen,
1887 tag: Mir.Inst.FixedTag,1895 tag: Mir.Inst.FixedTag,
1888 reg: Register,1896 reg: Register,
1889 m: Memory,1897 m: Memory,
...@@ -1928,7 +1936,7 @@ fn asmRegisterMemoryImmediate(...@@ -1928,7 +1936,7 @@ fn asmRegisterMemoryImmediate(
1928}1936}
19291937
1930fn asmRegisterRegisterMemoryImmediate(1938fn asmRegisterRegisterMemoryImmediate(
1931 self: *Self,1939 self: *CodeGen,
1932 tag: Mir.Inst.FixedTag,1940 tag: Mir.Inst.FixedTag,
1933 reg1: Register,1941 reg1: Register,
1934 reg2: Register,1942 reg2: Register,
...@@ -1948,7 +1956,7 @@ fn asmRegisterRegisterMemoryImmediate(...@@ -1948,7 +1956,7 @@ fn asmRegisterRegisterMemoryImmediate(
1948 });1956 });
1949}1957}
19501958
1951fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {1959fn asmMemoryRegister(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, reg: Register) !void {
1952 _ = try self.addInst(.{1960 _ = try self.addInst(.{
1953 .tag = tag[1],1961 .tag = tag[1],
1954 .ops = .mr,1962 .ops = .mr,
...@@ -1960,7 +1968,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Regist...@@ -1960,7 +1968,7 @@ fn asmMemoryRegister(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, reg: Regist
1960 });1968 });
1961}1969}
19621970
1963fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {1971fn asmMemoryImmediate(self: *CodeGen, tag: Mir.Inst.FixedTag, m: Memory, imm: Immediate) !void {
1964 const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {1972 const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
1965 .signed => |s| @bitCast(s),1973 .signed => |s| @bitCast(s),
1966 .unsigned => |u| @intCast(u),1974 .unsigned => |u| @intCast(u),
...@@ -1982,7 +1990,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed...@@ -1982,7 +1990,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed
1982}1990}
19831991
1984fn asmMemoryRegisterRegister(1992fn asmMemoryRegisterRegister(
1985 self: *Self,1993 self: *CodeGen,
1986 tag: Mir.Inst.FixedTag,1994 tag: Mir.Inst.FixedTag,
1987 m: Memory,1995 m: Memory,
1988 reg1: Register,1996 reg1: Register,
...@@ -2001,7 +2009,7 @@ fn asmMemoryRegisterRegister(...@@ -2001,7 +2009,7 @@ fn asmMemoryRegisterRegister(
2001}2009}
20022010
2003fn asmMemoryRegisterImmediate(2011fn asmMemoryRegisterImmediate(
2004 self: *Self,2012 self: *CodeGen,
2005 tag: Mir.Inst.FixedTag,2013 tag: Mir.Inst.FixedTag,
2006 m: Memory,2014 m: Memory,
2007 reg: Register,2015 reg: Register,
...@@ -2019,7 +2027,7 @@ fn asmMemoryRegisterImmediate(...@@ -2019,7 +2027,7 @@ fn asmMemoryRegisterImmediate(
2019 });2027 });
2020}2028}
20212029
2022fn gen(self: *Self) InnerError!void {2030fn gen(self: *CodeGen) InnerError!void {
2023 const pt = self.pt;2031 const pt = self.pt;
2024 const zcu = pt.zcu;2032 const zcu = pt.zcu;
2025 const fn_info = zcu.typeToFunc(self.fn_type).?;2033 const fn_info = zcu.typeToFunc(self.fn_type).?;
...@@ -2229,7 +2237,7 @@ fn gen(self: *Self) InnerError!void {...@@ -2229,7 +2237,7 @@ fn gen(self: *Self) InnerError!void {
2229 });2237 });
2230}2238}
22312239
2232fn checkInvariantsAfterAirInst(self: *Self) void {2240fn checkInvariantsAfterAirInst(self: *CodeGen) void {
2233 assert(!self.register_manager.lockedRegsExist());2241 assert(!self.register_manager.lockedRegsExist());
22342242
2235 if (std.debug.runtime_safety) {2243 if (std.debug.runtime_safety) {
...@@ -2245,13 +2253,13 @@ fn checkInvariantsAfterAirInst(self: *Self) void {...@@ -2245,13 +2253,13 @@ fn checkInvariantsAfterAirInst(self: *Self) void {
2245 }2253 }
2246}2254}
22472255
2248fn genBodyBlock(self: *Self, body: []const Air.Inst.Index) InnerError!void {2256fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2249 try self.asmPseudo(.pseudo_dbg_enter_block_none);2257 try self.asmPseudo(.pseudo_dbg_enter_block_none);
2250 try self.genBody(body);2258 try self.genBody(body);
2251 try self.asmPseudo(.pseudo_dbg_leave_block_none);2259 try self.asmPseudo(.pseudo_dbg_leave_block_none);
2252}2260}
22532261
2254fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {2262fn genBody(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2255 const pt = self.pt;2263 const pt = self.pt;
2256 const zcu = pt.zcu;2264 const zcu = pt.zcu;
2257 const ip = &zcu.intern_pool;2265 const ip = &zcu.intern_pool;
...@@ -2461,15 +2469,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2461,15 +2469,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2461 try slot.moveTo(inst, self);2469 try slot.moveTo(inst, self);
2462 },2470 },
2463 .assembly => try self.airAsm(inst),2471 .assembly => try self.airAsm(inst),
2464 inline .bit_and, .bit_or, .xor => |tag| if (use_old or self.typeOfIndex(inst).abiSize(zcu) > @as(2472 inline .bit_and, .bit_or, .xor => |tag| if (use_old) try self.airBinOp(inst, tag) else {
2465 u64,
2466 if (!self.typeOfIndex(inst).isVector(zcu))
2467 8
2468 else if (!self.hasFeature(.avx2))
2469 16
2470 else
2471 32,
2472 )) try self.airBinOp(inst, tag) else {
2473 const bin_op = air_datas[@intFromEnum(inst)].bin_op;2473 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
2474 var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });2474 var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
2475 try self.spillEflagsIfOccupied();2475 try self.spillEflagsIfOccupied();
...@@ -2481,30 +2481,109 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2481,30 +2481,109 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2481 };2481 };
2482 var res: [1]Temp = undefined;2482 var res: [1]Temp = undefined;
2483 try self.select(&res, &.{ &ops[0], &ops[1] }, &.{2483 try self.select(&res, &.{ &ops[0], &ops[1] }, &.{
2484 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .ymm, .mem }, .features = &.{.avx2} },2484 .{
2485 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 }, .features = &.{.avx2} },2485 .required_features = &.{.avx2},
2486 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .ymm, .ymm, .ymm }, .features = &.{.avx2} },2486 .mir_tag = .{ .vp_, mir_tag },
2487 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .xmm, .mem }, .features = &.{.avx} },2487 .patterns = &.{
2488 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 }, .features = &.{.avx} },2488 .{ .ops = &.{ .ymm, .ymm, .mem } },
2489 .{ .tag = .{ .vp_, mir_tag }, .ops = &.{ .xmm, .xmm, .xmm }, .features = &.{.avx} },2489 .{ .ops = &.{ .ymm, .mem, .ymm }, .commute = .{ 1, 2 } },
2490 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .{ .match = 0 }, .mem }, .features = &.{.sse2} },2490 .{ .ops = &.{ .ymm, .ymm, .ymm } },
2491 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .mem, .{ .match = 0 } }, .features = &.{.sse2} },2491 },
2492 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .{ .match = 0 }, .xmm }, .features = &.{.sse2} },2492 },
2493 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .xmm, .xmm, .{ .match = 0 } }, .features = &.{.sse2} },2493 .{
2494 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .{ .match = 0 }, .mem }, .features = &.{.mmx} },2494 .required_features = &.{.avx},
2495 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .mem, .{ .match = 0 } }, .features = &.{.mmx} },2495 .mir_tag = .{ .vp_, mir_tag },
2496 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .{ .match = 0 }, .mm }, .features = &.{.mmx} },2496 .patterns = &.{
2497 .{ .tag = .{ .p_, mir_tag }, .ops = &.{ .mm, .mm, .{ .match = 0 } }, .features = &.{.mmx} },2497 .{ .ops = &.{ .xmm, .xmm, .mem } },
2498 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .{ .match = 0 }, .simm32 } },2498 .{ .ops = &.{ .xmm, .mem, .xmm }, .commute = .{ 1, 2 } },
2499 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .simm32, .{ .match = 0 } } },2499 .{ .ops = &.{ .xmm, .xmm, .xmm } },
2500 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .{ .match = 0 }, .gpr } },2500 },
2501 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .mem, .gpr, .{ .match = 0 } } },2501 },
2502 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .simm32 } },2502 .{
2503 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .simm32, .{ .match = 0 } } },2503 .required_features = &.{.sse2},
2504 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .mem } },2504 .mir_tag = .{ .p_, mir_tag },
2505 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .mem, .{ .match = 0 } } },2505 .patterns = &.{
2506 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .{ .match = 0 }, .gpr } },2506 .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .mem } },
2507 .{ .tag = .{ ._, mir_tag }, .ops = &.{ .gpr, .gpr, .{ .match = 0 } } },2507 .{ .ops = &.{ .xmm, .mem, .{ .implicit = 0 } } },
2508 .{ .ops = &.{ .xmm, .{ .implicit = 0 }, .xmm } },
2509 .{ .ops = &.{ .xmm, .xmm, .{ .implicit = 0 } } },
2510 },
2511 },
2512 .{
2513 .required_features = &.{.mmx},
2514 .mir_tag = .{ .p_, mir_tag },
2515 .patterns = &.{
2516 .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mem } },
2517 .{ .ops = &.{ .mm, .mem, .{ .implicit = 0 } } },
2518 .{ .ops = &.{ .mm, .{ .implicit = 0 }, .mm } },
2519 .{ .ops = &.{ .mm, .mm, .{ .implicit = 0 } } },
2520 },
2521 },
2522 .{
2523 .mir_tag = .{ ._, mir_tag },
2524 .patterns = &.{
2525 .{ .ops = &.{ .mem, .{ .implicit = 0 }, .simm32 } },
2526 .{ .ops = &.{ .mem, .simm32, .{ .implicit = 0 } } },
2527 .{ .ops = &.{ .mem, .{ .implicit = 0 }, .gpr } },
2528 .{ .ops = &.{ .mem, .gpr, .{ .implicit = 0 } } },
2529 .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .simm32 } },
2530 .{ .ops = &.{ .gpr, .simm32, .{ .implicit = 0 } } },
2531 .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .mem } },
2532 .{ .ops = &.{ .gpr, .mem, .{ .implicit = 0 } } },
2533 .{ .ops = &.{ .gpr, .{ .implicit = 0 }, .gpr } },
2534 .{ .ops = &.{ .gpr, .gpr, .{ .implicit = 0 } } },
2535 },
2536 },
2537
2538 .{
2539 .required_features = &.{.avx2},
2540 .loop = .bitwise,
2541 .mir_tag = .{ .vp_, mir_tag },
2542 .patterns = &.{
2543 .{ .ops = &.{ .ymm_limb, .{ .explicit = 0 }, .mem_limb } },
2544 .{ .ops = &.{ .ymm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
2545 .{ .ops = &.{ .ymm_limb, .ymm_limb, .mem_limb } },
2546 },
2547 },
2548 .{
2549 .required_features = &.{.avx},
2550 .loop = .bitwise,
2551 .mir_tag = .{ .vp_, mir_tag },
2552 .patterns = &.{
2553 .{ .ops = &.{ .xmm_limb, .{ .explicit = 0 }, .mem_limb } },
2554 .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } },
2555 .{ .ops = &.{ .xmm_limb, .xmm_limb, .mem_limb } },
2556 },
2557 },
2558 .{
2559 .required_features = &.{.sse2},
2560 .loop = .bitwise,
2561 .mir_tag = .{ .p_, mir_tag },
2562 .patterns = &.{
2563 .{ .ops = &.{ .xmm_limb, .{ .implicit = 0 }, .mem_limb } },
2564 .{ .ops = &.{ .xmm_limb, .mem_limb, .{ .implicit = 0 } } },
2565 },
2566 },
2567 .{
2568 .required_features = &.{.mmx},
2569 .loop = .bitwise,
2570 .mir_tag = .{ .p_, mir_tag },
2571 .patterns = &.{
2572 .{ .ops = &.{ .mm_limb, .{ .implicit = 0 }, .mem_limb } },
2573 .{ .ops = &.{ .mm_limb, .mem_limb, .{ .implicit = 0 } } },
2574 },
2575 },
2576 .{
2577 .loop = .bitwise,
2578 .mir_tag = .{ ._, mir_tag },
2579 .patterns = &.{
2580 .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } },
2581 .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } },
2582 .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } },
2583 .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } },
2584 .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } },
2585 },
2586 },
2508 });2587 });
2509 if (ops[0].index != res[0].index) try ops[0].die(self);2588 if (ops[0].index != res[0].index) try ops[0].die(self);
2510 if (ops[1].index != res[0].index) try ops[1].die(self);2589 if (ops[1].index != res[0].index) try ops[1].die(self);
...@@ -2620,12 +2699,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2620,12 +2699,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2620 while (try ops[0].toLea(self)) {}2699 while (try ops[0].toLea(self)) {}
2621 try self.asmMemoryImmediate(2700 try self.asmMemoryImmediate(
2622 .{ ._, .cmp },2701 .{ ._, .cmp },
2623 try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl)2702 try ops[0].tracking(self).short.deref().mem(self, .{ .size = if (!opt_repr_is_pl)
2624 .byte2703 .byte
2625 else if (opt_child_ty.isSlice(zcu))2704 else if (opt_child_ty.isSlice(zcu))
2626 .qword2705 .qword
2627 else2706 else
2628 .fromSize(opt_child_abi_size)),2707 .fromSize(opt_child_abi_size) }),
2629 .u(0),2708 .u(0),
2630 );2709 );
2631 var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });2710 var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
...@@ -2643,12 +2722,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2643,12 +2722,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2643 while (try ops[0].toLea(self)) {}2722 while (try ops[0].toLea(self)) {}
2644 try self.asmMemoryImmediate(2723 try self.asmMemoryImmediate(
2645 .{ ._, .cmp },2724 .{ ._, .cmp },
2646 try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl)2725 try ops[0].tracking(self).short.deref().mem(self, .{ .size = if (!opt_repr_is_pl)
2647 .byte2726 .byte
2648 else if (opt_child_ty.isSlice(zcu))2727 else if (opt_child_ty.isSlice(zcu))
2649 .qword2728 .qword
2650 else2729 else
2651 .fromSize(opt_child_abi_size)),2730 .fromSize(opt_child_abi_size) }),
2652 .u(0),2731 .u(0),
2653 );2732 );
2654 var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });2733 var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
...@@ -2666,7 +2745,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2666,7 +2745,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2666 while (try ops[0].toLea(self)) {}2745 while (try ops[0].toLea(self)) {}
2667 try self.asmMemoryImmediate(2746 try self.asmMemoryImmediate(
2668 .{ ._, .cmp },2747 .{ ._, .cmp },
2669 try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)),2748 try ops[0].tracking(self).short.deref().mem(self, .{ .size = self.memSize(eu_err_ty) }),
2670 .u(0),2749 .u(0),
2671 );2750 );
2672 var is_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });2751 var is_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne });
...@@ -2684,7 +2763,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2684,7 +2763,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2684 while (try ops[0].toLea(self)) {}2763 while (try ops[0].toLea(self)) {}
2685 try self.asmMemoryImmediate(2764 try self.asmMemoryImmediate(
2686 .{ ._, .cmp },2765 .{ ._, .cmp },
2687 try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)),2766 try ops[0].tracking(self).short.deref().mem(self, .{ .size = self.memSize(eu_err_ty) }),
2688 .u(0),2767 .u(0),
2689 );2768 );
2690 var is_non_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });2769 var is_non_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e });
...@@ -2950,7 +3029,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -2950,7 +3029,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2950 verbose_tracking_log.debug("{}", .{self.fmtTracking()});3029 verbose_tracking_log.debug("{}", .{self.fmtTracking()});
2951}3030}
29523031
2953fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {3032fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
2954 const pt = self.pt;3033 const pt = self.pt;
2955 const zcu = pt.zcu;3034 const zcu = pt.zcu;
2956 const ip = &zcu.intern_pool;3035 const ip = &zcu.intern_pool;
...@@ -3017,7 +3096,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {...@@ -3017,7 +3096,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void {
3017 }3096 }
3018}3097}
30193098
3020fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {3099fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void {
3021 for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst);3100 for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst);
3022 switch (value) {3101 switch (value) {
3023 else => {},3102 else => {},
...@@ -3025,21 +3104,23 @@ fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {...@@ -3025,21 +3104,23 @@ fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void {
3025 }3104 }
3026}3105}
30273106
3028fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void {3107fn getValueIfFree(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) void {
3029 for (value.getRegs()) |reg| if (self.register_manager.isRegFree(reg))3108 for (value.getRegs()) |reg| if (self.register_manager.isRegFree(reg))
3030 self.register_manager.getRegAssumeFree(reg, inst);3109 self.register_manager.getRegAssumeFree(reg, inst);
3031}3110}
30323111
3033fn freeValue(self: *Self, value: MCValue) !void {3112fn freeReg(self: *CodeGen, reg: Register) !void {
3113 self.register_manager.freeReg(reg);
3114 if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);
3115}
3116
3117fn freeValue(self: *CodeGen, value: MCValue) !void {
3034 switch (value) {3118 switch (value) {
3035 .register => |reg| {3119 .register => |reg| try self.freeReg(reg),
3036 self.register_manager.freeReg(reg);3120 .register_pair => |regs| for (regs) |reg| try self.freeReg(reg),
3037 if (reg.class() == .x87) try self.asmRegister(.{ .f_, .free }, reg);3121 .register_offset, .indirect => |reg_off| try self.freeReg(reg_off.reg),
3038 },
3039 .register_pair => |regs| for (regs) |reg| self.register_manager.freeReg(reg),
3040 .register_offset => |reg_off| self.register_manager.freeReg(reg_off.reg),
3041 .register_overflow => |reg_ov| {3122 .register_overflow => |reg_ov| {
3042 self.register_manager.freeReg(reg_ov.reg);3123 try self.freeReg(reg_ov.reg);
3043 self.eflags_inst = null;3124 self.eflags_inst = null;
3044 },3125 },
3045 .eflags => self.eflags_inst = null,3126 .eflags => self.eflags_inst = null,
...@@ -3047,16 +3128,16 @@ fn freeValue(self: *Self, value: MCValue) !void {...@@ -3047,16 +3128,16 @@ fn freeValue(self: *Self, value: MCValue) !void {
3047 }3128 }
3048}3129}
30493130
3050fn feed(self: *Self, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {3131fn feed(self: *CodeGen, bt: *Liveness.BigTomb, operand: Air.Inst.Ref) !void {
3051 if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst);3132 if (bt.feed()) if (operand.toIndex()) |inst| try self.processDeath(inst);
3052}3133}
30533134
3054/// Asserts there is already capacity to insert into top branch inst_table.3135/// Asserts there is already capacity to insert into top branch inst_table.
3055fn processDeath(self: *Self, inst: Air.Inst.Index) !void {3136fn processDeath(self: *CodeGen, inst: Air.Inst.Index) !void {
3056 try self.inst_tracking.getPtr(inst).?.die(self, inst);3137 try self.inst_tracking.getPtr(inst).?.die(self, inst);
3057}3138}
30583139
3059fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {3140fn finishAirResult(self: *CodeGen, inst: Air.Inst.Index, result: MCValue) void {
3060 if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {3141 if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) {
3061 .none, .dead, .unreach => {},3142 .none, .dead, .unreach => {},
3062 else => unreachable, // Why didn't the result die?3143 else => unreachable, // Why didn't the result die?
...@@ -3071,7 +3152,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {...@@ -3071,7 +3152,7 @@ fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void {
3071}3152}
30723153
3073fn finishAir(3154fn finishAir(
3074 self: *Self,3155 self: *CodeGen,
3075 inst: Air.Inst.Index,3156 inst: Air.Inst.Index,
3076 result: MCValue,3157 result: MCValue,
3077 operands: [Liveness.bpi - 1]Air.Inst.Ref,3158 operands: [Liveness.bpi - 1]Air.Inst.Ref,
...@@ -3092,7 +3173,7 @@ const FrameLayout = struct {...@@ -3092,7 +3173,7 @@ const FrameLayout = struct {
3092};3173};
30933174
3094fn setFrameLoc(3175fn setFrameLoc(
3095 self: *Self,3176 self: *CodeGen,
3096 frame_index: FrameIndex,3177 frame_index: FrameIndex,
3097 base: Register,3178 base: Register,
3098 offset: *i32,3179 offset: *i32,
...@@ -3107,7 +3188,7 @@ fn setFrameLoc(...@@ -3107,7 +3188,7 @@ fn setFrameLoc(
3107 offset.* += self.frame_allocs.items(.abi_size)[frame_i];3188 offset.* += self.frame_allocs.items(.abi_size)[frame_i];
3108}3189}
31093190
3110fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayout {3191fn computeFrameLayout(self: *CodeGen, cc: std.builtin.CallingConvention) !FrameLayout {
3111 const frame_allocs_len = self.frame_allocs.len;3192 const frame_allocs_len = self.frame_allocs.len;
3112 try self.frame_locs.resize(self.gpa, frame_allocs_len);3193 try self.frame_locs.resize(self.gpa, frame_allocs_len);
3113 const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);3194 const stack_frame_order = try self.gpa.alloc(FrameIndex, frame_allocs_len - FrameIndex.named_count);
...@@ -3173,16 +3254,16 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo...@@ -3173,16 +3254,16 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo
3173 };3254 };
3174}3255}
31753256
3176fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) InternPool.Alignment {3257fn getFrameAddrAlignment(self: *CodeGen, frame_addr: bits.FrameAddr) InternPool.Alignment {
3177 const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;3258 const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align;
3178 return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));3259 return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off)));
3179}3260}
31803261
3181fn getFrameAddrSize(self: *Self, frame_addr: bits.FrameAddr) u32 {3262fn getFrameAddrSize(self: *CodeGen, frame_addr: bits.FrameAddr) u32 {
3182 return self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_size - @as(u31, @intCast(frame_addr.off));3263 return self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_size - @as(u31, @intCast(frame_addr.off));
3183}3264}
31843265
3185fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {3266fn allocFrameIndex(self: *CodeGen, alloc: FrameAlloc) !FrameIndex {
3186 const frame_allocs_slice = self.frame_allocs.slice();3267 const frame_allocs_slice = self.frame_allocs.slice();
3187 const frame_size = frame_allocs_slice.items(.abi_size);3268 const frame_size = frame_allocs_slice.items(.abi_size);
3188 const frame_align = frame_allocs_slice.items(.abi_align);3269 const frame_align = frame_allocs_slice.items(.abi_align);
...@@ -3205,7 +3286,7 @@ fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {...@@ -3205,7 +3286,7 @@ fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
3205}3286}
32063287
3207/// Use a pointer instruction as the basis for allocating stack memory.3288/// Use a pointer instruction as the basis for allocating stack memory.
3208fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {3289fn allocMemPtr(self: *CodeGen, inst: Air.Inst.Index) !FrameIndex {
3209 const pt = self.pt;3290 const pt = self.pt;
3210 const zcu = pt.zcu;3291 const zcu = pt.zcu;
3211 const ptr_ty = self.typeOfIndex(inst);3292 const ptr_ty = self.typeOfIndex(inst);
...@@ -3218,15 +3299,15 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {...@@ -3218,15 +3299,15 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex {
3218 }));3299 }));
3219}3300}
32203301
3221fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {3302fn allocRegOrMem(self: *CodeGen, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
3222 return self.allocRegOrMemAdvanced(self.typeOfIndex(inst), inst, reg_ok);3303 return self.allocRegOrMemAdvanced(self.typeOfIndex(inst), inst, reg_ok);
3223}3304}
32243305
3225fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue {3306fn allocTempRegOrMem(self: *CodeGen, elem_ty: Type, reg_ok: bool) !MCValue {
3226 return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);3307 return self.allocRegOrMemAdvanced(elem_ty, null, reg_ok);
3227}3308}
32283309
3229fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {3310fn allocRegOrMemAdvanced(self: *CodeGen, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue {
3230 const pt = self.pt;3311 const pt = self.pt;
3231 const zcu = pt.zcu;3312 const zcu = pt.zcu;
3232 const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {3313 const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
...@@ -3260,7 +3341,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b...@@ -3260,7 +3341,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
3260 return .{ .load_frame = .{ .index = frame_index } };3341 return .{ .load_frame = .{ .index = frame_index } };
3261}3342}
32623343
3263fn regClassForType(self: *Self, ty: Type) Register.Class {3344fn regClassForType(self: *CodeGen, ty: Type) Register.Class {
3264 const pt = self.pt;3345 const pt = self.pt;
3265 const zcu = pt.zcu;3346 const zcu = pt.zcu;
3266 return switch (ty.zigTypeTag(zcu)) {3347 return switch (ty.zigTypeTag(zcu)) {
...@@ -3289,7 +3370,7 @@ fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet {...@@ -3289,7 +3370,7 @@ fn regSetForRegClass(rc: Register.Class) RegisterManager.RegisterBitSet {
3289 };3370 };
3290}3371}
32913372
3292fn regSetForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {3373fn regSetForType(self: *CodeGen, ty: Type) RegisterManager.RegisterBitSet {
3293 return regSetForRegClass(self.regClassForType(ty));3374 return regSetForRegClass(self.regClassForType(ty));
3294}3375}
32953376
...@@ -3301,14 +3382,14 @@ const State = struct {...@@ -3301,14 +3382,14 @@ const State = struct {
3301 scope_generation: u32,3382 scope_generation: u32,
3302};3383};
33033384
3304fn initRetroactiveState(self: *Self) State {3385fn initRetroactiveState(self: *CodeGen) State {
3305 var state: State = undefined;3386 var state: State = undefined;
3306 state.inst_tracking_len = @intCast(self.inst_tracking.count());3387 state.inst_tracking_len = @intCast(self.inst_tracking.count());
3307 state.scope_generation = self.scope_generation;3388 state.scope_generation = self.scope_generation;
3308 return state;3389 return state;
3309}3390}
33103391
3311fn saveRetroactiveState(self: *Self, state: *State) !void {3392fn saveRetroactiveState(self: *CodeGen, state: *State) !void {
3312 try self.spillEflagsIfOccupied();3393 try self.spillEflagsIfOccupied();
3313 const free_registers = self.register_manager.free_registers;3394 const free_registers = self.register_manager.free_registers;
3314 var it = free_registers.iterator(.{ .kind = .unset });3395 var it = free_registers.iterator(.{ .kind = .unset });
...@@ -3320,13 +3401,13 @@ fn saveRetroactiveState(self: *Self, state: *State) !void {...@@ -3320,13 +3401,13 @@ fn saveRetroactiveState(self: *Self, state: *State) !void {
3320 state.free_registers = free_registers;3401 state.free_registers = free_registers;
3321}3402}
33223403
3323fn saveState(self: *Self) !State {3404fn saveState(self: *CodeGen) !State {
3324 var state = self.initRetroactiveState();3405 var state = self.initRetroactiveState();
3325 try self.saveRetroactiveState(&state);3406 try self.saveRetroactiveState(&state);
3326 return state;3407 return state;
3327}3408}
33283409
3329fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {3410fn restoreState(self: *CodeGen, state: State, deaths: []const Air.Inst.Index, comptime opts: struct {
3330 emit_instructions: bool,3411 emit_instructions: bool,
3331 update_tracking: bool,3412 update_tracking: bool,
3332 resurrect: bool,3413 resurrect: bool,
...@@ -3414,7 +3495,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt...@@ -3414,7 +3495,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
3414 }3495 }
3415}3496}
34163497
3417pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void {3498pub fn spillInstruction(self: *CodeGen, reg: Register, inst: Air.Inst.Index) !void {
3418 const tracking = self.inst_tracking.getPtr(inst) orelse return;3499 const tracking = self.inst_tracking.getPtr(inst) orelse return;
3419 for (tracking.getRegs()) |tracked_reg| {3500 for (tracking.getRegs()) |tracked_reg| {
3420 if (tracked_reg.id() == reg.id()) break;3501 if (tracked_reg.id() == reg.id()) break;
...@@ -3423,7 +3504,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void...@@ -3423,7 +3504,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void
3423 try tracking.trackSpill(self, inst);3504 try tracking.trackSpill(self, inst);
3424}3505}
34253506
3426pub fn spillEflagsIfOccupied(self: *Self) !void {3507pub fn spillEflagsIfOccupied(self: *CodeGen) !void {
3427 if (self.eflags_inst) |inst| {3508 if (self.eflags_inst) |inst| {
3428 self.eflags_inst = null;3509 self.eflags_inst = null;
3429 const tracking = self.inst_tracking.getPtr(inst).?;3510 const tracking = self.inst_tracking.getPtr(inst).?;
...@@ -3433,7 +3514,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {...@@ -3433,7 +3514,7 @@ pub fn spillEflagsIfOccupied(self: *Self) !void {
3433 }3514 }
3434}3515}
34353516
3436pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention) !void {3517pub fn spillCallerPreservedRegs(self: *CodeGen, cc: std.builtin.CallingConvention) !void {
3437 switch (cc) {3518 switch (cc) {
3438 .x86_64_sysv => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_sysv = .{} })),3519 .x86_64_sysv => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_sysv = .{} })),
3439 .x86_64_win => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_win = .{} })),3520 .x86_64_win => try self.spillRegisters(abi.getCallerPreservedRegs(.{ .x86_64_win = .{} })),
...@@ -3441,14 +3522,14 @@ pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention)...@@ -3441,14 +3522,14 @@ pub fn spillCallerPreservedRegs(self: *Self, cc: std.builtin.CallingConvention)
3441 }3522 }
3442}3523}
34433524
3444pub fn spillRegisters(self: *Self, comptime registers: []const Register) !void {3525pub fn spillRegisters(self: *CodeGen, comptime registers: []const Register) !void {
3445 inline for (registers) |reg| try self.register_manager.getKnownReg(reg, null);3526 inline for (registers) |reg| try self.register_manager.getKnownReg(reg, null);
3446}3527}
34473528
3448/// Copies a value to a register without tracking the register. The register is not considered3529/// Copies a value to a register without tracking the register. The register is not considered
3449/// allocated. A second call to `copyToTmpRegister` may return the same register.3530/// allocated. A second call to `copyToTmpRegister` may return the same register.
3450/// This can have a side effect of spilling instructions to the stack to free up a register.3531/// This can have a side effect of spilling instructions to the stack to free up a register.
3451fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {3532fn copyToTmpRegister(self: *CodeGen, ty: Type, mcv: MCValue) !Register {
3452 const reg = try self.register_manager.allocReg(null, self.regSetForType(ty));3533 const reg = try self.register_manager.allocReg(null, self.regSetForType(ty));
3453 try self.genSetReg(reg, ty, mcv, .{});3534 try self.genSetReg(reg, ty, mcv, .{});
3454 return reg;3535 return reg;
...@@ -3459,7 +3540,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {...@@ -3459,7 +3540,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
3459/// This can have a side effect of spilling instructions to the stack to free up a register.3540/// This can have a side effect of spilling instructions to the stack to free up a register.
3460/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!3541/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
3461fn copyToRegisterWithInstTracking(3542fn copyToRegisterWithInstTracking(
3462 self: *Self,3543 self: *CodeGen,
3463 reg_owner: Air.Inst.Index,3544 reg_owner: Air.Inst.Index,
3464 ty: Type,3545 ty: Type,
3465 mcv: MCValue,3546 mcv: MCValue,
...@@ -3469,12 +3550,12 @@ fn copyToRegisterWithInstTracking(...@@ -3469,12 +3550,12 @@ fn copyToRegisterWithInstTracking(
3469 return MCValue{ .register = reg };3550 return MCValue{ .register = reg };
3470}3551}
34713552
3472fn airAlloc(self: *Self, inst: Air.Inst.Index) !void {3553fn airAlloc(self: *CodeGen, inst: Air.Inst.Index) !void {
3473 const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };3554 const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
3474 return self.finishAir(inst, result, .{ .none, .none, .none });3555 return self.finishAir(inst, result, .{ .none, .none, .none });
3475}3556}
34763557
3477fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {3558fn airRetPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
3478 const result: MCValue = switch (self.ret_mcv.long) {3559 const result: MCValue = switch (self.ret_mcv.long) {
3479 else => unreachable,3560 else => unreachable,
3480 .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },3561 .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
...@@ -3490,7 +3571,7 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -3490,7 +3571,7 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void {
3490 return self.finishAir(inst, result, .{ .none, .none, .none });3571 return self.finishAir(inst, result, .{ .none, .none, .none });
3491}3572}
34923573
3493fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {3574fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
3494 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3575 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3495 const dst_ty = self.typeOfIndex(inst);3576 const dst_ty = self.typeOfIndex(inst);
3496 const dst_bits = dst_ty.floatBits(self.target.*);3577 const dst_bits = dst_ty.floatBits(self.target.*);
...@@ -3562,7 +3643,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3562,7 +3643,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3562 .{ .v_ss, .cvtsd2 },3643 .{ .v_ss, .cvtsd2 },
3563 dst_reg,3644 dst_reg,
3564 dst_reg,3645 dst_reg,
3565 try src_mcv.mem(self, .qword),3646 try src_mcv.mem(self, .{ .size = .qword }),
3566 ) else try self.asmRegisterRegisterRegister(3647 ) else try self.asmRegisterRegisterRegister(
3567 .{ .v_ss, .cvtsd2 },3648 .{ .v_ss, .cvtsd2 },
3568 dst_reg,3649 dst_reg,
...@@ -3574,7 +3655,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3574,7 +3655,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3574 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(3655 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
3575 .{ ._ss, .cvtsd2 },3656 .{ ._ss, .cvtsd2 },
3576 dst_reg,3657 dst_reg,
3577 try src_mcv.mem(self, .qword),3658 try src_mcv.mem(self, .{ .size = .qword }),
3578 ) else try self.asmRegisterRegister(3659 ) else try self.asmRegisterRegister(
3579 .{ ._ss, .cvtsd2 },3660 .{ ._ss, .cvtsd2 },
3580 dst_reg,3661 dst_reg,
...@@ -3589,7 +3670,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -3589,7 +3670,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
3589 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3670 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3590}3671}
35913672
3592fn airFpext(self: *Self, inst: Air.Inst.Index) !void {3673fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void {
3593 const pt = self.pt;3674 const pt = self.pt;
3594 const zcu = pt.zcu;3675 const zcu = pt.zcu;
3595 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3676 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -3674,7 +3755,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -3674,7 +3755,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3674 .{ .v_sd, .cvtss2 },3755 .{ .v_sd, .cvtss2 },
3675 dst_alias,3756 dst_alias,
3676 dst_alias,3757 dst_alias,
3677 try src_mcv.mem(self, self.memSize(src_ty)),3758 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
3678 ) else try self.asmRegisterRegisterRegister(3759 ) else try self.asmRegisterRegisterRegister(
3679 .{ .v_sd, .cvtss2 },3760 .{ .v_sd, .cvtss2 },
3680 dst_alias,3761 dst_alias,
...@@ -3687,7 +3768,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -3687,7 +3768,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3687 2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(3768 2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(
3688 .{ .v_pd, .cvtps2 },3769 .{ .v_pd, .cvtps2 },
3689 dst_alias,3770 dst_alias,
3690 try src_mcv.mem(self, self.memSize(src_ty)),3771 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
3691 ) else try self.asmRegisterRegister(3772 ) else try self.asmRegisterRegister(
3692 .{ .v_pd, .cvtps2 },3773 .{ .v_pd, .cvtps2 },
3693 dst_alias,3774 dst_alias,
...@@ -3704,7 +3785,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -3704,7 +3785,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3704 else => break :result null,3785 else => break :result null,
3705 },3786 },
3706 dst_alias,3787 dst_alias,
3707 try src_mcv.mem(self, self.memSize(src_ty)),3788 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
3708 ) else try self.asmRegisterRegister(3789 ) else try self.asmRegisterRegister(
3709 switch (vec_len) {3790 switch (vec_len) {
3710 1 => .{ ._sd, .cvtss2 },3791 1 => .{ ._sd, .cvtss2 },
...@@ -3725,7 +3806,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -3725,7 +3806,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
3725 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3806 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3726}3807}
37273808
3728fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {3809fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void {
3729 const pt = self.pt;3810 const pt = self.pt;
3730 const zcu = pt.zcu;3811 const zcu = pt.zcu;
3731 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3812 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -3841,7 +3922,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -3841,7 +3922,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3841 if (src_mcv.isMemory()) try self.asmRegisterMemory(3922 if (src_mcv.isMemory()) try self.asmRegisterMemory(
3842 mir_tag,3923 mir_tag,
3843 dst_alias,3924 dst_alias,
3844 try src_mcv.mem(self, self.memSize(src_ty)),3925 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
3845 ) else try self.asmRegisterRegister(3926 ) else try self.asmRegisterRegister(
3846 mir_tag,3927 mir_tag,
3847 dst_alias,3928 dst_alias,
...@@ -3984,7 +4065,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -3984,7 +4065,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
3984 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });4065 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3985}4066}
39864067
3987fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {4068fn airTrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
3988 const pt = self.pt;4069 const pt = self.pt;
3989 const zcu = pt.zcu;4070 const zcu = pt.zcu;
3990 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4071 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -4080,7 +4161,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -4080,7 +4161,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
4080 .{ .vp_, .@"and" },4161 .{ .vp_, .@"and" },
4081 dst_alias,4162 dst_alias,
4082 dst_alias,4163 dst_alias,
4083 try splat_addr_mcv.deref().mem(self, .fromSize(splat_abi_size)),4164 try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
4084 );4165 );
4085 if (src_abi_size > 16) {4166 if (src_abi_size > 16) {
4086 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);4167 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
...@@ -4104,7 +4185,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -4104,7 +4185,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
4104 try self.asmRegisterMemory(4185 try self.asmRegisterMemory(
4105 .{ .p_, .@"and" },4186 .{ .p_, .@"and" },
4106 dst_alias,4187 dst_alias,
4107 try splat_addr_mcv.deref().mem(self, .fromSize(splat_abi_size)),4188 try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
4108 );4189 );
4109 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);4190 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
4110 }4191 }
...@@ -4130,7 +4211,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -4130,7 +4211,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
4130 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });4211 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
4131}4212}
41324213
4133fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {4214fn airIntFromBool(self: *CodeGen, inst: Air.Inst.Index) !void {
4134 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;4215 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4135 const ty = self.typeOfIndex(inst);4216 const ty = self.typeOfIndex(inst);
41364217
...@@ -4143,7 +4224,7 @@ fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {...@@ -4143,7 +4224,7 @@ fn airIntFromBool(self: *Self, inst: Air.Inst.Index) !void {
4143 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });4224 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4144}4225}
41454226
4146fn airSlice(self: *Self, inst: Air.Inst.Index) !void {4227fn airSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
4147 const zcu = self.pt.zcu;4228 const zcu = self.pt.zcu;
4148 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;4229 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4149 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;4230 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
...@@ -4167,13 +4248,13 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -4167,13 +4248,13 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
4167 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });4248 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4168}4249}
41694250
4170fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {4251fn airUnOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4171 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4252 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4172 const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand);4253 const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand);
4173 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });4254 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
4174}4255}
41754256
4176fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {4257fn airBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4177 const pt = self.pt;4258 const pt = self.pt;
4178 const zcu = pt.zcu;4259 const zcu = pt.zcu;
4179 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4260 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -4208,14 +4289,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {...@@ -4208,14 +4289,14 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4208 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });4289 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
4209}4290}
42104291
4211fn airPtrArithmetic(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {4292fn airPtrArithmetic(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
4212 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;4293 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4213 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;4294 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
4214 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);4295 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
4215 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });4296 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
4216}4297}
42174298
4218fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {4299fn activeIntBits(self: *CodeGen, dst_air: Air.Inst.Ref) u16 {
4219 const pt = self.pt;4300 const pt = self.pt;
4220 const zcu = pt.zcu;4301 const zcu = pt.zcu;
4221 const air_tag = self.air.instructions.items(.tag);4302 const air_tag = self.air.instructions.items(.tag);
...@@ -4250,7 +4331,7 @@ fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {...@@ -4250,7 +4331,7 @@ fn activeIntBits(self: *Self, dst_air: Air.Inst.Ref) u16 {
4250 return dst_info.bits;4331 return dst_info.bits;
4251}4332}
42524333
4253fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {4334fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
4254 const pt = self.pt;4335 const pt = self.pt;
4255 const zcu = pt.zcu;4336 const zcu = pt.zcu;
4256 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4337 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -4314,7 +4395,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4314,7 +4395,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
4314 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(4395 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
4315 .{ ._, .mov },4396 .{ ._, .mov },
4316 tmp_reg,4397 tmp_reg,
4317 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),4398 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4318 ) else try self.asmRegisterRegister(4399 ) else try self.asmRegisterRegister(
4319 .{ ._, .mov },4400 .{ ._, .mov },
4320 tmp_reg,4401 tmp_reg,
...@@ -4338,7 +4419,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4338,7 +4419,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
4338 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(4419 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
4339 .{ ._, .xor },4420 .{ ._, .xor },
4340 tmp_reg,4421 tmp_reg,
4341 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),4422 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4342 ) else try self.asmRegisterRegister(4423 ) else try self.asmRegisterRegister(
4343 .{ ._, .xor },4424 .{ ._, .xor },
4344 tmp_reg,4425 tmp_reg,
...@@ -4441,12 +4522,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4441,12 +4522,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
4441 try self.asmRegisterMemory(4522 try self.asmRegisterMemory(
4442 .{ ._, .add },4523 .{ ._, .add },
4443 tmp_regs[0],4524 tmp_regs[0],
4444 try mat_rhs_mcv.mem(self, .qword),4525 try mat_rhs_mcv.mem(self, .{ .size = .qword }),
4445 );4526 );
4446 try self.asmRegisterMemory(4527 try self.asmRegisterMemory(
4447 .{ ._, .adc },4528 .{ ._, .adc },
4448 tmp_regs[1],4529 tmp_regs[1],
4449 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),4530 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4450 );4531 );
4451 } else for (4532 } else for (
4452 [_]Mir.Inst.Tag{ .add, .adc },4533 [_]Mir.Inst.Tag{ .add, .adc },
...@@ -4476,7 +4557,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4476,7 +4557,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
4476 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });4557 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4477}4558}
44784559
4479fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {4560fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void {
4480 const pt = self.pt;4561 const pt = self.pt;
4481 const zcu = pt.zcu;4562 const zcu = pt.zcu;
4482 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4563 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -4577,7 +4658,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -4577,7 +4658,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
4577 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });4658 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
4578}4659}
45794660
4580fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {4661fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void {
4581 const pt = self.pt;4662 const pt = self.pt;
4582 const zcu = pt.zcu;4663 const zcu = pt.zcu;
4583 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4664 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -4671,7 +4752,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -4671,7 +4752,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
4671 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });4752 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
4672}4753}
46734754
4674fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {4755fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
4675 const pt = self.pt;4756 const pt = self.pt;
4676 const zcu = pt.zcu;4757 const zcu = pt.zcu;
4677 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4758 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -4715,7 +4796,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -4715,7 +4796,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4715 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(4796 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
4716 .{ ._, .mov },4797 .{ ._, .mov },
4717 tmp_reg,4798 tmp_reg,
4718 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),4799 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4719 ) else try self.asmRegisterRegister(4800 ) else try self.asmRegisterRegister(
4720 .{ ._, .mov },4801 .{ ._, .mov },
4721 tmp_reg,4802 tmp_reg,
...@@ -4739,7 +4820,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -4739,7 +4820,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4739 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(4820 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
4740 .{ ._, .xor },4821 .{ ._, .xor },
4741 tmp_reg,4822 tmp_reg,
4742 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),4823 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
4743 ) else try self.asmRegisterRegister(4824 ) else try self.asmRegisterRegister(
4744 .{ ._, .xor },4825 .{ ._, .xor },
4745 tmp_reg,4826 tmp_reg,
...@@ -4748,7 +4829,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -4748,7 +4829,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
47484829
4749 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));4830 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
4750 try self.asmRegister(.{ ._, .not }, tmp_reg);4831 try self.asmRegister(.{ ._, .not }, tmp_reg);
4751 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), .s(0));4832 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .{ .size = .dword }), .s(0));
4752 try self.freeValue(overflow);4833 try self.freeValue(overflow);
4753 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);4834 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
4754 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63));4835 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63));
...@@ -4818,7 +4899,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -4818,7 +4899,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
4818 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });4899 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4819}4900}
48204901
4821fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {4902fn airAddSubWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
4822 const pt = self.pt;4903 const pt = self.pt;
4823 const zcu = pt.zcu;4904 const zcu = pt.zcu;
4824 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;4905 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -4883,7 +4964,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4883,7 +4964,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4883 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });4964 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4884}4965}
48854966
4886fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {4967fn airShlWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
4887 const pt = self.pt;4968 const pt = self.pt;
4888 const zcu = pt.zcu;4969 const zcu = pt.zcu;
4889 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;4970 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -4961,7 +5042,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4961,7 +5042,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4961}5042}
49625043
4963fn genSetFrameTruncatedOverflowCompare(5044fn genSetFrameTruncatedOverflowCompare(
4964 self: *Self,5045 self: *CodeGen,
4965 tuple_ty: Type,5046 tuple_ty: Type,
4966 frame_index: FrameIndex,5047 frame_index: FrameIndex,
4967 src_mcv: MCValue,5048 src_mcv: MCValue,
...@@ -5039,7 +5120,7 @@ fn genSetFrameTruncatedOverflowCompare(...@@ -5039,7 +5120,7 @@ fn genSetFrameTruncatedOverflowCompare(
5039 );5120 );
5040}5121}
50415122
5042fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {5123fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
5043 const pt = self.pt;5124 const pt = self.pt;
5044 const zcu = pt.zcu;5125 const zcu = pt.zcu;
5045 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;5126 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -5214,7 +5295,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5214,7 +5295,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5214 );5295 );
5215 try self.asmMemoryImmediate(5296 try self.asmMemoryImmediate(
5216 .{ ._, .cmp },5297 .{ ._, .cmp },
5217 try overflow.mem(self, self.memSize(Type.c_int)),5298 try overflow.mem(self, .{ .size = self.memSize(Type.c_int) }),
5218 .s(0),5299 .s(0),
5219 );5300 );
5220 try self.genSetMem(5301 try self.genSetMem(
...@@ -5270,7 +5351,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5270,7 +5351,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5270 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(5351 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
5271 .{ ._, .mov },5352 .{ ._, .mov },
5272 .rax,5353 .rax,
5273 try mat_lhs_mcv.mem(self, .qword),5354 try mat_lhs_mcv.mem(self, .{ .size = .qword }),
5274 ) else try self.asmRegisterRegister(5355 ) else try self.asmRegisterRegister(
5275 .{ ._, .mov },5356 .{ ._, .mov },
5276 .rax,5357 .rax,
...@@ -5279,7 +5360,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5279,7 +5360,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5279 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(5360 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
5280 .{ ._, .mov },5361 .{ ._, .mov },
5281 tmp_regs[0],5362 tmp_regs[0],
5282 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),5363 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
5283 ) else try self.asmRegisterRegister(5364 ) else try self.asmRegisterRegister(
5284 .{ ._, .mov },5365 .{ ._, .mov },
5285 tmp_regs[0],5366 tmp_regs[0],
...@@ -5290,7 +5371,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5290,7 +5371,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5290 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);5371 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
5291 try self.asmSetccRegister(.o, tmp_regs[2].to8());5372 try self.asmSetccRegister(.o, tmp_regs[2].to8());
5292 if (mat_rhs_mcv.isMemory())5373 if (mat_rhs_mcv.isMemory())
5293 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))5374 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
5294 else5375 else
5295 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);5376 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
5296 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);5377 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...@@ -5299,7 +5380,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5299,7 +5380,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5299 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(5380 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
5300 .{ ._, .mov },5381 .{ ._, .mov },
5301 tmp_regs[0],5382 tmp_regs[0],
5302 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),5383 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
5303 ) else try self.asmRegisterRegister(5384 ) else try self.asmRegisterRegister(
5304 .{ ._, .mov },5385 .{ ._, .mov },
5305 tmp_regs[0],5386 tmp_regs[0],
...@@ -5316,7 +5397,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5316,7 +5397,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5316 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(5397 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
5317 .{ .i_, .mul },5398 .{ .i_, .mul },
5318 tmp_regs[0],5399 tmp_regs[0],
5319 try mat_rhs_mcv.mem(self, .qword),5400 try mat_rhs_mcv.mem(self, .{ .size = .qword }),
5320 ) else try self.asmRegisterRegister(5401 ) else try self.asmRegisterRegister(
5321 .{ .i_, .mul },5402 .{ .i_, .mul },
5322 tmp_regs[0],5403 tmp_regs[0],
...@@ -5416,7 +5497,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -5416,7 +5497,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
5416/// Generates signed or unsigned integer multiplication/division.5497/// Generates signed or unsigned integer multiplication/division.
5417/// Clobbers .rax and .rdx registers.5498/// Clobbers .rax and .rdx registers.
5418/// Quotient is saved in .rax and remainder in .rdx.5499/// Quotient is saved in .rax and remainder in .rdx.
5419fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {5500fn genIntMulDivOpMir(self: *CodeGen, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
5420 const pt = self.pt;5501 const pt = self.pt;
5421 const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));5502 const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
5422 const bit_size: u32 = @intCast(self.regBitSize(ty));5503 const bit_size: u32 = @intCast(self.regBitSize(ty));
...@@ -5455,7 +5536,10 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue...@@ -5455,7 +5536,10 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
5455 };5536 };
5456 switch (mat_rhs) {5537 switch (mat_rhs) {
5457 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),5538 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
5458 .memory, .indirect, .load_frame => try self.asmMemory(tag, try mat_rhs.mem(self, .fromSize(abi_size))),5539 .memory, .indirect, .load_frame => try self.asmMemory(
5540 tag,
5541 try mat_rhs.mem(self, .{ .size = .fromSize(abi_size) }),
5542 ),
5459 else => unreachable,5543 else => unreachable,
5460 }5544 }
5461 if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah);5545 if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah);
...@@ -5463,7 +5547,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue...@@ -5463,7 +5547,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
54635547
5464/// Always returns a register.5548/// Always returns a register.
5465/// Clobbers .rax and .rdx registers.5549/// Clobbers .rax and .rdx registers.
5466fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {5550fn genInlineIntDivFloor(self: *CodeGen, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
5467 const pt = self.pt;5551 const pt = self.pt;
5468 const zcu = pt.zcu;5552 const zcu = pt.zcu;
5469 const abi_size: u32 = @intCast(ty.abiSize(zcu));5553 const abi_size: u32 = @intCast(ty.abiSize(zcu));
...@@ -5516,7 +5600,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa...@@ -5516,7 +5600,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
5516 return MCValue{ .register = divisor };5600 return MCValue{ .register = divisor };
5517}5601}
55185602
5519fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {5603fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
5520 const pt = self.pt;5604 const pt = self.pt;
5521 const zcu = pt.zcu;5605 const zcu = pt.zcu;
5522 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;5606 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -5784,14 +5868,14 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -5784,14 +5868,14 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
5784 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });5868 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
5785}5869}
57865870
5787fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {5871fn airShlSat(self: *CodeGen, inst: Air.Inst.Index) !void {
5788 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;5872 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5789 _ = bin_op;5873 _ = bin_op;
5790 return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});5874 return self.fail("TODO implement shl_sat for {}", .{self.target.cpu.arch});
5791 //return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });5875 //return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
5792}5876}
57935877
5794fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {5878fn airOptionalPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
5795 const zcu = self.pt.zcu;5879 const zcu = self.pt.zcu;
5796 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5880 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5797 const result: MCValue = result: {5881 const result: MCValue = result: {
...@@ -5824,7 +5908,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -5824,7 +5908,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
5824 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });5908 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5825}5909}
58265910
5827fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {5911fn airOptionalPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5828 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5912 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
58295913
5830 const dst_ty = self.typeOfIndex(inst);5914 const dst_ty = self.typeOfIndex(inst);
...@@ -5837,7 +5921,7 @@ fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -5837,7 +5921,7 @@ fn airOptionalPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
5837 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });5921 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
5838}5922}
58395923
5840fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {5924fn airOptionalPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
5841 const pt = self.pt;5925 const pt = self.pt;
5842 const zcu = pt.zcu;5926 const zcu = pt.zcu;
5843 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5927 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -5878,7 +5962,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -5878,7 +5962,7 @@ fn airOptionalPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
5878 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });5962 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5879}5963}
58805964
5881fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {5965fn airUnwrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
5882 const pt = self.pt;5966 const pt = self.pt;
5883 const zcu = pt.zcu;5967 const zcu = pt.zcu;
5884 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5968 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -5923,7 +6007,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -5923,7 +6007,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
5923 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });6007 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5924}6008}
59256009
5926fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {6010fn airUnwrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
5927 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6011 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5928 const operand_ty = self.typeOf(ty_op.operand);6012 const operand_ty = self.typeOf(ty_op.operand);
5929 const operand = try self.resolveInst(ty_op.operand);6013 const operand = try self.resolveInst(ty_op.operand);
...@@ -5932,7 +6016,7 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -5932,7 +6016,7 @@ fn airUnwrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
5932}6016}
59336017
5934// *(E!T) -> E6018// *(E!T) -> E
5935fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {6019fn airUnwrapErrUnionErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5936 const pt = self.pt;6020 const pt = self.pt;
5937 const zcu = pt.zcu;6021 const zcu = pt.zcu;
5938 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6022 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -5972,7 +6056,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -5972,7 +6056,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void {
5972}6056}
59736057
5974// *(E!T) -> *T6058// *(E!T) -> *T
5975fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {6059fn airUnwrapErrUnionPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
5976 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6060 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5977 const operand_ty = self.typeOf(ty_op.operand);6061 const operand_ty = self.typeOf(ty_op.operand);
5978 const operand = try self.resolveInst(ty_op.operand);6062 const operand = try self.resolveInst(ty_op.operand);
...@@ -5980,7 +6064,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -5980,7 +6064,7 @@ fn airUnwrapErrUnionPayloadPtr(self: *Self, inst: Air.Inst.Index) !void {
5980 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });6064 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
5981}6065}
59826066
5983fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {6067fn airErrUnionPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
5984 const pt = self.pt;6068 const pt = self.pt;
5985 const zcu = pt.zcu;6069 const zcu = pt.zcu;
5986 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6070 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -6037,7 +6121,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {...@@ -6037,7 +6121,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void {
6037}6121}
60386122
6039fn genUnwrapErrUnionPayloadMir(6123fn genUnwrapErrUnionPayloadMir(
6040 self: *Self,6124 self: *CodeGen,
6041 maybe_inst: ?Air.Inst.Index,6125 maybe_inst: ?Air.Inst.Index,
6042 err_union_ty: Type,6126 err_union_ty: Type,
6043 err_union: MCValue,6127 err_union: MCValue,
...@@ -6087,7 +6171,7 @@ fn genUnwrapErrUnionPayloadMir(...@@ -6087,7 +6171,7 @@ fn genUnwrapErrUnionPayloadMir(
6087}6171}
60886172
6089fn genUnwrapErrUnionPayloadPtrMir(6173fn genUnwrapErrUnionPayloadPtrMir(
6090 self: *Self,6174 self: *CodeGen,
6091 maybe_inst: ?Air.Inst.Index,6175 maybe_inst: ?Air.Inst.Index,
6092 ptr_ty: Type,6176 ptr_ty: Type,
6093 ptr_mcv: MCValue,6177 ptr_mcv: MCValue,
...@@ -6110,23 +6194,23 @@ fn genUnwrapErrUnionPayloadPtrMir(...@@ -6110,23 +6194,23 @@ fn genUnwrapErrUnionPayloadPtrMir(
6110 return result;6194 return result;
6111}6195}
61126196
6113fn airErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {6197fn airErrReturnTrace(self: *CodeGen, inst: Air.Inst.Index) !void {
6114 _ = inst;6198 _ = inst;
6115 return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});6199 return self.fail("TODO implement airErrReturnTrace for {}", .{self.target.cpu.arch});
6116 //return self.finishAir(inst, result, .{ .none, .none, .none });6200 //return self.finishAir(inst, result, .{ .none, .none, .none });
6117}6201}
61186202
6119fn airSetErrReturnTrace(self: *Self, inst: Air.Inst.Index) !void {6203fn airSetErrReturnTrace(self: *CodeGen, inst: Air.Inst.Index) !void {
6120 _ = inst;6204 _ = inst;
6121 return self.fail("TODO implement airSetErrReturnTrace for {}", .{self.target.cpu.arch});6205 return self.fail("TODO implement airSetErrReturnTrace for {}", .{self.target.cpu.arch});
6122}6206}
61236207
6124fn airSaveErrReturnTraceIndex(self: *Self, inst: Air.Inst.Index) !void {6208fn airSaveErrReturnTraceIndex(self: *CodeGen, inst: Air.Inst.Index) !void {
6125 _ = inst;6209 _ = inst;
6126 return self.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{self.target.cpu.arch});6210 return self.fail("TODO implement airSaveErrReturnTraceIndex for {}", .{self.target.cpu.arch});
6127}6211}
61286212
6129fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {6213fn airWrapOptional(self: *CodeGen, inst: Air.Inst.Index) !void {
6130 const pt = self.pt;6214 const pt = self.pt;
6131 const zcu = pt.zcu;6215 const zcu = pt.zcu;
6132 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6216 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -6181,7 +6265,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {...@@ -6181,7 +6265,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
6181}6265}
61826266
6183/// T to E!T6267/// T to E!T
6184fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {6268fn airWrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
6185 const pt = self.pt;6269 const pt = self.pt;
6186 const zcu = pt.zcu;6270 const zcu = pt.zcu;
6187 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6271 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -6205,7 +6289,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -6205,7 +6289,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
6205}6289}
62066290
6207/// E to E!T6291/// E to E!T
6208fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {6292fn airWrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
6209 const pt = self.pt;6293 const pt = self.pt;
6210 const zcu = pt.zcu;6294 const zcu = pt.zcu;
6211 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6295 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -6228,7 +6312,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6228,7 +6312,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
6228 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });6312 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
6229}6313}
62306314
6231fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {6315fn airSlicePtr(self: *CodeGen, inst: Air.Inst.Index) !void {
6232 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6316 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6233 const result = result: {6317 const result = result: {
6234 const src_mcv = try self.resolveInst(ty_op.operand);6318 const src_mcv = try self.resolveInst(ty_op.operand);
...@@ -6251,7 +6335,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6251,7 +6335,7 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
6251 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });6335 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
6252}6336}
62536337
6254fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {6338fn airSliceLen(self: *CodeGen, inst: Air.Inst.Index) !void {
6255 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6339 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6256 const result = result: {6340 const result = result: {
6257 const src_mcv = try self.resolveInst(ty_op.operand);6341 const src_mcv = try self.resolveInst(ty_op.operand);
...@@ -6279,7 +6363,7 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -6279,7 +6363,7 @@ fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void {
6279 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });6363 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
6280}6364}
62816365
6282fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {6366fn airPtrSliceLenPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
6283 const pt = self.pt;6367 const pt = self.pt;
6284 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6368 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
62856369
...@@ -6314,7 +6398,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6314,7 +6398,7 @@ fn airPtrSliceLenPtr(self: *Self, inst: Air.Inst.Index) !void {
6314 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });6398 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
6315}6399}
63166400
6317fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {6401fn airPtrSlicePtrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
6318 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6402 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
63196403
6320 const dst_ty = self.typeOfIndex(inst);6404 const dst_ty = self.typeOfIndex(inst);
...@@ -6327,7 +6411,7 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6327,7 +6411,7 @@ fn airPtrSlicePtrPtr(self: *Self, inst: Air.Inst.Index) !void {
6327 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });6411 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
6328}6412}
63296413
6330fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Register {6414fn elemOffset(self: *CodeGen, index_ty: Type, index: MCValue, elem_size: u64) !Register {
6331 const reg: Register = blk: {6415 const reg: Register = blk: {
6332 switch (index) {6416 switch (index) {
6333 .immediate => |imm| {6417 .immediate => |imm| {
...@@ -6347,7 +6431,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi...@@ -6347,7 +6431,7 @@ fn elemOffset(self: *Self, index_ty: Type, index: MCValue, elem_size: u64) !Regi
6347 return reg;6431 return reg;
6348}6432}
63496433
6350fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {6434fn genSliceElemPtr(self: *CodeGen, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
6351 const pt = self.pt;6435 const pt = self.pt;
6352 const zcu = pt.zcu;6436 const zcu = pt.zcu;
6353 const slice_ty = self.typeOf(lhs);6437 const slice_ty = self.typeOf(lhs);
...@@ -6384,7 +6468,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {...@@ -6384,7 +6468,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
6384 return MCValue{ .register = addr_reg.to64() };6468 return MCValue{ .register = addr_reg.to64() };
6385}6469}
63866470
6387fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {6471fn airSliceElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
6388 const pt = self.pt;6472 const pt = self.pt;
6389 const zcu = pt.zcu;6473 const zcu = pt.zcu;
6390 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6474 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -6403,14 +6487,14 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6403,14 +6487,14 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
6403 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });6487 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6404}6488}
64056489
6406fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {6490fn airSliceElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
6407 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;6491 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6408 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;6492 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
6409 const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs);6493 const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs);
6410 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });6494 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
6411}6495}
64126496
6413fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {6497fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
6414 const pt = self.pt;6498 const pt = self.pt;
6415 const zcu = pt.zcu;6499 const zcu = pt.zcu;
6416 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6500 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -6463,7 +6547,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6463,7 +6547,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
6463 },6547 },
6464 .load_frame => try self.asmMemoryRegister(6548 .load_frame => try self.asmMemoryRegister(
6465 .{ ._, .bt },6549 .{ ._, .bt },
6466 try array_mcv.mem(self, .qword),6550 try array_mcv.mem(self, .{ .size = .qword }),
6467 index_reg.to64(),6551 index_reg.to64(),
6468 ),6552 ),
6469 .memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister(6553 .memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister(
...@@ -6540,7 +6624,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6540,7 +6624,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
6540 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });6624 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6541}6625}
65426626
6543fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {6627fn airPtrElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
6544 const pt = self.pt;6628 const pt = self.pt;
6545 const zcu = pt.zcu;6629 const zcu = pt.zcu;
6546 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6630 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -6591,7 +6675,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6591,7 +6675,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void {
6591 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });6675 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
6592}6676}
65936677
6594fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {6678fn airPtrElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
6595 const pt = self.pt;6679 const pt = self.pt;
6596 const zcu = pt.zcu;6680 const zcu = pt.zcu;
6597 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;6681 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -6637,7 +6721,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -6637,7 +6721,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void {
6637 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });6721 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
6638}6722}
66396723
6640fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {6724fn airSetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
6641 const pt = self.pt;6725 const pt = self.pt;
6642 const zcu = pt.zcu;6726 const zcu = pt.zcu;
6643 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6727 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -6682,7 +6766,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -6682,7 +6766,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
6682 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });6766 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
6683}6767}
66846768
6685fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {6769fn airGetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
6686 const zcu = self.pt.zcu;6770 const zcu = self.pt.zcu;
6687 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6771 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
66886772
...@@ -6739,7 +6823,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {...@@ -6739,7 +6823,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void {
6739 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });6823 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
6740}6824}
67416825
6742fn airClz(self: *Self, inst: Air.Inst.Index) !void {6826fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
6743 const pt = self.pt;6827 const pt = self.pt;
6744 const zcu = pt.zcu;6828 const zcu = pt.zcu;
6745 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6829 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -6936,7 +7020,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -6936,7 +7020,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
6936 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });7020 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
6937}7021}
69387022
6939fn airCtz(self: *Self, inst: Air.Inst.Index) !void {7023fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
6940 const pt = self.pt;7024 const pt = self.pt;
6941 const zcu = pt.zcu;7025 const zcu = pt.zcu;
6942 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;7026 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -7098,7 +7182,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -7098,7 +7182,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
7098 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });7182 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
7099}7183}
71007184
7101fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {7185fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void {
7102 const pt = self.pt;7186 const pt = self.pt;
7103 const zcu = pt.zcu;7187 const zcu = pt.zcu;
7104 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;7188 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -7157,7 +7241,7 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {...@@ -7157,7 +7241,7 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
7157}7241}
71587242
7159fn genPopCount(7243fn genPopCount(
7160 self: *Self,7244 self: *CodeGen,
7161 dst_reg: Register,7245 dst_reg: Register,
7162 src_ty: Type,7246 src_ty: Type,
7163 src_mcv: MCValue,7247 src_mcv: MCValue,
...@@ -7249,7 +7333,7 @@ fn genPopCount(...@@ -7249,7 +7333,7 @@ fn genPopCount(
7249}7333}
72507334
7251fn genByteSwap(7335fn genByteSwap(
7252 self: *Self,7336 self: *CodeGen,
7253 inst: Air.Inst.Index,7337 inst: Air.Inst.Index,
7254 src_ty: Type,7338 src_ty: Type,
7255 src_mcv: MCValue,7339 src_mcv: MCValue,
...@@ -7308,7 +7392,7 @@ fn genByteSwap(...@@ -7308,7 +7392,7 @@ fn genByteSwap(
7308 try self.asmRegisterMemory(7392 try self.asmRegisterMemory(
7309 .{ ._, if (has_movbe) .movbe else .mov },7393 .{ ._, if (has_movbe) .movbe else .mov },
7310 dst_reg.to64(),7394 dst_reg.to64(),
7311 try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .qword),7395 try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .{ .size = .qword }),
7312 );7396 );
7313 if (!has_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());7397 if (!has_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
7314 } else {7398 } else {
...@@ -7414,7 +7498,7 @@ fn genByteSwap(...@@ -7414,7 +7498,7 @@ fn genByteSwap(
7414 return dst_mcv;7498 return dst_mcv;
7415}7499}
74167500
7417fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {7501fn airByteSwap(self: *CodeGen, inst: Air.Inst.Index) !void {
7418 const pt = self.pt;7502 const pt = self.pt;
7419 const zcu = pt.zcu;7503 const zcu = pt.zcu;
7420 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;7504 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -7437,7 +7521,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {...@@ -7437,7 +7521,7 @@ fn airByteSwap(self: *Self, inst: Air.Inst.Index) !void {
7437 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });7521 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
7438}7522}
74397523
7440fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {7524fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void {
7441 const pt = self.pt;7525 const pt = self.pt;
7442 const zcu = pt.zcu;7526 const zcu = pt.zcu;
7443 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;7527 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -7560,7 +7644,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -7560,7 +7644,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
7560 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });7644 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
7561}7645}
75627646
7563fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {7647fn floatSign(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type) !void {
7564 const pt = self.pt;7648 const pt = self.pt;
7565 const zcu = pt.zcu;7649 const zcu = pt.zcu;
7566 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];7650 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
...@@ -7623,7 +7707,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)...@@ -7623,7 +7707,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
7623 else => unreachable,7707 else => unreachable,
7624 });7708 });
7625 const sign_mem: Memory = if (sign_mcv.isMemory())7709 const sign_mem: Memory = if (sign_mcv.isMemory())
7626 try sign_mcv.mem(self, .fromSize(abi_size))7710 try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) })
7627 else7711 else
7628 .{7712 .{
7629 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },7713 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
...@@ -7688,7 +7772,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)...@@ -7688,7 +7772,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
7688 return self.finishAir(inst, result, .{ operand, .none, .none });7772 return self.finishAir(inst, result, .{ operand, .none, .none });
7689}7773}
76907774
7691fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {7775fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index) !void {
7692 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;7776 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7693 const ty = self.typeOf(un_op);7777 const ty = self.typeOf(un_op);
7694 return self.floatSign(inst, un_op, ty);7778 return self.floatSign(inst, un_op, ty);
...@@ -7713,7 +7797,7 @@ const RoundMode = packed struct(u5) {...@@ -7713,7 +7797,7 @@ const RoundMode = packed struct(u5) {
7713 } = .normal,7797 } = .normal,
7714};7798};
77157799
7716fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {7800fn airRound(self: *CodeGen, inst: Air.Inst.Index, mode: RoundMode) !void {
7717 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;7801 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7718 const ty = self.typeOf(un_op);7802 const ty = self.typeOf(un_op);
77197803
...@@ -7737,7 +7821,7 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {...@@ -7737,7 +7821,7 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void {
7737 return self.finishAir(inst, result, .{ un_op, .none, .none });7821 return self.finishAir(inst, result, .{ un_op, .none, .none });
7738}7822}
77397823
7740fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {7824fn getRoundTag(self: *CodeGen, ty: Type) ?Mir.Inst.FixedTag {
7741 const pt = self.pt;7825 const pt = self.pt;
7742 const zcu = pt.zcu;7826 const zcu = pt.zcu;
7743 return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) {7827 return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) {
...@@ -7770,7 +7854,7 @@ fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {...@@ -7770,7 +7854,7 @@ fn getRoundTag(self: *Self, ty: Type) ?Mir.Inst.FixedTag {
7770 } else null;7854 } else null;
7771}7855}
77727856
7773fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {7857fn genRoundLibcall(self: *CodeGen, ty: Type, src_mcv: MCValue, mode: RoundMode) !MCValue {
7774 const pt = self.pt;7858 const pt = self.pt;
7775 const zcu = pt.zcu;7859 const zcu = pt.zcu;
7776 if (self.getRoundTag(ty)) |_| return .none;7860 if (self.getRoundTag(ty)) |_| return .none;
...@@ -7795,7 +7879,7 @@ fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MC...@@ -7795,7 +7879,7 @@ fn genRoundLibcall(self: *Self, ty: Type, src_mcv: MCValue, mode: RoundMode) !MC
7795 } }, &.{ty}, &.{src_mcv});7879 } }, &.{ty}, &.{src_mcv});
7796}7880}
77977881
7798fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {7882fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void {
7799 const pt = self.pt;7883 const pt = self.pt;
7800 const mir_tag = self.getRoundTag(ty) orelse {7884 const mir_tag = self.getRoundTag(ty) orelse {
7801 const result = try self.genRoundLibcall(ty, src_mcv, mode);7885 const result = try self.genRoundLibcall(ty, src_mcv, mode);
...@@ -7808,7 +7892,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -7808,7 +7892,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7808 mir_tag,7892 mir_tag,
7809 dst_alias,7893 dst_alias,
7810 dst_alias,7894 dst_alias,
7811 try src_mcv.mem(self, .fromSize(abi_size)),7895 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
7812 .u(@as(u5, @bitCast(mode))),7896 .u(@as(u5, @bitCast(mode))),
7813 ) else try self.asmRegisterRegisterRegisterImmediate(7897 ) else try self.asmRegisterRegisterRegisterImmediate(
7814 mir_tag,7898 mir_tag,
...@@ -7823,7 +7907,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -7823,7 +7907,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7823 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(7907 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
7824 mir_tag,7908 mir_tag,
7825 dst_alias,7909 dst_alias,
7826 try src_mcv.mem(self, .fromSize(abi_size)),7910 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
7827 .u(@as(u5, @bitCast(mode))),7911 .u(@as(u5, @bitCast(mode))),
7828 ) else try self.asmRegisterRegisterImmediate(7912 ) else try self.asmRegisterRegisterImmediate(
7829 mir_tag,7913 mir_tag,
...@@ -7837,7 +7921,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -7837,7 +7921,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
7837 }7921 }
7838}7922}
78397923
7840fn airAbs(self: *Self, inst: Air.Inst.Index) !void {7924fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void {
7841 const pt = self.pt;7925 const pt = self.pt;
7842 const zcu = pt.zcu;7926 const zcu = pt.zcu;
7843 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;7927 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -7865,7 +7949,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -7865,7 +7949,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
7865 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(7949 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
7866 .l,7950 .l,
7867 registerAlias(dst_mcv.register, cmov_abi_size),7951 registerAlias(dst_mcv.register, cmov_abi_size),
7868 try src_mcv.mem(self, .fromSize(cmov_abi_size)),7952 try src_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
7869 ),7953 ),
7870 else => {7954 else => {
7871 const val_reg = try self.copyToTmpRegister(ty, src_mcv);7955 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...@@ -7931,7 +8015,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -7931,7 +8015,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
79318015
7932 try self.asmMemoryImmediate(8016 try self.asmMemoryImmediate(
7933 .{ ._, .cmp },8017 .{ ._, .cmp },
7934 try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .qword),8018 try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .{ .size = .qword }),
7935 .u(0),8019 .u(0),
7936 );8020 );
7937 const positive = try self.asmJccReloc(.ns, undefined);8021 const positive = try self.asmJccReloc(.ns, undefined);
...@@ -8024,7 +8108,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -8024,7 +8108,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
8024 if (src_mcv.isMemory()) try self.asmRegisterMemory(8108 if (src_mcv.isMemory()) try self.asmRegisterMemory(
8025 mir_tag,8109 mir_tag,
8026 dst_alias,8110 dst_alias,
8027 try src_mcv.mem(self, self.memSize(ty)),8111 try src_mcv.mem(self, .{ .size = self.memSize(ty) }),
8028 ) else try self.asmRegisterRegister(8112 ) else try self.asmRegisterRegister(
8029 mir_tag,8113 mir_tag,
8030 dst_alias,8114 dst_alias,
...@@ -8038,7 +8122,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -8038,7 +8122,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
8038 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });8122 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8039}8123}
80408124
8041fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {8125fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void {
8042 const pt = self.pt;8126 const pt = self.pt;
8043 const zcu = pt.zcu;8127 const zcu = pt.zcu;
8044 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;8128 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
...@@ -8131,9 +8215,9 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -8131,9 +8215,9 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
8131 if (src_mcv.isMemory()) try self.asmRegisterMemory(8215 if (src_mcv.isMemory()) try self.asmRegisterMemory(
8132 .{ .v_ps, .cvtph2 },8216 .{ .v_ps, .cvtph2 },
8133 wide_reg,8217 wide_reg,
8134 try src_mcv.mem(self, .fromSize(8218 try src_mcv.mem(self, .{ .size = .fromSize(
8135 @intCast(@divExact(wide_reg.bitSize(), 16)),8219 @intCast(@divExact(wide_reg.bitSize(), 16)),
8136 )),8220 ) }),
8137 ) else try self.asmRegisterRegister(8221 ) else try self.asmRegisterRegister(
8138 .{ .v_ps, .cvtph2 },8222 .{ .v_ps, .cvtph2 },
8139 wide_reg,8223 wide_reg,
...@@ -8177,7 +8261,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -8177,7 +8261,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
8177 mir_tag,8261 mir_tag,
8178 dst_reg,8262 dst_reg,
8179 dst_reg,8263 dst_reg,
8180 try src_mcv.mem(self, .fromSize(abi_size)),8264 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
8181 ) else try self.asmRegisterRegisterRegister(8265 ) else try self.asmRegisterRegisterRegister(
8182 mir_tag,8266 mir_tag,
8183 dst_reg,8267 dst_reg,
...@@ -8190,7 +8274,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -8190,7 +8274,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
8190 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(8274 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
8191 mir_tag,8275 mir_tag,
8192 dst_reg,8276 dst_reg,
8193 try src_mcv.mem(self, .fromSize(abi_size)),8277 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
8194 ) else try self.asmRegisterRegister(8278 ) else try self.asmRegisterRegister(
8195 mir_tag,8279 mir_tag,
8196 dst_reg,8280 dst_reg,
...@@ -8205,7 +8289,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -8205,7 +8289,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
8205 return self.finishAir(inst, result, .{ un_op, .none, .none });8289 return self.finishAir(inst, result, .{ un_op, .none, .none });
8206}8290}
82078291
8208fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {8292fn airUnaryMath(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
8209 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;8293 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
8210 const ty = self.typeOf(un_op);8294 const ty = self.typeOf(un_op);
8211 var callee_buf: ["__round?".len]u8 = undefined;8295 var callee_buf: ["__round?".len]u8 = undefined;
...@@ -8234,7 +8318,7 @@ fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {...@@ -8234,7 +8318,7 @@ fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
8234}8318}
82358319
8236fn reuseOperand(8320fn reuseOperand(
8237 self: *Self,8321 self: *CodeGen,
8238 inst: Air.Inst.Index,8322 inst: Air.Inst.Index,
8239 operand: Air.Inst.Ref,8323 operand: Air.Inst.Ref,
8240 op_index: Liveness.OperandInt,8324 op_index: Liveness.OperandInt,
...@@ -8244,7 +8328,7 @@ fn reuseOperand(...@@ -8244,7 +8328,7 @@ fn reuseOperand(
8244}8328}
82458329
8246fn reuseOperandAdvanced(8330fn reuseOperandAdvanced(
8247 self: *Self,8331 self: *CodeGen,
8248 inst: Air.Inst.Index,8332 inst: Air.Inst.Index,
8249 operand: Air.Inst.Ref,8333 operand: Air.Inst.Ref,
8250 op_index: Liveness.OperandInt,8334 op_index: Liveness.OperandInt,
...@@ -8282,7 +8366,7 @@ fn reuseOperandAdvanced(...@@ -8282,7 +8366,7 @@ fn reuseOperandAdvanced(
8282 return true;8366 return true;
8283}8367}
82848368
8285fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {8369fn packedLoad(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
8286 const pt = self.pt;8370 const pt = self.pt;
8287 const zcu = pt.zcu;8371 const zcu = pt.zcu;
82888372
...@@ -8390,7 +8474,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -8390,7 +8474,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
8390 try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg }, .{});8474 try self.genCopy(val_ty, dst_mcv, .{ .register = dst_reg }, .{});
8391}8475}
83928476
8393fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {8477fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void {
8394 const pt = self.pt;8478 const pt = self.pt;
8395 const zcu = pt.zcu;8479 const zcu = pt.zcu;
8396 const dst_ty = ptr_ty.childType(zcu);8480 const dst_ty = ptr_ty.childType(zcu);
...@@ -8433,7 +8517,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro...@@ -8433,7 +8517,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro
8433 }8517 }
8434}8518}
84358519
8436fn airLoad(self: *Self, inst: Air.Inst.Index) !void {8520fn airLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
8437 const pt = self.pt;8521 const pt = self.pt;
8438 const zcu = pt.zcu;8522 const zcu = pt.zcu;
8439 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;8523 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -8492,7 +8576,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -8492,7 +8576,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
8492 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });8576 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8493}8577}
84948578
8495fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {8579fn packedStore(self: *CodeGen, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void {
8496 const pt = self.pt;8580 const pt = self.pt;
8497 const zcu = pt.zcu;8581 const zcu = pt.zcu;
8498 const ptr_info = ptr_ty.ptrInfo(zcu);8582 const ptr_info = ptr_ty.ptrInfo(zcu);
...@@ -8593,7 +8677,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -8593,7 +8677,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
8593}8677}
85948678
8595fn store(8679fn store(
8596 self: *Self,8680 self: *CodeGen,
8597 ptr_ty: Type,8681 ptr_ty: Type,
8598 ptr_mcv: MCValue,8682 ptr_mcv: MCValue,
8599 src_mcv: MCValue,8683 src_mcv: MCValue,
...@@ -8641,7 +8725,7 @@ fn store(...@@ -8641,7 +8725,7 @@ fn store(
8641 }8725 }
8642}8726}
86438727
8644fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {8728fn airStore(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
8645 const pt = self.pt;8729 const pt = self.pt;
8646 const zcu = pt.zcu;8730 const zcu = pt.zcu;
8647 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;8731 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -8667,20 +8751,20 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8667,20 +8751,20 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8667 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });8751 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
8668}8752}
86698753
8670fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void {8754fn airStructFieldPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
8671 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;8755 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8672 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;8756 const extra = self.air.extraData(Air.StructField, ty_pl.payload).data;
8673 const result = try self.fieldPtr(inst, extra.struct_operand, extra.field_index);8757 const result = try self.fieldPtr(inst, extra.struct_operand, extra.field_index);
8674 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });8758 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
8675}8759}
86768760
8677fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, field_index: u8) !void {8761fn airStructFieldPtrIndex(self: *CodeGen, inst: Air.Inst.Index, field_index: u8) !void {
8678 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;8762 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
8679 const result = try self.fieldPtr(inst, ty_op.operand, field_index);8763 const result = try self.fieldPtr(inst, ty_op.operand, field_index);
8680 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });8764 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
8681}8765}
86828766
8683fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue {8767fn fieldPtr(self: *CodeGen, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue {
8684 const ptr_field_ty = self.typeOfIndex(inst);8768 const ptr_field_ty = self.typeOfIndex(inst);
86858769
8686 const src_mcv = try self.resolveInst(operand);8770 const src_mcv = try self.resolveInst(operand);
...@@ -8692,7 +8776,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_inde...@@ -8692,7 +8776,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_inde
8692 return dst_mcv.offset(self.fieldOffset(self.typeOf(operand), ptr_field_ty, field_index));8776 return dst_mcv.offset(self.fieldOffset(self.typeOf(operand), ptr_field_ty, field_index));
8693}8777}
86948778
8695fn fieldOffset(self: *Self, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 {8779fn fieldOffset(self: *CodeGen, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 {
8696 const pt = self.pt;8780 const pt = self.pt;
8697 const zcu = pt.zcu;8781 const zcu = pt.zcu;
8698 const agg_ty = ptr_agg_ty.childType(zcu);8782 const agg_ty = ptr_agg_ty.childType(zcu);
...@@ -8704,7 +8788,7 @@ fn fieldOffset(self: *Self, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u...@@ -8704,7 +8788,7 @@ fn fieldOffset(self: *Self, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u
8704 };8788 };
8705}8789}
87068790
8707fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {8791fn airStructFieldVal(self: *CodeGen, inst: Air.Inst.Index) !void {
8708 const pt = self.pt;8792 const pt = self.pt;
8709 const zcu = pt.zcu;8793 const zcu = pt.zcu;
8710 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;8794 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -8993,7 +9077,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -8993,7 +9077,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
8993 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });9077 return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none });
8994}9078}
89959079
8996fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {9080fn airFieldParentPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
8997 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;9081 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8998 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;9082 const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
89999083
...@@ -9008,7 +9092,7 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -9008,7 +9092,7 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
9008 return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });9092 return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none });
9009}9093}
90109094
9011fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {9095fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
9012 const pt = self.pt;9096 const pt = self.pt;
9013 const zcu = pt.zcu;9097 const zcu = pt.zcu;
9014 const src_ty = self.typeOf(src_air);9098 const src_ty = self.typeOf(src_air);
...@@ -9097,7 +9181,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:...@@ -9097,7 +9181,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
9097 return dst_mcv;9181 return dst_mcv;
9098}9182}
90999183
9100fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {9184fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
9101 const pt = self.pt;9185 const pt = self.pt;
9102 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));9186 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
9103 if (abi_size > 8) return self.fail("TODO implement {} for {}", .{ mir_tag, dst_ty.fmt(pt) });9187 if (abi_size > 8) return self.fail("TODO implement {} for {}", .{ mir_tag, dst_ty.fmt(pt) });
...@@ -9133,14 +9217,14 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC...@@ -9133,14 +9217,14 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
9133 },9217 },
9134 .indirect, .load_frame => try self.asmMemory(9218 .indirect, .load_frame => try self.asmMemory(
9135 mir_tag,9219 mir_tag,
9136 try dst_mcv.mem(self, .fromSize(abi_size)),9220 try dst_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
9137 ),9221 ),
9138 }9222 }
9139}9223}
91409224
9141/// Clobbers .rcx for non-immediate shift value.9225/// Clobbers .rcx for non-immediate shift value.
9142fn genShiftBinOpMir(9226fn genShiftBinOpMir(
9143 self: *Self,9227 self: *CodeGen,
9144 tag: Mir.Inst.FixedTag,9228 tag: Mir.Inst.FixedTag,
9145 lhs_ty: Type,9229 lhs_ty: Type,
9146 lhs_mcv: MCValue,9230 lhs_mcv: MCValue,
...@@ -9773,7 +9857,7 @@ fn genShiftBinOpMir(...@@ -9773,7 +9857,7 @@ fn genShiftBinOpMir(
9773/// Clobbers .rcx for non-immediate rhs, therefore care is needed to spill .rcx upfront.9857/// Clobbers .rcx for non-immediate rhs, therefore care is needed to spill .rcx upfront.
9774/// Asserts .rcx is free.9858/// Asserts .rcx is free.
9775fn genShiftBinOp(9859fn genShiftBinOp(
9776 self: *Self,9860 self: *CodeGen,
9777 air_tag: Air.Inst.Tag,9861 air_tag: Air.Inst.Tag,
9778 maybe_inst: ?Air.Inst.Index,9862 maybe_inst: ?Air.Inst.Index,
9779 lhs_mcv: MCValue,9863 lhs_mcv: MCValue,
...@@ -9832,7 +9916,7 @@ fn genShiftBinOp(...@@ -9832,7 +9916,7 @@ fn genShiftBinOp(
9832/// Clobbers .rax and .rdx therefore care is needed to spill .rax and .rdx upfront.9916/// Clobbers .rax and .rdx therefore care is needed to spill .rax and .rdx upfront.
9833/// Asserts .rax and .rdx are free.9917/// Asserts .rax and .rdx are free.
9834fn genMulDivBinOp(9918fn genMulDivBinOp(
9835 self: *Self,9919 self: *CodeGen,
9836 tag: Air.Inst.Tag,9920 tag: Air.Inst.Tag,
9837 maybe_inst: ?Air.Inst.Index,9921 maybe_inst: ?Air.Inst.Index,
9838 dst_ty: Type,9922 dst_ty: Type,
...@@ -9891,27 +9975,27 @@ fn genMulDivBinOp(...@@ -9891,27 +9975,27 @@ fn genMulDivBinOp(
9891 defer self.register_manager.unlockReg(tmp_lock);9975 defer self.register_manager.unlockReg(tmp_lock);
98929976
9893 if (mat_lhs_mcv.isMemory())9977 if (mat_lhs_mcv.isMemory())
9894 try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .qword))9978 try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .{ .size = .qword }))
9895 else9979 else
9896 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);9980 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
9897 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(9981 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
9898 .{ ._, .mov },9982 .{ ._, .mov },
9899 tmp_reg,9983 tmp_reg,
9900 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),9984 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
9901 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);9985 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);
9902 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);9986 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);
9903 if (mat_rhs_mcv.isMemory())9987 if (mat_rhs_mcv.isMemory())
9904 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))9988 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
9905 else9989 else
9906 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);9990 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
9907 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);9991 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
9908 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(9992 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
9909 .{ ._, .mov },9993 .{ ._, .mov },
9910 tmp_reg,9994 tmp_reg,
9911 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),9995 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
9912 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);9996 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);
9913 if (mat_rhs_mcv.isMemory())9997 if (mat_rhs_mcv.isMemory())
9914 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .qword))9998 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
9915 else9999 else
9916 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);10000 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);
9917 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);10001 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
...@@ -10238,7 +10322,7 @@ fn genMulDivBinOp(...@@ -10238,7 +10322,7 @@ fn genMulDivBinOp(
10238}10322}
1023910323
10240fn genBinOp(10324fn genBinOp(
10241 self: *Self,10325 self: *CodeGen,
10242 maybe_inst: ?Air.Inst.Index,10326 maybe_inst: ?Air.Inst.Index,
10243 air_tag: Air.Inst.Tag,10327 air_tag: Air.Inst.Tag,
10244 lhs_air: Air.Inst.Ref,10328 lhs_air: Air.Inst.Ref,
...@@ -10334,7 +10418,7 @@ fn genBinOp(...@@ -10334,7 +10418,7 @@ fn genBinOp(
10334 .{ .vp_w, .insr },10418 .{ .vp_w, .insr },
10335 dst_reg,10419 dst_reg,
10336 dst_reg,10420 dst_reg,
10337 try rhs_mcv.mem(self, .word),10421 try rhs_mcv.mem(self, .{ .size = .word }),
10338 .u(1),10422 .u(1),
10339 ) else try self.asmRegisterRegisterRegister(10423 ) else try self.asmRegisterRegisterRegister(
10340 .{ .vp_, .unpcklwd },10424 .{ .vp_, .unpcklwd },
...@@ -10359,7 +10443,7 @@ fn genBinOp(...@@ -10359,7 +10443,7 @@ fn genBinOp(
10359 mir_tag,10443 mir_tag,
10360 dst_reg,10444 dst_reg,
10361 dst_reg,10445 dst_reg,
10362 try src_mcv.mem(self, .fromBitSize(float_bits)),10446 try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
10363 ) else try self.asmRegisterRegisterRegister(10447 ) else try self.asmRegisterRegisterRegister(
10364 mir_tag,10448 mir_tag,
10365 dst_reg,10449 dst_reg,
...@@ -10378,7 +10462,7 @@ fn genBinOp(...@@ -10378,7 +10462,7 @@ fn genBinOp(
10378 if (src_mcv.isMemory()) try self.asmRegisterMemory(10462 if (src_mcv.isMemory()) try self.asmRegisterMemory(
10379 mir_tag,10463 mir_tag,
10380 dst_reg,10464 dst_reg,
10381 try src_mcv.mem(self, .fromBitSize(float_bits)),10465 try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
10382 ) else try self.asmRegisterRegister(10466 ) else try self.asmRegisterRegister(
10383 mir_tag,10467 mir_tag,
10384 dst_reg,10468 dst_reg,
...@@ -10651,22 +10735,22 @@ fn genBinOp(...@@ -10651,22 +10735,22 @@ fn genBinOp(
10651 try self.asmRegisterMemory(10735 try self.asmRegisterMemory(
10652 .{ ._, .cmp },10736 .{ ._, .cmp },
10653 dst_regs[0],10737 dst_regs[0],
10654 try src_mcv.mem(self, .qword),10738 try src_mcv.mem(self, .{ .size = .qword }),
10655 );10739 );
10656 try self.asmRegisterMemory(10740 try self.asmRegisterMemory(
10657 .{ ._, .sbb },10741 .{ ._, .sbb },
10658 tmp_reg,10742 tmp_reg,
10659 try src_mcv.address().offset(8).deref().mem(self, .qword),10743 try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
10660 );10744 );
10661 try self.asmCmovccRegisterMemory(10745 try self.asmCmovccRegisterMemory(
10662 cc,10746 cc,
10663 dst_regs[0],10747 dst_regs[0],
10664 try src_mcv.mem(self, .qword),10748 try src_mcv.mem(self, .{ .size = .qword }),
10665 );10749 );
10666 try self.asmCmovccRegisterMemory(10750 try self.asmCmovccRegisterMemory(
10667 cc,10751 cc,
10668 dst_regs[1],10752 dst_regs[1],
10669 try src_mcv.address().offset(8).deref().mem(self, .qword),10753 try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
10670 );10754 );
10671 } else {10755 } else {
10672 try self.asmRegisterRegister(10756 try self.asmRegisterRegister(
...@@ -10829,7 +10913,7 @@ fn genBinOp(...@@ -10829,7 +10913,7 @@ fn genBinOp(
10829 .{ .vp_w, .insr },10913 .{ .vp_w, .insr },
10830 dst_reg,10914 dst_reg,
10831 dst_reg,10915 dst_reg,
10832 try src_mcv.mem(self, .word),10916 try src_mcv.mem(self, .{ .size = .word }),
10833 .u(1),10917 .u(1),
10834 ) else try self.asmRegisterRegisterRegister(10918 ) else try self.asmRegisterRegisterRegister(
10835 .{ .vp_, .unpcklwd },10919 .{ .vp_, .unpcklwd },
...@@ -11275,7 +11359,7 @@ fn genBinOp(...@@ -11275,7 +11359,7 @@ fn genBinOp(
11275 .{ .vp_w, .insr },11359 .{ .vp_w, .insr },
11276 dst_reg,11360 dst_reg,
11277 dst_reg,11361 dst_reg,
11278 try src_mcv.mem(self, .word),11362 try src_mcv.mem(self, .{ .size = .word }),
11279 .u(1),11363 .u(1),
11280 ) else try self.asmRegisterRegisterRegister(11364 ) else try self.asmRegisterRegisterRegister(
11281 .{ .vp_, .unpcklwd },11365 .{ .vp_, .unpcklwd },
...@@ -11321,7 +11405,7 @@ fn genBinOp(...@@ -11321,7 +11405,7 @@ fn genBinOp(
11321 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(11405 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
11322 .{ .vp_d, .insr },11406 .{ .vp_d, .insr },
11323 dst_reg,11407 dst_reg,
11324 try src_mcv.mem(self, .dword),11408 try src_mcv.mem(self, .{ .size = .dword }),
11325 .u(1),11409 .u(1),
11326 ) else try self.asmRegisterRegisterRegister(11410 ) else try self.asmRegisterRegisterRegister(
11327 .{ .v_ps, .unpckl },11411 .{ .v_ps, .unpckl },
...@@ -11373,7 +11457,7 @@ fn genBinOp(...@@ -11373,7 +11457,7 @@ fn genBinOp(
11373 if (src_mcv.isMemory()) try self.asmRegisterMemory(11457 if (src_mcv.isMemory()) try self.asmRegisterMemory(
11374 .{ .v_ps, .cvtph2 },11458 .{ .v_ps, .cvtph2 },
11375 tmp_reg,11459 tmp_reg,
11376 try src_mcv.mem(self, .qword),11460 try src_mcv.mem(self, .{ .size = .qword }),
11377 ) else try self.asmRegisterRegister(11461 ) else try self.asmRegisterRegister(
11378 .{ .v_ps, .cvtph2 },11462 .{ .v_ps, .cvtph2 },
11379 tmp_reg,11463 tmp_reg,
...@@ -11416,7 +11500,7 @@ fn genBinOp(...@@ -11416,7 +11500,7 @@ fn genBinOp(
11416 if (src_mcv.isMemory()) try self.asmRegisterMemory(11500 if (src_mcv.isMemory()) try self.asmRegisterMemory(
11417 .{ .v_ps, .cvtph2 },11501 .{ .v_ps, .cvtph2 },
11418 tmp_reg,11502 tmp_reg,
11419 try src_mcv.mem(self, .xword),11503 try src_mcv.mem(self, .{ .size = .xword }),
11420 ) else try self.asmRegisterRegister(11504 ) else try self.asmRegisterRegister(
11421 .{ .v_ps, .cvtph2 },11505 .{ .v_ps, .cvtph2 },
11422 tmp_reg,11506 tmp_reg,
...@@ -11579,10 +11663,10 @@ fn genBinOp(...@@ -11579,10 +11663,10 @@ fn genBinOp(
11579 mir_tag,11663 mir_tag,
11580 dst_reg,11664 dst_reg,
11581 lhs_reg,11665 lhs_reg,
11582 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {11666 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
11583 else => .fromSize(abi_size),11667 else => .fromSize(abi_size),
11584 .vector => .fromBitSize(dst_reg.bitSize()),11668 .vector => .fromBitSize(dst_reg.bitSize()),
11585 }),11669 } }),
11586 ) else try self.asmRegisterRegisterRegister(11670 ) else try self.asmRegisterRegisterRegister(
11587 mir_tag,11671 mir_tag,
11588 dst_reg,11672 dst_reg,
...@@ -11597,10 +11681,10 @@ fn genBinOp(...@@ -11597,10 +11681,10 @@ fn genBinOp(
11597 if (src_mcv.isMemory()) try self.asmRegisterMemory(11681 if (src_mcv.isMemory()) try self.asmRegisterMemory(
11598 mir_tag,11682 mir_tag,
11599 dst_reg,11683 dst_reg,
11600 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {11684 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
11601 else => .fromSize(abi_size),11685 else => .fromSize(abi_size),
11602 .vector => .fromBitSize(dst_reg.bitSize()),11686 .vector => .fromBitSize(dst_reg.bitSize()),
11603 }),11687 } }),
11604 ) else try self.asmRegisterRegister(11688 ) else try self.asmRegisterRegister(
11605 mir_tag,11689 mir_tag,
11606 dst_reg,11690 dst_reg,
...@@ -11625,10 +11709,10 @@ fn genBinOp(...@@ -11625,10 +11709,10 @@ fn genBinOp(
11625 mir_tag,11709 mir_tag,
11626 dst_reg,11710 dst_reg,
11627 lhs_reg,11711 lhs_reg,
11628 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {11712 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
11629 else => .fromSize(abi_size),11713 else => .fromSize(abi_size),
11630 .vector => .fromBitSize(dst_reg.bitSize()),11714 .vector => .fromBitSize(dst_reg.bitSize()),
11631 }),11715 } }),
11632 imm,11716 imm,
11633 ) else try self.asmRegisterRegisterRegisterImmediate(11717 ) else try self.asmRegisterRegisterRegisterImmediate(
11634 mir_tag,11718 mir_tag,
...@@ -11645,10 +11729,10 @@ fn genBinOp(...@@ -11645,10 +11729,10 @@ fn genBinOp(
11645 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(11729 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
11646 mir_tag,11730 mir_tag,
11647 dst_reg,11731 dst_reg,
11648 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(zcu)) {11732 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
11649 else => .fromSize(abi_size),11733 else => .fromSize(abi_size),
11650 .vector => .fromBitSize(dst_reg.bitSize()),11734 .vector => .fromBitSize(dst_reg.bitSize()),
11651 }),11735 } }),
11652 imm,11736 imm,
11653 ) else try self.asmRegisterRegisterImmediate(11737 ) else try self.asmRegisterRegisterImmediate(
11654 mir_tag,11738 mir_tag,
...@@ -11854,7 +11938,7 @@ fn genBinOp(...@@ -11854,7 +11938,7 @@ fn genBinOp(
11854 const unsigned_ty = try lhs_ty.toUnsigned(pt);11938 const unsigned_ty = try lhs_ty.toUnsigned(pt);
11855 const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));11939 const not_mcv = try self.genTypedValue(try unsigned_ty.maxInt(pt, unsigned_ty));
11856 const not_mem: Memory = if (not_mcv.isMemory())11940 const not_mem: Memory = if (not_mcv.isMemory())
11857 try not_mcv.mem(self, .fromSize(abi_size))11941 try not_mcv.mem(self, .{ .size = .fromSize(abi_size) })
11858 else11942 else
11859 .{ .base = .{11943 .{ .base = .{
11860 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),11944 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
...@@ -11915,7 +11999,7 @@ fn genBinOp(...@@ -11915,7 +11999,7 @@ fn genBinOp(
11915}11999}
1191612000
11917fn genBinOpMir(12001fn genBinOpMir(
11918 self: *Self,12002 self: *CodeGen,
11919 mir_tag: Mir.Inst.FixedTag,12003 mir_tag: Mir.Inst.FixedTag,
11920 ty: Type,12004 ty: Type,
11921 dst_mcv: MCValue,12005 dst_mcv: MCValue,
...@@ -12341,7 +12425,7 @@ fn genBinOpMir(...@@ -12341,7 +12425,7 @@ fn genBinOpMir(
1234112425
12342/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.12426/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
12343/// Does not support byte-size operands.12427/// Does not support byte-size operands.
12344fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {12428fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
12345 const pt = self.pt;12429 const pt = self.pt;
12346 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));12430 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
12347 try self.spillEflagsIfOccupied();12431 try self.spillEflagsIfOccupied();
...@@ -12468,7 +12552,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M...@@ -12468,7 +12552,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
12468 }12552 }
12469}12553}
1247012554
12471fn airArg(self: *Self, inst: Air.Inst.Index) !void {12555fn airArg(self: *CodeGen, inst: Air.Inst.Index) !void {
12472 const pt = self.pt;12556 const pt = self.pt;
12473 const zcu = pt.zcu;12557 const zcu = pt.zcu;
12474 // skip zero-bit arguments as they don't have a corresponding arg instruction12558 // skip zero-bit arguments as they don't have a corresponding arg instruction
...@@ -12522,7 +12606,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -12522,7 +12606,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12522 const dst_mcv = try self.allocRegOrMem(inst, false);12606 const dst_mcv = try self.allocRegOrMem(inst, false);
12523 if (regs_frame_addr.regs > 0) try self.asmMemoryRegister(12607 if (regs_frame_addr.regs > 0) try self.asmMemoryRegister(
12524 .{ ._, .mov },12608 .{ ._, .mov },
12525 try dst_mcv.mem(self, .byte),12609 try dst_mcv.mem(self, .{ .size = .byte }),
12526 prev_reg.to8(),12610 prev_reg.to8(),
12527 );12611 );
12528 try self.genInlineMemset(12612 try self.genInlineMemset(
...@@ -12554,7 +12638,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -12554,7 +12638,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12554 const unset = try self.asmJccReloc(.e, undefined);12638 const unset = try self.asmJccReloc(.e, undefined);
12555 try self.asmMemoryRegister(12639 try self.asmMemoryRegister(
12556 .{ ._s, .bt },12640 .{ ._s, .bt },
12557 try dst_mcv.mem(self, .dword),12641 try dst_mcv.mem(self, .{ .size = .dword }),
12558 index_reg.to32(),12642 index_reg.to32(),
12559 );12643 );
12560 self.performReloc(unset);12644 self.performReloc(unset);
...@@ -12578,7 +12662,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -12578,7 +12662,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void {
12578 return self.finishAir(inst, result, .{ .none, .none, .none });12662 return self.finishAir(inst, result, .{ .none, .none, .none });
12579}12663}
1258012664
12581fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {12665fn airDbgArg(self: *CodeGen, inst: Air.Inst.Index) !void {
12582 // skip zero-bit arguments as they don't have a corresponding arg instruction12666 // skip zero-bit arguments as they don't have a corresponding arg instruction
12583 var arg_index = self.arg_index;12667 var arg_index = self.arg_index;
12584 while (self.args[arg_index] == .none) arg_index += 1;12668 while (self.args[arg_index] == .none) arg_index += 1;
...@@ -12594,13 +12678,13 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -12594,13 +12678,13 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void {
12594 } else try self.airDbgVarArgs();12678 } else try self.airDbgVarArgs();
12595}12679}
1259612680
12597fn airDbgVarArgs(self: *Self) !void {12681fn airDbgVarArgs(self: *CodeGen) !void {
12598 if (self.pt.zcu.typeToFunc(self.fn_type).?.is_var_args)12682 if (self.pt.zcu.typeToFunc(self.fn_type).?.is_var_args)
12599 try self.asmPseudo(.pseudo_dbg_var_args_none);12683 try self.asmPseudo(.pseudo_dbg_var_args_none);
12600}12684}
1260112685
12602fn genLocalDebugInfo(12686fn genLocalDebugInfo(
12603 self: *Self,12687 self: *CodeGen,
12604 inst: Air.Inst.Index,12688 inst: Air.Inst.Index,
12605 mcv: MCValue,12689 mcv: MCValue,
12606) !void {12690) !void {
...@@ -12656,19 +12740,19 @@ fn genLocalDebugInfo(...@@ -12656,19 +12740,19 @@ fn genLocalDebugInfo(
12656 }12740 }
12657}12741}
1265812742
12659fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void {12743fn airRetAddr(self: *CodeGen, inst: Air.Inst.Index) !void {
12660 const dst_mcv = try self.allocRegOrMem(inst, true);12744 const dst_mcv = try self.allocRegOrMem(inst, true);
12661 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});12745 try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{});
12662 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });12746 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
12663}12747}
1266412748
12665fn airFrameAddress(self: *Self, inst: Air.Inst.Index) !void {12749fn airFrameAddress(self: *CodeGen, inst: Air.Inst.Index) !void {
12666 const dst_mcv = try self.allocRegOrMem(inst, true);12750 const dst_mcv = try self.allocRegOrMem(inst, true);
12667 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});12751 try self.genCopy(Type.usize, dst_mcv, .{ .lea_frame = .{ .index = .base_ptr } }, .{});
12668 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });12752 return self.finishAir(inst, dst_mcv, .{ .none, .none, .none });
12669}12753}
1267012754
12671fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {12755fn airCall(self: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !void {
12672 if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{});12756 if (modifier == .always_tail) return self.fail("TODO implement tail calls for x86_64", .{});
1267312757
12674 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;12758 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
...@@ -12702,7 +12786,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier...@@ -12702,7 +12786,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
12702 return self.finishAirResult(inst, result);12786 return self.finishAirResult(inst, result);
12703}12787}
1270412788
12705fn genCall(self: *Self, info: union(enum) {12789fn genCall(self: *CodeGen, info: union(enum) {
12706 air: Air.Inst.Ref,12790 air: Air.Inst.Ref,
12707 lib: struct {12791 lib: struct {
12708 return_type: InternPool.Index,12792 return_type: InternPool.Index,
...@@ -12808,7 +12892,7 @@ fn genCall(self: *Self, info: union(enum) {...@@ -12808,7 +12892,7 @@ fn genCall(self: *Self, info: union(enum) {
12808 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);12892 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
12809 defer self.register_manager.unlockReg(index_lock);12893 defer self.register_manager.unlockReg(index_lock);
1281012894
12811 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .dword) else .{12895 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .{ .size = .dword }) else .{
12812 .base = .{ .reg = try self.copyToTmpRegister(12896 .base = .{ .reg = try self.copyToTmpRegister(
12813 Type.usize,12897 Type.usize,
12814 switch (src_arg) {12898 switch (src_arg) {
...@@ -12900,7 +12984,7 @@ fn genCall(self: *Self, info: union(enum) {...@@ -12900,7 +12984,7 @@ fn genCall(self: *Self, info: union(enum) {
12900 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },12984 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
12901 }, .{}),12985 }, .{}),
12902 .elementwise_regs_then_frame => |regs_frame_addr| {12986 .elementwise_regs_then_frame => |regs_frame_addr| {
12903 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .dword) else .{12987 const src_mem: Memory = if (src_arg.isMemory()) try src_arg.mem(self, .{ .size = .dword }) else .{
12904 .base = .{ .reg = try self.copyToTmpRegister(12988 .base = .{ .reg = try self.copyToTmpRegister(
12905 Type.usize,12989 Type.usize,
12906 switch (src_arg) {12990 switch (src_arg) {
...@@ -13006,7 +13090,7 @@ fn genCall(self: *Self, info: union(enum) {...@@ -13006,7 +13090,7 @@ fn genCall(self: *Self, info: union(enum) {
13006 return call_info.return_value.short;13090 return call_info.return_value.short;
13007}13091}
1300813092
13009fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {13093fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
13010 const pt = self.pt;13094 const pt = self.pt;
13011 const zcu = pt.zcu;13095 const zcu = pt.zcu;
13012 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;13096 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
...@@ -13042,7 +13126,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -13042,7 +13126,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
13042 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);13126 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
13043}13127}
1304413128
13045fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {13129fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
13046 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;13130 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
13047 const ptr = try self.resolveInst(un_op);13131 const ptr = try self.resolveInst(un_op);
1304813132
...@@ -13062,7 +13146,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -13062,7 +13146,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
13062 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);13146 try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc);
13063}13147}
1306413148
13065fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {13149fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {
13066 const pt = self.pt;13150 const pt = self.pt;
13067 const zcu = pt.zcu;13151 const zcu = pt.zcu;
13068 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;13152 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -13145,7 +13229,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void...@@ -13145,7 +13229,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
13145 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(13229 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
13146 .{ ._, .mov },13230 .{ ._, .mov },
13147 temp_lhs_reg.to8(),13231 temp_lhs_reg.to8(),
13148 try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte),13232 try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }),
13149 ) else {13233 ) else {
13150 try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});13234 try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
13151 try self.asmRegisterImmediate(13235 try self.asmRegisterImmediate(
...@@ -13158,7 +13242,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void...@@ -13158,7 +13242,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
13158 const payload_compare = payload_compare: {13242 const payload_compare = payload_compare: {
13159 if (rhs_mcv.isMemory()) {13243 if (rhs_mcv.isMemory()) {
13160 const rhs_mem =13244 const rhs_mem =
13161 try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte);13245 try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte });
13162 try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());13246 try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
13163 const payload_compare = try self.asmJccReloc(.nz, undefined);13247 const payload_compare = try self.asmJccReloc(.nz, undefined);
13164 try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);13248 try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
...@@ -13459,7 +13543,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void...@@ -13459,7 +13543,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
13459 .{ .vp_w, .insr },13543 .{ .vp_w, .insr },
13460 tmp1_reg,13544 tmp1_reg,
13461 dst_reg.to128(),13545 dst_reg.to128(),
13462 try src_mcv.mem(self, .word),13546 try src_mcv.mem(self, .{ .size = .word }),
13463 .u(1),13547 .u(1),
13464 ) else try self.asmRegisterRegisterRegister(13548 ) else try self.asmRegisterRegisterRegister(
13465 .{ .vp_, .unpcklwd },13549 .{ .vp_, .unpcklwd },
...@@ -13505,7 +13589,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void...@@ -13505,7 +13589,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void
13505 return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });13589 return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
13506}13590}
1350713591
13508fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {13592fn airCmpVector(self: *CodeGen, inst: Air.Inst.Index) !void {
13509 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;13593 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
13510 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;13594 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
13511 const dst_mcv = try self.genBinOp(13595 const dst_mcv = try self.genBinOp(
...@@ -13517,7 +13601,7 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {...@@ -13517,7 +13601,7 @@ fn airCmpVector(self: *Self, inst: Air.Inst.Index) !void {
13517 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });13601 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
13518}13602}
1351913603
13520fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {13604fn airCmpLtErrorsLen(self: *CodeGen, inst: Air.Inst.Index) !void {
13521 const pt = self.pt;13605 const pt = self.pt;
13522 const zcu = pt.zcu;13606 const zcu = pt.zcu;
13523 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;13607 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
...@@ -13550,7 +13634,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {...@@ -13550,7 +13634,7 @@ fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
13550 return self.finishAir(inst, .{ .eflags = .b }, .{ un_op, .none, .none });13634 return self.finishAir(inst, .{ .eflags = .b }, .{ un_op, .none, .none });
13551}13635}
1355213636
13553fn airTry(self: *Self, inst: Air.Inst.Index) !void {13637fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
13554 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;13638 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
13555 const extra = self.air.extraData(Air.Try, pl_op.payload);13639 const extra = self.air.extraData(Air.Try, pl_op.payload);
13556 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);13640 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
...@@ -13559,7 +13643,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {...@@ -13559,7 +13643,7 @@ fn airTry(self: *Self, inst: Air.Inst.Index) !void {
13559 return self.finishAir(inst, result, .{ .none, .none, .none });13643 return self.finishAir(inst, result, .{ .none, .none, .none });
13560}13644}
1356113645
13562fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {13646fn airTryPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
13563 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;13647 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
13564 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);13648 const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
13565 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);13649 const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
...@@ -13569,7 +13653,7 @@ fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -13569,7 +13653,7 @@ fn airTryPtr(self: *Self, inst: Air.Inst.Index) !void {
13569}13653}
1357013654
13571fn genTry(13655fn genTry(
13572 self: *Self,13656 self: *CodeGen,
13573 inst: Air.Inst.Index,13657 inst: Air.Inst.Index,
13574 operand: Air.Inst.Ref,13658 operand: Air.Inst.Ref,
13575 body: []const Air.Inst.Index,13659 body: []const Air.Inst.Index,
...@@ -13615,7 +13699,7 @@ fn genTry(...@@ -13615,7 +13699,7 @@ fn genTry(
13615 return result;13699 return result;
13616}13700}
1361713701
13618fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {13702fn airDbgStmt(self: *CodeGen, inst: Air.Inst.Index) !void {
13619 const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;13703 const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
13620 _ = try self.addInst(.{13704 _ = try self.addInst(.{
13621 .tag = .pseudo,13705 .tag = .pseudo,
...@@ -13627,14 +13711,14 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {...@@ -13627,14 +13711,14 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void {
13627 });13711 });
13628}13712}
1362913713
13630fn airDbgEmptyStmt(self: *Self) !void {13714fn airDbgEmptyStmt(self: *CodeGen) !void {
13631 if (self.mir_instructions.len > 0 and13715 if (self.mir_instructions.len > 0 and
13632 self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)13716 self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column)
13633 self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;13717 self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column;
13634 try self.asmOpOnly(.{ ._, .nop });13718 try self.asmOpOnly(.{ ._, .nop });
13635}13719}
1363613720
13637fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {13721fn airDbgInlineBlock(self: *CodeGen, inst: Air.Inst.Index) !void {
13638 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;13722 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
13639 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);13723 const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload);
13640 const old_inline_func = self.inline_func;13724 const old_inline_func = self.inline_func;
...@@ -13653,13 +13737,13 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {...@@ -13653,13 +13737,13 @@ fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void {
13653 });13737 });
13654}13738}
1365513739
13656fn airDbgVar(self: *Self, inst: Air.Inst.Index) !void {13740fn airDbgVar(self: *CodeGen, inst: Air.Inst.Index) !void {
13657 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;13741 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
13658 try self.genLocalDebugInfo(inst, try self.resolveInst(pl_op.operand));13742 try self.genLocalDebugInfo(inst, try self.resolveInst(pl_op.operand));
13659 return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });13743 return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
13660}13744}
1366113745
13662fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {13746fn genCondBrMir(self: *CodeGen, ty: Type, mcv: MCValue) !Mir.Inst.Index {
13663 const pt = self.pt;13747 const pt = self.pt;
13664 const abi_size = ty.abiSize(pt.zcu);13748 const abi_size = ty.abiSize(pt.zcu);
13665 switch (mcv) {13749 switch (mcv) {
...@@ -13686,7 +13770,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {...@@ -13686,7 +13770,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
13686 }13770 }
13687}13771}
1368813772
13689fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {13773fn airCondBr(self: *CodeGen, inst: Air.Inst.Index) !void {
13690 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;13774 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
13691 const cond = try self.resolveInst(pl_op.operand);13775 const cond = try self.resolveInst(pl_op.operand);
13692 const cond_ty = self.typeOf(pl_op.operand);13776 const cond_ty = self.typeOf(pl_op.operand);
...@@ -13731,7 +13815,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -13731,7 +13815,7 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void {
13731 // We already took care of pl_op.operand earlier, so there's nothing left to do.13815 // We already took care of pl_op.operand earlier, so there's nothing left to do.
13732}13816}
1373313817
13734fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {13818fn isNull(self: *CodeGen, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue {
13735 const pt = self.pt;13819 const pt = self.pt;
13736 const zcu = pt.zcu;13820 const zcu = pt.zcu;
13737 switch (opt_mcv) {13821 switch (opt_mcv) {
...@@ -13862,7 +13946,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC...@@ -13862,7 +13946,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
13862 }13946 }
13863}13947}
1386413948
13865fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {13949fn isNullPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
13866 const pt = self.pt;13950 const pt = self.pt;
13867 const zcu = pt.zcu;13951 const zcu = pt.zcu;
13868 const opt_ty = ptr_ty.childType(zcu);13952 const opt_ty = ptr_ty.childType(zcu);
...@@ -13899,7 +13983,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)...@@ -13899,7 +13983,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue)
13899 return .{ .eflags = .e };13983 return .{ .eflags = .e };
13900}13984}
1390113985
13902fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {13986fn isErr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
13903 const pt = self.pt;13987 const pt = self.pt;
13904 const zcu = pt.zcu;13988 const zcu = pt.zcu;
13905 const err_ty = eu_ty.errorUnionSet(zcu);13989 const err_ty = eu_ty.errorUnionSet(zcu);
...@@ -13948,7 +14032,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)...@@ -13948,7 +14032,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue)
13948 return MCValue{ .eflags = .a };14032 return MCValue{ .eflags = .a };
13949}14033}
1395014034
13951fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {14035fn isErrPtr(self: *CodeGen, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
13952 const pt = self.pt;14036 const pt = self.pt;
13953 const zcu = pt.zcu;14037 const zcu = pt.zcu;
13954 const eu_ty = ptr_ty.childType(zcu);14038 const eu_ty = ptr_ty.childType(zcu);
...@@ -13981,7 +14065,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV...@@ -13981,7 +14065,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV
13981 return MCValue{ .eflags = .a };14065 return MCValue{ .eflags = .a };
13982}14066}
1398314067
13984fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {14068fn isNonErr(self: *CodeGen, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MCValue {
13985 const is_err_res = try self.isErr(inst, eu_ty, eu_mcv);14069 const is_err_res = try self.isErr(inst, eu_ty, eu_mcv);
13986 switch (is_err_res) {14070 switch (is_err_res) {
13987 .eflags => |cc| {14071 .eflags => |cc| {
...@@ -13996,7 +14080,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC...@@ -13996,7 +14080,7 @@ fn isNonErr(self: *Self, inst: Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) !MC
13996 }14080 }
13997}14081}
1399814082
13999fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {14083fn isNonErrPtr(self: *CodeGen, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) !MCValue {
14000 const is_err_res = try self.isErrPtr(inst, ptr_ty, ptr_mcv);14084 const is_err_res = try self.isErrPtr(inst, ptr_ty, ptr_mcv);
14001 switch (is_err_res) {14085 switch (is_err_res) {
14002 .eflags => |cc| {14086 .eflags => |cc| {
...@@ -14011,7 +14095,7 @@ fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue...@@ -14011,7 +14095,7 @@ fn isNonErrPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue
14011 }14095 }
14012}14096}
1401314097
14014fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {14098fn airIsNull(self: *CodeGen, inst: Air.Inst.Index) !void {
14015 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;14099 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
14016 const operand = try self.resolveInst(un_op);14100 const operand = try self.resolveInst(un_op);
14017 const ty = self.typeOf(un_op);14101 const ty = self.typeOf(un_op);
...@@ -14019,7 +14103,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {...@@ -14019,7 +14103,7 @@ fn airIsNull(self: *Self, inst: Air.Inst.Index) !void {
14019 return self.finishAir(inst, result, .{ un_op, .none, .none });14103 return self.finishAir(inst, result, .{ un_op, .none, .none });
14020}14104}
1402114105
14022fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {14106fn airIsNullPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
14023 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;14107 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
14024 const operand = try self.resolveInst(un_op);14108 const operand = try self.resolveInst(un_op);
14025 const ty = self.typeOf(un_op);14109 const ty = self.typeOf(un_op);
...@@ -14027,7 +14111,7 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14027,7 +14111,7 @@ fn airIsNullPtr(self: *Self, inst: Air.Inst.Index) !void {
14027 return self.finishAir(inst, result, .{ un_op, .none, .none });14111 return self.finishAir(inst, result, .{ un_op, .none, .none });
14028}14112}
1402914113
14030fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {14114fn airIsNonNull(self: *CodeGen, inst: Air.Inst.Index) !void {
14031 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;14115 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
14032 const operand = try self.resolveInst(un_op);14116 const operand = try self.resolveInst(un_op);
14033 const ty = self.typeOf(un_op);14117 const ty = self.typeOf(un_op);
...@@ -14039,7 +14123,7 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {...@@ -14039,7 +14123,7 @@ fn airIsNonNull(self: *Self, inst: Air.Inst.Index) !void {
14039 return self.finishAir(inst, result, .{ un_op, .none, .none });14123 return self.finishAir(inst, result, .{ un_op, .none, .none });
14040}14124}
1404114125
14042fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {14126fn airIsNonNullPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
14043 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;14127 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
14044 const operand = try self.resolveInst(un_op);14128 const operand = try self.resolveInst(un_op);
14045 const ty = self.typeOf(un_op);14129 const ty = self.typeOf(un_op);
...@@ -14050,7 +14134,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14050,7 +14134,7 @@ fn airIsNonNullPtr(self: *Self, inst: Air.Inst.Index) !void {
14050 return self.finishAir(inst, result, .{ un_op, .none, .none });14134 return self.finishAir(inst, result, .{ un_op, .none, .none });
14051}14135}
1405214136
14053fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {14137fn airIsErr(self: *CodeGen, inst: Air.Inst.Index) !void {
14054 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;14138 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
14055 const operand = try self.resolveInst(un_op);14139 const operand = try self.resolveInst(un_op);
14056 const ty = self.typeOf(un_op);14140 const ty = self.typeOf(un_op);
...@@ -14058,7 +14142,7 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14058,7 +14142,7 @@ fn airIsErr(self: *Self, inst: Air.Inst.Index) !void {
14058 return self.finishAir(inst, result, .{ un_op, .none, .none });14142 return self.finishAir(inst, result, .{ un_op, .none, .none });
14059}14143}
1406014144
14061fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {14145fn airIsErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
14062 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;14146 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
14063 const operand = try self.resolveInst(un_op);14147 const operand = try self.resolveInst(un_op);
14064 const ty = self.typeOf(un_op);14148 const ty = self.typeOf(un_op);
...@@ -14066,7 +14150,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14066,7 +14150,7 @@ fn airIsErrPtr(self: *Self, inst: Air.Inst.Index) !void {
14066 return self.finishAir(inst, result, .{ un_op, .none, .none });14150 return self.finishAir(inst, result, .{ un_op, .none, .none });
14067}14151}
1406814152
14069fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {14153fn airIsNonErr(self: *CodeGen, inst: Air.Inst.Index) !void {
14070 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;14154 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
14071 const operand = try self.resolveInst(un_op);14155 const operand = try self.resolveInst(un_op);
14072 const ty = self.typeOf(un_op);14156 const ty = self.typeOf(un_op);
...@@ -14074,7 +14158,7 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14074,7 +14158,7 @@ fn airIsNonErr(self: *Self, inst: Air.Inst.Index) !void {
14074 return self.finishAir(inst, result, .{ un_op, .none, .none });14158 return self.finishAir(inst, result, .{ un_op, .none, .none });
14075}14159}
1407614160
14077fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {14161fn airIsNonErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
14078 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;14162 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
14079 const operand = try self.resolveInst(un_op);14163 const operand = try self.resolveInst(un_op);
14080 const ty = self.typeOf(un_op);14164 const ty = self.typeOf(un_op);
...@@ -14082,7 +14166,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14082,7 +14166,7 @@ fn airIsNonErrPtr(self: *Self, inst: Air.Inst.Index) !void {
14082 return self.finishAir(inst, result, .{ un_op, .none, .none });14166 return self.finishAir(inst, result, .{ un_op, .none, .none });
14083}14167}
1408414168
14085fn airLoop(self: *Self, inst: Air.Inst.Index) !void {14169fn airLoop(self: *CodeGen, inst: Air.Inst.Index) !void {
14086 // A loop is a setup to be able to jump back to the beginning.14170 // A loop is a setup to be able to jump back to the beginning.
14087 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;14171 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
14088 const loop = self.air.extraData(Air.Block, ty_pl.payload);14172 const loop = self.air.extraData(Air.Block, ty_pl.payload);
...@@ -14100,7 +14184,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {...@@ -14100,7 +14184,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void {
14100 try self.genBodyBlock(body);14184 try self.genBodyBlock(body);
14101}14185}
1410214186
14103fn airBlock(self: *Self, inst: Air.Inst.Index) !void {14187fn airBlock(self: *CodeGen, inst: Air.Inst.Index) !void {
14104 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;14188 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
14105 const extra = self.air.extraData(Air.Block, ty_pl.payload);14189 const extra = self.air.extraData(Air.Block, ty_pl.payload);
14106 try self.asmPseudo(.pseudo_dbg_enter_block_none);14190 try self.asmPseudo(.pseudo_dbg_enter_block_none);
...@@ -14108,7 +14192,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {...@@ -14108,7 +14192,7 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
14108 try self.asmPseudo(.pseudo_dbg_leave_block_none);14192 try self.asmPseudo(.pseudo_dbg_leave_block_none);
14109}14193}
1411014194
14111fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {14195fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !void {
14112 // A block is a setup to be able to jump to the end.14196 // A block is a setup to be able to jump to the end.
14113 const inst_tracking_i = self.inst_tracking.count();14197 const inst_tracking_i = self.inst_tracking.count();
14114 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach));14198 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(.unreach));
...@@ -14137,7 +14221,7 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !...@@ -14137,7 +14221,7 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
14137 self.getValueIfFree(tracking.short, inst);14221 self.getValueIfFree(tracking.short, inst);
14138}14222}
1413914223
14140fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {14224fn lowerSwitchBr(self: *CodeGen, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void {
14141 const zcu = self.pt.zcu;14225 const zcu = self.pt.zcu;
14142 const condition_ty = self.typeOf(switch_br.operand);14226 const condition_ty = self.typeOf(switch_br.operand);
14143 const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1);14227 const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1);
...@@ -14255,7 +14339,7 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit...@@ -14255,7 +14339,7 @@ fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwit
14255 }14339 }
14256}14340}
1425714341
14258fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {14342fn airSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
14259 const switch_br = self.air.unwrapSwitch(inst);14343 const switch_br = self.air.unwrapSwitch(inst);
14260 const condition = try self.resolveInst(switch_br.operand);14344 const condition = try self.resolveInst(switch_br.operand);
1426114345
...@@ -14271,7 +14355,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14271,7 +14355,7 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
14271 // We already took care of pl_op.operand earlier, so there's nothing left to do14355 // We already took care of pl_op.operand earlier, so there's nothing left to do
14272}14356}
1427314357
14274fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {14358fn airLoopSwitchBr(self: *CodeGen, inst: Air.Inst.Index) !void {
14275 const switch_br = self.air.unwrapSwitch(inst);14359 const switch_br = self.air.unwrapSwitch(inst);
14276 const condition = try self.resolveInst(switch_br.operand);14360 const condition = try self.resolveInst(switch_br.operand);
1427714361
...@@ -14309,7 +14393,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14309,7 +14393,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
14309 try self.processDeath(inst);14393 try self.processDeath(inst);
14310}14394}
1431114395
14312fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {14396fn airSwitchDispatch(self: *CodeGen, inst: Air.Inst.Index) !void {
14313 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;14397 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
1431414398
14315 const block_ty = self.typeOfIndex(br.block_inst);14399 const block_ty = self.typeOfIndex(br.block_inst);
...@@ -14354,7 +14438,7 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {...@@ -14354,7 +14438,7 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void {
14354 try self.freeValue(block_tracking.short);14438 try self.freeValue(block_tracking.short);
14355}14439}
1435614440
14357fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {14441fn performReloc(self: *CodeGen, reloc: Mir.Inst.Index) void {
14358 const next_inst: u32 = @intCast(self.mir_instructions.len);14442 const next_inst: u32 = @intCast(self.mir_instructions.len);
14359 switch (self.mir_instructions.items(.tag)[reloc]) {14443 switch (self.mir_instructions.items(.tag)[reloc]) {
14360 .j, .jmp => {},14444 .j, .jmp => {},
...@@ -14367,7 +14451,7 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {...@@ -14367,7 +14451,7 @@ fn performReloc(self: *Self, reloc: Mir.Inst.Index) void {
14367 self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;14451 self.mir_instructions.items(.data)[reloc].inst.inst = next_inst;
14368}14452}
1436914453
14370fn airBr(self: *Self, inst: Air.Inst.Index) !void {14454fn airBr(self: *CodeGen, inst: Air.Inst.Index) !void {
14371 const zcu = self.pt.zcu;14455 const zcu = self.pt.zcu;
14372 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;14456 const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
1437314457
...@@ -14426,7 +14510,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -14426,7 +14510,7 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
14426 try self.freeValue(block_tracking.short);14510 try self.freeValue(block_tracking.short);
14427}14511}
1442814512
14429fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {14513fn airRepeat(self: *CodeGen, inst: Air.Inst.Index) !void {
14430 const loop_inst = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst;14514 const loop_inst = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat.loop_inst;
14431 const repeat_info = self.loops.get(loop_inst).?;14515 const repeat_info = self.loops.get(loop_inst).?;
14432 try self.restoreState(repeat_info.state, &.{}, .{14516 try self.restoreState(repeat_info.state, &.{}, .{
...@@ -14438,7 +14522,7 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14438,7 +14522,7 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void {
14438 _ = try self.asmJmpReloc(repeat_info.target);14522 _ = try self.asmJmpReloc(repeat_info.target);
14439}14523}
1444014524
14441fn airAsm(self: *Self, inst: Air.Inst.Index) !void {14525fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
14442 const pt = self.pt;14526 const pt = self.pt;
14443 const zcu = pt.zcu;14527 const zcu = pt.zcu;
14444 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;14528 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -15110,7 +15194,7 @@ const MoveStrategy = union(enum) {...@@ -15110,7 +15194,7 @@ const MoveStrategy = union(enum) {
15110 extract: Mir.Inst.FixedTag,15194 extract: Mir.Inst.FixedTag,
15111 };15195 };
1511215196
15113 pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void {15197 pub fn read(strat: MoveStrategy, self: *CodeGen, dst_reg: Register, src_mem: Memory) !void {
15114 switch (strat) {15198 switch (strat) {
15115 .move => |tag| try self.asmRegisterMemory(tag, switch (tag[1]) {15199 .move => |tag| try self.asmRegisterMemory(tag, switch (tag[1]) {
15116 else => dst_reg,15200 else => dst_reg,
...@@ -15136,7 +15220,7 @@ const MoveStrategy = union(enum) {...@@ -15136,7 +15220,7 @@ const MoveStrategy = union(enum) {
15136 ),15220 ),
15137 }15221 }
15138 }15222 }
15139 pub fn write(strat: MoveStrategy, self: *Self, dst_mem: Memory, src_reg: Register) !void {15223 pub fn write(strat: MoveStrategy, self: *CodeGen, dst_mem: Memory, src_reg: Register) !void {
15140 switch (strat) {15224 switch (strat) {
15141 .move => |tag| try self.asmMemoryRegister(tag, dst_mem, src_reg),15225 .move => |tag| try self.asmMemoryRegister(tag, dst_mem, src_reg),
15142 .x87_load_store => {15226 .x87_load_store => {
...@@ -15152,7 +15236,7 @@ const MoveStrategy = union(enum) {...@@ -15152,7 +15236,7 @@ const MoveStrategy = union(enum) {
15152 }15236 }
15153 }15237 }
15154};15238};
15155fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy {15239fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) !MoveStrategy {
15156 const pt = self.pt;15240 const pt = self.pt;
15157 const zcu = pt.zcu;15241 const zcu = pt.zcu;
15158 switch (class) {15242 switch (class) {
...@@ -15383,7 +15467,7 @@ const CopyOptions = struct {...@@ -15383,7 +15467,7 @@ const CopyOptions = struct {
15383 safety: bool = false,15467 safety: bool = false,
15384};15468};
1538515469
15386fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {15470fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: CopyOptions) InnerError!void {
15387 const pt = self.pt;15471 const pt = self.pt;
1538815472
15389 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;15473 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
...@@ -15496,7 +15580,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy...@@ -15496,7 +15580,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy
15496}15580}
1549715581
15498fn genSetReg(15582fn genSetReg(
15499 self: *Self,15583 self: *CodeGen,
15500 dst_reg: Register,15584 dst_reg: Register,
15501 ty: Type,15585 ty: Type,
15502 src_mcv: MCValue,15586 src_mcv: MCValue,
...@@ -15781,7 +15865,7 @@ fn genSetReg(...@@ -15781,7 +15865,7 @@ fn genSetReg(
15781}15865}
1578215866
15783fn genSetMem(15867fn genSetMem(
15784 self: *Self,15868 self: *CodeGen,
15785 base: Memory.Base,15869 base: Memory.Base,
15786 disp: i32,15870 disp: i32,
15787 ty: Type,15871 ty: Type,
...@@ -16012,7 +16096,7 @@ fn genSetMem(...@@ -16012,7 +16096,7 @@ fn genSetMem(
16012 }16096 }
16013}16097}
1601416098
16015fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {16099fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
16016 try self.spillRegisters(&.{ .rsi, .rdi, .rcx });16100 try self.spillRegisters(&.{ .rsi, .rdi, .rcx });
16017 try self.genSetReg(.rsi, Type.usize, src_ptr, .{});16101 try self.genSetReg(.rsi, Type.usize, src_ptr, .{});
16018 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});16102 try self.genSetReg(.rdi, Type.usize, dst_ptr, .{});
...@@ -16021,7 +16105,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue...@@ -16021,7 +16105,7 @@ fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue
16021}16105}
1602216106
16023fn genInlineMemset(16107fn genInlineMemset(
16024 self: *Self,16108 self: *CodeGen,
16025 dst_ptr: MCValue,16109 dst_ptr: MCValue,
16026 value: MCValue,16110 value: MCValue,
16027 len: MCValue,16111 len: MCValue,
...@@ -16035,7 +16119,7 @@ fn genInlineMemset(...@@ -16035,7 +16119,7 @@ fn genInlineMemset(
16035}16119}
1603616120
16037fn genExternSymbolRef(16121fn genExternSymbolRef(
16038 self: *Self,16122 self: *CodeGen,
16039 comptime tag: Mir.Inst.Tag,16123 comptime tag: Mir.Inst.Tag,
16040 lib: ?[]const u8,16124 lib: ?[]const u8,
16041 callee: []const u8,16125 callee: []const u8,
...@@ -16061,7 +16145,7 @@ fn genExternSymbolRef(...@@ -16061,7 +16145,7 @@ fn genExternSymbolRef(
16061}16145}
1606216146
16063fn genLazySymbolRef(16147fn genLazySymbolRef(
16064 self: *Self,16148 self: *CodeGen,
16065 comptime tag: Mir.Inst.Tag,16149 comptime tag: Mir.Inst.Tag,
16066 reg: Register,16150 reg: Register,
16067 lazy_sym: link.File.LazySymbol,16151 lazy_sym: link.File.LazySymbol,
...@@ -16159,7 +16243,7 @@ fn genLazySymbolRef(...@@ -16159,7 +16243,7 @@ fn genLazySymbolRef(
16159 }16243 }
16160}16244}
1616116245
16162fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {16246fn airIntFromPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
16163 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;16247 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
16164 const result = result: {16248 const result = result: {
16165 // TODO: handle case where the operand is a slice not a raw pointer16249 // TODO: handle case where the operand is a slice not a raw pointer
...@@ -16174,7 +16258,7 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {...@@ -16174,7 +16258,7 @@ fn airIntFromPtr(self: *Self, inst: Air.Inst.Index) !void {
16174 return self.finishAir(inst, result, .{ un_op, .none, .none });16258 return self.finishAir(inst, result, .{ un_op, .none, .none });
16175}16259}
1617616260
16177fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {16261fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
16178 const pt = self.pt;16262 const pt = self.pt;
16179 const zcu = pt.zcu;16263 const zcu = pt.zcu;
16180 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;16264 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -16239,7 +16323,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -16239,7 +16323,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
16239 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });16323 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
16240}16324}
1624116325
16242fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {16326fn airArrayToSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
16243 const pt = self.pt;16327 const pt = self.pt;
16244 const zcu = pt.zcu;16328 const zcu = pt.zcu;
16245 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;16329 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -16264,7 +16348,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -16264,7 +16348,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
16264 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });16348 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
16265}16349}
1626616350
16267fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {16351fn airFloatFromInt(self: *CodeGen, inst: Air.Inst.Index) !void {
16268 const pt = self.pt;16352 const pt = self.pt;
16269 const zcu = pt.zcu;16353 const zcu = pt.zcu;
16270 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;16354 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -16344,7 +16428,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -16344,7 +16428,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void {
16344 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });16428 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
16345}16429}
1634616430
16347fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {16431fn airIntFromFloat(self: *CodeGen, inst: Air.Inst.Index) !void {
16348 const pt = self.pt;16432 const pt = self.pt;
16349 const zcu = pt.zcu;16433 const zcu = pt.zcu;
16350 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;16434 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -16416,7 +16500,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {...@@ -16416,7 +16500,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void {
16416 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });16500 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
16417}16501}
1641816502
16419fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {16503fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
16420 const pt = self.pt;16504 const pt = self.pt;
16421 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;16505 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
16422 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;16506 const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
...@@ -16463,7 +16547,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -16463,7 +16547,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
16463 const ptr_mcv = try self.resolveInst(extra.ptr);16547 const ptr_mcv = try self.resolveInst(extra.ptr);
16464 const mem_size: Memory.Size = .fromSize(val_abi_size);16548 const mem_size: Memory.Size = .fromSize(val_abi_size);
16465 const ptr_mem: Memory = switch (ptr_mcv) {16549 const ptr_mem: Memory = switch (ptr_mcv) {
16466 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),16550 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
16467 else => .{16551 else => .{
16468 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },16552 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
16469 .mod = .{ .rm = .{ .size = mem_size } },16553 .mod = .{ .rm = .{ .size = mem_size } },
...@@ -16504,7 +16588,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -16504,7 +16588,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
16504}16588}
1650516589
16506fn atomicOp(16590fn atomicOp(
16507 self: *Self,16591 self: *CodeGen,
16508 ptr_mcv: MCValue,16592 ptr_mcv: MCValue,
16509 val_mcv: MCValue,16593 val_mcv: MCValue,
16510 ptr_ty: Type,16594 ptr_ty: Type,
...@@ -16530,7 +16614,7 @@ fn atomicOp(...@@ -16530,7 +16614,7 @@ fn atomicOp(
16530 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));16614 const val_abi_size: u32 = @intCast(val_ty.abiSize(zcu));
16531 const mem_size: Memory.Size = .fromSize(val_abi_size);16615 const mem_size: Memory.Size = .fromSize(val_abi_size);
16532 const ptr_mem: Memory = switch (ptr_mcv) {16616 const ptr_mem: Memory = switch (ptr_mcv) {
16533 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),16617 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, .{ .size = mem_size }),
16534 else => .{16618 else => .{
16535 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },16619 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
16536 .mod = .{ .rm = .{ .size = mem_size } },16620 .mod = .{ .rm = .{ .size = mem_size } },
...@@ -16649,7 +16733,7 @@ fn atomicOp(...@@ -16649,7 +16733,7 @@ fn atomicOp(
16649 mir_tag,16733 mir_tag,
16650 sse_reg.to128(),16734 sse_reg.to128(),
16651 sse_reg.to128(),16735 sse_reg.to128(),
16652 try val_mcv.mem(self, self.memSize(val_ty)),16736 try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }),
16653 ) else try self.asmRegisterRegisterRegister(16737 ) else try self.asmRegisterRegisterRegister(
16654 mir_tag,16738 mir_tag,
16655 sse_reg.to128(),16739 sse_reg.to128(),
...@@ -16662,7 +16746,7 @@ fn atomicOp(...@@ -16662,7 +16746,7 @@ fn atomicOp(
16662 ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory(16746 ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory(
16663 mir_tag,16747 mir_tag,
16664 sse_reg.to128(),16748 sse_reg.to128(),
16665 try val_mcv.mem(self, self.memSize(val_ty)),16749 try val_mcv.mem(self, .{ .size = self.memSize(val_ty) }),
16666 ) else try self.asmRegisterRegister(16750 ) else try self.asmRegisterRegister(
16667 mir_tag,16751 mir_tag,
16668 sse_reg.to128(),16752 sse_reg.to128(),
...@@ -16717,7 +16801,7 @@ fn atomicOp(...@@ -16717,7 +16801,7 @@ fn atomicOp(
16717 try self.asmCmovccRegisterMemory(16801 try self.asmCmovccRegisterMemory(
16718 cc,16802 cc,
16719 registerAlias(tmp_reg, cmov_abi_size),16803 registerAlias(tmp_reg, cmov_abi_size),
16720 try val_mcv.mem(self, .fromSize(cmov_abi_size)),16804 try val_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
16721 );16805 );
16722 },16806 },
16723 else => {16807 else => {
...@@ -16773,8 +16857,8 @@ fn atomicOp(...@@ -16773,8 +16857,8 @@ fn atomicOp(
16773 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),16857 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
16774 } },16858 } },
16775 };16859 };
16776 const val_lo_mem = try val_mem_mcv.mem(self, .qword);16860 const val_lo_mem = try val_mem_mcv.mem(self, .{ .size = .qword });
16777 const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .qword);16861 const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .{ .size = .qword });
16778 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {16862 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
16779 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);16863 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
16780 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);16864 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
...@@ -16862,7 +16946,7 @@ fn atomicOp(...@@ -16862,7 +16946,7 @@ fn atomicOp(
16862 }16946 }
16863}16947}
1686416948
16865fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {16949fn airAtomicRmw(self: *CodeGen, inst: Air.Inst.Index) !void {
16866 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;16950 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
16867 const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;16951 const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
1686816952
...@@ -16883,7 +16967,7 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {...@@ -16883,7 +16967,7 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
16883 return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });16967 return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
16884}16968}
1688516969
16886fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {16970fn airAtomicLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
16887 const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;16971 const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
1688816972
16889 const ptr_ty = self.typeOf(atomic_load.ptr);16973 const ptr_ty = self.typeOf(atomic_load.ptr);
...@@ -16904,7 +16988,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -16904,7 +16988,7 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
16904 return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });16988 return self.finishAir(inst, dst_mcv, .{ atomic_load.ptr, .none, .none });
16905}16989}
1690616990
16907fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {16991fn airAtomicStore(self: *CodeGen, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
16908 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;16992 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1690916993
16910 const ptr_ty = self.typeOf(bin_op.lhs);16994 const ptr_ty = self.typeOf(bin_op.lhs);
...@@ -16917,7 +17001,7 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr...@@ -16917,7 +17001,7 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
16917 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });17001 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
16918}17002}
1691917003
16920fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {17004fn airMemset(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
16921 const pt = self.pt;17005 const pt = self.pt;
16922 const zcu = pt.zcu;17006 const zcu = pt.zcu;
16923 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;17007 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -17059,7 +17143,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -17059,7 +17143,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
17059 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });17143 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
17060}17144}
1706117145
17062fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {17146fn airMemcpy(self: *CodeGen, inst: Air.Inst.Index) !void {
17063 const pt = self.pt;17147 const pt = self.pt;
17064 const zcu = pt.zcu;17148 const zcu = pt.zcu;
17065 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;17149 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -17107,7 +17191,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -17107,7 +17191,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
17107 else => try self.asmRegisterMemoryImmediate(17191 else => try self.asmRegisterMemoryImmediate(
17108 .{ .i_, .mul },17192 .{ .i_, .mul },
17109 len_reg,17193 len_reg,
17110 try dst.address().offset(8).deref().mem(self, .qword),17194 try dst.address().offset(8).deref().mem(self, .{ .size = .qword }),
17111 .s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),17195 .s(@intCast(dst_ty.childType(zcu).abiSize(zcu))),
17112 ),17196 ),
17113 }17197 }
...@@ -17139,7 +17223,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -17139,7 +17223,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
17139 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });17223 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
17140}17224}
1714117225
17142fn airTagName(self: *Self, inst: Air.Inst.Index) !void {17226fn airTagName(self: *CodeGen, inst: Air.Inst.Index) !void {
17143 const pt = self.pt;17227 const pt = self.pt;
17144 const zcu = pt.zcu;17228 const zcu = pt.zcu;
17145 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;17229 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
...@@ -17179,7 +17263,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {...@@ -17179,7 +17263,7 @@ fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
17179 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });17263 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
17180}17264}
1718117265
17182fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {17266fn airErrorName(self: *CodeGen, inst: Air.Inst.Index) !void {
17183 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;17267 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1718417268
17185 const err_ty = self.typeOf(un_op);17269 const err_ty = self.typeOf(un_op);
...@@ -17281,7 +17365,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {...@@ -17281,7 +17365,7 @@ fn airErrorName(self: *Self, inst: Air.Inst.Index) !void {
17281 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });17365 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
17282}17366}
1728317367
17284fn airSplat(self: *Self, inst: Air.Inst.Index) !void {17368fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void {
17285 const pt = self.pt;17369 const pt = self.pt;
17286 const zcu = pt.zcu;17370 const zcu = pt.zcu;
17287 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;17371 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -17318,7 +17402,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17318,7 +17402,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17318 else => cc: {17402 else => cc: {
17319 try self.asmMemoryImmediate(17403 try self.asmMemoryImmediate(
17320 .{ ._, .@"test" },17404 .{ ._, .@"test" },
17321 try src_mcv.mem(self, .byte),17405 try src_mcv.mem(self, .{ .size = .byte }),
17322 .u(1),17406 .u(1),
17323 );17407 );
17324 break :cc .nz;17408 break :cc .nz;
...@@ -17362,7 +17446,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17362,7 +17446,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17362 if (src_mcv.isMemory()) try self.asmRegisterMemory(17446 if (src_mcv.isMemory()) try self.asmRegisterMemory(
17363 mir_tag,17447 mir_tag,
17364 registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))),17448 registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))),
17365 try src_mcv.mem(self, self.memSize(scalar_ty)),17449 try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }),
17366 ) else {17450 ) else {
17367 if (mir_tag[0] == .v_i128) break :avx2;17451 if (mir_tag[0] == .v_i128) break :avx2;
17368 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});17452 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
...@@ -17438,7 +17522,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17438,7 +17522,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17438 if (src_mcv.isMemory()) try self.asmRegisterMemory(17522 if (src_mcv.isMemory()) try self.asmRegisterMemory(
17439 .{ .v_ss, .broadcast },17523 .{ .v_ss, .broadcast },
17440 dst_reg.to128(),17524 dst_reg.to128(),
17441 try src_mcv.mem(self, .dword),17525 try src_mcv.mem(self, .{ .size = .dword }),
17442 ) else {17526 ) else {
17443 const src_reg = if (src_mcv.isRegister())17527 const src_reg = if (src_mcv.isRegister())
17444 src_mcv.getReg().?17528 src_mcv.getReg().?
...@@ -17475,7 +17559,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17475,7 +17559,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17475 if (src_mcv.isMemory()) try self.asmRegisterMemory(17559 if (src_mcv.isMemory()) try self.asmRegisterMemory(
17476 .{ .v_ss, .broadcast },17560 .{ .v_ss, .broadcast },
17477 dst_reg.to256(),17561 dst_reg.to256(),
17478 try src_mcv.mem(self, .dword),17562 try src_mcv.mem(self, .{ .size = .dword }),
17479 ) else {17563 ) else {
17480 const src_reg = if (src_mcv.isRegister())17564 const src_reg = if (src_mcv.isRegister())
17481 src_mcv.getReg().?17565 src_mcv.getReg().?
...@@ -17521,7 +17605,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17521,7 +17605,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17521 if (src_mcv.isMemory()) try self.asmRegisterMemory(17605 if (src_mcv.isMemory()) try self.asmRegisterMemory(
17522 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },17606 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
17523 dst_reg.to128(),17607 dst_reg.to128(),
17524 try src_mcv.mem(self, .qword),17608 try src_mcv.mem(self, .{ .size = .qword }),
17525 ) else try self.asmRegisterRegister(17609 ) else try self.asmRegisterRegister(
17526 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },17610 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
17527 dst_reg.to128(),17611 dst_reg.to128(),
...@@ -17546,7 +17630,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17546,7 +17630,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17546 if (src_mcv.isMemory()) try self.asmRegisterMemory(17630 if (src_mcv.isMemory()) try self.asmRegisterMemory(
17547 .{ .v_sd, .broadcast },17631 .{ .v_sd, .broadcast },
17548 dst_reg.to256(),17632 dst_reg.to256(),
17549 try src_mcv.mem(self, .qword),17633 try src_mcv.mem(self, .{ .size = .qword }),
17550 ) else {17634 ) else {
17551 const src_reg = if (src_mcv.isRegister())17635 const src_reg = if (src_mcv.isRegister())
17552 src_mcv.getReg().?17636 src_mcv.getReg().?
...@@ -17589,7 +17673,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17589,7 +17673,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17589 if (src_mcv.isMemory()) try self.asmRegisterMemory(17673 if (src_mcv.isMemory()) try self.asmRegisterMemory(
17590 .{ .v_f128, .broadcast },17674 .{ .v_f128, .broadcast },
17591 dst_reg.to256(),17675 dst_reg.to256(),
17592 try src_mcv.mem(self, .xword),17676 try src_mcv.mem(self, .{ .size = .xword }),
17593 ) else {17677 ) else {
17594 const src_reg = if (src_mcv.isRegister())17678 const src_reg = if (src_mcv.isRegister())
17595 src_mcv.getReg().?17679 src_mcv.getReg().?
...@@ -17616,7 +17700,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -17616,7 +17700,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
17616 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });17700 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
17617}17701}
1761817702
17619fn airSelect(self: *Self, inst: Air.Inst.Index) !void {17703fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
17620 const pt = self.pt;17704 const pt = self.pt;
17621 const zcu = pt.zcu;17705 const zcu = pt.zcu;
17622 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;17706 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
...@@ -17695,7 +17779,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -17695,7 +17779,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17695 else => unreachable,17779 else => unreachable,
17696 }, .broadcast },17780 }, .broadcast },
17697 mask_alias,17781 mask_alias,
17698 if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{17782 if (pred_mcv.isMemory()) try pred_mcv.mem(self, .{ .size = .byte }) else .{
17699 .base = .{ .reg = (try self.copyToTmpRegister(17783 .base = .{ .reg = (try self.copyToTmpRegister(
17700 Type.usize,17784 Type.usize,
17701 pred_mcv.address(),17785 pred_mcv.address(),
...@@ -17893,7 +17977,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -17893,7 +17977,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17893 mir_tag,17977 mir_tag,
17894 dst_alias,17978 dst_alias,
17895 rhs_alias,17979 rhs_alias,
17896 try lhs_mcv.mem(self, self.memSize(ty)),17980 try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }),
17897 mask_alias,17981 mask_alias,
17898 ) else try self.asmRegisterRegisterRegisterRegister(17982 ) else try self.asmRegisterRegisterRegisterRegister(
17899 mir_tag,17983 mir_tag,
...@@ -17908,7 +17992,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -17908,7 +17992,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17908 } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(17992 } else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
17909 mir_tag,17993 mir_tag,
17910 dst_alias,17994 dst_alias,
17911 try lhs_mcv.mem(self, self.memSize(ty)),17995 try lhs_mcv.mem(self, .{ .size = self.memSize(ty) }),
17912 mask_alias,17996 mask_alias,
17913 ) else try self.asmRegisterRegisterRegister(17997 ) else try self.asmRegisterRegisterRegister(
17914 mir_tag,17998 mir_tag,
...@@ -17933,7 +18017,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -17933,7 +18017,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17933 if (rhs_mcv.isMemory()) try self.asmRegisterMemory(18017 if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
17934 .{ mir_fixes, .andn },18018 .{ mir_fixes, .andn },
17935 mask_alias,18019 mask_alias,
17936 try rhs_mcv.mem(self, .fromSize(abi_size)),18020 try rhs_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
17937 ) else try self.asmRegisterRegister(18021 ) else try self.asmRegisterRegister(
17938 .{ mir_fixes, .andn },18022 .{ mir_fixes, .andn },
17939 mask_alias,18023 mask_alias,
...@@ -17949,7 +18033,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -17949,7 +18033,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
17949 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });18033 return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
17950}18034}
1795118035
17952fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {18036fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void {
17953 const pt = self.pt;18037 const pt = self.pt;
17954 const zcu = pt.zcu;18038 const zcu = pt.zcu;
17955 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;18039 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -18074,7 +18158,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18074,7 +18158,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18074 mir_tag,18158 mir_tag,
18075 dst_alias,18159 dst_alias,
18076 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),18160 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18077 try rhs_mcv.mem(self, .fromSize(max_abi_size)),18161 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
18078 ) else try self.asmRegisterRegisterRegister(18162 ) else try self.asmRegisterRegisterRegister(
18079 mir_tag,18163 mir_tag,
18080 dst_alias,18164 dst_alias,
...@@ -18086,7 +18170,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18086,7 +18170,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18086 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(18170 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
18087 mir_tag,18171 mir_tag,
18088 dst_alias,18172 dst_alias,
18089 try rhs_mcv.mem(self, .fromSize(max_abi_size)),18173 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
18090 ) else try self.asmRegisterRegister(18174 ) else try self.asmRegisterRegister(
18091 mir_tag,18175 mir_tag,
18092 dst_alias,18176 dst_alias,
...@@ -18135,7 +18219,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18135,7 +18219,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18135 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(18219 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18136 .{ if (has_avx) .vp_d else .p_d, .shuf },18220 .{ if (has_avx) .vp_d else .p_d, .shuf },
18137 dst_alias,18221 dst_alias,
18138 try src_mcv.mem(self, .fromSize(max_abi_size)),18222 try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
18139 .u(control),18223 .u(control),
18140 ) else try self.asmRegisterRegisterImmediate(18224 ) else try self.asmRegisterRegisterImmediate(
18141 .{ if (has_avx) .vp_d else .p_d, .shuf },18225 .{ if (has_avx) .vp_d else .p_d, .shuf },
...@@ -18192,7 +18276,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18192,7 +18276,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18192 .{ .v_ps, .shuf },18276 .{ .v_ps, .shuf },
18193 dst_alias,18277 dst_alias,
18194 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),18278 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18195 try rhs_mcv.mem(self, .fromSize(max_abi_size)),18279 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
18196 .u(control),18280 .u(control),
18197 ) else try self.asmRegisterRegisterRegisterImmediate(18281 ) else try self.asmRegisterRegisterRegisterImmediate(
18198 .{ .v_ps, .shuf },18282 .{ .v_ps, .shuf },
...@@ -18206,7 +18290,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18206,7 +18290,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18206 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(18290 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18207 .{ ._ps, .shuf },18291 .{ ._ps, .shuf },
18208 dst_alias,18292 dst_alias,
18209 try rhs_mcv.mem(self, .fromSize(max_abi_size)),18293 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
18210 .u(control),18294 .u(control),
18211 ) else try self.asmRegisterRegisterImmediate(18295 ) else try self.asmRegisterRegisterImmediate(
18212 .{ ._ps, .shuf },18296 .{ ._ps, .shuf },
...@@ -18259,7 +18343,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18259,7 +18343,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18259 .{ .v_pd, .shuf },18343 .{ .v_pd, .shuf },
18260 dst_alias,18344 dst_alias,
18261 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),18345 registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
18262 try rhs_mcv.mem(self, .fromSize(max_abi_size)),18346 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
18263 .u(control),18347 .u(control),
18264 ) else try self.asmRegisterRegisterRegisterImmediate(18348 ) else try self.asmRegisterRegisterRegisterImmediate(
18265 .{ .v_pd, .shuf },18349 .{ .v_pd, .shuf },
...@@ -18273,7 +18357,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18273,7 +18357,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18273 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(18357 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18274 .{ ._pd, .shuf },18358 .{ ._pd, .shuf },
18275 dst_alias,18359 dst_alias,
18276 try rhs_mcv.mem(self, .fromSize(max_abi_size)),18360 try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
18277 .u(control),18361 .u(control),
18278 ) else try self.asmRegisterRegisterImmediate(18362 ) else try self.asmRegisterRegisterImmediate(
18279 .{ ._pd, .shuf },18363 .{ ._pd, .shuf },
...@@ -18329,7 +18413,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18329,7 +18413,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18329 .{ .vp_d, .blend },18413 .{ .vp_d, .blend },
18330 registerAlias(dst_reg, dst_abi_size),18414 registerAlias(dst_reg, dst_abi_size),
18331 registerAlias(lhs_reg, dst_abi_size),18415 registerAlias(lhs_reg, dst_abi_size),
18332 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),18416 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18333 .u(expanded_control),18417 .u(expanded_control),
18334 ) else try self.asmRegisterRegisterRegisterImmediate(18418 ) else try self.asmRegisterRegisterRegisterImmediate(
18335 .{ .vp_d, .blend },18419 .{ .vp_d, .blend },
...@@ -18384,7 +18468,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18384,7 +18468,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18384 lhs_mcv.getReg().?18468 lhs_mcv.getReg().?
18385 else18469 else
18386 dst_reg, dst_abi_size),18470 dst_reg, dst_abi_size),
18387 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),18471 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18388 .u(expanded_control),18472 .u(expanded_control),
18389 ) else try self.asmRegisterRegisterRegisterImmediate(18473 ) else try self.asmRegisterRegisterRegisterImmediate(
18390 .{ .vp_w, .blend },18474 .{ .vp_w, .blend },
...@@ -18401,7 +18485,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18401,7 +18485,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18401 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(18485 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
18402 .{ .p_w, .blend },18486 .{ .p_w, .blend },
18403 registerAlias(dst_reg, dst_abi_size),18487 registerAlias(dst_reg, dst_abi_size),
18404 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),18488 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18405 .u(expanded_control),18489 .u(expanded_control),
18406 ) else try self.asmRegisterRegisterImmediate(18490 ) else try self.asmRegisterRegisterImmediate(
18407 .{ .p_w, .blend },18491 .{ .p_w, .blend },
...@@ -18445,7 +18529,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18445,7 +18529,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18445 lhs_mcv.getReg().?18529 lhs_mcv.getReg().?
18446 else18530 else
18447 dst_reg, dst_abi_size),18531 dst_reg, dst_abi_size),
18448 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),18532 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18449 .u(expanded_control),18533 .u(expanded_control),
18450 ) else try self.asmRegisterRegisterRegisterImmediate(18534 ) else try self.asmRegisterRegisterRegisterImmediate(
18451 switch (elem_abi_size) {18535 switch (elem_abi_size) {
...@@ -18470,7 +18554,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18470,7 +18554,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18470 else => unreachable,18554 else => unreachable,
18471 },18555 },
18472 registerAlias(dst_reg, dst_abi_size),18556 registerAlias(dst_reg, dst_abi_size),
18473 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),18557 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18474 .u(expanded_control),18558 .u(expanded_control),
18475 ) else try self.asmRegisterRegisterImmediate(18559 ) else try self.asmRegisterRegisterImmediate(
18476 switch (elem_abi_size) {18560 switch (elem_abi_size) {
...@@ -18560,7 +18644,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18560,7 +18644,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18560 registerAlias(lhs_mcv.getReg().?, dst_abi_size)18644 registerAlias(lhs_mcv.getReg().?, dst_abi_size)
18561 else18645 else
18562 dst_alias,18646 dst_alias,
18563 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),18647 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18564 select_mask_alias,18648 select_mask_alias,
18565 ) else try self.asmRegisterRegisterRegisterRegister(18649 ) else try self.asmRegisterRegisterRegisterRegister(
18566 mir_tag,18650 mir_tag,
...@@ -18577,7 +18661,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18577,7 +18661,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18577 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(18661 ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
18578 mir_tag,18662 mir_tag,
18579 dst_alias,18663 dst_alias,
18580 try rhs_mcv.mem(self, .fromSize(dst_abi_size)),18664 try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18581 select_mask_alias,18665 select_mask_alias,
18582 ) else try self.asmRegisterRegisterRegister(18666 ) else try self.asmRegisterRegisterRegister(
18583 mir_tag,18667 mir_tag,
...@@ -18620,7 +18704,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18620,7 +18704,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18620 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(18704 if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
18621 .{ mir_fixes, .andn },18705 .{ mir_fixes, .andn },
18622 mask_alias,18706 mask_alias,
18623 try lhs_mcv.mem(self, .fromSize(dst_abi_size)),18707 try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
18624 ) else try self.asmRegisterRegister(18708 ) else try self.asmRegisterRegister(
18625 .{ mir_fixes, .andn },18709 .{ mir_fixes, .andn },
18626 mask_alias,18710 mask_alias,
...@@ -18757,7 +18841,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {...@@ -18757,7 +18841,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
18757 return self.finishAir(inst, result, .{ extra.a, extra.b, .none });18841 return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
18758}18842}
1875918843
18760fn airReduce(self: *Self, inst: Air.Inst.Index) !void {18844fn airReduce(self: *CodeGen, inst: Air.Inst.Index) !void {
18761 const pt = self.pt;18845 const pt = self.pt;
18762 const zcu = pt.zcu;18846 const zcu = pt.zcu;
18763 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;18847 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
...@@ -18776,7 +18860,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {...@@ -18776,7 +18860,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18776 .Or => {18860 .Or => {
18777 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(18861 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
18778 .{ ._, .@"test" },18862 .{ ._, .@"test" },
18779 try operand_mcv.mem(self, .fromSize(abi_size)),18863 try operand_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
18780 .u(mask),18864 .u(mask),
18781 ) else {18865 ) else {
18782 const operand_reg = registerAlias(if (operand_mcv.isRegister())18866 const operand_reg = registerAlias(if (operand_mcv.isRegister())
...@@ -18815,7 +18899,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {...@@ -18815,7 +18899,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
18815 return self.finishAir(inst, result, .{ reduce.operand, .none, .none });18899 return self.finishAir(inst, result, .{ reduce.operand, .none, .none });
18816}18900}
1881718901
18818fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {18902fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
18819 const pt = self.pt;18903 const pt = self.pt;
18820 const zcu = pt.zcu;18904 const zcu = pt.zcu;
18821 const result_ty = self.typeOfIndex(inst);18905 const result_ty = self.typeOfIndex(inst);
...@@ -19007,7 +19091,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -19007,7 +19091,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
19007 return self.finishAirResult(inst, result);19091 return self.finishAirResult(inst, result);
19008}19092}
1900919093
19010fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {19094fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void {
19011 const pt = self.pt;19095 const pt = self.pt;
19012 const zcu = pt.zcu;19096 const zcu = pt.zcu;
19013 const ip = &zcu.intern_pool;19097 const ip = &zcu.intern_pool;
...@@ -19053,12 +19137,12 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -19053,12 +19137,12 @@ fn airUnionInit(self: *Self, inst: Air.Inst.Index) !void {
19053 return self.finishAir(inst, result, .{ extra.init, .none, .none });19137 return self.finishAir(inst, result, .{ extra.init, .none, .none });
19054}19138}
1905519139
19056fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {19140fn airPrefetch(self: *CodeGen, inst: Air.Inst.Index) !void {
19057 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;19141 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
19058 return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });19142 return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
19059}19143}
1906019144
19061fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {19145fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void {
19062 const pt = self.pt;19146 const pt = self.pt;
19063 const zcu = pt.zcu;19147 const zcu = pt.zcu;
19064 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;19148 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
...@@ -19219,14 +19303,14 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {...@@ -19219,14 +19303,14 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
19219 mir_tag,19303 mir_tag,
19220 mop1_reg,19304 mop1_reg,
19221 mop2_reg,19305 mop2_reg,
19222 try mops[2].mem(self, .fromSize(abi_size)),19306 try mops[2].mem(self, .{ .size = .fromSize(abi_size) }),
19223 );19307 );
19224 break :result mops[0];19308 break :result mops[0];
19225 };19309 };
19226 return self.finishAir(inst, result, ops);19310 return self.finishAir(inst, result, ops);
19227}19311}
1922819312
19229fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {19313fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void {
19230 const pt = self.pt;19314 const pt = self.pt;
19231 const zcu = pt.zcu;19315 const zcu = pt.zcu;
19232 const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty;19316 const va_list_ty = self.air.instructions.items(.data)[@intFromEnum(inst)].ty;
...@@ -19284,7 +19368,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {...@@ -19284,7 +19368,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
19284 return self.finishAir(inst, result, .{ .none, .none, .none });19368 return self.finishAir(inst, result, .{ .none, .none, .none });
19285}19369}
1928619370
19287fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {19371fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
19288 const pt = self.pt;19372 const pt = self.pt;
19289 const zcu = pt.zcu;19373 const zcu = pt.zcu;
19290 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;19374 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -19451,7 +19535,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -19451,7 +19535,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19451 .{ .v_ss, .cvtsd2 },19535 .{ .v_ss, .cvtsd2 },
19452 dst_reg,19536 dst_reg,
19453 dst_reg,19537 dst_reg,
19454 try promote_mcv.mem(self, .qword),19538 try promote_mcv.mem(self, .{ .size = .qword }),
19455 ) else try self.asmRegisterRegisterRegister(19539 ) else try self.asmRegisterRegisterRegister(
19456 .{ .v_ss, .cvtsd2 },19540 .{ .v_ss, .cvtsd2 },
19457 dst_reg,19541 dst_reg,
...@@ -19463,7 +19547,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -19463,7 +19547,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19463 ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(19547 ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
19464 .{ ._ss, .cvtsd2 },19548 .{ ._ss, .cvtsd2 },
19465 dst_reg,19549 dst_reg,
19466 try promote_mcv.mem(self, .qword),19550 try promote_mcv.mem(self, .{ .size = .qword }),
19467 ) else try self.asmRegisterRegister(19551 ) else try self.asmRegisterRegister(
19468 .{ ._ss, .cvtsd2 },19552 .{ ._ss, .cvtsd2 },
19469 dst_reg,19553 dst_reg,
...@@ -19480,7 +19564,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -19480,7 +19564,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
19480 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });19564 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
19481}19565}
1948219566
19483fn airVaCopy(self: *Self, inst: Air.Inst.Index) !void {19567fn airVaCopy(self: *CodeGen, inst: Air.Inst.Index) !void {
19484 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;19568 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
19485 const ptr_va_list_ty = self.typeOf(ty_op.operand);19569 const ptr_va_list_ty = self.typeOf(ty_op.operand);
1948619570
...@@ -19489,12 +19573,12 @@ fn airVaCopy(self: *Self, inst: Air.Inst.Index) !void {...@@ -19489,12 +19573,12 @@ fn airVaCopy(self: *Self, inst: Air.Inst.Index) !void {
19489 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });19573 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
19490}19574}
1949119575
19492fn airVaEnd(self: *Self, inst: Air.Inst.Index) !void {19576fn airVaEnd(self: *CodeGen, inst: Air.Inst.Index) !void {
19493 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;19577 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
19494 return self.finishAir(inst, .unreach, .{ un_op, .none, .none });19578 return self.finishAir(inst, .unreach, .{ un_op, .none, .none });
19495}19579}
1949619580
19497fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {19581fn resolveInst(self: *CodeGen, ref: Air.Inst.Ref) InnerError!MCValue {
19498 const zcu = self.pt.zcu;19582 const zcu = self.pt.zcu;
19499 const ty = self.typeOf(ref);19583 const ty = self.typeOf(ref);
1950019584
...@@ -19543,7 +19627,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {...@@ -19543,7 +19627,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
19543 }19627 }
19544}19628}
1954519629
19546fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) *InstTracking {19630fn getResolvedInstValue(self: *CodeGen, inst: Air.Inst.Index) *InstTracking {
19547 const tracking = self.inst_tracking.getPtr(inst).?;19631 const tracking = self.inst_tracking.getPtr(inst).?;
19548 return switch (tracking.short) {19632 return switch (tracking.short) {
19549 .none, .unreach, .dead => unreachable,19633 .none, .unreach, .dead => unreachable,
...@@ -19556,7 +19640,7 @@ fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) *InstTracking {...@@ -19556,7 +19640,7 @@ fn getResolvedInstValue(self: *Self, inst: Air.Inst.Index) *InstTracking {
19556/// A potential opportunity for future optimization here would be keeping track19640/// A potential opportunity for future optimization here would be keeping track
19557/// of the fact that the instruction is available both as an immediate19641/// of the fact that the instruction is available both as an immediate
19558/// and as a register.19642/// and as a register.
19559fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCValue {19643fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) !MCValue {
19560 const mcv = try self.resolveInst(operand);19644 const mcv = try self.resolveInst(operand);
19561 const ti = @typeInfo(T).int;19645 const ti = @typeInfo(T).int;
19562 switch (mcv) {19646 switch (mcv) {
...@@ -19572,7 +19656,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV...@@ -19572,7 +19656,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV
19572 return mcv;19656 return mcv;
19573}19657}
1957419658
19575fn genTypedValue(self: *Self, val: Value) InnerError!MCValue {19659fn genTypedValue(self: *CodeGen, val: Value) InnerError!MCValue {
19576 const pt = self.pt;19660 const pt = self.pt;
19577 return switch (try codegen.genTypedValue(self.bin_file, pt, self.src_loc, val, self.target.*)) {19661 return switch (try codegen.genTypedValue(self.bin_file, pt, self.src_loc, val, self.target.*)) {
19578 .mcv => |mcv| switch (mcv) {19662 .mcv => |mcv| switch (mcv) {
...@@ -19599,7 +19683,7 @@ const CallMCValues = struct {...@@ -19599,7 +19683,7 @@ const CallMCValues = struct {
19599 gp_count: u32,19683 gp_count: u32,
19600 fp_count: u32,19684 fp_count: u32,
1960119685
19602 fn deinit(self: *CallMCValues, func: *Self) void {19686 fn deinit(self: *CallMCValues, func: *CodeGen) void {
19603 func.gpa.free(self.args);19687 func.gpa.free(self.args);
19604 self.* = undefined;19688 self.* = undefined;
19605 }19689 }
...@@ -19607,7 +19691,7 @@ const CallMCValues = struct {...@@ -19607,7 +19691,7 @@ const CallMCValues = struct {
1960719691
19608/// Caller must call `CallMCValues.deinit`.19692/// Caller must call `CallMCValues.deinit`.
19609fn resolveCallingConventionValues(19693fn resolveCallingConventionValues(
19610 self: *Self,19694 self: *CodeGen,
19611 fn_info: InternPool.Key.FuncType,19695 fn_info: InternPool.Key.FuncType,
19612 var_args: []const Type,19696 var_args: []const Type,
19613 stack_frame_base: FrameIndex,19697 stack_frame_base: FrameIndex,
...@@ -19889,7 +19973,7 @@ fn resolveCallingConventionValues(...@@ -19889,7 +19973,7 @@ fn resolveCallingConventionValues(
19889 return result;19973 return result;
19890}19974}
1989119975
19892fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {19976fn fail(self: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMemory, CodegenFail } {
19893 @branchHint(.cold);19977 @branchHint(.cold);
19894 const zcu = self.pt.zcu;19978 const zcu = self.pt.zcu;
19895 switch (self.owner) {19979 switch (self.owner) {
...@@ -19899,7 +19983,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMem...@@ -19899,7 +19983,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMem
19899 return error.CodegenFail;19983 return error.CodegenFail;
19900}19984}
1990119985
19902fn failMsg(self: *Self, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {19986fn failMsg(self: *CodeGen, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
19903 @branchHint(.cold);19987 @branchHint(.cold);
19904 const zcu = self.pt.zcu;19988 const zcu = self.pt.zcu;
19905 switch (self.owner) {19989 switch (self.owner) {
...@@ -19960,7 +20044,7 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {...@@ -19960,7 +20044,7 @@ fn registerAlias(reg: Register, size_bytes: u32) Register {
19960 };20044 };
19961}20045}
1996220046
19963fn memSize(self: *Self, ty: Type) Memory.Size {20047fn memSize(self: *CodeGen, ty: Type) Memory.Size {
19964 const zcu = self.pt.zcu;20048 const zcu = self.pt.zcu;
19965 return switch (ty.zigTypeTag(zcu)) {20049 return switch (ty.zigTypeTag(zcu)) {
19966 .float => .fromBitSize(ty.floatBits(self.target.*)),20050 .float => .fromBitSize(ty.floatBits(self.target.*)),
...@@ -19968,7 +20052,7 @@ fn memSize(self: *Self, ty: Type) Memory.Size {...@@ -19968,7 +20052,7 @@ fn memSize(self: *Self, ty: Type) Memory.Size {
19968 };20052 };
19969}20053}
1997020054
19971fn splitType(self: *Self, ty: Type) ![2]Type {20055fn splitType(self: *CodeGen, ty: Type) ![2]Type {
19972 const pt = self.pt;20056 const pt = self.pt;
19973 const zcu = pt.zcu;20057 const zcu = pt.zcu;
19974 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);20058 const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none);
...@@ -19998,7 +20082,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {...@@ -19998,7 +20082,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {
1999820082
19999/// Truncates the value in the register in place.20083/// Truncates the value in the register in place.
20000/// Clobbers any remaining bits.20084/// Clobbers any remaining bits.
20001fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {20085fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
20002 const pt = self.pt;20086 const pt = self.pt;
20003 const zcu = pt.zcu;20087 const zcu = pt.zcu;
20004 const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{20088 const int_info = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else std.builtin.Type.Int{
...@@ -20046,7 +20130,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {...@@ -20046,7 +20130,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
20046 }20130 }
20047}20131}
2004820132
20049fn regBitSize(self: *Self, ty: Type) u64 {20133fn regBitSize(self: *CodeGen, ty: Type) u64 {
20050 const zcu = self.pt.zcu;20134 const zcu = self.pt.zcu;
20051 const abi_size = ty.abiSize(zcu);20135 const abi_size = ty.abiSize(zcu);
20052 return switch (ty.zigTypeTag(zcu)) {20136 return switch (ty.zigTypeTag(zcu)) {
...@@ -20065,27 +20149,27 @@ fn regBitSize(self: *Self, ty: Type) u64 {...@@ -20065,27 +20149,27 @@ fn regBitSize(self: *Self, ty: Type) u64 {
20065 };20149 };
20066}20150}
2006720151
20068fn regExtraBits(self: *Self, ty: Type) u64 {20152fn regExtraBits(self: *CodeGen, ty: Type) u64 {
20069 return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);20153 return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
20070}20154}
2007120155
20072fn hasFeature(self: *Self, feature: std.Target.x86.Feature) bool {20156fn hasFeature(self: *CodeGen, feature: std.Target.x86.Feature) bool {
20073 return std.Target.x86.featureSetHas(self.target.cpu.features, feature);20157 return std.Target.x86.featureSetHas(self.target.cpu.features, feature);
20074}20158}
20075fn hasAnyFeatures(self: *Self, features: anytype) bool {20159fn hasAnyFeatures(self: *CodeGen, features: anytype) bool {
20076 return std.Target.x86.featureSetHasAny(self.target.cpu.features, features);20160 return std.Target.x86.featureSetHasAny(self.target.cpu.features, features);
20077}20161}
20078fn hasAllFeatures(self: *Self, features: anytype) bool {20162fn hasAllFeatures(self: *CodeGen, features: anytype) bool {
20079 return std.Target.x86.featureSetHasAll(self.target.cpu.features, features);20163 return std.Target.x86.featureSetHasAll(self.target.cpu.features, features);
20080}20164}
2008120165
20082fn typeOf(self: *Self, inst: Air.Inst.Ref) Type {20166fn typeOf(self: *CodeGen, inst: Air.Inst.Ref) Type {
20083 const pt = self.pt;20167 const pt = self.pt;
20084 const zcu = pt.zcu;20168 const zcu = pt.zcu;
20085 return self.air.typeOf(inst, &zcu.intern_pool);20169 return self.air.typeOf(inst, &zcu.intern_pool);
20086}20170}
2008720171
20088fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type {20172fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type {
20089 const pt = self.pt;20173 const pt = self.pt;
20090 const zcu = pt.zcu;20174 const zcu = pt.zcu;
20091 const temp: Temp = .{ .index = inst };20175 const temp: Temp = .{ .index = inst };
...@@ -20118,7 +20202,7 @@ fn floatCompilerRtAbiName(float_bits: u32) u8 {...@@ -20118,7 +20202,7 @@ fn floatCompilerRtAbiName(float_bits: u32) u8 {
20118 };20202 };
20119}20203}
2012020204
20121fn floatCompilerRtAbiType(self: *Self, ty: Type, other_ty: Type) Type {20205fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type {
20122 if (ty.toIntern() == .f16_type and20206 if (ty.toIntern() == .f16_type and
20123 (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and20207 (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and
20124 self.target.isDarwin()) return Type.u16;20208 self.target.isDarwin()) return Type.u16;
...@@ -20145,7 +20229,7 @@ fn floatLibcAbiSuffix(ty: Type) []const u8 {...@@ -20145,7 +20229,7 @@ fn floatLibcAbiSuffix(ty: Type) []const u8 {
20145 };20229 };
20146}20230}
2014720231
20148fn promoteInt(self: *Self, ty: Type) Type {20232fn promoteInt(self: *CodeGen, ty: Type) Type {
20149 const pt = self.pt;20233 const pt = self.pt;
20150 const zcu = pt.zcu;20234 const zcu = pt.zcu;
20151 const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {20235 const int_info: InternPool.Key.IntType = switch (ty.toIntern()) {
...@@ -20165,7 +20249,7 @@ fn promoteInt(self: *Self, ty: Type) Type {...@@ -20165,7 +20249,7 @@ fn promoteInt(self: *Self, ty: Type) Type {
20165 return ty;20249 return ty;
20166}20250}
2016720251
20168fn promoteVarArg(self: *Self, ty: Type) Type {20252fn promoteVarArg(self: *CodeGen, ty: Type) Type {
20169 if (!ty.isRuntimeFloat()) return self.promoteInt(ty);20253 if (!ty.isRuntimeFloat()) return self.promoteInt(ty);
20170 switch (ty.floatBits(self.target.*)) {20254 switch (ty.floatBits(self.target.*)) {
20171 32, 64 => return Type.f64,20255 32, 64 => return Type.f64,
...@@ -20181,7 +20265,7 @@ fn promoteVarArg(self: *Self, ty: Type) Type {...@@ -20181,7 +20265,7 @@ fn promoteVarArg(self: *Self, ty: Type) Type {
20181const Temp = struct {20265const Temp = struct {
20182 index: Air.Inst.Index,20266 index: Air.Inst.Index,
2018320267
20184 fn unwrap(temp: Temp, self: *Self) union(enum) {20268 fn unwrap(temp: Temp, cg: *CodeGen) union(enum) {
20185 ref: Air.Inst.Ref,20269 ref: Air.Inst.Ref,
20186 temp: Index,20270 temp: Index,
20187 } {20271 } {
...@@ -20189,38 +20273,69 @@ const Temp = struct {...@@ -20189,38 +20273,69 @@ const Temp = struct {
20189 .ref => |ref| return .{ .ref = ref },20273 .ref => |ref| return .{ .ref = ref },
20190 .target => |target_index| {20274 .target => |target_index| {
20191 const temp_index: Index = @enumFromInt(target_index);20275 const temp_index: Index = @enumFromInt(target_index);
20192 assert(temp_index.isValid(self));20276 assert(temp_index.isValid(cg));
20193 return .{ .temp = temp_index };20277 return .{ .temp = temp_index };
20194 },20278 },
20195 }20279 }
20196 }20280 }
2019720281
20198 fn typeOf(temp: Temp, self: *Self) Type {20282 fn typeOf(temp: Temp, cg: *CodeGen) Type {
20199 return switch (temp.unwrap(self)) {20283 return switch (temp.unwrap(cg)) {
20200 .ref => |ref| self.typeOf(ref),20284 .ref => |ref| cg.typeOf(ref),
20201 .temp => |temp_index| temp_index.typeOf(self),20285 .temp => |temp_index| temp_index.typeOf(cg),
20202 };20286 };
20203 }20287 }
2020420288
20205 fn isMut(temp: Temp, self: *Self) bool {20289 fn isMut(temp: Temp, cg: *CodeGen) bool {
20206 return temp.unwrap(self) == .temp;20290 return switch (temp.unwrap(cg)) {
20291 .ref => false,
20292 .temp => |temp_index| switch (temp_index.tracking(cg).short) {
20293 .none,
20294 .unreach,
20295 .dead,
20296 .undef,
20297 .immediate,
20298 .eflags,
20299 .register_offset,
20300 .memory,
20301 .load_symbol,
20302 .lea_symbol,
20303 .indirect,
20304 .load_direct,
20305 .lea_direct,
20306 .load_got,
20307 .lea_got,
20308 .load_tlv,
20309 .lea_tlv,
20310 .lea_frame,
20311 .elementwise_regs_then_frame,
20312 .reserved_frame,
20313 .air_ref,
20314 => false,
20315 .register,
20316 .register_pair,
20317 .register_overflow,
20318 => true,
20319 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
20320 },
20321 };
20207 }20322 }
2020820323
20209 fn tracking(temp: Temp, self: *Self) InstTracking {20324 fn tracking(temp: Temp, cg: *CodeGen) InstTracking {
20210 return self.inst_tracking.get(temp.index).?;20325 return cg.inst_tracking.get(temp.index).?;
20211 }20326 }
2021220327
20213 fn getOffset(temp: Temp, off: i32, self: *Self) !Temp {20328 fn getOffset(temp: Temp, off: i32, cg: *CodeGen) !Temp {
20214 const new_temp_index = self.next_temp_index;20329 const new_temp_index = cg.next_temp_index;
20215 self.temp_type[@intFromEnum(new_temp_index)] = Type.usize;20330 cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
20216 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);20331 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20217 switch (temp.tracking(self).short) {20332 switch (temp.tracking(cg).short) {
20218 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),20333 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
20219 .register => |reg| {20334 .register => |reg| {
20220 const new_reg =20335 const new_reg =
20221 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20336 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20222 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20337 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20223 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{20338 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20224 .base = .{ .reg = reg.to64() },20339 .base = .{ .reg = reg.to64() },
20225 .mod = .{ .rm = .{20340 .mod = .{ .rm = .{
20226 .size = .qword,20341 .size = .qword,
...@@ -20230,9 +20345,9 @@ const Temp = struct {...@@ -20230,9 +20345,9 @@ const Temp = struct {
20230 },20345 },
20231 .register_offset => |reg_off| {20346 .register_offset => |reg_off| {
20232 const new_reg =20347 const new_reg =
20233 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20348 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20234 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20349 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20235 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{20350 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20236 .base = .{ .reg = reg_off.reg.to64() },20351 .base = .{ .reg = reg_off.reg.to64() },
20237 .mod = .{ .rm = .{20352 .mod = .{ .rm = .{
20238 .size = .qword,20353 .size = .qword,
...@@ -20240,18 +20355,18 @@ const Temp = struct {...@@ -20240,18 +20355,18 @@ const Temp = struct {
20240 } },20355 } },
20241 });20356 });
20242 },20357 },
20243 .lea_symbol => |sym_off| new_temp_index.tracking(self).* = .init(.{ .lea_symbol = .{20358 .lea_symbol => |sym_off| new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = .{
20244 .sym_index = sym_off.sym_index,20359 .sym_index = sym_off.sym_index,
20245 .off = sym_off.off + off,20360 .off = sym_off.off + off,
20246 } }),20361 } }),
20247 .load_frame => |frame_addr| {20362 .load_frame => |frame_addr| {
20248 const new_reg =20363 const new_reg =
20249 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20364 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20250 new_temp_index.tracking(self).* = .init(.{ .register_offset = .{20365 new_temp_index.tracking(cg).* = .init(.{ .register_offset = .{
20251 .reg = new_reg,20366 .reg = new_reg,
20252 .off = off,20367 .off = off,
20253 } });20368 } });
20254 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{20369 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
20255 .base = .{ .frame = frame_addr.index },20370 .base = .{ .frame = frame_addr.index },
20256 .mod = .{ .rm = .{20371 .mod = .{ .rm = .{
20257 .size = .qword,20372 .size = .qword,
...@@ -20259,7 +20374,7 @@ const Temp = struct {...@@ -20259,7 +20374,7 @@ const Temp = struct {
20259 } },20374 } },
20260 });20375 });
20261 },20376 },
20262 .lea_frame => |frame_addr| new_temp_index.tracking(self).* = .init(.{ .lea_frame = .{20377 .lea_frame => |frame_addr| new_temp_index.tracking(cg).* = .init(.{ .lea_frame = .{
20263 .index = frame_addr.index,20378 .index = frame_addr.index,
20264 .off = frame_addr.off + off,20379 .off = frame_addr.off + off,
20265 } }),20380 } }),
...@@ -20267,16 +20382,16 @@ const Temp = struct {...@@ -20267,16 +20382,16 @@ const Temp = struct {
20267 return .{ .index = new_temp_index.toIndex() };20382 return .{ .index = new_temp_index.toIndex() };
20268 }20383 }
2026920384
20270 fn toOffset(temp: *Temp, off: i32, self: *Self) !void {20385 fn toOffset(temp: *Temp, off: i32, cg: *CodeGen) !void {
20271 if (off == 0) return;20386 if (off == 0) return;
20272 switch (temp.unwrap(self)) {20387 switch (temp.unwrap(cg)) {
20273 .ref => {},20388 .ref => {},
20274 .temp => |temp_index| {20389 .temp => |temp_index| {
20275 const temp_tracking = temp_index.tracking(self);20390 const temp_tracking = temp_index.tracking(cg);
20276 switch (temp_tracking.short) {20391 switch (temp_tracking.short) {
20277 else => {},20392 else => {},
20278 .register => |reg| {20393 .register => |reg| {
20279 try self.freeValue(temp_tracking.long);20394 try cg.freeValue(temp_tracking.long);
20280 temp_tracking.* = .init(.{ .register_offset = .{20395 temp_tracking.* = .init(.{ .register_offset = .{
20281 .reg = reg,20396 .reg = reg,
20282 .off = off,20397 .off = off,
...@@ -20284,7 +20399,7 @@ const Temp = struct {...@@ -20284,7 +20399,7 @@ const Temp = struct {
20284 return;20399 return;
20285 },20400 },
20286 .register_offset => |reg_off| {20401 .register_offset => |reg_off| {
20287 try self.freeValue(temp_tracking.long);20402 try cg.freeValue(temp_tracking.long);
20288 temp_tracking.* = .init(.{ .register_offset = .{20403 temp_tracking.* = .init(.{ .register_offset = .{
20289 .reg = reg_off.reg,20404 .reg = reg_off.reg,
20290 .off = reg_off.off + off,20405 .off = reg_off.off + off,
...@@ -20310,39 +20425,39 @@ const Temp = struct {...@@ -20310,39 +20425,39 @@ const Temp = struct {
20310 }20425 }
20311 },20426 },
20312 }20427 }
20313 const new_temp = try temp.getOffset(off, self);20428 const new_temp = try temp.getOffset(off, cg);
20314 try temp.die(self);20429 try temp.die(cg);
20315 temp.* = new_temp;20430 temp.* = new_temp;
20316 }20431 }
2031720432
20318 fn getLimb(temp: Temp, limb_index: u28, self: *Self) !Temp {20433 fn getLimb(temp: Temp, limb_index: u28, cg: *CodeGen) !Temp {
20319 const new_temp_index = self.next_temp_index;20434 const new_temp_index = cg.next_temp_index;
20320 self.temp_type[@intFromEnum(new_temp_index)] = Type.usize;20435 cg.temp_type[@intFromEnum(new_temp_index)] = Type.usize;
20321 switch (temp.tracking(self).short) {20436 switch (temp.tracking(cg).short) {
20322 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),20437 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
20323 .immediate => |imm| {20438 .immediate => |imm| {
20324 assert(limb_index == 0);20439 assert(limb_index == 0);
20325 new_temp_index.tracking(self).* = .init(.{ .immediate = imm });20440 new_temp_index.tracking(cg).* = .init(.{ .immediate = imm });
20326 },20441 },
20327 .register => |reg| {20442 .register => |reg| {
20328 assert(limb_index == 0);20443 assert(limb_index == 0);
20329 const new_reg =20444 const new_reg =
20330 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20445 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20331 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20446 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20332 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());20447 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64());
20333 },20448 },
20334 .register_pair => |regs| {20449 .register_pair => |regs| {
20335 const new_reg =20450 const new_reg =
20336 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20451 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20337 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20452 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20338 try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());20453 try cg.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64());
20339 },20454 },
20340 .register_offset => |reg_off| {20455 .register_offset => |reg_off| {
20341 assert(limb_index == 0);20456 assert(limb_index == 0);
20342 const new_reg =20457 const new_reg =
20343 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20458 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20344 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20459 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20345 try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{20460 try cg.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{
20346 .base = .{ .reg = reg_off.reg.to64() },20461 .base = .{ .reg = reg_off.reg.to64() },
20347 .mod = .{ .rm = .{20462 .mod = .{ .rm = .{
20348 .size = .qword,20463 .size = .qword,
...@@ -20352,9 +20467,9 @@ const Temp = struct {...@@ -20352,9 +20467,9 @@ const Temp = struct {
20352 },20467 },
20353 .load_symbol => |sym_off| {20468 .load_symbol => |sym_off| {
20354 const new_reg =20469 const new_reg =
20355 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20470 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20356 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20471 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20357 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{20472 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
20358 .base = .{ .reloc = sym_off.sym_index },20473 .base = .{ .reloc = sym_off.sym_index },
20359 .mod = .{ .rm = .{20474 .mod = .{ .rm = .{
20360 .size = .qword,20475 .size = .qword,
...@@ -20364,13 +20479,13 @@ const Temp = struct {...@@ -20364,13 +20479,13 @@ const Temp = struct {
20364 },20479 },
20365 .lea_symbol => |sym_off| {20480 .lea_symbol => |sym_off| {
20366 assert(limb_index == 0);20481 assert(limb_index == 0);
20367 new_temp_index.tracking(self).* = .init(.{ .lea_symbol = sym_off });20482 new_temp_index.tracking(cg).* = .init(.{ .lea_symbol = sym_off });
20368 },20483 },
20369 .load_frame => |frame_addr| {20484 .load_frame => |frame_addr| {
20370 const new_reg =20485 const new_reg =
20371 try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);20486 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20372 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20487 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20373 try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{20488 try cg.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{
20374 .base = .{ .frame = frame_addr.index },20489 .base = .{ .frame = frame_addr.index },
20375 .mod = .{ .rm = .{20490 .mod = .{ .rm = .{
20376 .size = .qword,20491 .size = .qword,
...@@ -20380,23 +20495,23 @@ const Temp = struct {...@@ -20380,23 +20495,23 @@ const Temp = struct {
20380 },20495 },
20381 .lea_frame => |frame_addr| {20496 .lea_frame => |frame_addr| {
20382 assert(limb_index == 0);20497 assert(limb_index == 0);
20383 new_temp_index.tracking(self).* = .init(.{ .lea_frame = frame_addr });20498 new_temp_index.tracking(cg).* = .init(.{ .lea_frame = frame_addr });
20384 },20499 },
20385 }20500 }
20386 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);20501 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20387 return .{ .index = new_temp_index.toIndex() };20502 return .{ .index = new_temp_index.toIndex() };
20388 }20503 }
2038920504
20390 fn toLimb(temp: *Temp, limb_index: u28, self: *Self) !void {20505 fn toLimb(temp: *Temp, limb_index: u28, cg: *CodeGen) !void {
20391 switch (temp.unwrap(self)) {20506 switch (temp.unwrap(cg)) {
20392 .ref => {},20507 .ref => {},
20393 .temp => |temp_index| {20508 .temp => |temp_index| {
20394 const temp_tracking = temp_index.tracking(self);20509 const temp_tracking = temp_index.tracking(cg);
20395 switch (temp_tracking.short) {20510 switch (temp_tracking.short) {
20396 else => {},20511 else => {},
20397 .register, .lea_symbol, .lea_frame => {20512 .register, .lea_symbol, .lea_frame => {
20398 assert(limb_index == 0);20513 assert(limb_index == 0);
20399 self.temp_type[@intFromEnum(temp_index)] = Type.usize;20514 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
20400 return;20515 return;
20401 },20516 },
20402 .register_pair => |regs| {20517 .register_pair => |regs| {
...@@ -20406,9 +20521,9 @@ const Temp = struct {...@@ -20406,9 +20521,9 @@ const Temp = struct {
20406 temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),20521 temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(),
20407 }20522 }
20408 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)20523 for (regs, 0..) |reg, reg_index| if (reg_index != limb_index)
20409 self.register_manager.freeReg(reg);20524 cg.register_manager.freeReg(reg);
20410 temp_tracking.* = .init(.{ .register = regs[limb_index] });20525 temp_tracking.* = .init(.{ .register = regs[limb_index] });
20411 self.temp_type[@intFromEnum(temp_index)] = Type.usize;20526 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
20412 return;20527 return;
20413 },20528 },
20414 .load_symbol => |sym_off| {20529 .load_symbol => |sym_off| {
...@@ -20417,7 +20532,7 @@ const Temp = struct {...@@ -20417,7 +20532,7 @@ const Temp = struct {
20417 .sym_index = sym_off.sym_index,20532 .sym_index = sym_off.sym_index,
20418 .off = sym_off.off + @as(u31, limb_index) * 8,20533 .off = sym_off.off + @as(u31, limb_index) * 8,
20419 } });20534 } });
20420 self.temp_type[@intFromEnum(temp_index)] = Type.usize;20535 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
20421 return;20536 return;
20422 },20537 },
20423 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {20538 .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) {
...@@ -20426,102 +20541,102 @@ const Temp = struct {...@@ -20426,102 +20541,102 @@ const Temp = struct {
20426 .index = frame_addr.index,20541 .index = frame_addr.index,
20427 .off = frame_addr.off + @as(u31, limb_index) * 8,20542 .off = frame_addr.off + @as(u31, limb_index) * 8,
20428 } });20543 } });
20429 self.temp_type[@intFromEnum(temp_index)] = Type.usize;20544 cg.temp_type[@intFromEnum(temp_index)] = Type.usize;
20430 return;20545 return;
20431 },20546 },
20432 }20547 }
20433 },20548 },
20434 }20549 }
20435 const new_temp = try temp.getLimb(limb_index, self);20550 const new_temp = try temp.getLimb(limb_index, cg);
20436 try temp.die(self);20551 try temp.die(cg);
20437 temp.* = new_temp;20552 temp.* = new_temp;
20438 }20553 }
2043920554
20440 fn toReg(temp: *Temp, new_reg: Register, self: *Self) !bool {20555 fn toReg(temp: *Temp, new_reg: Register, cg: *CodeGen) !bool {
20441 const val, const ty = switch (temp.unwrap(self)) {20556 const val, const ty = val_ty: switch (temp.unwrap(cg)) {
20442 .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) },20557 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
20443 .temp => |temp_index| val: {20558 .temp => |temp_index| {
20444 const temp_tracking = temp_index.tracking(self);20559 const temp_tracking = temp_index.tracking(cg);
20445 if (temp_tracking.short == .register and20560 if (temp_tracking.short == .register and
20446 temp_tracking.short.register == new_reg) return false;20561 temp_tracking.short.register == new_reg) return false;
20447 break :val .{ temp_tracking.short, temp_index.typeOf(self) };20562 break :val_ty .{ temp_tracking.short, temp_index.typeOf(cg) };
20448 },20563 },
20449 };20564 };
20450 const new_temp_index = self.next_temp_index;20565 const new_temp_index = cg.next_temp_index;
20451 self.temp_type[@intFromEnum(new_temp_index)] = ty;20566 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
20452 try self.genSetReg(new_reg, ty, val, .{});20567 try cg.genSetReg(new_reg, ty, val, .{});
20453 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20568 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20454 try temp.die(self);20569 try temp.die(cg);
20455 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);20570 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20456 temp.* = .{ .index = new_temp_index.toIndex() };20571 temp.* = .{ .index = new_temp_index.toIndex() };
20457 return true;20572 return true;
20458 }20573 }
2045920574
20460 fn toAnyReg(temp: *Temp, self: *Self) !bool {20575 fn toAnyReg(temp: *Temp, cg: *CodeGen) !bool {
20461 const val, const ty = switch (temp.unwrap(self)) {20576 const val, const ty = switch (temp.unwrap(cg)) {
20462 .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) },20577 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
20463 .temp => |temp_index| val: {20578 .temp => |temp_index| val: {
20464 const temp_tracking = temp_index.tracking(self);20579 const temp_tracking = temp_index.tracking(cg);
20465 if (temp_tracking.short == .register) return false;20580 if (temp_tracking.short == .register) return false;
20466 break :val .{ temp_tracking.short, temp_index.typeOf(self) };20581 break :val .{ temp_tracking.short, temp_index.typeOf(cg) };
20467 },20582 },
20468 };20583 };
20469 const new_temp_index = self.next_temp_index;20584 const new_temp_index = cg.next_temp_index;
20470 self.temp_type[@intFromEnum(new_temp_index)] = ty;20585 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
20471 const new_reg =20586 const new_reg =
20472 try self.register_manager.allocReg(new_temp_index.toIndex(), self.regSetForType(ty));20587 try cg.register_manager.allocReg(new_temp_index.toIndex(), cg.regSetForType(ty));
20473 try self.genSetReg(new_reg, ty, val, .{});20588 try cg.genSetReg(new_reg, ty, val, .{});
20474 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20589 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20475 try temp.die(self);20590 try temp.die(cg);
20476 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);20591 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20477 temp.* = .{ .index = new_temp_index.toIndex() };20592 temp.* = .{ .index = new_temp_index.toIndex() };
20478 return true;20593 return true;
20479 }20594 }
2048020595
20481 fn toRegClass(temp: *Temp, rc: Register.Class, self: *Self) !bool {20596 fn toRegClass(temp: *Temp, rc: Register.Class, cg: *CodeGen) !bool {
20482 const val, const ty = switch (temp.unwrap(self)) {20597 const val, const ty = switch (temp.unwrap(cg)) {
20483 .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) },20598 .ref => |ref| .{ temp.tracking(cg).short, cg.typeOf(ref) },
20484 .temp => |temp_index| val: {20599 .temp => |temp_index| val: {
20485 const temp_tracking = temp_index.tracking(self);20600 const temp_tracking = temp_index.tracking(cg);
20486 switch (temp_tracking.short) {20601 switch (temp_tracking.short) {
20487 else => {},20602 else => {},
20488 .register => |reg| if (reg.class() == rc) return false,20603 .register => |reg| if (reg.class() == rc) return false,
20489 }20604 }
20490 break :val .{ temp_tracking.short, temp_index.typeOf(self) };20605 break :val .{ temp_tracking.short, temp_index.typeOf(cg) };
20491 },20606 },
20492 };20607 };
20493 const new_temp_index = self.next_temp_index;20608 const new_temp_index = cg.next_temp_index;
20494 self.temp_type[@intFromEnum(new_temp_index)] = ty;20609 cg.temp_type[@intFromEnum(new_temp_index)] = ty;
20495 const new_reg = try self.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));20610 const new_reg = try cg.register_manager.allocReg(new_temp_index.toIndex(), regSetForRegClass(rc));
20496 try self.genSetReg(new_reg, ty, val, .{});20611 try cg.genSetReg(new_reg, ty, val, .{});
20497 new_temp_index.tracking(self).* = .init(.{ .register = new_reg });20612 new_temp_index.tracking(cg).* = .init(.{ .register = new_reg });
20498 try temp.die(self);20613 try temp.die(cg);
20499 self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);20614 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20500 temp.* = .{ .index = new_temp_index.toIndex() };20615 temp.* = .{ .index = new_temp_index.toIndex() };
20501 return true;20616 return true;
20502 }20617 }
2050320618
20504 fn toPair(first_temp: *Temp, second_temp: *Temp, self: *Self) !void {20619 fn toPair(first_temp: *Temp, second_temp: *Temp, cg: *CodeGen) !void {
20505 while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| {20620 while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| {
20506 if (try part_temp.toAnyReg(self)) break;20621 if (try part_temp.toAnyReg(cg)) break;
20507 } else break;20622 } else break;
20508 const first_temp_tracking = first_temp.unwrap(self).temp.tracking(self);20623 const first_temp_tracking = first_temp.unwrap(cg).temp.tracking(cg);
20509 const second_temp_tracking = second_temp.unwrap(self).temp.tracking(self);20624 const second_temp_tracking = second_temp.unwrap(cg).temp.tracking(cg);
20510 const result: MCValue = .{ .register_pair = .{20625 const result: MCValue = .{ .register_pair = .{
20511 first_temp_tracking.short.register,20626 first_temp_tracking.short.register,
20512 second_temp_tracking.short.register,20627 second_temp_tracking.short.register,
20513 } };20628 } };
20514 const result_temp_index = self.next_temp_index;20629 const result_temp_index = cg.next_temp_index;
20515 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };20630 const result_temp: Temp = .{ .index = result_temp_index.toIndex() };
20516 assert(self.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));20631 assert(cg.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking));
20517 assert(self.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));20632 assert(cg.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking));
20518 self.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;20633 cg.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8;
20519 result_temp_index.tracking(self).* = .init(result);20634 result_temp_index.tracking(cg).* = .init(result);
20520 first_temp.* = result_temp;20635 first_temp.* = result_temp;
20521 }20636 }
2052220637
20523 fn toLea(temp: *Temp, self: *Self) !bool {20638 fn toLea(temp: *Temp, cg: *CodeGen) !bool {
20524 switch (temp.tracking(self).short) {20639 switch (temp.tracking(cg).short) {
20525 .none,20640 .none,
20526 .unreach,20641 .unreach,
20527 .dead,20642 .dead,
...@@ -20548,65 +20663,83 @@ const Temp = struct {...@@ -20548,65 +20663,83 @@ const Temp = struct {
20548 .load_got,20663 .load_got,
20549 .load_tlv,20664 .load_tlv,
20550 .load_frame,20665 .load_frame,
20551 => return temp.toAnyReg(self),20666 => return temp.toAnyReg(cg),
20552 .lea_symbol => |sym_off| {20667 .lea_symbol => |sym_off| {
20553 const off = sym_off.off;20668 const off = sym_off.off;
20554 if (off == 0) return false;20669 if (off == 0) return false;
20555 try temp.toOffset(-off, self);20670 try temp.toOffset(-off, cg);
20556 while (try temp.toAnyReg(self)) {}20671 while (try temp.toAnyReg(cg)) {}
20557 try temp.toOffset(off, self);20672 try temp.toOffset(off, cg);
20558 return true;20673 return true;
20559 },20674 },
20560 }20675 }
20561 }20676 }
2056220677
20563 fn load(ptr: *Temp, val_ty: Type, self: *Self) !Temp {20678 fn toBase(temp: *Temp, cg: *CodeGen) !bool {
20564 const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu));20679 const temp_tracking = temp.tracking(cg);
20565 const val = try self.tempAlloc(val_ty);20680 switch (temp_tracking.short) {
20566 switch (val.tracking(self).short) {20681 else => {},
20682 .indirect, .load_frame => return false,
20683 }
20684 const new_temp_index = cg.next_temp_index;
20685 cg.temp_type[@intFromEnum(new_temp_index)] = temp.typeOf(cg);
20686 const new_reg =
20687 try cg.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp);
20688 try cg.genSetReg(new_reg, Type.usize, temp_tracking.short.address(), .{});
20689 new_temp_index.tracking(cg).* = .init(.{ .indirect = .{ .reg = new_reg } });
20690 try temp.die(cg);
20691 cg.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1);
20692 temp.* = .{ .index = new_temp_index.toIndex() };
20693 return true;
20694 }
20695
20696 fn load(ptr: *Temp, val_ty: Type, cg: *CodeGen) !Temp {
20697 const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu));
20698 const val = try cg.tempAlloc(val_ty);
20699 switch (val.tracking(cg).short) {
20567 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),20700 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
20568 .register => |val_reg| {20701 .register => |val_reg| {
20569 while (try ptr.toLea(self)) {}20702 while (try ptr.toLea(cg)) {}
20570 switch (val_reg.class()) {20703 switch (val_reg.class()) {
20571 .general_purpose => try self.asmRegisterMemory(20704 .general_purpose => try cg.asmRegisterMemory(
20572 .{ ._, .mov },20705 .{ ._, .mov },
20573 registerAlias(val_reg, val_abi_size),20706 registerAlias(val_reg, val_abi_size),
20574 try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),20707 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }),
20575 ),20708 ),
20576 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),20709 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
20577 }20710 }
20578 },20711 },
20579 .load_frame => |val_frame_addr| {20712 .load_frame => |val_frame_addr| {
20580 var val_ptr = try self.tempFromValue(Type.usize, .{ .lea_frame = val_frame_addr });20713 var val_ptr = try cg.tempFromValue(Type.usize, .{ .lea_frame = val_frame_addr });
20581 var len = try self.tempFromValue(Type.usize, .{ .immediate = val_abi_size });20714 var len = try cg.tempFromValue(Type.usize, .{ .immediate = val_abi_size });
20582 try val_ptr.memcpy(ptr, &len, self);20715 try val_ptr.memcpy(ptr, &len, cg);
20583 try val_ptr.die(self);20716 try val_ptr.die(cg);
20584 try len.die(self);20717 try len.die(cg);
20585 },20718 },
20586 }20719 }
20587 return val;20720 return val;
20588 }20721 }
2058920722
20590 fn store(ptr: *Temp, val: *Temp, self: *Self) !void {20723 fn store(ptr: *Temp, val: *Temp, cg: *CodeGen) !void {
20591 const val_ty = val.typeOf(self);20724 const val_ty = val.typeOf(cg);
20592 const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu));20725 const val_abi_size: u32 = @intCast(val_ty.abiSize(cg.pt.zcu));
20593 val: switch (val.tracking(self).short) {20726 val: switch (val.tracking(cg).short) {
20594 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),20727 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
20595 .immediate => |imm| if (std.math.cast(i32, imm)) |s| {20728 .immediate => |imm| if (std.math.cast(i32, imm)) |s| {
20596 while (try ptr.toLea(self)) {}20729 while (try ptr.toLea(cg)) {}
20597 try self.asmMemoryImmediate(20730 try cg.asmMemoryImmediate(
20598 .{ ._, .mov },20731 .{ ._, .mov },
20599 try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),20732 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }),
20600 .s(s),20733 .s(s),
20601 );20734 );
20602 } else continue :val .{ .register = undefined },20735 } else continue :val .{ .register = undefined },
20603 .register => {20736 .register => {
20604 while (try ptr.toLea(self) or try val.toAnyReg(self)) {}20737 while (try ptr.toLea(cg) or try val.toAnyReg(cg)) {}
20605 const val_reg = val.tracking(self).short.register;20738 const val_reg = val.tracking(cg).short.register;
20606 switch (val_reg.class()) {20739 switch (val_reg.class()) {
20607 .general_purpose => try self.asmMemoryRegister(20740 .general_purpose => try cg.asmMemoryRegister(
20608 .{ ._, .mov },20741 .{ ._, .mov },
20609 try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)),20742 try ptr.tracking(cg).short.deref().mem(cg, .{ .size = cg.memSize(val_ty) }),
20610 registerAlias(val_reg, val_abi_size),20743 registerAlias(val_reg, val_abi_size),
20611 ),20744 ),
20612 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),20745 else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }),
...@@ -20615,64 +20748,64 @@ const Temp = struct {...@@ -20615,64 +20748,64 @@ const Temp = struct {
20615 }20748 }
20616 }20749 }
2061720750
20618 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, self: *Self) !void {20751 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) !void {
20619 while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {20752 while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {
20620 if (try temp.toReg(reg, self)) break;20753 if (try temp.toReg(reg, cg)) break;
20621 } else break;20754 } else break;
20622 try self.asmOpOnly(.{ .@"rep _sb", .mov });20755 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
20623 }20756 }
2062420757
20625 // i, m, r20758 // i, m, r
20626 fn add(lhs: *Temp, rhs: *Temp, self: *Self) !Temp {20759 fn add(lhs: *Temp, rhs: *Temp, cg: *CodeGen) !Temp {
20627 const res_index = self.next_temp_index;20760 const res_index = cg.next_temp_index;
20628 var res: Temp = .{ .index = res_index.toIndex() };20761 var res: Temp = .{ .index = res_index.toIndex() };
20629 try self.select(&.{ &res, lhs, rhs }, .{ ._, .add }, &.{20762 try cg.select(&.{ &res, lhs, rhs }, .{ ._, .add }, &.{
20630 .{ .ops = &.{ .{ .match = 1 }, .r, .i } },20763 .{ .ops = &.{ .{ .match = 1 }, .r, .i } },
20631 .{ .ops = &.{ .{ .match = 1 }, .m, .i } },20764 .{ .ops = &.{ .{ .match = 1 }, .m, .i } },
20632 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },20765 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
20633 .{ .ops = &.{ .{ .match = 1 }, .m, .r } },20766 .{ .ops = &.{ .{ .match = 1 }, .m, .r } },
20634 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },20767 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
20635 });20768 });
20636 self.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);20769 cg.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20637 self.temp_type[@intFromEnum(res_index)] = lhs.typeOf(self);20770 cg.temp_type[@intFromEnum(res_index)] = lhs.typeOf(cg);
20638 return res;20771 return res;
20639 }20772 }
2064020773
20641 fn mul(lhs: *Temp, rhs: *Temp, self: *Self) !Temp {20774 fn mul(lhs: *Temp, rhs: *Temp, cg: *CodeGen) !Temp {
20642 const res_index = self.next_temp_index;20775 const res_index = cg.next_temp_index;
20643 var res: Temp = .{ .index = self.next_temp_index.toIndex() };20776 var res: Temp = .{ .index = cg.next_temp_index.toIndex() };
20644 try self.select(&.{ &res, lhs, rhs }, .{ .i_, .mul }, &.{20777 try cg.select(&.{ &res, lhs, rhs }, .{ .i_, .mul }, &.{
20645 .{ .ops = &.{ .r, .m, .i } },20778 .{ .ops = &.{ .r, .m, .i } },
20646 .{ .ops = &.{ .r, .r, .i } },20779 .{ .ops = &.{ .r, .r, .i } },
20647 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },20780 .{ .ops = &.{ .{ .match = 1 }, .r, .m } },
20648 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },20781 .{ .ops = &.{ .{ .match = 1 }, .r, .r } },
20649 });20782 });
20650 self.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);20783 cg.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1);
20651 self.temp_type[@intFromEnum(res_index)] = lhs.typeOf(self);20784 cg.temp_type[@intFromEnum(res_index)] = lhs.typeOf(cg);
20652 return res;20785 return res;
20653 }20786 }
2065420787
20655 fn moveTo(temp: Temp, inst: Air.Inst.Index, self: *Self) !void {20788 fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void {
20656 if (self.liveness.isUnused(inst)) try temp.die(self) else switch (temp.unwrap(self)) {20789 if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) {
20657 .ref => {20790 .ref => {
20658 const result = try self.allocRegOrMem(inst, true);20791 const result = try cg.allocRegOrMem(inst, true);
20659 try self.genCopy(self.typeOfIndex(inst), result, temp.tracking(self).short, .{});20792 try cg.genCopy(cg.typeOfIndex(inst), result, temp.tracking(cg).short, .{});
20660 tracking_log.debug("{} => {} (birth)", .{ inst, result });20793 tracking_log.debug("{} => {} (birth)", .{ inst, result });
20661 self.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));20794 cg.inst_tracking.putAssumeCapacityNoClobber(inst, .init(result));
20662 },20795 },
20663 .temp => |temp_index| {20796 .temp => |temp_index| {
20664 const temp_tracking = temp_index.tracking(self);20797 const temp_tracking = temp_index.tracking(cg);
20665 tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short });20798 tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short });
20666 self.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);20799 cg.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*);
20667 assert(self.reuseTemp(inst, temp_index.toIndex(), temp_tracking));20800 assert(cg.reuseTemp(inst, temp_index.toIndex(), temp_tracking));
20668 },20801 },
20669 }20802 }
20670 }20803 }
2067120804
20672 fn die(temp: Temp, self: *Self) !void {20805 fn die(temp: Temp, cg: *CodeGen) !void {
20673 switch (temp.unwrap(self)) {20806 switch (temp.unwrap(cg)) {
20674 .ref => {},20807 .ref => {},
20675 .temp => |temp_index| try temp_index.tracking(self).die(self, temp_index.toIndex()),20808 .temp => |temp_index| try temp_index.tracking(cg).die(cg, temp_index.toIndex()),
20676 }20809 }
20677 }20810 }
2067820811
...@@ -20687,17 +20820,17 @@ const Temp = struct {...@@ -20687,17 +20820,17 @@ const Temp = struct {
20687 return @enumFromInt(index.toTargetIndex());20820 return @enumFromInt(index.toTargetIndex());
20688 }20821 }
2068920822
20690 fn tracking(index: Index, self: *Self) *InstTracking {20823 fn tracking(index: Index, cg: *CodeGen) *InstTracking {
20691 return &self.inst_tracking.values()[@intFromEnum(index)];20824 return &cg.inst_tracking.values()[@intFromEnum(index)];
20692 }20825 }
2069320826
20694 fn isValid(index: Index, self: *Self) bool {20827 fn isValid(index: Index, cg: *CodeGen) bool {
20695 return index.tracking(self).short != .dead;20828 return index.tracking(cg).short != .dead;
20696 }20829 }
2069720830
20698 fn typeOf(index: Index, self: *Self) Type {20831 fn typeOf(index: Index, cg: *CodeGen) Type {
20699 assert(index.isValid(self));20832 assert(index.isValid(cg));
20700 return self.temp_type[@intFromEnum(index)];20833 return cg.temp_type[@intFromEnum(index)];
20701 }20834 }
2070220835
20703 const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type);20836 const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type);
...@@ -20723,17 +20856,17 @@ const Temp = struct {...@@ -20723,17 +20856,17 @@ const Temp = struct {
20723 };20856 };
20724};20857};
2072520858
20726fn resetTemps(self: *Self) void {20859fn resetTemps(cg: *CodeGen) void {
20727 for (0..@intFromEnum(self.next_temp_index)) |temp_index| {20860 for (0..@intFromEnum(cg.next_temp_index)) |temp_index| {
20728 const temp: Temp.Index = @enumFromInt(temp_index);20861 const temp: Temp.Index = @enumFromInt(temp_index);
20729 assert(!temp.isValid(self));20862 assert(!temp.isValid(cg));
20730 self.temp_type[temp_index] = undefined;20863 cg.temp_type[temp_index] = undefined;
20731 }20864 }
20732 self.next_temp_index = @enumFromInt(0);20865 cg.next_temp_index = @enumFromInt(0);
20733}20866}
2073420867
20735fn reuseTemp(20868fn reuseTemp(
20736 self: *Self,20869 cg: *CodeGen,
20737 new_inst: Air.Inst.Index,20870 new_inst: Air.Inst.Index,
20738 old_inst: Air.Inst.Index,20871 old_inst: Air.Inst.Index,
20739 tracking: *InstTracking,20872 tracking: *InstTracking,
...@@ -20743,79 +20876,80 @@ fn reuseTemp(...@@ -20743,79 +20876,80 @@ fn reuseTemp(
20743 .register_pair,20876 .register_pair,
20744 .register_offset,20877 .register_offset,
20745 .register_overflow,20878 .register_overflow,
20879 .indirect,
20746 => for (tracking.short.getRegs()) |tracked_reg| {20880 => for (tracking.short.getRegs()) |tracked_reg| {
20747 if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| {20881 if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| {
20748 self.register_manager.registers[tracked_index] = new_inst;20882 cg.register_manager.registers[tracked_index] = new_inst;
20749 }20883 }
20750 },20884 },
20751 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,20885 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
20752 else => {},20886 else => {},
20753 }20887 }
20754 switch (tracking.short) {20888 switch (tracking.short) {
20755 .eflags, .register_overflow => self.eflags_inst = new_inst,20889 .eflags, .register_overflow => cg.eflags_inst = new_inst,
20756 else => {},20890 else => {},
20757 }20891 }
20758 tracking.reuse(self, new_inst, old_inst);20892 tracking.reuse(cg, new_inst, old_inst);
20759 return true;20893 return true;
20760}20894}
2076120895
20762fn tempAlloc(self: *Self, ty: Type) !Temp {20896fn tempAlloc(cg: *CodeGen, ty: Type) !Temp {
20763 const temp_index = self.next_temp_index;20897 const temp_index = cg.next_temp_index;
20764 temp_index.tracking(self).* = .init(20898 temp_index.tracking(cg).* = .init(
20765 try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true),20899 try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true),
20766 );20900 );
20767 self.temp_type[@intFromEnum(temp_index)] = ty;20901 cg.temp_type[@intFromEnum(temp_index)] = ty;
20768 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);20902 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20769 return .{ .index = temp_index.toIndex() };20903 return .{ .index = temp_index.toIndex() };
20770}20904}
2077120905
20772fn tempAllocReg(self: *Self, ty: Type, rc: RegisterManager.RegisterBitSet) !Temp {20906fn tempAllocReg(cg: *CodeGen, ty: Type, rc: RegisterManager.RegisterBitSet) !Temp {
20773 const temp_index = self.next_temp_index;20907 const temp_index = cg.next_temp_index;
20774 temp_index.tracking(self).* = .init(20908 temp_index.tracking(cg).* = .init(
20775 .{ .register = try self.register_manager.allocReg(temp_index.toIndex(), rc) },20909 .{ .register = try cg.register_manager.allocReg(temp_index.toIndex(), rc) },
20776 );20910 );
20777 self.temp_type[@intFromEnum(temp_index)] = ty;20911 cg.temp_type[@intFromEnum(temp_index)] = ty;
20778 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);20912 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20779 return .{ .index = temp_index.toIndex() };20913 return .{ .index = temp_index.toIndex() };
20780}20914}
2078120915
20782fn tempFromValue(self: *Self, ty: Type, value: MCValue) !Temp {20916fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp {
20783 const temp_index = self.next_temp_index;20917 const temp_index = cg.next_temp_index;
20784 temp_index.tracking(self).* = .init(value);20918 temp_index.tracking(cg).* = .init(value);
20785 self.temp_type[@intFromEnum(temp_index)] = ty;20919 cg.temp_type[@intFromEnum(temp_index)] = ty;
20786 try self.getValue(value, temp_index.toIndex());20920 try cg.getValue(value, temp_index.toIndex());
20787 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);20921 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20788 return .{ .index = temp_index.toIndex() };20922 return .{ .index = temp_index.toIndex() };
20789}20923}
2079020924
20791fn tempFromOperand(20925fn tempFromOperand(
20792 self: *Self,20926 cg: *CodeGen,
20793 inst: Air.Inst.Index,20927 inst: Air.Inst.Index,
20794 op_index: Liveness.OperandInt,20928 op_index: Liveness.OperandInt,
20795 op_ref: Air.Inst.Ref,20929 op_ref: Air.Inst.Ref,
20796) !Temp {20930) !Temp {
20797 const zcu = self.pt.zcu;20931 const zcu = cg.pt.zcu;
20798 const ip = &zcu.intern_pool;20932 const ip = &zcu.intern_pool;
2079920933
20800 if (!self.liveness.operandDies(inst, op_index)) {20934 if (!cg.liveness.operandDies(inst, op_index)) {
20801 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };20935 if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
20802 const val = op_ref.toInterned().?;20936 const val = op_ref.toInterned().?;
20803 const gop = try self.const_tracking.getOrPut(self.gpa, val);20937 const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
20804 if (!gop.found_existing) gop.value_ptr.* = .init(init: {20938 if (!gop.found_existing) gop.value_ptr.* = .init(init: {
20805 const const_mcv = try self.genTypedValue(.fromInterned(val));20939 const const_mcv = try cg.genTypedValue(.fromInterned(val));
20806 switch (const_mcv) {20940 switch (const_mcv) {
20807 .lea_tlv => |tlv_sym| switch (self.bin_file.tag) {20941 .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) {
20808 .elf, .macho => {20942 .elf, .macho => {
20809 if (self.mod.pic) {20943 if (cg.mod.pic) {
20810 try self.spillRegisters(&.{ .rdi, .rax });20944 try cg.spillRegisters(&.{ .rdi, .rax });
20811 } else {20945 } else {
20812 try self.spillRegisters(&.{.rax});20946 try cg.spillRegisters(&.{.rax});
20813 }20947 }
20814 const frame_index = try self.allocFrameIndex(.init(.{20948 const frame_index = try cg.allocFrameIndex(.init(.{
20815 .size = 8,20949 .size = 8,
20816 .alignment = .@"8",20950 .alignment = .@"8",
20817 }));20951 }));
20818 try self.genSetMem(20952 try cg.genSetMem(
20819 .{ .frame = frame_index },20953 .{ .frame = frame_index },
20820 0,20954 0,
20821 Type.usize,20955 Type.usize,
...@@ -20829,24 +20963,24 @@ fn tempFromOperand(...@@ -20829,24 +20963,24 @@ fn tempFromOperand(
20829 else => break :init const_mcv,20963 else => break :init const_mcv,
20830 }20964 }
20831 });20965 });
20832 return self.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);20966 return cg.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
20833 }20967 }
2083420968
20835 const temp_index = self.next_temp_index;20969 const temp_index = cg.next_temp_index;
20836 const temp: Temp = .{ .index = temp_index.toIndex() };20970 const temp: Temp = .{ .index = temp_index.toIndex() };
20837 const op_inst = op_ref.toIndex().?;20971 const op_inst = op_ref.toIndex().?;
20838 const tracking = self.getResolvedInstValue(op_inst);20972 const tracking = cg.getResolvedInstValue(op_inst);
20839 temp_index.tracking(self).* = tracking.*;20973 temp_index.tracking(cg).* = tracking.*;
20840 if (!self.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };20974 if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
20841 self.temp_type[@intFromEnum(temp_index)] = self.typeOf(op_ref);20975 cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
20842 self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);20976 cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
20843 return temp;20977 return temp;
20844}20978}
2084520979
20846inline fn tempsFromOperands(self: *Self, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {20980inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp {
20847 var temps: [op_refs.len]Temp = undefined;20981 var temps: [op_refs.len]Temp = undefined;
20848 inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| {20982 inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| {
20849 temp.* = try self.tempFromOperand(inst, op_index, op_ref);20983 temp.* = try cg.tempFromOperand(inst, op_index, op_ref);
20850 }20984 }
20851 return temps;20985 return temps;
20852}20986}
...@@ -20859,15 +20993,55 @@ const Operand = union(enum) {...@@ -20859,15 +20993,55 @@ const Operand = union(enum) {
20859 inst: Mir.Inst.Index,20993 inst: Mir.Inst.Index,
20860};20994};
2086120995
20996const SelectLoop = struct {
20997 element_reloc: Mir.Inst.Index,
20998 element_offset: union(enum) {
20999 unused,
21000 known: u31,
21001 temp: Temp,
21002 },
21003 element_size: ?u13,
21004 limb_reloc: Mir.Inst.Index,
21005 limb_offset: union(enum) {
21006 unused,
21007 known: u31,
21008 temp: Temp,
21009 },
21010 limb_size: ?u8,
21011 remaining_size: ?u64,
21012};
21013
20862const Pattern = struct {21014const Pattern = struct {
20863 tag: Mir.Inst.FixedTag,
20864 ops: []const Op,21015 ops: []const Op,
20865 commute: struct { u8, u8 } = .{ 0, 0 },21016 commute: struct { u8, u8 } = .{ 0, 0 },
20866 features: []const std.Target.x86.Feature = &.{},21017
21018 const Set = struct {
21019 required_features: []const std.Target.x86.Feature = &.{},
21020 loop: enum {
21021 /// only execute the instruction once
21022 once,
21023 /// execute the instruction on all groups of non-overlapping bits in the entire value
21024 bitwise,
21025 /// for each element, execute the instruction on each limb, propogating the carry flag
21026 limbwise_carry,
21027 /// for each element, execute the instruction on pairs of limbs, starting from the
21028 /// least significant, propogating a limb
21029 limbwise_pairs_forward,
21030 /// for each element, execute the instruction on pairs of limbs, starting from the
21031 /// most significant, propogating a limb
21032 limbwise_pairs_reverse,
21033 /// for each element, execute the instruction
21034 elementwise,
21035 } = .once,
21036 mir_tag: Mir.Inst.FixedTag,
21037 patterns: []const Pattern,
21038 };
2086721039
20868 const Op = union(enum) {21040 const Op = union(enum) {
20869 /// match another operand21041 /// reuse another operand
20870 match: u8,21042 implicit: u8,
21043 /// repeat another operand
21044 explicit: u8,
20871 /// any general purpose register21045 /// any general purpose register
20872 gpr,21046 gpr,
20873 /// any 64-bit mmx register21047 /// any 64-bit mmx register
...@@ -20878,165 +21052,449 @@ const Pattern = struct {...@@ -20878,165 +21052,449 @@ const Pattern = struct {
20878 ymm,21052 ymm,
20879 /// any memory21053 /// any memory
20880 mem,21054 mem,
21055 /// a limb stored in a gpr
21056 gpr_limb,
21057 /// a limb stored in a 64-bit mmx register
21058 mm_limb,
21059 /// a limb stored in a 128-bit sse register
21060 xmm_limb,
21061 /// a limb stored in a 256-bit sse register
21062 ymm_limb,
21063 /// a limb stored in memory
21064 mem_limb,
20881 /// specific immediate21065 /// specific immediate
20882 imm: i8,21066 imm: i8,
20883 /// any immediate signed extended from 32 bits21067 /// any immediate signed extended from 32 bits
20884 simm32,21068 simm32,
2088521069
20886 fn matches(op: Op, is_mut: bool, temp: Temp, self: *Self) bool {21070 fn matches(op: Op, is_mut: bool, temp: Temp, cg: *CodeGen) bool {
21071 const abi_size = temp.typeOf(cg).abiSize(cg.pt.zcu);
20887 return switch (op) {21072 return switch (op) {
20888 .match => unreachable,21073 .implicit, .explicit => unreachable,
20889 .gpr => switch (temp.tracking(self).short) {21074 .gpr => abi_size <= 8 and switch (temp.tracking(cg).short) {
20890 .register => |reg| reg.class() == .general_purpose,21075 .register => |reg| reg.class() == .general_purpose,
20891 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and21076 .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and
20892 reg_off.off == 0,21077 reg_off.off == 0,
20893 else => self.regClassForType(temp.typeOf(self)) == .general_purpose,21078 else => cg.regClassForType(temp.typeOf(cg)) == .general_purpose,
20894 },21079 },
20895 .mm => switch (temp.tracking(self).short) {21080 .mm => abi_size <= 8 and switch (temp.tracking(cg).short) {
20896 .register => |reg| reg.class() == .mmx,21081 .register => |reg| reg.class() == .mmx,
20897 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,21082 .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0,
20898 else => self.regClassForType(temp.typeOf(self)) == .mmx,21083 else => cg.regClassForType(temp.typeOf(cg)) == .mmx,
20899 },21084 },
20900 .xmm => switch (temp.tracking(self).short) {21085 .xmm => abi_size > 8 and abi_size <= 16 and switch (temp.tracking(cg).short) {
20901 .register => |reg| reg.class() == .sse,21086 .register => |reg| reg.class() == .sse,
20902 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,21087 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
20903 else => self.regClassForType(temp.typeOf(self)) == .sse,21088 else => cg.regClassForType(temp.typeOf(cg)) == .sse,
20904 },21089 },
20905 .ymm => switch (temp.tracking(self).short) {21090 .ymm => abi_size > 16 and abi_size <= 32 and switch (temp.tracking(cg).short) {
20906 .register => |reg| reg.class() == .sse,21091 .register => |reg| reg.class() == .sse,
20907 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,21092 .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0,
20908 else => self.regClassForType(temp.typeOf(self)) == .sse,21093 else => cg.regClassForType(temp.typeOf(cg)) == .sse,
20909 } and temp.typeOf(self).abiSize(self.pt.zcu) > 16,21094 },
20910 .mem => (!is_mut or temp.isMut(self)) and temp.tracking(self).short.isMemory(),21095 .mem, .mem_limb => (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(),
20911 .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(self).short) {21096 .gpr_limb => abi_size > 8,
21097 .mm_limb => abi_size > 8 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory() and cg.regClassForType(temp.typeOf(cg)) == .mmx,
21098 .xmm_limb => abi_size > 16 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(),
21099 .ymm_limb => abi_size > 32 and (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(),
21100 .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(cg).short) {
20912 .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm,21101 .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm,
20913 else => false,21102 else => false,
20914 },21103 },
20915 .simm32 => if (is_mut) unreachable else switch (temp.tracking(self).short) {21104 .simm32 => if (is_mut) unreachable else switch (temp.tracking(cg).short) {
20916 .immediate => |imm| temp.typeOf(self).abiSize(self.pt.zcu) <= 4 or21105 .immediate => |imm| abi_size <= 4 or std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
20917 std.math.cast(i32, @as(i64, @bitCast(imm))) != null,
20918 else => false,21106 else => false,
20919 },21107 },
20920 };21108 };
20921 }21109 }
20922 };21110 };
20923};21111};
20924fn select(self: *Self, dst_temps: []Temp, src_temps: []const *Temp, patterns: []const Pattern) !void {21112fn select(cg: *CodeGen, dst_temps: []Temp, src_temps: []const *Temp, pattern_sets: []const Pattern.Set) !void {
20925 patterns: for (patterns) |pattern| {21113 var loop: SelectLoop = .{
20926 for (pattern.features) |feature| if (!self.hasFeature(feature)) continue :patterns;21114 .element_reloc = undefined,
20927 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| if (!switch (src_op) {21115 .element_offset = .unused,
20928 .match => |match_index| pattern.ops[match_index],21116 .element_size = null,
20929 else => src_op,21117 .limb_reloc = undefined,
20930 }.matches(src_op == .match, src_temp.*, self)) continue :patterns;21118 .limb_offset = .unused,
20931 while (true) for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {21119 .limb_size = null,
20932 if (switch (switch (src_op) {21120 .remaining_size = null,
20933 .match => |match_index| pattern.ops[match_index],21121 };
20934 else => src_op,21122 var extra_temps: [4]?Temp = @splat(null);
20935 }) {21123 pattern_sets: for (pattern_sets) |pattern_set| {
20936 .match => unreachable,21124 for (pattern_set.required_features) |required_feature| if (!cg.hasFeature(required_feature)) continue :pattern_sets;
20937 .gpr => try src_temp.toRegClass(.general_purpose, self),21125 patterns: for (pattern_set.patterns) |pattern| {
20938 .mm => try src_temp.toRegClass(.mmx, self),21126 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20939 .xmm, .ymm => try src_temp.toRegClass(.sse, self),21127 const ref_src_op, const is_mut = switch (src_op) {
20940 .mem, .imm, .simm32 => false,21128 .implicit, .explicit => |op_index| .{ pattern.ops[op_index], true },
20941 }) break;21129 else => .{ src_op, false },
20942 } else break;21130 };
20943 var mir_ops: [4]Operand = @splat(.none);21131 if (!ref_src_op.matches(is_mut, src_temp.*, cg)) continue :patterns;
20944 var mir_ops_len = dst_temps.len;21132 }
20945 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {21133 while (true) for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20946 const mir_op, const matched_src_op = op: switch (src_op) {21134 if (switch (switch (src_op) {
20947 .match => |match_index| {21135 .implicit, .explicit => |op_index| pattern.ops[op_index],
20948 dst_temps[match_index] = src_temp.*;21136 else => src_op,
20949 break :op .{ &mir_ops[match_index], pattern.ops[match_index] };21137 }) {
20950 },21138 .implicit, .explicit => unreachable,
20951 else => {21139 .gpr => try src_temp.toRegClass(.general_purpose, cg),
20952 defer mir_ops_len += 1;21140 .mm => try src_temp.toRegClass(.mmx, cg),
20953 break :op .{ &mir_ops[mir_ops_len], src_op };21141 .xmm, .ymm => try src_temp.toRegClass(.sse, cg),
20954 },21142 .mem, .imm, .simm32 => false,
20955 };21143 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb, .mem_limb => switch (src_temp.tracking(cg).short) {
20956 const src_mcv = src_temp.tracking(self).short;21144 .register_pair => false,
20957 mir_op.* = switch (matched_src_op) {21145 else => try src_temp.toBase(cg),
20958 .match => unreachable,21146 },
20959 .gpr => .{ .reg = registerAlias(21147 }) break;
20960 src_mcv.register,21148 } else break;
20961 @intCast(src_temp.typeOf(self).abiSize(self.pt.zcu)),21149 var mir_ops_len = dst_temps.len;
20962 ) },21150 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
20963 .mm => .{ .reg = src_mcv.register },21151 const ref_src_op, const extra_temp = op: switch (src_op) {
20964 .xmm => .{ .reg = src_mcv.register.to128() },21152 .implicit => |op_index| {
20965 .ymm => .{ .reg = src_mcv.register.to256() },21153 dst_temps[op_index] = if (src_temp.isMut(cg))
20966 .mem => .{ .mem = try src_mcv.mem(self, self.memSize(src_temp.typeOf(self))) },21154 src_temp.*
20967 .imm => |imm| .{ .imm = .s(imm) },21155 else
20968 .simm32 => switch (src_temp.typeOf(self).abiSize(self.pt.zcu)) {21156 try cg.tempAlloc(src_temp.typeOf(cg));
20969 else => unreachable,21157 break :op .{ pattern.ops[op_index], &extra_temps[op_index] };
20970 1 => .{ .imm = if (std.math.cast(i8, @as(i64, @bitCast(src_mcv.immediate)))) |small|21158 },
20971 .s(small)21159 .explicit => |op_index| {
20972 else21160 dst_temps[op_index] = if (src_temp.isMut(cg))
20973 .u(@as(u8, @intCast(src_mcv.immediate))) },21161 src_temp.*
20974 2 => .{ .imm = if (std.math.cast(i16, @as(i64, @bitCast(src_mcv.immediate)))) |small|21162 else
20975 .s(small)21163 try cg.tempAlloc(src_temp.typeOf(cg));
20976 else21164 defer mir_ops_len += 1;
20977 .u(@as(u16, @intCast(src_mcv.immediate))) },21165 break :op .{ pattern.ops[op_index], &extra_temps[mir_ops_len] };
20978 3...8 => .{ .imm = if (std.math.cast(i32, @as(i64, @bitCast(src_mcv.immediate)))) |small|21166 },
20979 .s(small)21167 else => {
20980 else21168 defer mir_ops_len += 1;
20981 .u(@as(u32, @intCast(src_mcv.immediate))) },21169 break :op .{ src_op, &extra_temps[mir_ops_len] };
20982 },21170 },
20983 };21171 };
20984 }21172 const limb_size: u8, const rc = switch (ref_src_op) {
20985 for (21173 else => continue,
20986 pattern.ops[0..dst_temps.len],21174 .gpr_limb => .{ 8, abi.RegisterClass.gp },
20987 dst_temps,21175 .mm_limb => .{ 8, @panic("TODO") },
20988 mir_ops[0..dst_temps.len],21176 .xmm_limb => .{ 16, abi.RegisterClass.sse },
20989 ) |dst_op, *dst_temp, *mir_op| {21177 .ymm_limb => .{ 32, abi.RegisterClass.sse },
20990 if (mir_op.* != .none) continue;21178 };
20991 const ty = src_temps[0].typeOf(self);21179 assert(loop.limb_size == null or loop.limb_size == limb_size);
20992 switch (dst_op) {21180 loop.limb_size = limb_size;
20993 .match => |match_index| {21181 loop.remaining_size = loop.remaining_size orelse src_temp.typeOf(cg).abiSize(cg.pt.zcu);
20994 dst_temp.* = dst_temps[match_index];21182 switch (src_temp.tracking(cg).short) {
20995 mir_op.* = mir_ops[match_index];21183 .register_pair => switch (loop.limb_offset) {
20996 },21184 .unused, .temp => loop.limb_offset = .{ .known = 0 },
20997 .gpr => {21185 .known => {},
20998 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.gp);21186 },
20999 mir_op.* = .{ .reg = registerAlias(21187 else => {
21000 dst_temp.tracking(self).short.register,21188 switch (loop.limb_offset) {
21001 @intCast(ty.abiSize(self.pt.zcu)),21189 .unused => loop.limb_offset = .{ .temp = undefined },
21002 ) };21190 .known, .temp => {},
21003 },21191 }
21004 .mm => @panic("TODO"),21192 extra_temp.* = try cg.tempAllocReg(Type.usize, rc);
21005 .xmm => {21193 },
21006 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.sse);21194 }
21007 mir_op.* = .{ .reg = dst_temp.tracking(self).short.register.to128() };21195 }
21008 },21196 switch (loop.element_offset) {
21009 .ymm => {21197 .unused, .known => {},
21010 dst_temp.* = try self.tempAllocReg(ty, abi.RegisterClass.sse);21198 .temp => |*element_offset| {
21011 mir_op.* = .{ .reg = dst_temp.tracking(self).short.register.to256() };21199 element_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
21200 const element_offset_reg = element_offset.tracking(cg).short.register;
21201 try cg.asmRegisterRegister(.{ ._, .xor }, element_offset_reg.to32(), element_offset_reg.to32());
21202 loop.element_reloc = @intCast(cg.mir_instructions.len);
21012 },21203 },
21013 .mem => @panic("TODO"),
21014 .imm, .simm32 => unreachable, // unmodifiable destination
21015 }21204 }
21016 }21205 while (true) {
21017 std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]);21206 switch (loop.limb_offset) {
21018 self.asmOps(pattern.tag, mir_ops) catch |err| switch (err) {21207 .unused, .known => {},
21019 error.InvalidInstruction => {21208 .temp => |*limb_offset| {
21020 const fixes = @tagName(pattern.tag[0]);21209 limb_offset.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
21021 const fixes_replace = std.mem.indexOfScalar(u8, fixes, '_').?;21210 const limb_offset_reg = limb_offset.tracking(cg).short.register;
21022 return self.fail(21211 try cg.asmRegisterRegister(.{ ._, .xor }, limb_offset_reg.to32(), limb_offset_reg.to32());
21023 "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",21212 loop.limb_reloc = @intCast(cg.mir_instructions.len);
21024 .{
21025 fixes[0..fixes_replace],
21026 @tagName(pattern.tag[1]),
21027 fixes[fixes_replace + 1 ..],
21028 @tagName(mir_ops[0]),
21029 @tagName(mir_ops[1]),
21030 @tagName(mir_ops[2]),
21031 @tagName(mir_ops[3]),
21032 },21213 },
21033 );21214 }
21034 },21215 while (true) {
21035 else => |e| return e,21216 var mir_ops: [4]Operand = @splat(.none);
21036 };21217 mir_ops_len = dst_temps.len;
21037 return;21218 for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| {
21219 const mir_op, const ref_src_op, const extra_temp = op: switch (src_op) {
21220 .implicit => |op_index| .{ &mir_ops[op_index], pattern.ops[op_index], &extra_temps[op_index] },
21221 .explicit => |op_index| {
21222 defer mir_ops_len += 1;
21223 break :op .{ &mir_ops[mir_ops_len], pattern.ops[op_index], &extra_temps[mir_ops_len] };
21224 },
21225 else => {
21226 defer mir_ops_len += 1;
21227 break :op .{ &mir_ops[mir_ops_len], src_op, &extra_temps[mir_ops_len] };
21228 },
21229 };
21230 const src_mcv = src_temp.tracking(cg).short;
21231 switch (ref_src_op) {
21232 else => {},
21233 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (src_mcv) {
21234 .register_pair => {},
21235 else => try cg.asmRegisterMemory(
21236 switch (ref_src_op) {
21237 else => unreachable,
21238 .gpr_limb => .{ ._, .mov },
21239 .mm_limb => .{ ._q, .mov },
21240 .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu },
21241 },
21242 registerAlias(extra_temp.*.?.tracking(cg).short.register, loop.limb_size.?),
21243 try src_mcv.mem(cg, switch (loop.limb_offset) {
21244 .unused => unreachable,
21245 .known => |limb_offset| .{
21246 .size = .fromSize(loop.limb_size.?),
21247 .disp = limb_offset,
21248 },
21249 .temp => |limb_offset| .{
21250 .size = .fromSize(loop.limb_size.?),
21251 .index = limb_offset.tracking(cg).short.register.to64(),
21252 },
21253 }),
21254 ),
21255 },
21256 }
21257 mir_op.* = switch (ref_src_op) {
21258 .implicit, .explicit => unreachable,
21259 .gpr => .{ .reg = registerAlias(
21260 src_mcv.register,
21261 @intCast(src_temp.typeOf(cg).abiSize(cg.pt.zcu)),
21262 ) },
21263 .mm => .{ .reg = src_mcv.register },
21264 .xmm => .{ .reg = src_mcv.register.to128() },
21265 .ymm => .{ .reg = src_mcv.register.to256() },
21266 .mem => .{ .mem = try src_mcv.mem(cg, .{ .size = cg.memSize(src_temp.typeOf(cg)) }) },
21267 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (src_mcv) {
21268 .register_pair => |src_regs| switch (loop.limb_offset) {
21269 .unused => unreachable,
21270 .known => |limb_offset| .{ .reg = registerAlias(
21271 src_regs[@divExact(limb_offset, loop.limb_size.?)],
21272 loop.limb_size.?,
21273 ) },
21274 .temp => unreachable,
21275 },
21276 else => .{ .reg = registerAlias(
21277 extra_temp.*.?.tracking(cg).short.register,
21278 loop.limb_size.?,
21279 ) },
21280 },
21281 .mem_limb => .{ .mem = switch (src_mcv) {
21282 .register_pair => unreachable,
21283 else => switch (loop.limb_offset) {
21284 .unused => unreachable,
21285 .known => |limb_offset| try src_mcv.mem(cg, .{
21286 .size = .fromSize(loop.limb_size.?),
21287 .disp = limb_offset,
21288 }),
21289 .temp => |limb_offset| try src_mcv.mem(cg, .{
21290 .size = .fromSize(loop.limb_size.?),
21291 .index = limb_offset.tracking(cg).short.register.to64(),
21292 }),
21293 },
21294 } },
21295 .imm => |imm| .{ .imm = .s(imm) },
21296 .simm32 => switch (src_temp.typeOf(cg).abiSize(cg.pt.zcu)) {
21297 else => unreachable,
21298 1 => .{ .imm = if (std.math.cast(i8, @as(i64, @bitCast(src_mcv.immediate)))) |small|
21299 .s(small)
21300 else
21301 .u(@as(u8, @intCast(src_mcv.immediate))) },
21302 2 => .{ .imm = if (std.math.cast(i16, @as(i64, @bitCast(src_mcv.immediate)))) |small|
21303 .s(small)
21304 else
21305 .u(@as(u16, @intCast(src_mcv.immediate))) },
21306 3...8 => .{ .imm = if (std.math.cast(i32, @as(i64, @bitCast(src_mcv.immediate)))) |small|
21307 .s(small)
21308 else
21309 .u(@as(u32, @intCast(src_mcv.immediate))) },
21310 },
21311 };
21312 }
21313 for (
21314 pattern.ops[0..dst_temps.len],
21315 dst_temps,
21316 mir_ops[0..dst_temps.len],
21317 extra_temps[0..dst_temps.len],
21318 ) |dst_op, *dst_temp, *mir_op, *extra_temp| {
21319 if (mir_op.* != .none) continue;
21320 const ty = src_temps[0].typeOf(cg);
21321 switch (dst_op) {
21322 .implicit => unreachable,
21323 .explicit => |op_index| {
21324 dst_temp.* = dst_temps[op_index];
21325 mir_op.* = mir_ops[op_index];
21326 },
21327 .gpr => {
21328 dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.gp);
21329 mir_op.* = .{ .reg = registerAlias(
21330 dst_temp.tracking(cg).short.register,
21331 @intCast(ty.abiSize(cg.pt.zcu)),
21332 ) };
21333 },
21334 .mm => @panic("TODO"),
21335 .xmm => {
21336 dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.sse);
21337 mir_op.* = .{ .reg = dst_temp.tracking(cg).short.register.to128() };
21338 },
21339 .ymm => {
21340 dst_temp.* = try cg.tempAllocReg(ty, abi.RegisterClass.sse);
21341 mir_op.* = .{ .reg = dst_temp.tracking(cg).short.register.to256() };
21342 },
21343 .mem => @panic("TODO"),
21344 .gpr_limb => {
21345 dst_temp.* = try cg.tempAlloc(ty);
21346 extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.gp);
21347 mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to64() };
21348 },
21349 .mm_limb => {
21350 dst_temp.* = try cg.tempAlloc(ty);
21351 extra_temp.* = try cg.tempAllocReg(Type.usize, @panic("TODO"));
21352 mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register };
21353 },
21354 .xmm_limb => {
21355 dst_temp.* = try cg.tempAlloc(ty);
21356 extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.sse);
21357 mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to128() };
21358 },
21359 .ymm_limb => {
21360 dst_temp.* = try cg.tempAlloc(ty);
21361 extra_temp.* = try cg.tempAllocReg(Type.usize, abi.RegisterClass.sse);
21362 mir_op.* = .{ .reg = extra_temp.*.?.tracking(cg).short.register.to256() };
21363 },
21364 .mem_limb => {
21365 dst_temp.* = try cg.tempAlloc(ty);
21366 mir_op.* = .{ .mem = try dst_temp.tracking(cg).short.mem(cg, switch (loop.limb_offset) {
21367 .unused => unreachable,
21368 .known => |limb_offset| .{
21369 .size = .fromSize(loop.limb_size.?),
21370 .disp = limb_offset,
21371 },
21372 .temp => |limb_offset| .{
21373 .size = .fromSize(loop.limb_size.?),
21374 .index = limb_offset.tracking(cg).short.register.to64(),
21375 },
21376 }) };
21377 },
21378 .imm, .simm32 => unreachable, // unmodifiable destination
21379 }
21380 }
21381 std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]);
21382 cg.asmOps(pattern_set.mir_tag, mir_ops) catch |err| switch (err) {
21383 error.InvalidInstruction => {
21384 const fixes = @tagName(pattern_set.mir_tag[0]);
21385 const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?;
21386 return cg.fail(
21387 "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'",
21388 .{
21389 fixes[0..fixes_blank],
21390 @tagName(pattern_set.mir_tag[1]),
21391 fixes[fixes_blank + 1 ..],
21392 @tagName(mir_ops[0]),
21393 @tagName(mir_ops[1]),
21394 @tagName(mir_ops[2]),
21395 @tagName(mir_ops[3]),
21396 },
21397 );
21398 },
21399 else => |e| return e,
21400 };
21401 for (
21402 extra_temps[0..dst_temps.len],
21403 pattern.ops[0..dst_temps.len],
21404 dst_temps,
21405 ) |maybe_extra_temp, dst_op, dst_temp| if (maybe_extra_temp) |extra_temp| switch (dst_op) {
21406 else => {},
21407 .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb => switch (dst_temp.tracking(cg).short) {
21408 .register_pair => |dst_regs| switch (loop.limb_offset) {
21409 .unused => unreachable,
21410 .known => |limb_offset| try cg.asmRegisterRegister(
21411 .{ ._, .mov },
21412 dst_regs[@divExact(limb_offset, loop.limb_size.?)].to64(),
21413 extra_temp.tracking(cg).short.register.to64(),
21414 ),
21415 .temp => unreachable,
21416 },
21417 else => |dst_mcv| try cg.asmMemoryRegister(
21418 switch (dst_op) {
21419 else => unreachable,
21420 .gpr_limb => .{ ._, .mov },
21421 .mm_limb => .{ ._q, .mov },
21422 .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu },
21423 },
21424 try dst_mcv.mem(cg, switch (loop.limb_offset) {
21425 .unused => unreachable,
21426 .known => |limb_offset| .{
21427 .size = .fromSize(loop.limb_size.?),
21428 .disp = limb_offset,
21429 },
21430 .temp => |limb_offset| .{
21431 .size = .fromSize(loop.limb_size.?),
21432 .index = limb_offset.tracking(cg).short.register.to64(),
21433 },
21434 }),
21435 registerAlias(extra_temp.tracking(cg).short.register, loop.limb_size.?),
21436 ),
21437 },
21438 };
21439 switch (pattern_set.loop) {
21440 .once => break :pattern_sets,
21441 .bitwise => {},
21442 .limbwise_carry => @panic("TODO"),
21443 .limbwise_pairs_forward => @panic("TODO"),
21444 .limbwise_pairs_reverse => @panic("TODO"),
21445 .elementwise => @panic("TODO"),
21446 }
21447 switch (loop.limb_offset) {
21448 .unused => break,
21449 .known => |*limb_offset| {
21450 limb_offset.* += loop.limb_size.?;
21451 loop.remaining_size.? -= loop.limb_size.?;
21452 if (loop.remaining_size.? < loop.limb_size.? or
21453 (loop.element_size != null and limb_offset.* >= loop.element_size.?))
21454 {
21455 limb_offset.* = 0;
21456 break;
21457 }
21458 },
21459 .temp => |limb_offset| {
21460 const limb_offset_reg = limb_offset.tracking(cg).short.register;
21461 try cg.asmRegisterMemory(.{ ._, .lea }, limb_offset_reg.to32(), .{
21462 .base = .{ .reg = limb_offset_reg.to64() },
21463 .mod = .{ .rm = .{
21464 .size = .qword,
21465 .disp = loop.limb_size.?,
21466 } },
21467 });
21468 try cg.asmRegisterImmediate(
21469 .{ ._, .cmp },
21470 limb_offset_reg.to32(),
21471 .u(loop.element_size orelse loop.remaining_size.?),
21472 );
21473 _ = try cg.asmJccReloc(.b, loop.limb_reloc);
21474 try limb_offset.die(cg);
21475 break;
21476 },
21477 }
21478 }
21479 switch (loop.element_offset) {
21480 .unused => break :pattern_sets,
21481 .known => |*element_offset| {
21482 if (loop.remaining_size.? == 0) break :pattern_sets;
21483 element_offset.* += loop.element_size.?;
21484 },
21485 .temp => |element_offset| {
21486 if (true) @panic("TODO");
21487 try element_offset.die(cg);
21488 if (loop.remaining_size.? == 0) break :pattern_sets;
21489 break;
21490 },
21491 }
21492 }
21493 }
21494 } else {
21495 log.err("failed to select:", .{});
21496 for (src_temps) |src_temp| log.err("{}", .{src_temp.tracking(cg)});
21497 return cg.fail("failed to select", .{});
21038 }21498 }
21039 log.err("failed to select:", .{});21499 for (extra_temps) |extra_temp| if (extra_temp) |temp| try temp.die(cg);
21040 for (src_temps) |src_temp| log.err("{}", .{src_temp.tracking(self)});
21041 return self.fail("failed to select", .{});
21042}21500}
src/arch/x86_64/bits.zig+5-3
...@@ -474,13 +474,15 @@ pub const Memory = struct {...@@ -474,13 +474,15 @@ pub const Memory = struct {
474 };474 };
475475
476 pub const Mod = union(enum(u1)) {476 pub const Mod = union(enum(u1)) {
477 rm: struct {477 rm: Rm,
478 off: u64,
479
480 pub const Rm = struct {
478 size: Size,481 size: Size,
479 index: Register = .none,482 index: Register = .none,
480 scale: Scale = .@"1",483 scale: Scale = .@"1",
481 disp: i32 = 0,484 disp: i32 = 0,
482 },485 };
483 off: u64,
484 };486 };
485487
486 pub const Size = enum(u4) {488 pub const Size = enum(u4) {