Commit 91fb211faa3f37d08da55b0c8df92a6475624316 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-02-25 21:43:20-08:00
committer	GitHub <noreply@github.com> 2024-02-25 21:43:20-08:00
log	91fb211faa3f37d08da55b0c8df92a6475624316
tree	ad824c8fcdd386e378669c21a3e65923b52d0133
parent	d656c2a7abe90d00ef6dbc3731b82bd26180038a
parent	4fcc750ba58f51606c49310bdd7c81c156d48cfc
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #18906 from jacobly0/x86_64-tests

x86_64: pass more tests

24 files changed, 2239 insertions(+), 666 deletions(-)

lib
- std
  - crypto
    aes.zig
    blake3.zig
    salsa20.zig
    sha2.zig
  - meta.zig
  - unicode.zig
  - zig
    c_translation.zig
src
- InternPool.zig
- Sema.zig
- arch
  - x86_64
    CodeGen.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    encodings.zig
- codegen.zig
- codegen
  - c.zig
  - llvm.zig
- type.zig
test
- behavior

lib/std/crypto/aes.zig+1-1

...	@@ -6,7 +6,7 @@ const has_aesni = std.Target.x86.featureSetHas(builtin.cpu.features, .aes);	...	@@ -6,7 +6,7 @@ const has_aesni = std.Target.x86.featureSetHas(builtin.cpu.features, .aes);
6	const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);	6	const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);
7	const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);	7	const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);
8	// C backend doesn't currently support passing vectors to inline asm.	8	// C backend doesn't currently support passing vectors to inline asm.
9	const impl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and has_aesni and has_avx) impl: {	9	const impl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_c and has_aesni and has_avx) impl: {
10	break :impl @import("aes/aesni.zig");	10	break :impl @import("aes/aesni.zig");
11	} else if (builtin.cpu.arch == .aarch64 and builtin.zig_backend != .stage2_c and has_armaes)	11	} else if (builtin.cpu.arch == .aarch64 and builtin.zig_backend != .stage2_c and has_armaes)
12	impl: {	12	impl: {

lib/std/crypto/blake3.zig+1-1

...	@@ -200,7 +200,7 @@ const CompressGeneric = struct {	...	@@ -200,7 +200,7 @@ const CompressGeneric = struct {
200	}	200	}
201	};	201	};
202		202
203	const compress = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64)	203	const compress = if (builtin.cpu.arch == .x86_64)
204	CompressVectorized.compress	204	CompressVectorized.compress
205	else	205	else
206	CompressGeneric.compress;	206	CompressGeneric.compress;

lib/std/crypto/salsa20.zig+4-1

 @@ -302,7 +302,10 @@ fn SalsaNonVecImpl(comptime rounds: comptime_int) type {
     };
 }
-const SalsaImpl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64) SalsaVecImpl else SalsaNonVecImpl;
+const SalsaImpl = if (builtin.cpu.arch == .x86_64)
+    SalsaVecImpl
+else
+    SalsaNonVecImpl;
 fn keyToWords(key: [32]u8) [8]u32 {
     var k: [8]u32 = undefined;

lib/std/crypto/sha2.zig+1-1

...	@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {	...	@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
238	return;	238	return;
239	},	239	},
240	// C backend doesn't currently support passing vectors to inline asm.	240	// C backend doesn't currently support passing vectors to inline asm.
241	.x86_64 => if (builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {	241	.x86_64 => if (builtin.zig_backend != .stage2_c and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
242	var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };	242	var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
243	var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };	243	var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
244	const s_v = @as(*[16]v4u32, @ptrCast(&s));	244	const s_v = @as(*[16]v4u32, @ptrCast(&s));

lib/std/meta.zig+2-1

...	@@ -1286,5 +1286,6 @@ test "hasUniqueRepresentation" {	...	@@ -1286,5 +1286,6 @@ test "hasUniqueRepresentation" {
1286	try testing.expect(!hasUniqueRepresentation([]u8));	1286	try testing.expect(!hasUniqueRepresentation([]u8));
1287	try testing.expect(!hasUniqueRepresentation([]const u8));	1287	try testing.expect(!hasUniqueRepresentation([]const u8));
1288		1288
1289	try testing.expect(hasUniqueRepresentation(@Vector(4, u16)));	1289	try testing.expect(hasUniqueRepresentation(@Vector(std.simd.suggestVectorLength(u8) orelse 1, u8)));
		1290	try testing.expect(@sizeOf(@Vector(3, u8)) == 3 or !hasUniqueRepresentation(@Vector(3, u8)));
1290	}	1291	}

lib/std/unicode.zig+147-152

 @@ -239,18 +239,19 @@ pub fn utf8ValidateSlice(input: []const u8) bool {
 fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {
     var remaining = input;
-    const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
+    if (std.simd.suggestVectorLength(u8)) |chunk_len| {
-    const Chunk = @Vector(chunk_len, u8);
+        const Chunk = @Vector(chunk_len, u8);
-    // Fast path. Check for and skip ASCII characters at the start of the input.
+        // Fast path. Check for and skip ASCII characters at the start of the input.
-    while (remaining.len >= chunk_len) {
+        while (remaining.len >= chunk_len) {
-        const chunk: Chunk = remaining[0..chunk_len].*;
+            const chunk: Chunk = remaining[0..chunk_len].*;
-        const mask: Chunk = @splat(0x80);
+            const mask: Chunk = @splat(0x80);
-        if (@reduce(.Or, chunk & mask == mask)) {
+            if (@reduce(.Or, chunk & mask == mask)) {
-            // found a non ASCII byte
+                // found a non ASCII byte
-            break;
+                break;
+            }
+            remaining = remaining[chunk_len..];
         }
-        remaining = remaining[chunk_len..];
     }
     // default lowest and highest continuation byte
 @@ -601,9 +602,9 @@ fn testUtf8IteratorOnAscii() !void {
     const s = Utf8View.initComptime("abc");
     var it1 = s.iterator();
-    try testing.expect(std.mem.eql(u8, "a", it1.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "a", it1.nextCodepointSlice().?));
-    try testing.expect(std.mem.eql(u8, "b", it1.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "b", it1.nextCodepointSlice().?));
-    try testing.expect(std.mem.eql(u8, "c", it1.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "c", it1.nextCodepointSlice().?));
     try testing.expect(it1.nextCodepointSlice() == null);
     var it2 = s.iterator();
 @@ -631,9 +632,9 @@ fn testUtf8ViewOk() !void {
     const s = Utf8View.initComptime("東京市");
     var it1 = s.iterator();
-    try testing.expect(std.mem.eql(u8, "東", it1.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "東", it1.nextCodepointSlice().?));
-    try testing.expect(std.mem.eql(u8, "京", it1.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "京", it1.nextCodepointSlice().?));
-    try testing.expect(std.mem.eql(u8, "市", it1.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "市", it1.nextCodepointSlice().?));
     try testing.expect(it1.nextCodepointSlice() == null);
     var it2 = s.iterator();
 @@ -771,20 +772,20 @@ fn testUtf8Peeking() !void {
     const s = Utf8View.initComptime("noël");
     var it = s.iterator();
-    try testing.expect(std.mem.eql(u8, "n", it.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "n", it.nextCodepointSlice().?));
-    try testing.expect(std.mem.eql(u8, "o", it.peek(1)));
+    try testing.expect(mem.eql(u8, "o", it.peek(1)));
-    try testing.expect(std.mem.eql(u8, "oë", it.peek(2)));
+    try testing.expect(mem.eql(u8, "oë", it.peek(2)));
-    try testing.expect(std.mem.eql(u8, "oël", it.peek(3)));
+    try testing.expect(mem.eql(u8, "oël", it.peek(3)));
-    try testing.expect(std.mem.eql(u8, "oël", it.peek(4)));
+    try testing.expect(mem.eql(u8, "oël", it.peek(4)));
-    try testing.expect(std.mem.eql(u8, "oël", it.peek(10)));
+    try testing.expect(mem.eql(u8, "oël", it.peek(10)));
-    try testing.expect(std.mem.eql(u8, "o", it.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "o", it.nextCodepointSlice().?));
-    try testing.expect(std.mem.eql(u8, "ë", it.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "ë", it.nextCodepointSlice().?));
-    try testing.expect(std.mem.eql(u8, "l", it.nextCodepointSlice().?));
+    try testing.expect(mem.eql(u8, "l", it.nextCodepointSlice().?));
     try testing.expect(it.nextCodepointSlice() == null);
-    try testing.expect(std.mem.eql(u8, &[_]u8{}, it.peek(1)));
+    try testing.expect(mem.eql(u8, &[_]u8{}, it.peek(1)));
 }
 fn testError(bytes: []const u8, expected_err: anyerror) !void {
 @@ -926,59 +927,50 @@ test "fmtUtf8" {
 }
 fn utf16LeToUtf8ArrayListImpl(
-    array_list: *std.ArrayList(u8),
+    result: *std.ArrayList(u8),
     utf16le: []const u16,
     comptime surrogates: Surrogates,
 ) (switch (surrogates) {
     .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
     .can_encode_surrogate_half => mem.Allocator.Error,
 })!void {
-    // optimistically guess that it will all be ascii.
+    assert(result.capacity >= utf16le.len);
-    try array_list.ensureTotalCapacityPrecise(utf16le.len);
     var remaining = utf16le;
-    if (builtin.zig_backend != .stage2_x86_64) {
+    vectorized: {
-        const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;
+        const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
         const Chunk = @Vector(chunk_len, u16);
         // Fast path. Check for and encode ASCII characters at the start of the input.
         while (remaining.len >= chunk_len) {
             const chunk: Chunk = remaining[0..chunk_len].*;
-            const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));
+            const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
             if (@reduce(.Or, chunk | mask != mask)) {
                 // found a non ASCII code unit
                 break;
             }
-            const chunk_byte_len = chunk_len * 2;
+            const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
-            const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
-            const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
-            const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
             // We allocated enough space to encode every UTF-16 code unit
             // as ASCII, so if the entire string is ASCII then we are
             // guaranteed to have enough space allocated
-            array_list.appendSliceAssumeCapacity(&ascii_bytes);
+            result.addManyAsArrayAssumeCapacity(chunk_len).* = ascii_chunk;
             remaining = remaining[chunk_len..];
         }
     }
-    var out_index: usize = array_list.items.len;
     switch (surrogates) {
         .cannot_encode_surrogate_half => {
             var it = Utf16LeIterator.init(remaining);
             while (try it.nextCodepoint()) |codepoint| {
                 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
-                try array_list.resize(array_list.items.len + utf8_len);
+                assert((utf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
-                assert((utf8Encode(codepoint, array_list.items[out_index..]) catch unreachable) == utf8_len);
-                out_index += utf8_len;
             }
         },
         .can_encode_surrogate_half => {
             var it = Wtf16LeIterator.init(remaining);
             while (it.nextCodepoint()) |codepoint| {
                 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
-                try array_list.resize(array_list.items.len + utf8_len);
+                assert((wtf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
-                assert((wtf8Encode(codepoint, array_list.items[out_index..]) catch unreachable) == utf8_len);
-                out_index += utf8_len;
             }
         },
     }
 @@ -986,8 +978,9 @@ fn utf16LeToUtf8ArrayListImpl(
 pub const Utf16LeToUtf8AllocError = mem.Allocator.Error || Utf16LeToUtf8Error;
-pub fn utf16LeToUtf8ArrayList(array_list: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
+pub fn utf16LeToUtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
-    return utf16LeToUtf8ArrayListImpl(array_list, utf16le, .cannot_encode_surrogate_half);
+    try result.ensureTotalCapacityPrecise(utf16le.len);
+    return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
 }
 /// Deprecated; renamed to utf16LeToUtf8Alloc
 @@ -999,8 +992,7 @@ pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16L
     var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);
     errdefer result.deinit();
-    try utf16LeToUtf8ArrayList(&result, utf16le);
+    try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
     return result.toOwnedSlice();
 }
 @@ -1013,8 +1005,7 @@ pub fn utf16LeToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) Utf16
     var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);
     errdefer result.deinit();
-    try utf16LeToUtf8ArrayList(&result, utf16le);
+    try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
     return result.toOwnedSliceSentinel(0);
 }
 @@ -1026,27 +1017,24 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
     .cannot_encode_surrogate_half => Utf16LeToUtf8Error,
     .can_encode_surrogate_half => error{},
 })!usize {
-    var end_index: usize = 0;
+    var dest_index: usize = 0;
     var remaining = utf16le;
-    if (builtin.zig_backend != .stage2_x86_64) {
+    vectorized: {
-        const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;
+        const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
         const Chunk = @Vector(chunk_len, u16);
         // Fast path. Check for and encode ASCII characters at the start of the input.
         while (remaining.len >= chunk_len) {
             const chunk: Chunk = remaining[0..chunk_len].*;
-            const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));
+            const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
             if (@reduce(.Or, chunk | mask != mask)) {
                 // found a non ASCII code unit
                 break;
             }
-            const chunk_byte_len = chunk_len * 2;
+            const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
-            const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
+            utf8[dest_index..][0..chunk_len].* = ascii_chunk;
-            const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
+            dest_index += chunk_len;
-            const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
-            @memcpy(utf8[end_index .. end_index + chunk_len], &ascii_bytes);
-            end_index += chunk_len;
             remaining = remaining[chunk_len..];
         }
     }
 @@ -1055,7 +1043,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
         .cannot_encode_surrogate_half => {
             var it = Utf16LeIterator.init(remaining);
             while (try it.nextCodepoint()) |codepoint| {
-                end_index += utf8Encode(codepoint, utf8[end_index..]) catch |err| switch (err) {
+                dest_index += utf8Encode(codepoint, utf8[dest_index..]) catch |err| switch (err) {
                     // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
                     // which is within the valid codepoint range.
                     error.CodepointTooLarge => unreachable,
 @@ -1068,7 +1056,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
         .can_encode_surrogate_half => {
             var it = Wtf16LeIterator.init(remaining);
             while (it.nextCodepoint()) |codepoint| {
-                end_index += wtf8Encode(codepoint, utf8[end_index..]) catch |err| switch (err) {
+                dest_index += wtf8Encode(codepoint, utf8[dest_index..]) catch |err| switch (err) {
                     // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
                     // which is within the valid codepoint range.
                     error.CodepointTooLarge => unreachable,
 @@ -1076,7 +1064,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
             }
         },
     }
-    return end_index;
+    return dest_index;
 }
 /// Deprecated; renamed to utf16LeToUtf8
 @@ -1149,14 +1137,12 @@ test utf16LeToUtf8 {
     }
 }
-fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
+fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
-    // optimistically guess that it will not require surrogate pairs
+    assert(result.capacity >= utf8.len);
-    try array_list.ensureTotalCapacityPrecise(utf8.len);
     var remaining = utf8;
-    // Need support for std.simd.interlace
+    vectorized: {
-    if (builtin.zig_backend != .stage2_x86_64 and comptime !builtin.cpu.arch.isMIPS()) {
+        const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
-        const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
         const Chunk = @Vector(chunk_len, u8);
         // Fast path. Check for and encode ASCII characters at the start of the input.
 @@ -1167,9 +1153,8 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,
                 // found a non ASCII code unit
                 break;
             }
-            const zeroes: Chunk = @splat(0);
+            const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
-            const utf16_chunk: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });
+            result.addManyAsArrayAssumeCapacity(chunk_len).* = utf16_chunk;
-            array_list.appendSliceAssumeCapacity(std.mem.bytesAsSlice(u16, &utf16_chunk));
             remaining = remaining[chunk_len..];
         }
     }
 @@ -1181,21 +1166,18 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,
     var it = view.iterator();
     while (it.nextCodepoint()) |codepoint| {
         if (codepoint < 0x10000) {
-            const short = @as(u16, @intCast(codepoint));
+            try result.append(mem.nativeToLittle(u16, @intCast(codepoint)));
-            try array_list.append(mem.nativeToLittle(u16, short));
         } else {
             const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
             const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
-            var out: [2]u16 = undefined;
+            try result.appendSlice(&.{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) });
-            out[0] = mem.nativeToLittle(u16, high);
-            out[1] = mem.nativeToLittle(u16, low);
-            try array_list.appendSlice(out[0..]);
         }
     }
 }
-pub fn utf8ToUtf16LeArrayList(array_list: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
+pub fn utf8ToUtf16LeArrayList(result: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
-    return utf8ToUtf16LeArrayListImpl(array_list, utf8, .cannot_encode_surrogate_half);
+    try result.ensureTotalCapacityPrecise(utf8.len);
+    return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
 }
 pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {
 @@ -1204,7 +1186,6 @@ pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ Inv
     errdefer result.deinit();
     try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
     return result.toOwnedSlice();
 }
 @@ -1217,7 +1198,6 @@ pub fn utf8ToUtf16LeAllocZ(allocator: mem.Allocator, utf8: []const u8) error{ In
     errdefer result.deinit();
     try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
     return result.toOwnedSliceSentinel(0);
 }
 @@ -1228,12 +1208,11 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) error{InvalidUtf8}!usize
 }
 pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates: Surrogates) !usize {
-    var dest_i: usize = 0;
+    var dest_index: usize = 0;
     var remaining = utf8;
-    // Need support for std.simd.interlace
+    vectorized: {
-    if (builtin.zig_backend != .stage2_x86_64 and comptime !builtin.cpu.arch.isMIPS()) {
+        const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
-        const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
         const Chunk = @Vector(chunk_len, u8);
         // Fast path. Check for and encode ASCII characters at the start of the input.
 @@ -1244,57 +1223,60 @@ pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates:
                 // found a non ASCII code unit
                 break;
             }
-            const zeroes: Chunk = @splat(0);
+            const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
-            const utf16_bytes: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });
+            utf16le[dest_index..][0..chunk_len].* = utf16_chunk;
-            @memcpy(utf16le[dest_i..][0..chunk_len], std.mem.bytesAsSlice(u16, &utf16_bytes));
+            dest_index += chunk_len;
-            dest_i += chunk_len;
             remaining = remaining[chunk_len..];
         }
     }
-    var src_i: usize = 0;
+    const view = switch (surrogates) {
-    while (src_i < remaining.len) {
+        .cannot_encode_surrogate_half => try Utf8View.init(remaining),
-        const n = utf8ByteSequenceLength(remaining[src_i]) catch return switch (surrogates) {
+        .can_encode_surrogate_half => try Wtf8View.init(remaining),
-            .cannot_encode_surrogate_half => error.InvalidUtf8,
+    };
-            .can_encode_surrogate_half => error.InvalidWtf8,
+    var it = view.iterator();
-        };
+    while (it.nextCodepoint()) |codepoint| {
-        const next_src_i = src_i + n;
-        const codepoint = switch (surrogates) {
-            .cannot_encode_surrogate_half => utf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidUtf8,
-            .can_encode_surrogate_half => wtf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidWtf8,
-        };
         if (codepoint < 0x10000) {
-            const short = @as(u16, @intCast(codepoint));
+            utf16le[dest_index] = mem.nativeToLittle(u16, @intCast(codepoint));
-            utf16le[dest_i] = mem.nativeToLittle(u16, short);
+            dest_index += 1;
-            dest_i += 1;
         } else {
             const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
             const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
-            utf16le[dest_i] = mem.nativeToLittle(u16, high);
+            utf16le[dest_index..][0..2].* = .{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) };
-            utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);
+            dest_index += 2;
-            dest_i += 2;
         }
-        src_i = next_src_i;
     }
-    return dest_i;
+    return dest_index;
 }
 test "utf8ToUtf16Le" {
-    var utf16le: [2]u16 = [_]u16{0} ** 2;
+    var utf16le: [128]u16 = undefined;
     {
         const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
-        try testing.expectEqual(@as(usize, 2), length);
+        try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..length]));
-        try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..]));
     }
     {
         const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
-        try testing.expectEqual(@as(usize, 2), length);
+        try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..length]));
-        try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..]));
     }
     {
         const result = utf8ToUtf16Le(utf16le[0..], "\xf4\x90\x80\x80");
         try testing.expectError(error.InvalidUtf8, result);
     }
+    {
+        const length = try utf8ToUtf16Le(utf16le[0..], "This string has been designed to test the vectorized implementat" ++
+            "ion by beginning with one hundred twenty-seven ASCII characters¡");
+        try testing.expectEqualSlices(u8, &.{
+            'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ',  0,
+            'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o',  0,
+            ' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r',  0,
+            'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't',  0,
+            'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g',  0,
+            ' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e',  0,
+            'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A',  0,
+            'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0, '¡', 0,
+        }, mem.sliceAsBytes(utf16le[0..length]));
+    }
 }
 test utf8ToUtf16LeArrayList {
 @@ -1339,25 +1321,40 @@ test utf8ToUtf16LeAllocZ {
     {
         const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "𐐷");
         defer testing.allocator.free(utf16);
-        try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));
+        try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16));
         try testing.expect(utf16[2] == 0);
     }
     {
         const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "\u{10FFFF}");
         defer testing.allocator.free(utf16);
-        try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));
+        try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16));
         try testing.expect(utf16[2] == 0);
     }
     {
         const result = utf8ToUtf16LeAllocZ(testing.allocator, "\xf4\x90\x80\x80");
         try testing.expectError(error.InvalidUtf8, result);
     }
+    {
+        const utf16 = try utf8ToUtf16LeWithNull(testing.allocator, "This string has been designed to test the vectorized implementat" ++
+            "ion by beginning with one hundred twenty-seven ASCII characters¡");
+        defer testing.allocator.free(utf16);
+        try testing.expectEqualSlices(u8, &.{
+            'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ',  0,
+            'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o',  0,
+            ' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r',  0,
+            'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't',  0,
+            'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g',  0,
+            ' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e',  0,
+            'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A',  0,
+            'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0, '¡', 0,
+        }, mem.sliceAsBytes(utf16));
+    }
 }
 /// Converts a UTF-8 string literal into a UTF-16LE string literal.
-pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch unreachable:0]u16 {
+pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch |err| @compileError(err):0]u16 {
     return comptime blk: {
-        const len: usize = calcUtf16LeLen(utf8) catch |err| @compileError(err);
+        const len: usize = calcUtf16LeLen(utf8) catch unreachable;
         var utf16le: [len:0]u16 = [_:0]u16{0} ** len;
         const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch |err| @compileError(err);
         assert(len == utf16le_len);
 @@ -1438,12 +1435,12 @@ test "fmtUtf16Le" {
     try expectFmt("", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral(""))});
     try expectFmt("foo", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("foo"))});
     try expectFmt("𐐷", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("𐐷"))});
-    try expectFmt("퟿", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xd7", native_endian)})});
+    try expectFmt("퟿", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xd7", native_endian)})});
-    try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xd8", native_endian)})});
+    try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xd8", native_endian)})});
-    try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xdb", native_endian)})});
+    try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdb", native_endian)})});
-    try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xdc", native_endian)})});
+    try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xdc", native_endian)})});
-    try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xdf", native_endian)})});
+    try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdf", native_endian)})});
-    try expectFmt("", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xe0", native_endian)})});
+    try expectFmt("", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xe0", native_endian)})});
 }
 test "utf8ToUtf16LeStringLiteral" {
 @@ -1686,8 +1683,9 @@ pub const Wtf8Iterator = struct {
     }
 };
-pub fn wtf16LeToWtf8ArrayList(array_list: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {
+pub fn wtf16LeToWtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {
-    return utf16LeToUtf8ArrayListImpl(array_list, utf16le, .can_encode_surrogate_half);
+    try result.ensureTotalCapacityPrecise(utf16le.len);
+    return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
 }
 /// Caller must free returned memory.
 @@ -1696,8 +1694,7 @@ pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Al
     var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len);
     errdefer result.deinit();
-    try wtf16LeToWtf8ArrayList(&result, wtf16le);
+    try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
     return result.toOwnedSlice();
 }
 @@ -1707,8 +1704,7 @@ pub fn wtf16LeToWtf8AllocZ(allocator: mem.Allocator, wtf16le: []const u16) mem.A
     var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len + 1);
     errdefer result.deinit();
-    try wtf16LeToWtf8ArrayList(&result, wtf16le);
+    try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
     return result.toOwnedSliceSentinel(0);
 }
 @@ -1716,8 +1712,9 @@ pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {
     return utf16LeToUtf8Impl(wtf8, wtf16le, .can_encode_surrogate_half) catch |err| switch (err) {};
 }
-pub fn wtf8ToWtf16LeArrayList(array_list: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
+pub fn wtf8ToWtf16LeArrayList(result: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
-    return utf8ToUtf16LeArrayListImpl(array_list, wtf8, .can_encode_surrogate_half);
+    try result.ensureTotalCapacityPrecise(wtf8.len);
+    return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
 }
 pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {
 @@ -1726,7 +1723,6 @@ pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ Inv
     errdefer result.deinit();
     try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
     return result.toOwnedSlice();
 }
 @@ -1736,7 +1732,6 @@ pub fn wtf8ToWtf16LeAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ In
     errdefer result.deinit();
     try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
     return result.toOwnedSliceSentinel(0);
 }
 @@ -1895,7 +1890,7 @@ pub const Wtf16LeIterator = struct {
     pub fn init(s: []const u16) Wtf16LeIterator {
         return Wtf16LeIterator{
-            .bytes = std.mem.sliceAsBytes(s),
+            .bytes = mem.sliceAsBytes(s),
             .i = 0,
         };
     }
 @@ -1908,12 +1903,12 @@ pub const Wtf16LeIterator = struct {
         assert(it.i <= it.bytes.len);
         if (it.i == it.bytes.len) return null;
         var code_units: [2]u16 = undefined;
-        code_units[0] = std.mem.readInt(u16, it.bytes[it.i..][0..2], .little);
+        code_units[0] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
         it.i += 2;
         surrogate_pair: {
             if (utf16IsHighSurrogate(code_units[0])) {
                 if (it.i >= it.bytes.len) break :surrogate_pair;
-                code_units[1] = std.mem.readInt(u16, it.bytes[it.i..][0..2], .little);
+                code_units[1] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
                 const codepoint = utf16DecodeSurrogatePair(&code_units) catch break :surrogate_pair;
                 it.i += 2;
                 return codepoint;
 @@ -2030,31 +2025,31 @@ fn testRoundtripWtf16(wtf16le: []const u16) !void {
 test "well-formed WTF-16 roundtrips" {
     try testRoundtripWtf16(&[_]u16{
-        std.mem.nativeToLittle(u16, 0xD83D), // high surrogate
+        mem.nativeToLittle(u16, 0xD83D), // high surrogate
-        std.mem.nativeToLittle(u16, 0xDCA9), // low surrogate
+        mem.nativeToLittle(u16, 0xDCA9), // low surrogate
     });
     try testRoundtripWtf16(&[_]u16{
-        std.mem.nativeToLittle(u16, 0xD83D), // high surrogate
+        mem.nativeToLittle(u16, 0xD83D), // high surrogate
-        std.mem.nativeToLittle(u16, ' '), // not surrogate
+        mem.nativeToLittle(u16, ' '), // not surrogate
-        std.mem.nativeToLittle(u16, 0xDCA9), // low surrogate
+        mem.nativeToLittle(u16, 0xDCA9), // low surrogate
     });
     try testRoundtripWtf16(&[_]u16{
-        std.mem.nativeToLittle(u16, 0xD800), // high surrogate
+        mem.nativeToLittle(u16, 0xD800), // high surrogate
-        std.mem.nativeToLittle(u16, 0xDBFF), // high surrogate
+        mem.nativeToLittle(u16, 0xDBFF), // high surrogate
     });
     try testRoundtripWtf16(&[_]u16{
-        std.mem.nativeToLittle(u16, 0xD800), // high surrogate
+        mem.nativeToLittle(u16, 0xD800), // high surrogate
-        std.mem.nativeToLittle(u16, 0xE000), // not surrogate
+        mem.nativeToLittle(u16, 0xE000), // not surrogate
     });
     try testRoundtripWtf16(&[_]u16{
-        std.mem.nativeToLittle(u16, 0xD7FF), // not surrogate
+        mem.nativeToLittle(u16, 0xD7FF), // not surrogate
-        std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
+        mem.nativeToLittle(u16, 0xDC00), // low surrogate
     });
     try testRoundtripWtf16(&[_]u16{
-        std.mem.nativeToLittle(u16, 0x61), // not surrogate
+        mem.nativeToLittle(u16, 0x61), // not surrogate
-        std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
+        mem.nativeToLittle(u16, 0xDC00), // low surrogate
     });
     try testRoundtripWtf16(&[_]u16{
-        std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
+        mem.nativeToLittle(u16, 0xDC00), // low surrogate
     });
 }

lib/std/zig/c_translation.zig+3-5

 @@ -308,14 +308,12 @@ test "promoteIntLiteral" {
 /// Convert from clang __builtin_shufflevector index to Zig @shuffle index
 /// clang requires __builtin_shufflevector index arguments to be integer constants.
-/// negative values for `this_index` indicate "don't care" so we arbitrarily choose 0
+/// negative values for `this_index` indicate "don't care".
 /// clang enforces that `this_index` is less than the total number of vector elements
 /// See https://ziglang.org/documentation/master/#shuffle
 /// See https://clang.llvm.org/docs/LanguageExtensions.html#langext-builtin-shufflevector
 pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len: usize) i32 {
-    if (this_index <= 0) return 0;
+    const positive_index = std.math.cast(usize, this_index) orelse return undefined;
-    const positive_index = @as(usize, @intCast(this_index));
     if (positive_index < source_vector_len) return @as(i32, @intCast(this_index));
     const b_index = positive_index - source_vector_len;
     return ~@as(i32, @intCast(b_index));
 @@ -324,7 +322,7 @@ pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len
 test "shuffleVectorIndex" {
     const vector_len: usize = 4;
-    try testing.expect(shuffleVectorIndex(-1, vector_len) == 0);
+    _ = shuffleVectorIndex(-1, vector_len);
     try testing.expect(shuffleVectorIndex(0, vector_len) == 0);
     try testing.expect(shuffleVectorIndex(1, vector_len) == 1);

src/InternPool.zig+7-2

 @@ -3587,6 +3587,7 @@ pub const Alignment = enum(u6) {
     @"8" = 3,
     @"16" = 4,
     @"32" = 5,
+    @"64" = 6,
     none = std.math.maxInt(u6),
     _,
 @@ -7403,10 +7404,14 @@ pub fn isIntegerType(ip: *const InternPool, ty: Index) bool {
         .c_ulong_type,
         .c_longlong_type,
         .c_ulonglong_type,
-        .c_longdouble_type,
         .comptime_int_type,
         => true,
-        else => ip.indexToKey(ty) == .int_type,
+        else => switch (ip.items.items(.tag)[@intFromEnum(ty)]) {
+            .type_int_signed,
+            .type_int_unsigned,
+            => true,
+            else => false,
+        },
     };
 }

src/Sema.zig+31-2

...	@@ -23315,7 +23315,8 @@ fn checkVectorElemType(	...	@@ -23315,7 +23315,8 @@ fn checkVectorElemType(
23315	const mod = sema.mod;	23315	const mod = sema.mod;
23316	switch (ty.zigTypeTag(mod)) {	23316	switch (ty.zigTypeTag(mod)) {
23317	.Int, .Float, .Bool => return,	23317	.Int, .Float, .Bool => return,
23318	else => if (ty.isPtrAtRuntime(mod)) return,	23318	.Optional, .Pointer => if (ty.isPtrAtRuntime(mod)) return,
		23319	else => {},
23319	}	23320	}
23320	return sema.fail(block, ty_src, "expected integer, float, bool, or pointer for the vector element type; found '{}'", .{ty.fmt(mod)});	23321	return sema.fail(block, ty_src, "expected integer, float, bool, or pointer for the vector element type; found '{}'", .{ty.fmt(mod)});
23321	}	23322	}
...	@@ -28442,7 +28443,7 @@ const CoerceOpts = struct {	...	@@ -28442,7 +28443,7 @@ const CoerceOpts = struct {
28442	report_err: bool = true,	28443	report_err: bool = true,
28443	/// Ignored if `report_err == false`.	28444	/// Ignored if `report_err == false`.
28444	is_ret: bool = false,	28445	is_ret: bool = false,
28445	/// Should coercion to comptime_int ermit an error message.	28446	/// Should coercion to comptime_int emit an error message.
28446	no_cast_to_comptime_int: bool = false,	28447	no_cast_to_comptime_int: bool = false,
28447		28448
28448	param_src: struct {	28449	param_src: struct {
...	@@ -31845,6 +31846,34 @@ fn coerceArrayLike(	...	@@ -31845,6 +31846,34 @@ fn coerceArrayLike(
31845	}	31846	}
31846		31847
31847	const dest_elem_ty = dest_ty.childType(mod);	31848	const dest_elem_ty = dest_ty.childType(mod);
		31849	if (dest_ty.isVector(mod) and inst_ty.isVector(mod) and (try sema.resolveValue(inst)) == null) {
		31850	const inst_elem_ty = inst_ty.childType(mod);
		31851	switch (dest_elem_ty.zigTypeTag(mod)) {
		31852	.Int => if (inst_elem_ty.isInt(mod)) {
		31853	// integer widening
		31854	const dst_info = dest_elem_ty.intInfo(mod);
		31855	const src_info = inst_elem_ty.intInfo(mod);
		31856	if ((src_info.signedness == dst_info.signedness and dst_info.bits >= src_info.bits) or
		31857	// small enough unsigned ints can get casted to large enough signed ints
		31858	(dst_info.signedness == .signed and dst_info.bits > src_info.bits))
		31859	{
		31860	try sema.requireRuntimeBlock(block, inst_src, null);
		31861	return block.addTyOp(.intcast, dest_ty, inst);
		31862	}
		31863	},
		31864	.Float => if (inst_elem_ty.isRuntimeFloat()) {
		31865	// float widening
		31866	const src_bits = inst_elem_ty.floatBits(target);
		31867	const dst_bits = dest_elem_ty.floatBits(target);
		31868	if (dst_bits >= src_bits) {
		31869	try sema.requireRuntimeBlock(block, inst_src, null);
		31870	return block.addTyOp(.fpext, dest_ty, inst);
		31871	}
		31872	},
		31873	else => {},
		31874	}
		31875	}
		31876
31848	const element_vals = try sema.arena.alloc(InternPool.Index, dest_len);	31877	const element_vals = try sema.arena.alloc(InternPool.Index, dest_len);
31849	const element_refs = try sema.arena.alloc(Air.Inst.Ref, dest_len);	31878	const element_refs = try sema.arena.alloc(Air.Inst.Ref, dest_len);
31850	var runtime_src: ?LazySrcLoc = null;	31879	var runtime_src: ?LazySrcLoc = null;

src/arch/x86_64/CodeGen.zig+1659-327

...	@@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory(	...	@@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory(
1547	});	1547	});
1548	}	1548	}
1549		1549
		1550	fn asmRegisterRegisterMemoryRegister(
		1551	self: *Self,
		1552	tag: Mir.Inst.FixedTag,
		1553	reg1: Register,
		1554	reg2: Register,
		1555	m: Memory,
		1556	reg3: Register,
		1557	) !void {
		1558	_ = try self.addInst(.{
		1559	.tag = tag[1],
		1560	.ops = .rrmr,
		1561	.data = .{ .rrrx = .{
		1562	.fixes = tag[0],
		1563	.r1 = reg1,
		1564	.r2 = reg2,
		1565	.r3 = reg3,
		1566	.payload = try self.addExtra(Mir.Memory.encode(m)),
		1567	} },
		1568	});
		1569	}
		1570
1550	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {	1571	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1551	_ = try self.addInst(.{	1572	_ = try self.addInst(.{
1552	.tag = tag[1],	1573	.tag = tag[1],
...	@@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo	...	@@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo
1570	});	1591	});
1571	}	1592	}
1572		1593
		1594	fn asmRegisterMemoryRegister(
		1595	self: *Self,
		1596	tag: Mir.Inst.FixedTag,
		1597	reg1: Register,
		1598	m: Memory,
		1599	reg2: Register,
		1600	) !void {
		1601	_ = try self.addInst(.{
		1602	.tag = tag[1],
		1603	.ops = .rmr,
		1604	.data = .{ .rrx = .{
		1605	.fixes = tag[0],
		1606	.r1 = reg1,
		1607	.r2 = reg2,
		1608	.payload = try self.addExtra(Mir.Memory.encode(m)),
		1609	} },
		1610	});
		1611	}
		1612
1573	fn asmRegisterMemoryImmediate(	1613	fn asmRegisterMemoryImmediate(
1574	self: *Self,	1614	self: *Self,
1575	tag: Mir.Inst.FixedTag,	1615	tag: Mir.Inst.FixedTag,
...	@@ -2570,7 +2610,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt	...	@@ -2570,7 +2610,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2570		2610
2571	const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;	2611	const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
2572	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =	2612	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
2573	if (opts.update_tracking) ({}) else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);	2613	if (opts.update_tracking)
		2614	{} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
2574		2615
2575	var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(	2616	var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
2576	stack.get(),	2617	stack.get(),
...	@@ -2812,11 +2853,14 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2812,11 +2853,14 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2812	}	2853	}
2813		2854
2814	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	2855	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
		2856	const mod = self.bin_file.comp.module.?;
2815	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2857	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2816	const dst_ty = self.typeOfIndex(inst);	2858	const dst_ty = self.typeOfIndex(inst);
2817	const dst_bits = dst_ty.floatBits(self.target.*);	2859	const dst_scalar_ty = dst_ty.scalarType(mod);
		2860	const dst_bits = dst_scalar_ty.floatBits(self.target.*);
2818	const src_ty = self.typeOf(ty_op.operand);	2861	const src_ty = self.typeOf(ty_op.operand);
2819	const src_bits = src_ty.floatBits(self.target.*);	2862	const src_scalar_ty = src_ty.scalarType(mod);
		2863	const src_bits = src_scalar_ty.floatBits(self.target.*);
2820		2864
2821	const result = result: {	2865	const result = result: {
2822	if (switch (src_bits) {	2866	if (switch (src_bits) {
...	@@ -2840,94 +2884,290 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2840,94 +2884,290 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2840	},	2884	},
2841	else => unreachable,	2885	else => unreachable,
2842	}) {	2886	}) {
		2887	if (dst_ty.isVector(mod)) break :result null;
2843	var callee_buf: ["__extend?f?f2".len]u8 = undefined;	2888	var callee_buf: ["__extend?f?f2".len]u8 = undefined;
2844	break :result try self.genCall(.{ .lib = .{	2889	break :result try self.genCall(.{ .lib = .{
2845	.return_type = self.floatCompilerRtAbiType(dst_ty, src_ty).toIntern(),	2890	.return_type = self.floatCompilerRtAbiType(dst_scalar_ty, src_scalar_ty).toIntern(),
2846	.param_types = &.{self.floatCompilerRtAbiType(src_ty, dst_ty).toIntern()},	2891	.param_types = &.{self.floatCompilerRtAbiType(src_scalar_ty, dst_scalar_ty).toIntern()},
2847	.callee = std.fmt.bufPrint(&callee_buf, "__extend{c}f{c}f2", .{	2892	.callee = std.fmt.bufPrint(&callee_buf, "__extend{c}f{c}f2", .{
2848	floatCompilerRtAbiName(src_bits),	2893	floatCompilerRtAbiName(src_bits),
2849	floatCompilerRtAbiName(dst_bits),	2894	floatCompilerRtAbiName(dst_bits),
2850	}) catch unreachable,	2895	}) catch unreachable,
2851	} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});	2896	} }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }});
2852	}	2897	}
2853		2898
		2899	const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
2854	const src_mcv = try self.resolveInst(ty_op.operand);	2900	const src_mcv = try self.resolveInst(ty_op.operand);
2855	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))	2901	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2856	src_mcv	2902	src_mcv
2857	else	2903	else
2858	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);	2904	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
2859	const dst_reg = dst_mcv.getReg().?.to128();	2905	const dst_reg = dst_mcv.getReg().?;
		2906	const dst_alias = registerAlias(dst_reg, @intCast(@max(dst_ty.abiSize(mod), 16)));
2860	const dst_lock = self.register_manager.lockReg(dst_reg);	2907	const dst_lock = self.register_manager.lockReg(dst_reg);
2861	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);	2908	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
2862		2909
		2910	const vec_len = if (dst_ty.isVector(mod)) dst_ty.vectorLen(mod) else 1;
2863	if (src_bits == 16) {	2911	if (src_bits == 16) {
2864	assert(self.hasFeature(.f16c));	2912	assert(self.hasFeature(.f16c));
2865	const mat_src_reg = if (src_mcv.isRegister())	2913	const mat_src_reg = if (src_mcv.isRegister())
2866	src_mcv.getReg().?	2914	src_mcv.getReg().?
2867	else	2915	else
2868	try self.copyToTmpRegister(src_ty, src_mcv);	2916	try self.copyToTmpRegister(src_ty, src_mcv);
2869	try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128());	2917	try self.asmRegisterRegister(
		2918	.{ .v_ps, .cvtph2 },
		2919	dst_alias,
		2920	registerAlias(mat_src_reg, src_abi_size),
		2921	);
2870	switch (dst_bits) {	2922	switch (dst_bits) {
2871	32 => {},	2923	32 => {},
2872	64 => try self.asmRegisterRegisterRegister(	2924	64 => try self.asmRegisterRegisterRegister(
2873	.{ .v_sd, .cvtss2 },	2925	.{ .v_sd, .cvtss2 },
2874	dst_reg,	2926	dst_alias,
2875	dst_reg,	2927	dst_alias,
2876	dst_reg,	2928	dst_alias,
2877	),	2929	),
2878	else => unreachable,	2930	else => unreachable,
2879	}	2931	}
2880	} else {	2932	} else {
2881	assert(src_bits == 32 and dst_bits == 64);	2933	assert(src_bits == 32 and dst_bits == 64);
2882	if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(	2934	if (self.hasFeature(.avx)) switch (vec_len) {
2883	.{ .v_sd, .cvtss2 },	2935	1 => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2884	dst_reg,	2936	.{ .v_sd, .cvtss2 },
2885	dst_reg,	2937	dst_alias,
2886	try src_mcv.mem(self, .dword),	2938	dst_alias,
2887	) else try self.asmRegisterRegisterRegister(	2939	try src_mcv.mem(self, self.memSize(src_ty)),
2888	.{ .v_sd, .cvtss2 },	2940	) else try self.asmRegisterRegisterRegister(
2889	dst_reg,	2941	.{ .v_sd, .cvtss2 },
2890	dst_reg,	2942	dst_alias,
2891	(if (src_mcv.isRegister())	2943	dst_alias,
2892	src_mcv.getReg().?	2944	registerAlias(if (src_mcv.isRegister())
2893	else	2945	src_mcv.getReg().?
2894	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),	2946	else
2895	) else if (src_mcv.isMemory()) try self.asmRegisterMemory(	2947	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
2896	.{ ._sd, .cvtss2 },	2948	),
2897	dst_reg,	2949	2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(
2898	try src_mcv.mem(self, .dword),	2950	.{ .v_pd, .cvtps2 },
		2951	dst_alias,
		2952	try src_mcv.mem(self, self.memSize(src_ty)),
		2953	) else try self.asmRegisterRegister(
		2954	.{ .v_pd, .cvtps2 },
		2955	dst_alias,
		2956	registerAlias(if (src_mcv.isRegister())
		2957	src_mcv.getReg().?
		2958	else
		2959	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
		2960	),
		2961	else => break :result null,
		2962	} else if (src_mcv.isMemory()) try self.asmRegisterMemory(
		2963	switch (vec_len) {
		2964	1 => .{ ._sd, .cvtss2 },
		2965	2 => .{ ._pd, .cvtps2 },
		2966	else => break :result null,
		2967	},
		2968	dst_alias,
		2969	try src_mcv.mem(self, self.memSize(src_ty)),
2899	) else try self.asmRegisterRegister(	2970	) else try self.asmRegisterRegister(
2900	.{ ._sd, .cvtss2 },	2971	switch (vec_len) {
2901	dst_reg,	2972	1 => .{ ._sd, .cvtss2 },
2902	(if (src_mcv.isRegister())	2973	2 => .{ ._pd, .cvtps2 },
		2974	else => break :result null,
		2975	},
		2976	dst_alias,
		2977	registerAlias(if (src_mcv.isRegister())
2903	src_mcv.getReg().?	2978	src_mcv.getReg().?
2904	else	2979	else
2905	try self.copyToTmpRegister(src_ty, src_mcv)).to128(),	2980	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
2906	);	2981	);
2907	}	2982	}
2908	break :result dst_mcv;	2983	break :result dst_mcv;
2909	};	2984	} orelse return self.fail("TODO implement airFpext from {} to {}", .{
		2985	src_ty.fmt(mod), dst_ty.fmt(mod),
		2986	});
2910	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	2987	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2911	}	2988	}
2912		2989
2913	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	2990	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2914	const mod = self.bin_file.comp.module.?;	2991	const mod = self.bin_file.comp.module.?;
2915	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	2992	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2916	const result: MCValue = result: {	2993	const src_ty = self.typeOf(ty_op.operand);
2917	const src_ty = self.typeOf(ty_op.operand);	2994	const dst_ty = self.typeOfIndex(inst);
2918	const src_int_info = src_ty.intInfo(mod);
2919		2995
2920	const dst_ty = self.typeOfIndex(inst);	2996	const result = @as(?MCValue, result: {
2921	const dst_int_info = dst_ty.intInfo(mod);	2997	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
2922	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
2923		2998
2924	const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;	2999	const src_int_info = src_ty.intInfo(mod);
		3000	const dst_int_info = dst_ty.intInfo(mod);
2925	const extend = switch (src_int_info.signedness) {	3001	const extend = switch (src_int_info.signedness) {
2926	.signed => dst_int_info,	3002	.signed => dst_int_info,
2927	.unsigned => src_int_info,	3003	.unsigned => src_int_info,
2928	}.signedness;	3004	}.signedness;
2929		3005
2930	const src_mcv = try self.resolveInst(ty_op.operand);	3006	const src_mcv = try self.resolveInst(ty_op.operand);
		3007	if (dst_ty.isVector(mod)) {
		3008	const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
		3009	const max_abi_size = @max(dst_abi_size, src_abi_size);
		3010	if (max_abi_size > @as(u32, if (self.hasFeature(.avx2)) 32 else 16)) break :result null;
		3011	const has_avx = self.hasFeature(.avx);
		3012
		3013	const dst_elem_abi_size = dst_ty.childType(mod).abiSize(mod);
		3014	const src_elem_abi_size = src_ty.childType(mod).abiSize(mod);
		3015	switch (math.order(dst_elem_abi_size, src_elem_abi_size)) {
		3016	.lt => {
		3017	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
		3018	else => break :result null,
		3019	1 => switch (src_elem_abi_size) {
		3020	else => break :result null,
		3021	2 => switch (dst_int_info.signedness) {
		3022	.signed => if (has_avx) .{ .vp_b, .ackssw } else .{ .p_b, .ackssw },
		3023	.unsigned => if (has_avx) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
		3024	},
		3025	},
		3026	2 => switch (src_elem_abi_size) {
		3027	else => break :result null,
		3028	4 => switch (dst_int_info.signedness) {
		3029	.signed => if (has_avx) .{ .vp_w, .ackssd } else .{ .p_w, .ackssd },
		3030	.unsigned => if (has_avx)
		3031	.{ .vp_w, .ackusd }
		3032	else if (self.hasFeature(.sse4_1))
		3033	.{ .p_w, .ackusd }
		3034	else
		3035	break :result null,
		3036	},
		3037	},
		3038	};
		3039
		3040	const dst_mcv: MCValue = if (src_mcv.isRegister() and
		3041	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
		3042	src_mcv
		3043	else if (has_avx and src_mcv.isRegister())
		3044	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		3045	else
		3046	try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv);
		3047	const dst_reg = dst_mcv.getReg().?;
		3048	const dst_alias = registerAlias(dst_reg, dst_abi_size);
		3049
		3050	if (has_avx) try self.asmRegisterRegisterRegister(
		3051	mir_tag,
		3052	dst_alias,
		3053	registerAlias(if (src_mcv.isRegister())
		3054	src_mcv.getReg().?
		3055	else
		3056	dst_reg, src_abi_size),
		3057	dst_alias,
		3058	) else try self.asmRegisterRegister(
		3059	mir_tag,
		3060	dst_alias,
		3061	dst_alias,
		3062	);
		3063	break :result dst_mcv;
		3064	},
		3065	.eq => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
		3066	break :result src_mcv
		3067	else {
		3068	const dst_mcv = try self.allocRegOrMem(inst, true);
		3069	try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
		3070	break :result dst_mcv;
		3071	},
		3072	.gt => if (self.hasFeature(.sse4_1)) {
		3073	const mir_tag: Mir.Inst.FixedTag = .{ switch (dst_elem_abi_size) {
		3074	else => break :result null,
		3075	2 => if (has_avx) .vp_w else .p_w,
		3076	4 => if (has_avx) .vp_d else .p_d,
		3077	8 => if (has_avx) .vp_q else .p_q,
		3078	}, switch (src_elem_abi_size) {
		3079	else => break :result null,
		3080	1 => switch (extend) {
		3081	.signed => .movsxb,
		3082	.unsigned => .movzxb,
		3083	},
		3084	2 => switch (extend) {
		3085	.signed => .movsxw,
		3086	.unsigned => .movzxw,
		3087	},
		3088	4 => switch (extend) {
		3089	.signed => .movsxd,
		3090	.unsigned => .movzxd,
		3091	},
		3092	} };
		3093
		3094	const dst_mcv: MCValue = if (src_mcv.isRegister() and
		3095	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
		3096	src_mcv
		3097	else
		3098	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) };
		3099	const dst_reg = dst_mcv.getReg().?;
		3100	const dst_alias = registerAlias(dst_reg, dst_abi_size);
		3101
		3102	if (src_mcv.isMemory()) try self.asmRegisterMemory(
		3103	mir_tag,
		3104	dst_alias,
		3105	try src_mcv.mem(self, self.memSize(src_ty)),
		3106	) else try self.asmRegisterRegister(
		3107	mir_tag,
		3108	dst_alias,
		3109	registerAlias(if (src_mcv.isRegister())
		3110	src_mcv.getReg().?
		3111	else
		3112	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
		3113	);
		3114	break :result dst_mcv;
		3115	} else {
		3116	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
		3117	else => break :result null,
		3118	2 => switch (src_elem_abi_size) {
		3119	else => break :result null,
		3120	1 => .{ .p_, .unpcklbw },
		3121	},
		3122	4 => switch (src_elem_abi_size) {
		3123	else => break :result null,
		3124	2 => .{ .p_, .unpcklwd },
		3125	},
		3126	8 => switch (src_elem_abi_size) {
		3127	else => break :result null,
		3128	2 => .{ .p_, .unpckldq },
		3129	},
		3130	};
		3131
		3132	const dst_mcv: MCValue = if (src_mcv.isRegister() and
		3133	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
		3134	src_mcv
		3135	else
		3136	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
		3137	const dst_reg = dst_mcv.getReg().?;
		3138
		3139	const ext_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
		3140	const ext_alias = registerAlias(ext_reg, src_abi_size);
		3141	const ext_lock = self.register_manager.lockRegAssumeUnused(ext_reg);
		3142	defer self.register_manager.unlockReg(ext_lock);
		3143
		3144	try self.asmRegisterRegister(.{ .p_, .xor }, ext_alias, ext_alias);
		3145	switch (extend) {
		3146	.signed => try self.asmRegisterRegister(
		3147	.{ switch (src_elem_abi_size) {
		3148	else => unreachable,
		3149	1 => .p_b,
		3150	2 => .p_w,
		3151	4 => .p_d,
		3152	}, .cmpgt },
		3153	ext_alias,
		3154	registerAlias(dst_reg, src_abi_size),
		3155	),
		3156	.unsigned => {},
		3157	}
		3158	try self.asmRegisterRegister(
		3159	mir_tag,
		3160	registerAlias(dst_reg, dst_abi_size),
		3161	registerAlias(ext_reg, dst_abi_size),
		3162	);
		3163	break :result dst_mcv;
		3164	},
		3165	}
		3166	@compileError("unreachable");
		3167	}
		3168
		3169	const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
		3170
2931	const src_storage_bits: u16 = switch (src_mcv) {	3171	const src_storage_bits: u16 = switch (src_mcv) {
2932	.register, .register_offset => 64,	3172	.register, .register_offset => 64,
2933	.register_pair => 128,	3173	.register_pair => 128,
...	@@ -2945,13 +3185,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2945,13 +3185,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2945	};	3185	};
2946		3186
2947	if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister())	3187	if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister())
2948	.{ .register = registerAlias(dst_mcv.getReg().?, abi_size) }	3188	.{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) }
2949	else	3189	else
2950	dst_mcv;	3190	dst_mcv;
2951		3191
2952	if (dst_mcv.isRegister()) {	3192	if (dst_mcv.isRegister()) {
2953	try self.truncateRegister(src_ty, dst_mcv.getReg().?);	3193	try self.truncateRegister(src_ty, dst_mcv.getReg().?);
2954	break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };	3194	break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
2955	}	3195	}
2956		3196
2957	const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;	3197	const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
...	@@ -2999,7 +3239,9 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -2999,7 +3239,9 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2999	);	3239	);
3000		3240
3001	break :result dst_mcv;	3241	break :result dst_mcv;
3002	};	3242	}) orelse return self.fail("TODO implement airIntCast from {} to {}", .{
		3243	src_ty.fmt(mod), dst_ty.fmt(mod),
		3244	});
3003	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	3245	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3004	}	3246	}
3005		3247
...	@@ -3022,7 +3264,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3022,7 +3264,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3022	src_mcv	3264	src_mcv
3023	else if (dst_abi_size <= 8)	3265	else if (dst_abi_size <= 8)
3024	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)	3266	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)
3025	else if (dst_abi_size <= 16) dst: {	3267	else if (dst_abi_size <= 16 and !dst_ty.isVector(mod)) dst: {
3026	const dst_regs =	3268	const dst_regs =
3027	try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);	3269	try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);
3028	const dst_mcv: MCValue = .{ .register_pair = dst_regs };	3270	const dst_mcv: MCValue = .{ .register_pair = dst_regs };
...	@@ -3032,26 +3274,29 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3032,26 +3274,29 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3032	try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});	3274	try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
3033	break :dst dst_mcv;	3275	break :dst dst_mcv;
3034	} else dst: {	3276	} else dst: {
3035	const dst_mcv = try self.allocRegOrMem(inst, true);	3277	const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true);
3036	try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});	3278	try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
3037	break :dst dst_mcv;	3279	break :dst dst_mcv;
3038	};	3280	};
3039		3281
3040	if (dst_ty.zigTypeTag(mod) == .Vector) {	3282	if (dst_ty.zigTypeTag(mod) == .Vector) {
3041	assert(src_ty.zigTypeTag(mod) == .Vector and dst_ty.vectorLen(mod) == src_ty.vectorLen(mod));	3283	assert(src_ty.zigTypeTag(mod) == .Vector and dst_ty.vectorLen(mod) == src_ty.vectorLen(mod));
3042	const dst_info = dst_ty.childType(mod).intInfo(mod);	3284	const dst_elem_ty = dst_ty.childType(mod);
3043	const src_info = src_ty.childType(mod).intInfo(mod);	3285	const dst_elem_abi_size: u32 = @intCast(dst_elem_ty.abiSize(mod));
3044	const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_info.bits) {	3286	const src_elem_ty = src_ty.childType(mod);
3045	8 => switch (src_info.bits) {	3287	const src_elem_abi_size: u32 = @intCast(src_elem_ty.abiSize(mod));
3046	16 => switch (dst_ty.vectorLen(mod)) {	3288
		3289	const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_elem_abi_size) {
		3290	1 => switch (src_elem_abi_size) {
		3291	2 => switch (dst_ty.vectorLen(mod)) {
3047	1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },	3292	1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
3048	9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null,	3293	9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null,
3049	else => null,	3294	else => null,
3050	},	3295	},
3051	else => null,	3296	else => null,
3052	},	3297	},
3053	16 => switch (src_info.bits) {	3298	2 => switch (src_elem_abi_size) {
3054	32 => switch (dst_ty.vectorLen(mod)) {	3299	4 => switch (dst_ty.vectorLen(mod)) {
3055	1...4 => if (self.hasFeature(.avx))	3300	1...4 => if (self.hasFeature(.avx))
3056	.{ .vp_w, .ackusd }	3301	.{ .vp_w, .ackusd }
3057	else if (self.hasFeature(.sse4_1))	3302	else if (self.hasFeature(.sse4_1))
...	@@ -3066,12 +3311,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3066,12 +3311,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3066	else => null,	3311	else => null,
3067	}) orelse return self.fail("TODO implement airTrunc for {}", .{dst_ty.fmt(mod)});	3312	}) orelse return self.fail("TODO implement airTrunc for {}", .{dst_ty.fmt(mod)});
3068		3313
3069	const elem_ty = src_ty.childType(mod);	3314	const dst_info = dst_elem_ty.intInfo(mod);
3070	const mask_val = try mod.intValue(elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));	3315	const src_info = src_elem_ty.intInfo(mod);
		3316
		3317	const mask_val = try mod.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
3071		3318
3072	const splat_ty = try mod.vectorType(.{	3319	const splat_ty = try mod.vectorType(.{
3073	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),	3320	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
3074	.child = elem_ty.ip_index,	3321	.child = src_elem_ty.ip_index,
3075	});	3322	});
3076	const splat_abi_size: u32 = @intCast(splat_ty.abiSize(mod));	3323	const splat_abi_size: u32 = @intCast(splat_ty.abiSize(mod));
3077		3324
...	@@ -3086,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -3086,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3086	else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },	3333	else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
3087	};	3334	};
3088		3335
3089	const dst_reg = registerAlias(dst_mcv.getReg().?, src_abi_size);	3336	const dst_reg = dst_mcv.getReg().?;
		3337	const dst_alias = registerAlias(dst_reg, src_abi_size);
3090	if (self.hasFeature(.avx)) {	3338	if (self.hasFeature(.avx)) {
3091	try self.asmRegisterRegisterMemory(	3339	try self.asmRegisterRegisterMemory(
3092	.{ .vp_, .@"and" },	3340	.{ .vp_, .@"and" },
3093	dst_reg,	3341	dst_alias,
3094	dst_reg,	3342	dst_alias,
3095	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),	3343	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3096	);	3344	);
3097	try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);	3345	if (src_abi_size > 16) {
		3346	const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
		3347	const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
		3348	defer self.register_manager.unlockReg(temp_lock);
		3349
		3350	try self.asmRegisterRegisterImmediate(
		3351	.{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
		3352	registerAlias(temp_reg, dst_abi_size),
		3353	dst_alias,
		3354	Immediate.u(1),
		3355	);
		3356	try self.asmRegisterRegisterRegister(
		3357	mir_tag,
		3358	registerAlias(dst_reg, dst_abi_size),
		3359	registerAlias(dst_reg, dst_abi_size),
		3360	registerAlias(temp_reg, dst_abi_size),
		3361	);
		3362	} else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias);
3098	} else {	3363	} else {
3099	try self.asmRegisterMemory(	3364	try self.asmRegisterMemory(
3100	.{ .p_, .@"and" },	3365	.{ .p_, .@"and" },
3101	dst_reg,	3366	dst_alias,
3102	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),	3367	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3103	);	3368	);
3104	try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);	3369	try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
3105	}	3370	}
3106	break :result dst_mcv;	3371	break :result dst_mcv;
3107	}	3372	}
...	@@ -4045,7 +4310,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4045,7 +4310,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4045	if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {	4310	if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
4046	const slow_inc = self.hasFeature(.slow_incdec);	4311	const slow_inc = self.hasFeature(.slow_incdec);
4047	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));	4312	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
4048	const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;	4313	const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable;
4049		4314
4050	try self.spillRegisters(&.{ .rax, .rcx, .rdx });	4315	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
4051	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });	4316	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
...	@@ -4534,7 +4799,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -4534,7 +4799,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
4534	switch (lhs_ty.zigTypeTag(mod)) {	4799	switch (lhs_ty.zigTypeTag(mod)) {
4535	.Int => {	4800	.Int => {
4536	try self.spillRegisters(&.{.rcx});	4801	try self.spillRegisters(&.{.rcx});
4537	try self.register_manager.getReg(.rcx, null);	4802	try self.register_manager.getKnownReg(.rcx, null);
4538	const lhs_mcv = try self.resolveInst(bin_op.lhs);	4803	const lhs_mcv = try self.resolveInst(bin_op.lhs);
4539	const rhs_mcv = try self.resolveInst(bin_op.rhs);	4804	const rhs_mcv = try self.resolveInst(bin_op.rhs);
4540		4805
...	@@ -6560,7 +6825,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)	...	@@ -6560,7 +6825,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
6560		6825
6561	const dst_mcv: MCValue = .{ .register = .st0 };	6826	const dst_mcv: MCValue = .{ .register = .st0 };
6562	if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))	6827	if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))
6563	try self.register_manager.getReg(.st0, inst);	6828	try self.register_manager.getKnownReg(.st0, inst);
6564		6829
6565	try self.genCopy(ty, dst_mcv, src_mcv, .{});	6830	try self.genCopy(ty, dst_mcv, src_mcv, .{});
6566	switch (tag) {	6831	switch (tag) {
...	@@ -6894,7 +7159,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -6894,7 +7159,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6894	},	7159	},
6895	else => {	7160	else => {
6896	const abi_size: u31 = @intCast(ty.abiSize(mod));	7161	const abi_size: u31 = @intCast(ty.abiSize(mod));
6897	const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;	7162	const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable;
6898		7163
6899	const tmp_regs =	7164	const tmp_regs =
6900	try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);	7165	try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
...	@@ -8181,7 +8446,7 @@ fn genShiftBinOpMir(	...	@@ -8181,7 +8446,7 @@ fn genShiftBinOpMir(
8181	try self.asmRegisterImmediate(	8446	try self.asmRegisterImmediate(
8182	.{ ._, .@"and" },	8447	.{ ._, .@"and" },
8183	.cl,	8448	.cl,
8184	Immediate.u(std.math.maxInt(u6)),	8449	Immediate.u(math.maxInt(u6)),
8185	);	8450	);
8186	try self.asmRegisterImmediate(	8451	try self.asmRegisterImmediate(
8187	.{ ._r, .sh },	8452	.{ ._r, .sh },
...	@@ -8218,7 +8483,7 @@ fn genShiftBinOpMir(	...	@@ -8218,7 +8483,7 @@ fn genShiftBinOpMir(
8218	try self.asmRegisterImmediate(	8483	try self.asmRegisterImmediate(
8219	.{ ._, .@"and" },	8484	.{ ._, .@"and" },
8220	.cl,	8485	.cl,
8221	Immediate.u(std.math.maxInt(u6)),	8486	Immediate.u(math.maxInt(u6)),
8222	);	8487	);
8223	try self.asmRegisterImmediate(	8488	try self.asmRegisterImmediate(
8224	.{ ._r, .sh },	8489	.{ ._r, .sh },
...	@@ -8283,7 +8548,7 @@ fn genShiftBinOpMir(	...	@@ -8283,7 +8548,7 @@ fn genShiftBinOpMir(
8283	}, .sh },	8548	}, .sh },
8284	temp_regs[2].to64(),	8549	temp_regs[2].to64(),
8285	temp_regs[3].to64(),	8550	temp_regs[3].to64(),
8286	Immediate.u(shift_imm & std.math.maxInt(u6)),	8551	Immediate.u(shift_imm & math.maxInt(u6)),
8287	),	8552	),
8288	else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {	8553	else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {
8289	._l => ._ld,	8554	._l => ._ld,
...	@@ -8338,7 +8603,7 @@ fn genShiftBinOpMir(	...	@@ -8338,7 +8603,7 @@ fn genShiftBinOpMir(
8338	.immediate => \|shift_imm\| try self.asmRegisterImmediate(	8603	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8339	tag,	8604	tag,
8340	temp_regs[2].to64(),	8605	temp_regs[2].to64(),
8341	Immediate.u(shift_imm & std.math.maxInt(u6)),	8606	Immediate.u(shift_imm & math.maxInt(u6)),
8342	),	8607	),
8343	else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),	8608	else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),
8344	}	8609	}
...	@@ -8794,7 +9059,7 @@ fn genShiftBinOp(	...	@@ -8794,7 +9059,7 @@ fn genShiftBinOp(
8794	lhs_ty.fmt(mod),	9059	lhs_ty.fmt(mod),
8795	});	9060	});
8796		9061
8797	try self.register_manager.getReg(.rcx, null);	9062	try self.register_manager.getKnownReg(.rcx, null);
8798	const rcx_lock = self.register_manager.lockReg(.rcx);	9063	const rcx_lock = self.register_manager.lockReg(.rcx);
8799	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);	9064	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
8800		9065
...	@@ -8933,7 +9198,7 @@ fn genMulDivBinOp(	...	@@ -8933,7 +9198,7 @@ fn genMulDivBinOp(
8933	switch (tag) {	9198	switch (tag) {
8934	.mul, .mul_wrap => {	9199	.mul, .mul_wrap => {
8935	const slow_inc = self.hasFeature(.slow_incdec);	9200	const slow_inc = self.hasFeature(.slow_incdec);
8936	const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable;	9201	const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable;
8937		9202
8938	try self.spillRegisters(&.{ .rax, .rcx, .rdx });	9203	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
8939	const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });	9204	const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });
...	@@ -9117,8 +9382,8 @@ fn genMulDivBinOp(	...	@@ -9117,8 +9382,8 @@ fn genMulDivBinOp(
9117	.rem => maybe_inst,	9382	.rem => maybe_inst,
9118	else => null,	9383	else => null,
9119	};	9384	};
9120	try self.register_manager.getReg(.rax, track_inst_rax);	9385	try self.register_manager.getKnownReg(.rax, track_inst_rax);
9121	try self.register_manager.getReg(.rdx, track_inst_rdx);	9386	try self.register_manager.getKnownReg(.rdx, track_inst_rdx);
9122		9387
9123	try self.genIntMulDivOpMir(switch (signedness) {	9388	try self.genIntMulDivOpMir(switch (signedness) {
9124	.signed => switch (tag) {	9389	.signed => switch (tag) {
...	@@ -9158,8 +9423,11 @@ fn genMulDivBinOp(	...	@@ -9158,8 +9423,11 @@ fn genMulDivBinOp(
9158	},	9423	},
9159		9424
9160	.mod => {	9425	.mod => {
9161	try self.register_manager.getReg(.rax, null);	9426	try self.register_manager.getKnownReg(.rax, null);
9162	try self.register_manager.getReg(.rdx, if (signedness == .unsigned) maybe_inst else null);	9427	try self.register_manager.getKnownReg(
		9428	.rdx,
		9429	if (signedness == .unsigned) maybe_inst else null,
		9430	);
9163		9431
9164	switch (signedness) {	9432	switch (signedness) {
9165	.signed => {	9433	.signed => {
...	@@ -9200,8 +9468,11 @@ fn genMulDivBinOp(	...	@@ -9200,8 +9468,11 @@ fn genMulDivBinOp(
9200	},	9468	},
9201		9469
9202	.div_floor => {	9470	.div_floor => {
9203	try self.register_manager.getReg(.rax, if (signedness == .unsigned) maybe_inst else null);	9471	try self.register_manager.getKnownReg(
9204	try self.register_manager.getReg(.rdx, null);	9472	.rax,
		9473	if (signedness == .unsigned) maybe_inst else null,
		9474	);
		9475	try self.register_manager.getKnownReg(.rdx, null);
9205		9476
9206	const lhs_lock: ?RegisterLock = switch (lhs_mcv) {	9477	const lhs_lock: ?RegisterLock = switch (lhs_mcv) {
9207	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),	9478	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
...	@@ -9445,7 +9716,7 @@ fn genBinOp(	...	@@ -9445,7 +9716,7 @@ fn genBinOp(
9445	.rem, .mod => unreachable,	9716	.rem, .mod => unreachable,
9446	.max, .min => if (lhs_ty.scalarType(mod).isRuntimeFloat()) registerAlias(	9717	.max, .min => if (lhs_ty.scalarType(mod).isRuntimeFloat()) registerAlias(
9447	if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {	9718	if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {
9448	try self.register_manager.getReg(.xmm0, null);	9719	try self.register_manager.getKnownReg(.xmm0, null);
9449	break :mask .xmm0;	9720	break :mask .xmm0;
9450	} else try self.register_manager.allocReg(null, abi.RegisterClass.sse),	9721	} else try self.register_manager.allocReg(null, abi.RegisterClass.sse),
9451	abi_size,	9722	abi_size,
...	@@ -10820,96 +11091,35 @@ fn genBinOp(	...	@@ -10820,96 +11091,35 @@ fn genBinOp(
10820	lhs_copy_reg.?,	11091	lhs_copy_reg.?,
10821	mask_reg,	11092	mask_reg,
10822	) else {	11093	) else {
10823	try self.asmRegisterRegister(	11094	const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(mod)) {
10824	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {	11095	.Float => switch (lhs_ty.floatBits(self.target.*)) {
10825	.Float => switch (lhs_ty.floatBits(self.target.*)) {	11096	32 => ._ps,
10826	32 => .{ ._ps, .@"and" },	11097	64 => ._pd,
10827	64 => .{ ._pd, .@"and" },	11098	16, 80, 128 => null,
10828	16, 80, 128 => null,
10829	else => unreachable,
10830	},
10831	.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10832	.Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10833	32 => switch (lhs_ty.vectorLen(mod)) {
10834	1...4 => .{ ._ps, .@"and" },
10835	else => null,
10836	},
10837	64 => switch (lhs_ty.vectorLen(mod)) {
10838	1...2 => .{ ._pd, .@"and" },
10839	else => null,
10840	},
10841	16, 80, 128 => null,
10842	else => unreachable,
10843	},
10844	else => unreachable,
10845	},
10846	else => unreachable,	11099	else => unreachable,
10847	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{	11100	},
10848	@tagName(air_tag), lhs_ty.fmt(mod),	11101	.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10849	}),	11102	.Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10850	dst_reg,	11103	32 => switch (lhs_ty.vectorLen(mod)) {
10851	mask_reg,	11104	1...4 => ._ps,
10852	);	11105	else => null,
10853	try self.asmRegisterRegister(
10854	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
10855	.Float => switch (lhs_ty.floatBits(self.target.*)) {
10856	32 => .{ ._ps, .andn },
10857	64 => .{ ._pd, .andn },
10858	16, 80, 128 => null,
10859	else => unreachable,
10860	},
10861	.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10862	.Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10863	32 => switch (lhs_ty.vectorLen(mod)) {
10864	1...4 => .{ ._ps, .andn },
10865	else => null,
10866	},
10867	64 => switch (lhs_ty.vectorLen(mod)) {
10868	1...2 => .{ ._pd, .andn },
10869	else => null,
10870	},
10871	16, 80, 128 => null,
10872	else => unreachable,
10873	},	11106	},
10874	else => unreachable,	11107	64 => switch (lhs_ty.vectorLen(mod)) {
10875	},	11108	1...2 => ._pd,
10876	else => unreachable,	11109	else => null,
10877	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
10878	@tagName(air_tag), lhs_ty.fmt(mod),
10879	}),
10880	mask_reg,
10881	lhs_copy_reg.?,
10882	);
10883	try self.asmRegisterRegister(
10884	@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
10885	.Float => switch (lhs_ty.floatBits(self.target.*)) {
10886	32 => .{ ._ps, .@"or" },
10887	64 => .{ ._pd, .@"or" },
10888	16, 80, 128 => null,
10889	else => unreachable,
10890	},
10891	.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10892	.Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10893	32 => switch (lhs_ty.vectorLen(mod)) {
10894	1...4 => .{ ._ps, .@"or" },
10895	else => null,
10896	},
10897	64 => switch (lhs_ty.vectorLen(mod)) {
10898	1...2 => .{ ._pd, .@"or" },
10899	else => null,
10900	},
10901	16, 80, 128 => null,
10902	else => unreachable,
10903	},	11110	},
		11111	16, 80, 128 => null,
10904	else => unreachable,	11112	else => unreachable,
10905	},	11113	},
10906	else => unreachable,	11114	else => unreachable,
10907	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{	11115	},
10908	@tagName(air_tag), lhs_ty.fmt(mod),	11116	else => unreachable,
10909	}),	11117	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
10910	dst_reg,	11118	@tagName(air_tag), lhs_ty.fmt(mod),
10911	mask_reg,	11119	});
10912	);	11120	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg);
		11121	try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?);
		11122	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg);
10913	}	11123	}
10914	},	11124	},
10915	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {	11125	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
...	@@ -12192,48 +12402,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -12192,48 +12402,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
12192	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	12402	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12193	const mod = self.bin_file.comp.module.?;	12403	const mod = self.bin_file.comp.module.?;
12194	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	12404	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
12195	const ty = self.typeOf(bin_op.lhs);	12405	var ty = self.typeOf(bin_op.lhs);
		12406	var null_compare: ?Mir.Inst.Index = null;
12196		12407
12197	const result: Condition = result: {	12408	const result: Condition = result: {
12198	switch (ty.zigTypeTag(mod)) {
12199	.Float => {
12200	const float_bits = ty.floatBits(self.target.*);
12201	if (switch (float_bits) {
12202	16 => !self.hasFeature(.f16c),
12203	32, 64 => false,
12204	80, 128 => true,
12205	else => unreachable,
12206	}) {
12207	var callee_buf: ["__???f2".len]u8 = undefined;
12208	const ret = try self.genCall(.{ .lib = .{
12209	.return_type = .i32_type,
12210	.param_types = &.{ ty.toIntern(), ty.toIntern() },
12211	.callee = std.fmt.bufPrint(&callee_buf, "__{s}{c}f2", .{
12212	switch (op) {
12213	.eq => "eq",
12214	.neq => "ne",
12215	.lt => "lt",
12216	.lte => "le",
12217	.gt => "gt",
12218	.gte => "ge",
12219	},
12220	floatCompilerRtAbiName(float_bits),
12221	}) catch unreachable,
12222	} }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });
12223	try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);
12224	break :result switch (op) {
12225	.eq => .e,
12226	.neq => .ne,
12227	.lt => .l,
12228	.lte => .le,
12229	.gt => .g,
12230	.gte => .ge,
12231	};
12232	}
12233	},
12234	else => {},
12235	}
12236
12237	try self.spillEflagsIfOccupied();	12409	try self.spillEflagsIfOccupied();
12238		12410
12239	const lhs_mcv = try self.resolveInst(bin_op.lhs);	12411	const lhs_mcv = try self.resolveInst(bin_op.lhs);
...	@@ -12260,6 +12432,103 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12260,6 +12432,103 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12260	};	12432	};
12261	defer for (rhs_locks) \|rhs_lock\| if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);	12433	defer for (rhs_locks) \|rhs_lock\| if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
12262		12434
		12435	switch (ty.zigTypeTag(mod)) {
		12436	.Float => {
		12437	const float_bits = ty.floatBits(self.target.*);
		12438	if (switch (float_bits) {
		12439	16 => !self.hasFeature(.f16c),
		12440	32, 64 => false,
		12441	80, 128 => true,
		12442	else => unreachable,
		12443	}) {
		12444	var callee_buf: ["__???f2".len]u8 = undefined;
		12445	const ret = try self.genCall(.{ .lib = .{
		12446	.return_type = .i32_type,
		12447	.param_types = &.{ ty.toIntern(), ty.toIntern() },
		12448	.callee = std.fmt.bufPrint(&callee_buf, "__{s}{c}f2", .{
		12449	switch (op) {
		12450	.eq => "eq",
		12451	.neq => "ne",
		12452	.lt => "lt",
		12453	.lte => "le",
		12454	.gt => "gt",
		12455	.gte => "ge",
		12456	},
		12457	floatCompilerRtAbiName(float_bits),
		12458	}) catch unreachable,
		12459	} }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });
		12460	try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);
		12461	break :result switch (op) {
		12462	.eq => .e,
		12463	.neq => .ne,
		12464	.lt => .l,
		12465	.lte => .le,
		12466	.gt => .g,
		12467	.gte => .ge,
		12468	};
		12469	}
		12470	},
		12471	.Optional => if (!ty.optionalReprIsPayload(mod)) {
		12472	const opt_ty = ty;
		12473	const opt_abi_size: u31 = @intCast(opt_ty.abiSize(mod));
		12474	ty = opt_ty.optionalChild(mod);
		12475	const payload_abi_size: u31 = @intCast(ty.abiSize(mod));
		12476
		12477	const temp_lhs_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
		12478	const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg);
		12479	defer self.register_manager.unlockReg(temp_lhs_lock);
		12480
		12481	if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
		12482	.{ ._, .mov },
		12483	temp_lhs_reg.to8(),
		12484	try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte),
		12485	) else {
		12486	try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
		12487	try self.asmRegisterImmediate(
		12488	.{ ._r, .sh },
		12489	registerAlias(temp_lhs_reg, opt_abi_size),
		12490	Immediate.u(payload_abi_size * 8),
		12491	);
		12492	}
		12493
		12494	const payload_compare = payload_compare: {
		12495	if (rhs_mcv.isMemory()) {
		12496	const rhs_mem =
		12497	try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte);
		12498	try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
		12499	const payload_compare = try self.asmJccReloc(.nz, undefined);
		12500	try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
		12501	break :payload_compare payload_compare;
		12502	}
		12503
		12504	const temp_rhs_reg = try self.copyToTmpRegister(opt_ty, rhs_mcv);
		12505	const temp_rhs_lock = self.register_manager.lockRegAssumeUnused(temp_rhs_reg);
		12506	defer self.register_manager.unlockReg(temp_rhs_lock);
		12507
		12508	try self.asmRegisterImmediate(
		12509	.{ ._r, .sh },
		12510	registerAlias(temp_rhs_reg, opt_abi_size),
		12511	Immediate.u(payload_abi_size * 8),
		12512	);
		12513	try self.asmRegisterRegister(
		12514	.{ ._, .@"test" },
		12515	temp_lhs_reg.to8(),
		12516	temp_rhs_reg.to8(),
		12517	);
		12518	const payload_compare = try self.asmJccReloc(.nz, undefined);
		12519	try self.asmRegisterRegister(
		12520	.{ ._, .cmp },
		12521	temp_lhs_reg.to8(),
		12522	temp_rhs_reg.to8(),
		12523	);
		12524	break :payload_compare payload_compare;
		12525	};
		12526	null_compare = try self.asmJmpReloc(undefined);
		12527	self.performReloc(payload_compare);
		12528	},
		12529	else => {},
		12530	}
		12531
12263	switch (ty.zigTypeTag(mod)) {	12532	switch (ty.zigTypeTag(mod)) {
12264	else => {	12533	else => {
12265	const abi_size: u16 = @intCast(ty.abiSize(mod));	12534	const abi_size: u16 = @intCast(ty.abiSize(mod));
...	@@ -12571,6 +12840,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {	...	@@ -12571,6 +12840,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12571	}	12840	}
12572	};	12841	};
12573		12842
		12843	if (null_compare) \|reloc\| self.performReloc(reloc);
12574	self.eflags_inst = inst;	12844	self.eflags_inst = inst;
12575	return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });	12845	return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
12576	}	12846	}
...	@@ -13521,6 +13791,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13521,6 +13791,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
13521	} else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: {	13791	} else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: {
13522	const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable;	13792	const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable;
13523	if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint});	13793	if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint});
		13794	try self.genCopy(ty, args.items[index], input_mcv, .{});
13524	break :arg args.items[index];	13795	break :arg args.items[index];
13525	} else return self.fail("invalid constraint: '{s}'", .{constraint});	13796	} else return self.fail("invalid constraint: '{s}'", .{constraint});
13526	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {	13797	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
...	@@ -13619,25 +13890,26 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -13619,25 +13890,26 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
13619	label_gop.value_ptr.target = @intCast(self.mir_instructions.len);	13890	label_gop.value_ptr.target = @intCast(self.mir_instructions.len);
13620	} else continue;	13891	} else continue;
13621		13892
13622	var mnem_size: ?Memory.Size = null;	13893	var mnem_size: ?Memory.Size = if (mem.endsWith(u8, mnem_str, "b"))
13623	const mnem_tag = mnem: {	13894	.byte
13624	mnem_size = if (mem.endsWith(u8, mnem_str, "b"))	13895	else if (mem.endsWith(u8, mnem_str, "w"))
13625	.byte	13896	.word
13626	else if (mem.endsWith(u8, mnem_str, "w"))	13897	else if (mem.endsWith(u8, mnem_str, "l"))
13627	.word	13898	.dword
13628	else if (mem.endsWith(u8, mnem_str, "l"))	13899	else if (mem.endsWith(u8, mnem_str, "q") and
13629	.dword	13900	(std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq")))
13630	else if (mem.endsWith(u8, mnem_str, "q"))	13901	.qword
13631	.qword	13902	else if (mem.endsWith(u8, mnem_str, "t"))
13632	else if (mem.endsWith(u8, mnem_str, "t"))	13903	.tbyte
13633	.tbyte	13904	else
13634	else	13905	null;
13635	break :mnem null;	13906	const mnem_tag = while (true) break std.meta.stringToEnum(
13636	break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str[0 .. mnem_str.len - 1]);	13907	Instruction.Mnemonic,
13637	} orelse mnem: {	13908	mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],
		13909	) orelse if (mnem_size) \|_\| {
13638	mnem_size = null;	13910	mnem_size = null;
13639	break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str);	13911	continue;
13640	} orelse return self.fail("invalid mnemonic: '{s}'", .{mnem_str});	13912	} else return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
13641	if (@as(?Memory.Size, switch (mnem_tag) {	13913	if (@as(?Memory.Size, switch (mnem_tag) {
13642	.clflush => .byte,	13914	.clflush => .byte,
13643	.fldenv, .fnstenv, .fstenv => .none,	13915	.fldenv, .fnstenv, .fstenv => .none,
...	@@ -14135,30 +14407,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14135,30 +14407,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14135	else => {},	14407	else => {},
14136	},	14408	},
14137	.Int => switch (ty.childType(mod).intInfo(mod).bits) {	14409	.Int => switch (ty.childType(mod).intInfo(mod).bits) {
14138	8 => switch (ty.vectorLen(mod)) {	14410	1...8 => switch (ty.vectorLen(mod)) {
14139	1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{	14411	1...16 => return .{ .move = if (self.hasFeature(.avx))
14140	.insert = .{ .vp_b, .insr },
14141	.extract = .{ .vp_b, .extr },
14142	} } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
14143	.insert = .{ .p_b, .insr },
14144	.extract = .{ .p_b, .extr },
14145	} },
14146	2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
14147	.insert = .{ .vp_w, .insr },
14148	.extract = .{ .vp_w, .extr },
14149	} } else .{ .insert_extract = .{
14150	.insert = .{ .p_w, .insr },
14151	.extract = .{ .p_w, .extr },
14152	} },
14153	3...4 => return .{ .move = if (self.hasFeature(.avx))
14154	.{ .v_d, .mov }
14155	else
14156	.{ ._d, .mov } },
14157	5...8 => return .{ .move = if (self.hasFeature(.avx))
14158	.{ .v_q, .mov }
14159	else
14160	.{ ._q, .mov } },
14161	9...16 => return .{ .move = if (self.hasFeature(.avx))
14162	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	14412	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14163	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	14413	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14164	17...32 => if (self.hasFeature(.avx))	14414	17...32 => if (self.hasFeature(.avx))
...	@@ -14168,23 +14418,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14168,23 +14418,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14168	.{ .v_, .movdqu } },	14418	.{ .v_, .movdqu } },
14169	else => {},	14419	else => {},
14170	},	14420	},
14171	16 => switch (ty.vectorLen(mod)) {	14421	9...16 => switch (ty.vectorLen(mod)) {
14172	1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{	14422	1...8 => return .{ .move = if (self.hasFeature(.avx))
14173	.insert = .{ .vp_w, .insr },
14174	.extract = .{ .vp_w, .extr },
14175	} } else .{ .insert_extract = .{
14176	.insert = .{ .p_w, .insr },
14177	.extract = .{ .p_w, .extr },
14178	} },
14179	2 => return .{ .move = if (self.hasFeature(.avx))
14180	.{ .v_d, .mov }
14181	else
14182	.{ ._d, .mov } },
14183	3...4 => return .{ .move = if (self.hasFeature(.avx))
14184	.{ .v_q, .mov }
14185	else
14186	.{ ._q, .mov } },
14187	5...8 => return .{ .move = if (self.hasFeature(.avx))
14188	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	14423	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14189	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	14424	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14190	9...16 => if (self.hasFeature(.avx))	14425	9...16 => if (self.hasFeature(.avx))
...	@@ -14194,16 +14429,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14194,16 +14429,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14194	.{ .v_, .movdqu } },	14429	.{ .v_, .movdqu } },
14195	else => {},	14430	else => {},
14196	},	14431	},
14197	32 => switch (ty.vectorLen(mod)) {	14432	17...32 => switch (ty.vectorLen(mod)) {
14198	1 => return .{ .move = if (self.hasFeature(.avx))	14433	1...4 => return .{ .move = if (self.hasFeature(.avx))
14199	.{ .v_d, .mov }
14200	else
14201	.{ ._d, .mov } },
14202	2 => return .{ .move = if (self.hasFeature(.avx))
14203	.{ .v_q, .mov }
14204	else
14205	.{ ._q, .mov } },
14206	3...4 => return .{ .move = if (self.hasFeature(.avx))
14207	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	14434	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14208	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	14435	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14209	5...8 => if (self.hasFeature(.avx))	14436	5...8 => if (self.hasFeature(.avx))
...	@@ -14213,12 +14440,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14213,12 +14440,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14213	.{ .v_, .movdqu } },	14440	.{ .v_, .movdqu } },
14214	else => {},	14441	else => {},
14215	},	14442	},
14216	64 => switch (ty.vectorLen(mod)) {	14443	33...64 => switch (ty.vectorLen(mod)) {
14217	1 => return .{ .move = if (self.hasFeature(.avx))	14444	1...2 => return .{ .move = if (self.hasFeature(.avx))
14218	.{ .v_q, .mov }
14219	else
14220	.{ ._q, .mov } },
14221	2 => return .{ .move = if (self.hasFeature(.avx))
14222	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	14445	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14223	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	14446	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14224	3...4 => if (self.hasFeature(.avx))	14447	3...4 => if (self.hasFeature(.avx))
...	@@ -14228,7 +14451,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14228,7 +14451,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14228	.{ .v_, .movdqu } },	14451	.{ .v_, .movdqu } },
14229	else => {},	14452	else => {},
14230	},	14453	},
14231	128 => switch (ty.vectorLen(mod)) {	14454	65...128 => switch (ty.vectorLen(mod)) {
14232	1 => return .{ .move = if (self.hasFeature(.avx))	14455	1 => return .{ .move = if (self.hasFeature(.avx))
14233	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	14456	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14234	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	14457	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
...	@@ -14239,7 +14462,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14239,7 +14462,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14239	.{ .v_, .movdqu } },	14462	.{ .v_, .movdqu } },
14240	else => {},	14463	else => {},
14241	},	14464	},
14242	256 => switch (ty.vectorLen(mod)) {	14465	129...256 => switch (ty.vectorLen(mod)) {
14243	1 => if (self.hasFeature(.avx))	14466	1 => if (self.hasFeature(.avx))
14244	return .{ .move = if (aligned)	14467	return .{ .move = if (aligned)
14245	.{ .v_, .movdqa }	14468	.{ .v_, .movdqa }
...	@@ -14251,11 +14474,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14251,11 +14474,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14251	},	14474	},
14252	.Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod))	14475	.Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod))
14253	switch (ty.vectorLen(mod)) {	14476	switch (ty.vectorLen(mod)) {
14254	1 => return .{ .move = if (self.hasFeature(.avx))	14477	1...2 => return .{ .move = if (self.hasFeature(.avx))
14255	.{ .v_q, .mov }
14256	else
14257	.{ ._q, .mov } },
14258	2 => return .{ .move = if (self.hasFeature(.avx))
14259	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	14478	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14260	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	14479	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14261	3...4 => if (self.hasFeature(.avx))	14480	3...4 => if (self.hasFeature(.avx))
...	@@ -14269,22 +14488,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14269,22 +14488,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14269	unreachable,	14488	unreachable,
14270	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {	14489	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
14271	16 => switch (ty.vectorLen(mod)) {	14490	16 => switch (ty.vectorLen(mod)) {
14272	1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{	14491	1...8 => return .{ .move = if (self.hasFeature(.avx))
14273	.insert = .{ .vp_w, .insr },
14274	.extract = .{ .vp_w, .extr },
14275	} } else .{ .insert_extract = .{
14276	.insert = .{ .p_w, .insr },
14277	.extract = .{ .p_w, .extr },
14278	} },
14279	2 => return .{ .move = if (self.hasFeature(.avx))
14280	.{ .v_d, .mov }
14281	else
14282	.{ ._d, .mov } },
14283	3...4 => return .{ .move = if (self.hasFeature(.avx))
14284	.{ .v_q, .mov }
14285	else
14286	.{ ._q, .mov } },
14287	5...8 => return .{ .move = if (self.hasFeature(.avx))
14288	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }	14492	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14289	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },	14493	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14290	9...16 => if (self.hasFeature(.avx))	14494	9...16 => if (self.hasFeature(.avx))
...	@@ -14295,15 +14499,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14295,15 +14499,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14295	else => {},	14499	else => {},
14296	},	14500	},
14297	32 => switch (ty.vectorLen(mod)) {	14501	32 => switch (ty.vectorLen(mod)) {
14298	1 => return .{ .move = if (self.hasFeature(.avx))	14502	1...4 => return .{ .move = if (self.hasFeature(.avx))
14299	.{ .v_ss, .mov }
14300	else
14301	.{ ._ss, .mov } },
14302	2 => return .{ .move = if (self.hasFeature(.avx))
14303	.{ .v_sd, .mov }
14304	else
14305	.{ ._sd, .mov } },
14306	3...4 => return .{ .move = if (self.hasFeature(.avx))
14307	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }	14503	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
14308	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },	14504	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
14309	5...8 => if (self.hasFeature(.avx))	14505	5...8 => if (self.hasFeature(.avx))
...	@@ -14314,11 +14510,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo	...	@@ -14314,11 +14510,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14314	else => {},	14510	else => {},
14315	},	14511	},
14316	64 => switch (ty.vectorLen(mod)) {	14512	64 => switch (ty.vectorLen(mod)) {
14317	1 => return .{ .move = if (self.hasFeature(.avx))	14513	1...2 => return .{ .move = if (self.hasFeature(.avx))
14318	.{ .v_sd, .mov }
14319	else
14320	.{ ._sd, .mov } },
14321	2 => return .{ .move = if (self.hasFeature(.avx))
14322	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }	14514	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
14323	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },	14515	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
14324	3...4 => if (self.hasFeature(.avx))	14516	3...4 => if (self.hasFeature(.avx))
...	@@ -14633,7 +14825,7 @@ fn genSetReg(	...	@@ -14633,7 +14825,7 @@ fn genSetReg(
14633	ty,	14825	ty,
14634	dst_reg.class(),	14826	dst_reg.class(),
14635	self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(	14827	self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(
14636	std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),	14828	math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
14637	)),	14829	)),
14638	),	14830	),
14639	.lea_frame => .{ .move = .{ ._, .lea } },	14831	.lea_frame => .{ .move = .{ ._, .lea } },
...	@@ -16296,7 +16488,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16296,7 +16488,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16296	},	16488	},
16297	65...128 => switch (vector_len) {	16489	65...128 => switch (vector_len) {
16298	else => null,	16490	else => null,
16299	1...2 => .{ .vp_i128, .broadcast },	16491	1...2 => .{ .v_i128, .broadcast },
16300	},	16492	},
16301	}) orelse break :avx2;	16493	}) orelse break :avx2;
16302		16494
...	@@ -16310,7 +16502,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16310,7 +16502,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16310	registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),	16502	registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
16311	try src_mcv.mem(self, self.memSize(scalar_ty)),	16503	try src_mcv.mem(self, self.memSize(scalar_ty)),
16312	) else {	16504	) else {
16313	if (mir_tag[0] == .vp_i128) break :avx2;	16505	if (mir_tag[0] == .v_i128) break :avx2;
16314	try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});	16506	try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
16315	try self.asmRegisterRegister(	16507	try self.asmRegisterRegister(
16316	mir_tag,	16508	mir_tag,
...	@@ -16352,7 +16544,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16352,7 +16544,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16352	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },	16544	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
16353	dst_alias,	16545	dst_alias,
16354	dst_alias,	16546	dst_alias,
16355	Immediate.u(0),	16547	Immediate.u(0b00_00_00_00),
16356	);	16548	);
16357	if (switch (scalar_bits) {	16549	if (switch (scalar_bits) {
16358	1...8 => vector_len > 4,	16550	1...8 => vector_len > 4,
...	@@ -16563,18 +16755,1158 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16563,18 +16755,1158 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16563	}	16755	}
16564		16756
16565	fn airSelect(self: *Self, inst: Air.Inst.Index) !void {	16757	fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
		16758	const mod = self.bin_file.comp.module.?;
16566	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;	16759	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
16567	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;	16760	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
16568	_ = extra;	16761	const ty = self.typeOfIndex(inst);
16569	return self.fail("TODO implement airSelect for x86_64", .{});	16762	const vec_len = ty.vectorLen(mod);
16570	//return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });	16763	const elem_ty = ty.childType(mod);
		16764	const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
		16765	const abi_size: u32 = @intCast(ty.abiSize(mod));
		16766	const pred_ty = self.typeOf(pl_op.operand);
		16767
		16768	const result = result: {
		16769	const has_blend = self.hasFeature(.sse4_1);
		16770	const has_avx = self.hasFeature(.avx);
		16771	const need_xmm0 = has_blend and !has_avx;
		16772	const pred_mcv = try self.resolveInst(pl_op.operand);
		16773	const mask_reg = mask: {
		16774	switch (pred_mcv) {
		16775	.register => \|pred_reg\| switch (pred_reg.class()) {
		16776	.general_purpose => {},
		16777	.sse => if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) {
		16778	try self.register_manager.getKnownReg(.xmm0, null);
		16779	try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{});
		16780	break :mask .xmm0;
		16781	} else break :mask if (has_blend)
		16782	pred_reg
		16783	else
		16784	try self.copyToTmpRegister(pred_ty, pred_mcv),
		16785	else => unreachable,
		16786	},
		16787	else => {},
		16788	}
		16789	const mask_reg: Register = if (need_xmm0) mask_reg: {
		16790	try self.register_manager.getKnownReg(.xmm0, null);
		16791	break :mask_reg .xmm0;
		16792	} else try self.register_manager.allocReg(null, abi.RegisterClass.sse);
		16793	const mask_alias = registerAlias(mask_reg, abi_size);
		16794	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
		16795	defer self.register_manager.unlockReg(mask_lock);
		16796
		16797	const pred_fits_in_elem = vec_len <= elem_abi_size;
		16798	if (self.hasFeature(.avx2) and abi_size <= 32) {
		16799	if (pred_mcv.isRegister()) broadcast: {
		16800	try self.asmRegisterRegister(
		16801	.{ .v_d, .mov },
		16802	mask_reg.to128(),
		16803	pred_mcv.getReg().?.to32(),
		16804	);
		16805	if (pred_fits_in_elem and vec_len > 1) try self.asmRegisterRegister(
		16806	.{ switch (elem_abi_size) {
		16807	1 => .vp_b,
		16808	2 => .vp_w,
		16809	3...4 => .vp_d,
		16810	5...8 => .vp_q,
		16811	9...16 => {
		16812	try self.asmRegisterRegisterRegisterImmediate(
		16813	.{ .v_f128, .insert },
		16814	mask_alias,
		16815	mask_alias,
		16816	mask_reg.to128(),
		16817	Immediate.u(1),
		16818	);
		16819	break :broadcast;
		16820	},
		16821	17...32 => break :broadcast,
		16822	else => unreachable,
		16823	}, .broadcast },
		16824	mask_alias,
		16825	mask_reg.to128(),
		16826	);
		16827	} else try self.asmRegisterMemory(
		16828	.{ switch (vec_len) {
		16829	1...8 => .vp_b,
		16830	9...16 => .vp_w,
		16831	17...32 => .vp_d,
		16832	else => unreachable,
		16833	}, .broadcast },
		16834	mask_alias,
		16835	if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{
		16836	.base = .{ .reg = (try self.copyToTmpRegister(
		16837	Type.usize,
		16838	pred_mcv.address(),
		16839	)).to64() },
		16840	.mod = .{ .rm = .{ .size = .byte } },
		16841	},
		16842	);
		16843	} else if (abi_size <= 16) broadcast: {
		16844	try self.asmRegisterRegister(
		16845	.{ if (has_avx) .v_d else ._d, .mov },
		16846	mask_alias,
		16847	(if (pred_mcv.isRegister())
		16848	pred_mcv.getReg().?
		16849	else
		16850	try self.copyToTmpRegister(pred_ty, pred_mcv.address())).to32(),
		16851	);
		16852	if (!pred_fits_in_elem or vec_len == 1) break :broadcast;
		16853	if (elem_abi_size <= 1) {
		16854	if (has_avx) try self.asmRegisterRegisterRegister(
		16855	.{ .vp_, .unpcklbw },
		16856	mask_alias,
		16857	mask_alias,
		16858	mask_alias,
		16859	) else try self.asmRegisterRegister(
		16860	.{ .p_, .unpcklbw },
		16861	mask_alias,
		16862	mask_alias,
		16863	);
		16864	if (abi_size <= 2) break :broadcast;
		16865	}
		16866	if (elem_abi_size <= 2) {
		16867	try self.asmRegisterRegisterImmediate(
		16868	.{ if (has_avx) .vp_w else .p_w, .shufl },
		16869	mask_alias,
		16870	mask_alias,
		16871	Immediate.u(0b00_00_00_00),
		16872	);
		16873	if (abi_size <= 8) break :broadcast;
		16874	}
		16875	try self.asmRegisterRegisterImmediate(
		16876	.{ if (has_avx) .vp_d else .p_d, .shuf },
		16877	mask_alias,
		16878	mask_alias,
		16879	Immediate.u(switch (elem_abi_size) {
		16880	1...2, 5...8 => 0b01_00_01_00,
		16881	3...4 => 0b00_00_00_00,
		16882	else => unreachable,
		16883	}),
		16884	);
		16885	} else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
		16886	const elem_bits: u16 = @intCast(elem_abi_size * 8);
		16887	const mask_elem_ty = try mod.intType(.unsigned, elem_bits);
		16888	const mask_ty = try mod.vectorType(.{ .len = vec_len, .child = mask_elem_ty.toIntern() });
		16889	if (!pred_fits_in_elem) if (self.hasFeature(.ssse3)) {
		16890	var mask_elems: [32]InternPool.Index = undefined;
		16891	for (mask_elems[0..vec_len], 0..) \|elem, bit\| elem. = try mod.intern(.{ .int = .{
		16892	.ty = mask_elem_ty.toIntern(),
		16893	.storage = .{ .u64 = bit / elem_bits },
		16894	} });
		16895	const mask_mcv = try self.genTypedValue(.{
		16896	.ty = mask_ty,
		16897	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
		16898	.ty = mask_ty.toIntern(),
		16899	.storage = .{ .elems = mask_elems[0..vec_len] },
		16900	} })),
		16901	});
		16902	const mask_mem: Memory = .{
		16903	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
		16904	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
		16905	};
		16906	if (has_avx) try self.asmRegisterRegisterMemory(
		16907	.{ .vp_b, .shuf },
		16908	mask_alias,
		16909	mask_alias,
		16910	mask_mem,
		16911	) else try self.asmRegisterMemory(
		16912	.{ .p_b, .shuf },
		16913	mask_alias,
		16914	mask_mem,
		16915	);
		16916	} else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
		16917	{
		16918	var mask_elems: [32]InternPool.Index = undefined;
		16919	for (mask_elems[0..vec_len], 0..) \|elem, bit\| elem. = try mod.intern(.{ .int = .{
		16920	.ty = mask_elem_ty.toIntern(),
		16921	.storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },
		16922	} });
		16923	const mask_mcv = try self.genTypedValue(.{
		16924	.ty = mask_ty,
		16925	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
		16926	.ty = mask_ty.toIntern(),
		16927	.storage = .{ .elems = mask_elems[0..vec_len] },
		16928	} })),
		16929	});
		16930	const mask_mem: Memory = .{
		16931	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
		16932	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
		16933	};
		16934	if (has_avx) {
		16935	try self.asmRegisterRegisterMemory(
		16936	.{ .vp_, .@"and" },
		16937	mask_alias,
		16938	mask_alias,
		16939	mask_mem,
		16940	);
		16941	try self.asmRegisterRegisterMemory(
		16942	.{ .vp_d, .cmpeq },
		16943	mask_alias,
		16944	mask_alias,
		16945	mask_mem,
		16946	);
		16947	} else {
		16948	try self.asmRegisterMemory(
		16949	.{ .p_, .@"and" },
		16950	mask_alias,
		16951	mask_mem,
		16952	);
		16953	try self.asmRegisterMemory(
		16954	.{ .p_d, .cmpeq },
		16955	mask_alias,
		16956	mask_mem,
		16957	);
		16958	}
		16959	}
		16960	break :mask mask_reg;
		16961	};
		16962	const mask_alias = registerAlias(mask_reg, abi_size);
		16963	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
		16964	defer self.register_manager.unlockReg(mask_lock);
		16965
		16966	const lhs_mcv = try self.resolveInst(extra.lhs);
		16967	const lhs_lock = switch (lhs_mcv) {
		16968	.register => \|lhs_reg\| self.register_manager.lockRegAssumeUnused(lhs_reg),
		16969	else => null,
		16970	};
		16971	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
		16972
		16973	const rhs_mcv = try self.resolveInst(extra.rhs);
		16974	const rhs_lock = switch (rhs_mcv) {
		16975	.register => \|rhs_reg\| self.register_manager.lockReg(rhs_reg),
		16976	else => null,
		16977	};
		16978	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
		16979
		16980	const reuse_mcv = if (has_blend) rhs_mcv else lhs_mcv;
		16981	const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand(
		16982	inst,
		16983	if (has_blend) extra.rhs else extra.lhs,
		16984	@intFromBool(has_blend),
		16985	reuse_mcv,
		16986	)) reuse_mcv else if (has_avx)
		16987	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		16988	else
		16989	try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv);
		16990	const dst_reg = dst_mcv.getReg().?;
		16991	const dst_alias = registerAlias(dst_reg, abi_size);
		16992	const dst_lock = self.register_manager.lockReg(dst_reg);
		16993	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
		16994
		16995	const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.childType(mod).zigTypeTag(mod)) {
		16996	else => null,
		16997	.Int => switch (abi_size) {
		16998	0 => unreachable,
		16999	1...16 => if (has_avx)
		17000	.{ .vp_b, .blendv }
		17001	else if (has_blend)
		17002	.{ .p_b, .blendv }
		17003	else
		17004	.{ .p_, undefined },
		17005	17...32 => if (self.hasFeature(.avx2))
		17006	.{ .vp_b, .blendv }
		17007	else
		17008	null,
		17009	else => null,
		17010	},
		17011	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
		17012	else => unreachable,
		17013	16, 80, 128 => null,
		17014	32 => switch (vec_len) {
		17015	0 => unreachable,
		17016	1...4 => if (has_avx) .{ .v_ps, .blendv } else .{ ._ps, .blendv },
		17017	5...8 => if (has_avx) .{ .v_ps, .blendv } else null,
		17018	else => null,
		17019	},
		17020	64 => switch (vec_len) {
		17021	0 => unreachable,
		17022	1...2 => if (has_avx) .{ .v_pd, .blendv } else .{ ._pd, .blendv },
		17023	3...4 => if (has_avx) .{ .v_pd, .blendv } else null,
		17024	else => null,
		17025	},
		17026	},
		17027	}) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
		17028	if (has_avx) {
		17029	const rhs_alias = if (rhs_mcv.isRegister())
		17030	registerAlias(rhs_mcv.getReg().?, abi_size)
		17031	else rhs: {
		17032	try self.genSetReg(dst_reg, ty, rhs_mcv, .{});
		17033	break :rhs dst_alias;
		17034	};
		17035	if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
		17036	mir_tag,
		17037	dst_alias,
		17038	rhs_alias,
		17039	try lhs_mcv.mem(self, self.memSize(ty)),
		17040	mask_alias,
		17041	) else try self.asmRegisterRegisterRegisterRegister(
		17042	mir_tag,
		17043	dst_alias,
		17044	rhs_alias,
		17045	registerAlias(if (lhs_mcv.isRegister())
		17046	lhs_mcv.getReg().?
		17047	else
		17048	try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
		17049	mask_alias,
		17050	);
		17051	} else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
		17052	mir_tag,
		17053	dst_alias,
		17054	try lhs_mcv.mem(self, self.memSize(ty)),
		17055	mask_alias,
		17056	) else try self.asmRegisterRegisterRegister(
		17057	mir_tag,
		17058	dst_alias,
		17059	registerAlias(if (lhs_mcv.isRegister())
		17060	lhs_mcv.getReg().?
		17061	else
		17062	try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
		17063	mask_alias,
		17064	) else {
		17065	const mir_fixes = @as(?Mir.Inst.Fixes, switch (elem_ty.zigTypeTag(mod)) {
		17066	else => null,
		17067	.Int => .p_,
		17068	.Float => switch (elem_ty.floatBits(self.target.*)) {
		17069	32 => ._ps,
		17070	64 => ._pd,
		17071	16, 80, 128 => null,
		17072	else => unreachable,
		17073	},
		17074	}) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
		17075	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
		17076	if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
		17077	.{ mir_fixes, .andn },
		17078	mask_alias,
		17079	try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)),
		17080	) else try self.asmRegisterRegister(
		17081	.{ mir_fixes, .andn },
		17082	mask_alias,
		17083	if (rhs_mcv.isRegister())
		17084	rhs_mcv.getReg().?
		17085	else
		17086	try self.copyToTmpRegister(ty, rhs_mcv),
		17087	);
		17088	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
		17089	}
		17090	break :result dst_mcv;
		17091	};
		17092	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
16571	}	17093	}
16572		17094
16573	fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {	17095	fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
		17096	const mod = self.bin_file.comp.module.?;
16574	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	17097	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
16575	_ = ty_pl;	17098	const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
16576	return self.fail("TODO implement airShuffle for x86_64", .{});	17099
16577	//return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });	17100	const dst_ty = self.typeOfIndex(inst);
		17101	const elem_ty = dst_ty.childType(mod);
		17102	const elem_abi_size: u16 = @intCast(elem_ty.abiSize(mod));
		17103	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
		17104	const lhs_ty = self.typeOf(extra.a);
		17105	const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
		17106	const rhs_ty = self.typeOf(extra.b);
		17107	const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(mod));
		17108	const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size);
		17109
		17110	const ExpectedContents = [32]?i32;
		17111	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
		17112	std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
		17113	const allocator = stack.get();
		17114
		17115	const mask_elems = try allocator.alloc(?i32, extra.mask_len);
		17116	defer allocator.free(mask_elems);
		17117	for (mask_elems, 0..) \|*mask_elem, elem_index\| {
		17118	const mask_elem_val =
		17119	Value.fromInterned(extra.mask).elemValue(mod, elem_index) catch unreachable;
		17120	mask_elem.* = if (mask_elem_val.isUndef(mod))
		17121	null
		17122	else
		17123	@intCast(mask_elem_val.toSignedInt(mod));
		17124	}
		17125
		17126	const has_avx = self.hasFeature(.avx);
		17127	const result = @as(?MCValue, result: {
		17128	for (mask_elems) \|mask_elem\| {
		17129	if (mask_elem) \|_\| break;
		17130	} else break :result try self.allocRegOrMem(inst, true);
		17131
		17132	for (mask_elems, 0..) \|mask_elem, elem_index\| {
		17133	if (mask_elem orelse continue != elem_index) break;
		17134	} else {
		17135	const lhs_mcv = try self.resolveInst(extra.a);
		17136	if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv;
		17137	const dst_mcv = try self.allocRegOrMem(inst, true);
		17138	try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{});
		17139	break :result dst_mcv;
		17140	}
		17141
		17142	for (mask_elems, 0..) \|mask_elem, elem_index\| {
		17143	if (~(mask_elem orelse continue) != elem_index) break;
		17144	} else {
		17145	const rhs_mcv = try self.resolveInst(extra.b);
		17146	if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv;
		17147	const dst_mcv = try self.allocRegOrMem(inst, true);
		17148	try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{});
		17149	break :result dst_mcv;
		17150	}
		17151
		17152	for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) \|variant\| unpck: {
		17153	if (elem_abi_size > 8) break :unpck;
		17154	if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
		17155	has_avx
		17156	else
		17157	self.hasFeature(.avx2)) 32 else 16)) break :unpck;
		17158
		17159	var sources = [1]?u1{null} ** 2;
		17160	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
		17161	const mask_elem = maybe_mask_elem orelse continue;
		17162	const mask_elem_index =
		17163	math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
		17164	const elem_byte = (elem_index >> 1) * elem_abi_size;
		17165	if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {
		17166	.unpckl => 0b0000,
		17167	.unpckh => 0b1000,
		17168	else => unreachable,
		17169	}) \| (elem_byte << 1 & 0b10000)) break :unpck;
		17170
		17171	const source = @intFromBool(mask_elem < 0);
		17172	if (sources[elem_index & 0b00001]) \|prev_source\| {
		17173	if (source != prev_source) break :unpck;
		17174	} else sources[elem_index & 0b00001] = source;
		17175	}
		17176	if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck;
		17177
		17178	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
		17179	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
		17180	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
		17181	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
		17182
		17183	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
		17184	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
		17185	lhs_mcv
		17186	else if (has_avx and lhs_mcv.isRegister())
		17187	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		17188	else
		17189	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
		17190	const dst_reg = dst_mcv.getReg().?;
		17191	const dst_alias = registerAlias(dst_reg, max_abi_size);
		17192
		17193	const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
		17194	(dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) {
		17195	4 => if (has_avx) .v_ps else ._ps,
		17196	8 => if (has_avx) .v_pd else ._pd,
		17197	else => unreachable,
		17198	}, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) {
		17199	.unpckl => switch (elem_abi_size) {
		17200	1 => .unpcklbw,
		17201	2 => .unpcklwd,
		17202	4 => .unpckldq,
		17203	8 => .unpcklqdq,
		17204	else => unreachable,
		17205	},
		17206	.unpckh => switch (elem_abi_size) {
		17207	1 => .unpckhbw,
		17208	2 => .unpckhwd,
		17209	4 => .unpckhdq,
		17210	8 => .unpckhqdq,
		17211	else => unreachable,
		17212	},
		17213	else => unreachable,
		17214	} };
		17215	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory(
		17216	mir_tag,
		17217	dst_alias,
		17218	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
		17219	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
		17220	) else try self.asmRegisterRegisterRegister(
		17221	mir_tag,
		17222	dst_alias,
		17223	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
		17224	registerAlias(if (rhs_mcv.isRegister())
		17225	rhs_mcv.getReg().?
		17226	else
		17227	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
		17228	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
		17229	mir_tag,
		17230	dst_alias,
		17231	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
		17232	) else try self.asmRegisterRegister(
		17233	mir_tag,
		17234	dst_alias,
		17235	registerAlias(if (rhs_mcv.isRegister())
		17236	rhs_mcv.getReg().?
		17237	else
		17238	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
		17239	);
		17240	break :result dst_mcv;
		17241	}
		17242
		17243	pshufd: {
		17244	if (elem_abi_size != 4) break :pshufd;
		17245	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;
		17246
		17247	var control: u8 = 0b00_00_00_00;
		17248	var sources = [1]?u1{null} ** 1;
		17249	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
		17250	const mask_elem = maybe_mask_elem orelse continue;
		17251	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
		17252	if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd;
		17253
		17254	const source = @intFromBool(mask_elem < 0);
		17255	if (sources[0]) \|prev_source\| {
		17256	if (source != prev_source) break :pshufd;
		17257	} else sources[(elem_index & 0b010) >> 1] = source;
		17258
		17259	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
		17260	const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
		17261	if (elem_index & 0b100 == 0)
		17262	control \|= select
		17263	else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd;
		17264	}
		17265
		17266	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
		17267	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
		17268	const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]);
		17269
		17270	const dst_reg = if (src_mcv.isRegister() and
		17271	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv))
		17272	src_mcv.getReg().?
		17273	else
		17274	try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
		17275	const dst_alias = registerAlias(dst_reg, max_abi_size);
		17276
		17277	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
		17278	.{ if (has_avx) .vp_d else .p_d, .shuf },
		17279	dst_alias,
		17280	try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
		17281	Immediate.u(control),
		17282	) else try self.asmRegisterRegisterImmediate(
		17283	.{ if (has_avx) .vp_d else .p_d, .shuf },
		17284	dst_alias,
		17285	registerAlias(if (src_mcv.isRegister())
		17286	src_mcv.getReg().?
		17287	else
		17288	try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
		17289	Immediate.u(control),
		17290	);
		17291	break :result .{ .register = dst_reg };
		17292	}
		17293
		17294	shufps: {
		17295	if (elem_abi_size != 4) break :shufps;
		17296	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;
		17297
		17298	var control: u8 = 0b00_00_00_00;
		17299	var sources = [1]?u1{null} ** 2;
		17300	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
		17301	const mask_elem = maybe_mask_elem orelse continue;
		17302	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
		17303	if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps;
		17304
		17305	const source = @intFromBool(mask_elem < 0);
		17306	if (sources[(elem_index & 0b010) >> 1]) \|prev_source\| {
		17307	if (source != prev_source) break :shufps;
		17308	} else sources[(elem_index & 0b010) >> 1] = source;
		17309
		17310	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
		17311	const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
		17312	if (elem_index & 0b100 == 0)
		17313	control \|= select
		17314	else if (control & @as(u8, 0b11) << select_bit != select) break :shufps;
		17315	}
		17316	if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
		17317
		17318	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
		17319	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
		17320	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
		17321	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
		17322
		17323	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
		17324	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
		17325	lhs_mcv
		17326	else if (has_avx and lhs_mcv.isRegister())
		17327	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		17328	else
		17329	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
		17330	const dst_reg = dst_mcv.getReg().?;
		17331	const dst_alias = registerAlias(dst_reg, max_abi_size);
		17332
		17333	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		17334	.{ .v_ps, .shuf },
		17335	dst_alias,
		17336	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
		17337	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
		17338	Immediate.u(control),
		17339	) else try self.asmRegisterRegisterRegisterImmediate(
		17340	.{ .v_ps, .shuf },
		17341	dst_alias,
		17342	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
		17343	registerAlias(if (rhs_mcv.isRegister())
		17344	rhs_mcv.getReg().?
		17345	else
		17346	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
		17347	Immediate.u(control),
		17348	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
		17349	.{ ._ps, .shuf },
		17350	dst_alias,
		17351	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
		17352	Immediate.u(control),
		17353	) else try self.asmRegisterRegisterImmediate(
		17354	.{ ._ps, .shuf },
		17355	dst_alias,
		17356	registerAlias(if (rhs_mcv.isRegister())
		17357	rhs_mcv.getReg().?
		17358	else
		17359	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
		17360	Immediate.u(control),
		17361	);
		17362	break :result dst_mcv;
		17363	}
		17364
		17365	shufpd: {
		17366	if (elem_abi_size != 8) break :shufpd;
		17367	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;
		17368
		17369	var control: u4 = 0b0_0_0_0;
		17370	var sources = [1]?u1{null} ** 2;
		17371	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
		17372	const mask_elem = maybe_mask_elem orelse continue;
		17373	const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
		17374	if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd;
		17375
		17376	const source = @intFromBool(mask_elem < 0);
		17377	if (sources[elem_index & 0b01]) \|prev_source\| {
		17378	if (source != prev_source) break :shufpd;
		17379	} else sources[elem_index & 0b01] = source;
		17380
		17381	control \|= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index);
		17382	}
		17383	if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd;
		17384
		17385	const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b };
		17386	const operand_tys: [2]Type = .{ lhs_ty, rhs_ty };
		17387	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
		17388	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
		17389
		17390	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
		17391	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
		17392	lhs_mcv
		17393	else if (has_avx and lhs_mcv.isRegister())
		17394	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		17395	else
		17396	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
		17397	const dst_reg = dst_mcv.getReg().?;
		17398	const dst_alias = registerAlias(dst_reg, max_abi_size);
		17399
		17400	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		17401	.{ .v_pd, .shuf },
		17402	dst_alias,
		17403	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
		17404	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
		17405	Immediate.u(control),
		17406	) else try self.asmRegisterRegisterRegisterImmediate(
		17407	.{ .v_pd, .shuf },
		17408	dst_alias,
		17409	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
		17410	registerAlias(if (rhs_mcv.isRegister())
		17411	rhs_mcv.getReg().?
		17412	else
		17413	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
		17414	Immediate.u(control),
		17415	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
		17416	.{ ._pd, .shuf },
		17417	dst_alias,
		17418	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
		17419	Immediate.u(control),
		17420	) else try self.asmRegisterRegisterImmediate(
		17421	.{ ._pd, .shuf },
		17422	dst_alias,
		17423	registerAlias(if (rhs_mcv.isRegister())
		17424	rhs_mcv.getReg().?
		17425	else
		17426	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
		17427	Immediate.u(control),
		17428	);
		17429	break :result dst_mcv;
		17430	}
		17431
		17432	blend: {
		17433	if (elem_abi_size < 2) break :blend;
		17434	if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend;
		17435	if (!self.hasFeature(.sse4_1)) break :blend;
		17436
		17437	var control: u8 = 0b0_0_0_0_0_0_0_0;
		17438	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
		17439	const mask_elem = maybe_mask_elem orelse continue;
		17440	const mask_elem_index =
		17441	math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
		17442	if (mask_elem_index != elem_index) break :blend;
		17443
		17444	const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
		17445	if (elem_index & 0b1000 == 0)
		17446	control \|= select
		17447	else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend;
		17448	}
		17449
		17450	if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
		17451	const expanded_control = switch (elem_abi_size) {
		17452	4 => control,
		17453	8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) \|
		17454	@as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) \|
		17455	@as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) \|
		17456	@as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00),
		17457	else => break :vpblendd,
		17458	};
		17459
		17460	const lhs_mcv = try self.resolveInst(extra.a);
		17461	const lhs_reg = if (lhs_mcv.isRegister())
		17462	lhs_mcv.getReg().?
		17463	else
		17464	try self.copyToTmpRegister(dst_ty, lhs_mcv);
		17465	const lhs_lock = self.register_manager.lockReg(lhs_reg);
		17466	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
		17467
		17468	const rhs_mcv = try self.resolveInst(extra.b);
		17469	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
		17470	if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		17471	.{ .vp_d, .blend },
		17472	registerAlias(dst_reg, dst_abi_size),
		17473	registerAlias(lhs_reg, dst_abi_size),
		17474	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
		17475	Immediate.u(expanded_control),
		17476	) else try self.asmRegisterRegisterRegisterImmediate(
		17477	.{ .vp_d, .blend },
		17478	registerAlias(dst_reg, dst_abi_size),
		17479	registerAlias(lhs_reg, dst_abi_size),
		17480	registerAlias(if (rhs_mcv.isRegister())
		17481	rhs_mcv.getReg().?
		17482	else
		17483	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		17484	Immediate.u(expanded_control),
		17485	);
		17486	break :result .{ .register = dst_reg };
		17487	}
		17488
		17489	if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: {
		17490	const expanded_control = switch (elem_abi_size) {
		17491	2 => control,
		17492	4 => if (dst_abi_size <= 16 or
		17493	@as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0)))
		17494	@as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) \|
		17495	@as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) \|
		17496	@as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) \|
		17497	@as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00)
		17498	else
		17499	break :pblendw,
		17500	8 => if (dst_abi_size <= 16 or
		17501	@as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0)))
		17502	@as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) \|
		17503	@as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000)
		17504	else
		17505	break :pblendw,
		17506	16 => break :pblendw,
		17507	else => unreachable,
		17508	};
		17509
		17510	const lhs_mcv = try self.resolveInst(extra.a);
		17511	const rhs_mcv = try self.resolveInst(extra.b);
		17512
		17513	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
		17514	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
		17515	lhs_mcv
		17516	else if (has_avx and lhs_mcv.isRegister())
		17517	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		17518	else
		17519	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
		17520	const dst_reg = dst_mcv.getReg().?;
		17521
		17522	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		17523	.{ .vp_w, .blend },
		17524	registerAlias(dst_reg, dst_abi_size),
		17525	registerAlias(if (lhs_mcv.isRegister())
		17526	lhs_mcv.getReg().?
		17527	else
		17528	dst_reg, dst_abi_size),
		17529	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
		17530	Immediate.u(expanded_control),
		17531	) else try self.asmRegisterRegisterRegisterImmediate(
		17532	.{ .vp_w, .blend },
		17533	registerAlias(dst_reg, dst_abi_size),
		17534	registerAlias(if (lhs_mcv.isRegister())
		17535	lhs_mcv.getReg().?
		17536	else
		17537	dst_reg, dst_abi_size),
		17538	registerAlias(if (rhs_mcv.isRegister())
		17539	rhs_mcv.getReg().?
		17540	else
		17541	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		17542	Immediate.u(expanded_control),
		17543	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
		17544	.{ .p_w, .blend },
		17545	registerAlias(dst_reg, dst_abi_size),
		17546	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
		17547	Immediate.u(expanded_control),
		17548	) else try self.asmRegisterRegisterImmediate(
		17549	.{ .p_w, .blend },
		17550	registerAlias(dst_reg, dst_abi_size),
		17551	registerAlias(if (rhs_mcv.isRegister())
		17552	rhs_mcv.getReg().?
		17553	else
		17554	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		17555	Immediate.u(expanded_control),
		17556	);
		17557	break :result .{ .register = dst_reg };
		17558	}
		17559
		17560	const expanded_control = switch (elem_abi_size) {
		17561	4, 8 => control,
		17562	16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) \|
		17563	@as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
		17564	else => unreachable,
		17565	};
		17566
		17567	const lhs_mcv = try self.resolveInst(extra.a);
		17568	const rhs_mcv = try self.resolveInst(extra.b);
		17569
		17570	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
		17571	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
		17572	lhs_mcv
		17573	else if (has_avx and lhs_mcv.isRegister())
		17574	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		17575	else
		17576	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
		17577	const dst_reg = dst_mcv.getReg().?;
		17578
		17579	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
		17580	switch (elem_abi_size) {
		17581	4 => .{ .v_ps, .blend },
		17582	8, 16 => .{ .v_pd, .blend },
		17583	else => unreachable,
		17584	},
		17585	registerAlias(dst_reg, dst_abi_size),
		17586	registerAlias(if (lhs_mcv.isRegister())
		17587	lhs_mcv.getReg().?
		17588	else
		17589	dst_reg, dst_abi_size),
		17590	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
		17591	Immediate.u(expanded_control),
		17592	) else try self.asmRegisterRegisterRegisterImmediate(
		17593	switch (elem_abi_size) {
		17594	4 => .{ .v_ps, .blend },
		17595	8, 16 => .{ .v_pd, .blend },
		17596	else => unreachable,
		17597	},
		17598	registerAlias(dst_reg, dst_abi_size),
		17599	registerAlias(if (lhs_mcv.isRegister())
		17600	lhs_mcv.getReg().?
		17601	else
		17602	dst_reg, dst_abi_size),
		17603	registerAlias(if (rhs_mcv.isRegister())
		17604	rhs_mcv.getReg().?
		17605	else
		17606	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		17607	Immediate.u(expanded_control),
		17608	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
		17609	switch (elem_abi_size) {
		17610	4 => .{ ._ps, .blend },
		17611	8, 16 => .{ ._pd, .blend },
		17612	else => unreachable,
		17613	},
		17614	registerAlias(dst_reg, dst_abi_size),
		17615	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
		17616	Immediate.u(expanded_control),
		17617	) else try self.asmRegisterRegisterImmediate(
		17618	switch (elem_abi_size) {
		17619	4 => .{ ._ps, .blend },
		17620	8, 16 => .{ ._pd, .blend },
		17621	else => unreachable,
		17622	},
		17623	registerAlias(dst_reg, dst_abi_size),
		17624	registerAlias(if (rhs_mcv.isRegister())
		17625	rhs_mcv.getReg().?
		17626	else
		17627	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		17628	Immediate.u(expanded_control),
		17629	);
		17630	break :result .{ .register = dst_reg };
		17631	}
		17632
		17633	blendv: {
		17634	if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
		17635	has_avx
		17636	else
		17637	self.hasFeature(.avx2)) 32 else 16)) break :blendv;
		17638
		17639	const select_mask_elem_ty = try mod.intType(.unsigned, elem_abi_size * 8);
		17640	const select_mask_ty = try mod.vectorType(.{
		17641	.len = @intCast(mask_elems.len),
		17642	.child = select_mask_elem_ty.toIntern(),
		17643	});
		17644	var select_mask_elems: [32]InternPool.Index = undefined;
		17645	for (
		17646	select_mask_elems[0..mask_elems.len],
		17647	mask_elems,
		17648	0..,
		17649	) \|*select_mask_elem, maybe_mask_elem, elem_index\| {
		17650	const mask_elem = maybe_mask_elem orelse continue;
		17651	const mask_elem_index =
		17652	math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
		17653	if (mask_elem_index != elem_index) break :blendv;
		17654
		17655	select_mask_elem.* = (if (mask_elem < 0)
		17656	try select_mask_elem_ty.maxIntScalar(mod, select_mask_elem_ty)
		17657	else
		17658	try select_mask_elem_ty.minIntScalar(mod, select_mask_elem_ty)).toIntern();
		17659	}
		17660	const select_mask_mcv = try self.genTypedValue(.{
		17661	.ty = select_mask_ty,
		17662	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
		17663	.ty = select_mask_ty.toIntern(),
		17664	.storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
		17665	} })),
		17666	});
		17667
		17668	if (self.hasFeature(.sse4_1)) {
		17669	const mir_tag: Mir.Inst.FixedTag = .{
		17670	if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
		17671	(dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
		17672	4 => if (has_avx) .v_ps else ._ps,
		17673	8 => if (has_avx) .v_pd else ._pd,
		17674	else => unreachable,
		17675	} else if (has_avx) .vp_b else .p_b,
		17676	.blendv,
		17677	};
		17678
		17679	const select_mask_reg = if (!has_avx) reg: {
		17680	try self.register_manager.getKnownReg(.xmm0, null);
		17681	try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
		17682	break :reg .xmm0;
		17683	} else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
		17684	const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
		17685	const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
		17686	defer self.register_manager.unlockReg(select_mask_lock);
		17687
		17688	const lhs_mcv = try self.resolveInst(extra.a);
		17689	const rhs_mcv = try self.resolveInst(extra.b);
		17690
		17691	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
		17692	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
		17693	lhs_mcv
		17694	else if (has_avx and lhs_mcv.isRegister())
		17695	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
		17696	else
		17697	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
		17698	const dst_reg = dst_mcv.getReg().?;
		17699	const dst_alias = registerAlias(dst_reg, dst_abi_size);
		17700
		17701	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
		17702	mir_tag,
		17703	dst_alias,
		17704	if (lhs_mcv.isRegister())
		17705	registerAlias(lhs_mcv.getReg().?, dst_abi_size)
		17706	else
		17707	dst_alias,
		17708	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
		17709	select_mask_alias,
		17710	) else try self.asmRegisterRegisterRegisterRegister(
		17711	mir_tag,
		17712	dst_alias,
		17713	if (lhs_mcv.isRegister())
		17714	registerAlias(lhs_mcv.getReg().?, dst_abi_size)
		17715	else
		17716	dst_alias,
		17717	registerAlias(if (rhs_mcv.isRegister())
		17718	rhs_mcv.getReg().?
		17719	else
		17720	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		17721	select_mask_alias,
		17722	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
		17723	mir_tag,
		17724	dst_alias,
		17725	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
		17726	select_mask_alias,
		17727	) else try self.asmRegisterRegisterRegister(
		17728	mir_tag,
		17729	dst_alias,
		17730	registerAlias(if (rhs_mcv.isRegister())
		17731	rhs_mcv.getReg().?
		17732	else
		17733	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
		17734	select_mask_alias,
		17735	);
		17736	break :result dst_mcv;
		17737	}
		17738
		17739	const lhs_mcv = try self.resolveInst(extra.a);
		17740	const rhs_mcv = try self.resolveInst(extra.b);
		17741
		17742	const dst_mcv: MCValue = if (rhs_mcv.isRegister() and
		17743	self.reuseOperand(inst, extra.b, 1, rhs_mcv))
		17744	rhs_mcv
		17745	else
		17746	try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);
		17747	const dst_reg = dst_mcv.getReg().?;
		17748	const dst_alias = registerAlias(dst_reg, dst_abi_size);
		17749
		17750	const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
		17751	const mask_alias = registerAlias(mask_reg, dst_abi_size);
		17752	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
		17753	defer self.register_manager.unlockReg(mask_lock);
		17754
		17755	const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())
		17756	switch (elem_ty.floatBits(self.target.*)) {
		17757	16, 80, 128 => .p_,
		17758	32 => ._ps,
		17759	64 => ._pd,
		17760	else => unreachable,
		17761	}
		17762	else
		17763	.p_;
		17764	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
		17765	if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
		17766	.{ mir_fixes, .andn },
		17767	mask_alias,
		17768	try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
		17769	) else try self.asmRegisterRegister(
		17770	.{ mir_fixes, .andn },
		17771	mask_alias,
		17772	if (lhs_mcv.isRegister())
		17773	lhs_mcv.getReg().?
		17774	else
		17775	try self.copyToTmpRegister(dst_ty, lhs_mcv),
		17776	);
		17777	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
		17778	break :result dst_mcv;
		17779	}
		17780
		17781	pshufb: {
		17782	if (max_abi_size > 16) break :pshufb;
		17783	if (!self.hasFeature(.ssse3)) break :pshufb;
		17784
		17785	const temp_regs =
		17786	try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
		17787	const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
		17788	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
		17789
		17790	const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
		17791	try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
		17792
		17793	const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
		17794	try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
		17795
		17796	var lhs_mask_elems: [16]InternPool.Index = undefined;
		17797	for (lhs_mask_elems[0..max_abi_size], 0..) \|*lhs_mask_elem, byte_index\| {
		17798	const elem_index = byte_index / elem_abi_size;
		17799	lhs_mask_elem.* = try mod.intern(.{ .int = .{
		17800	.ty = .u8_type,
		17801	.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
		17802	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
		17803	if (mask_elem < 0) break :elem 0b1_00_00000;
		17804	const mask_elem_index: u31 = @intCast(mask_elem);
		17805	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
		17806	break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
		17807	} },
		17808	} });
		17809	}
		17810	const lhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
		17811	const lhs_mask_mcv = try self.genTypedValue(.{
		17812	.ty = lhs_mask_ty,
		17813	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
		17814	.ty = lhs_mask_ty.toIntern(),
		17815	.storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
		17816	} })),
		17817	});
		17818	const lhs_mask_mem: Memory = .{
		17819	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
		17820	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
		17821	};
		17822	if (has_avx) try self.asmRegisterRegisterMemory(
		17823	.{ .vp_b, .shuf },
		17824	lhs_temp_alias,
		17825	lhs_temp_alias,
		17826	lhs_mask_mem,
		17827	) else try self.asmRegisterMemory(
		17828	.{ .p_b, .shuf },
		17829	lhs_temp_alias,
		17830	lhs_mask_mem,
		17831	);
		17832
		17833	var rhs_mask_elems: [16]InternPool.Index = undefined;
		17834	for (rhs_mask_elems[0..max_abi_size], 0..) \|*rhs_mask_elem, byte_index\| {
		17835	const elem_index = byte_index / elem_abi_size;
		17836	rhs_mask_elem.* = try mod.intern(.{ .int = .{
		17837	.ty = .u8_type,
		17838	.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
		17839	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
		17840	if (mask_elem >= 0) break :elem 0b1_00_00000;
		17841	const mask_elem_index: u31 = @intCast(~mask_elem);
		17842	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
		17843	break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
		17844	} },
		17845	} });
		17846	}
		17847	const rhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
		17848	const rhs_mask_mcv = try self.genTypedValue(.{
		17849	.ty = rhs_mask_ty,
		17850	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
		17851	.ty = rhs_mask_ty.toIntern(),
		17852	.storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
		17853	} })),
		17854	});
		17855	const rhs_mask_mem: Memory = .{
		17856	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
		17857	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
		17858	};
		17859	if (has_avx) try self.asmRegisterRegisterMemory(
		17860	.{ .vp_b, .shuf },
		17861	rhs_temp_alias,
		17862	rhs_temp_alias,
		17863	rhs_mask_mem,
		17864	) else try self.asmRegisterMemory(
		17865	.{ .p_b, .shuf },
		17866	rhs_temp_alias,
		17867	rhs_mask_mem,
		17868	);
		17869
		17870	if (has_avx) try self.asmRegisterRegisterRegister(
		17871	.{ switch (elem_ty.zigTypeTag(mod)) {
		17872	else => break :result null,
		17873	.Int => .vp_,
		17874	.Float => switch (elem_ty.floatBits(self.target.*)) {
		17875	32 => .v_ps,
		17876	64 => .v_pd,
		17877	16, 80, 128 => break :result null,
		17878	else => unreachable,
		17879	},
		17880	}, .@"or" },
		17881	lhs_temp_alias,
		17882	lhs_temp_alias,
		17883	rhs_temp_alias,
		17884	) else try self.asmRegisterRegister(
		17885	.{ switch (elem_ty.zigTypeTag(mod)) {
		17886	else => break :result null,
		17887	.Int => .p_,
		17888	.Float => switch (elem_ty.floatBits(self.target.*)) {
		17889	32 => ._ps,
		17890	64 => ._pd,
		17891	16, 80, 128 => break :result null,
		17892	else => unreachable,
		17893	},
		17894	}, .@"or" },
		17895	lhs_temp_alias,
		17896	rhs_temp_alias,
		17897	);
		17898	break :result .{ .register = temp_regs[0] };
		17899	}
		17900
		17901	break :result null;
		17902	}) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
		17903	lhs_ty.fmt(mod), rhs_ty.fmt(mod), dst_ty.fmt(mod),
		17904	Value.fromInterned(extra.mask).fmtValue(
		17905	Type.fromInterned(mod.intern_pool.typeOf(extra.mask)),
		17906	mod,
		17907	),
		17908	});
		17909	return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
16578	}	17910	}
16579		17911
16580	fn airReduce(self: *Self, inst: Air.Inst.Index) !void {	17912	fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
...	@@ -16751,7 +18083,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {	...	@@ -16751,7 +18083,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
16751	},	18083	},
16752	.Array, .Vector => {	18084	.Array, .Vector => {
16753	const elem_ty = result_ty.childType(mod);	18085	const elem_ty = result_ty.childType(mod);
16754	if (result_ty.isVector(mod) and elem_ty.bitSize(mod) == 1) {	18086	if (result_ty.isVector(mod) and elem_ty.toIntern() == .bool_type) {
16755	const result_size: u32 = @intCast(result_ty.abiSize(mod));	18087	const result_size: u32 = @intCast(result_ty.abiSize(mod));
16756	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);	18088	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
16757	try self.asmRegisterRegister(	18089	try self.asmRegisterRegister(
...	@@ -17801,7 +19133,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {	...	@@ -17801,7 +19133,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {
17801	else => unreachable,	19133	else => unreachable,
17802	},	19134	},
17803	.float => Type.f32,	19135	.float => Type.f32,
17804	.float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),	19136	.float_combine => try mod.arrayType(.{ .len = 2, .child = .f32_type }),
17805	.sse => Type.f64,	19137	.sse => Type.f64,
17806	else => break,	19138	else => break,
17807	};	19139	};

src/arch/x86_64/Encoding.zig+11-4

 @@ -324,16 +324,19 @@ pub const Mnemonic = enum {
     // SSE3
     movddup, movshdup, movsldup,
     // SSSE3
-    pabsb, pabsd, pabsw, palignr,
+    pabsb, pabsd, pabsw, palignr, pshufb,
     // SSE4.1
     blendpd, blendps, blendvpd, blendvps,
     extractps,
     insertps,
     packusdw,
+    pblendvb, pblendw,
     pcmpeqq,
     pextrb, pextrd, pextrq,
     pinsrb, pinsrd, pinsrq,
     pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,
+    pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq,
+    pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq,
     pmulld,
     roundpd, roundps, roundsd, roundss,
     // SSE4.2
 @@ -377,7 +380,8 @@ pub const Mnemonic = enum {
     vpabsb, vpabsd, vpabsw,
     vpackssdw, vpacksswb, vpackusdw, vpackuswb,
     vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,
-    vpalignr, vpand, vpandn, vpclmulqdq,
+    vpalignr, vpand, vpandn,
+    vpblendvb, vpblendw, vpclmulqdq,
     vpcmpeqb, vpcmpeqd, vpcmpeqq, vpcmpeqw,
     vpcmpgtb, vpcmpgtd, vpcmpgtq, vpcmpgtw,
     vpextrb, vpextrd, vpextrq, vpextrw,
 @@ -385,9 +389,11 @@ pub const Mnemonic = enum {
     vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw,
     vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw,
     vpmovmskb,
+    vpmovsxbd, vpmovsxbq, vpmovsxbw, vpmovsxdq, vpmovsxwd, vpmovsxwq,
+    vpmovzxbd, vpmovzxbq, vpmovzxbw, vpmovzxdq, vpmovzxwd, vpmovzxwq,
     vpmulhw, vpmulld, vpmullw,
     vpor,
-    vpshufd, vpshufhw, vpshuflw,
+    vpshufb, vpshufd, vpshufhw, vpshuflw,
     vpslld, vpslldq, vpsllq, vpsllw,
     vpsrad, vpsraq, vpsraw,
     vpsrld, vpsrldq, vpsrlq, vpsrlw,
 @@ -409,7 +415,8 @@ pub const Mnemonic = enum {
     vfmadd132sd, vfmadd213sd, vfmadd231sd,
     vfmadd132ss, vfmadd213ss, vfmadd231ss,
     // AVX2
-    vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,
+    vbroadcasti128, vpbroadcastb, vpbroadcastd, vpbroadcastq, vpbroadcastw,
+    vextracti128, vinserti128, vpblendd,
     // zig fmt: on
 };

src/arch/x86_64/Lower.zig+13-1

 @@ -477,8 +477,9 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
         .rri_s, .rri_u => inst.data.rri.fixes,
         .ri_s, .ri_u => inst.data.ri.fixes,
         .ri64, .rm, .rmi_s, .mr => inst.data.rx.fixes,
-        .mrr, .rrm => inst.data.rrx.fixes,
+        .mrr, .rrm, .rmr => inst.data.rrx.fixes,
         .rmi, .mri => inst.data.rix.fixes,
+        .rrmr => inst.data.rrrx.fixes,
         .rrmi => inst.data.rrix.fixes,
         .mi_u, .mi_s => inst.data.x.fixes,
         .m => inst.data.x.fixes,
 @@ -565,6 +566,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rx.r1 },
             .{ .mem = lower.mem(inst.data.rx.payload) },
         },
+        .rmr => &.{
+            .{ .reg = inst.data.rrx.r1 },
+            .{ .mem = lower.mem(inst.data.rrx.payload) },
+            .{ .reg = inst.data.rrx.r2 },
+        },
         .rmi => &.{
             .{ .reg = inst.data.rix.r1 },
             .{ .mem = lower.mem(inst.data.rix.payload) },
 @@ -597,6 +603,12 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = inst.data.rrx.r2 },
             .{ .mem = lower.mem(inst.data.rrx.payload) },
         },
+        .rrmr => &.{
+            .{ .reg = inst.data.rrrx.r1 },
+            .{ .reg = inst.data.rrrx.r2 },
+            .{ .mem = lower.mem(inst.data.rrrx.payload) },
+            .{ .reg = inst.data.rrrx.r3 },
+        },
         .rrmi => &.{
             .{ .reg = inst.data.rrix.r1 },
             .{ .reg = inst.data.rrix.r2 },

src/arch/x86_64/Mir.zig+27-3

 @@ -230,6 +230,8 @@ pub const Inst = struct {
         v_d,
         /// VEX-Encoded ___ QuadWord
         v_q,
+        /// VEX-Encoded ___ Integer Data
+        v_i128,
         /// VEX-Encoded Packed ___
         vp_,
         /// VEX-Encoded Packed ___ Byte
 @@ -242,8 +244,6 @@ pub const Inst = struct {
         vp_q,
         /// VEX-Encoded Packed ___ Double Quadword
         vp_dq,
-        /// VEX-Encoded Packed ___ Integer Data
-        vp_i128,
         /// VEX-Encoded ___ Scalar Single-Precision Values
         v_ss,
         /// VEX-Encoded ___ Packed Single-Precision Values
 @@ -654,10 +654,19 @@ pub const Inst = struct {
         /// Variable blend scalar double-precision floating-point values
         blendv,
         /// Extract packed floating-point values
+        /// Extract packed integer values
         extract,
         /// Insert scalar single-precision floating-point value
         /// Insert packed floating-point values
         insert,
+        /// Packed move with sign extend
+        movsxb,
+        movsxd,
+        movsxw,
+        /// Packed move with zero extend
+        movzxb,
+        movzxd,
+        movzxw,
         /// Round packed single-precision floating-point values
         /// Round scalar single-precision floating-point value
         /// Round packed double-precision floating-point values
 @@ -688,6 +697,7 @@ pub const Inst = struct {
         sha256rnds2,
         /// Load with broadcast floating-point data
+        /// Load integer and broadcast
         broadcast,
         /// Convert 16-bit floating-point values to single-precision floating-point values
 @@ -762,8 +772,11 @@ pub const Inst = struct {
         /// Uses `imm` payload.
         rel,
         /// Register, memory operands.
-        /// Uses `rx` payload.
+        /// Uses `rx` payload with extra data of type `Memory`.
         rm,
+        /// Register, memory, register operands.
+        /// Uses `rrx` payload with extra data of type `Memory`.
+        rmr,
         /// Register, memory, immediate (word) operands.
         /// Uses `rix` payload with extra data of type `Memory`.
         rmi,
 @@ -776,6 +789,9 @@ pub const Inst = struct {
         /// Register, register, memory.
         /// Uses `rrix` payload with extra data of type `Memory`.
         rrm,
+        /// Register, register, memory, register.
+        /// Uses `rrrx` payload with extra data of type `Memory`.
+        rrmr,
         /// Register, register, memory, immediate (byte) operands.
         /// Uses `rrix` payload with extra data of type `Memory`.
         rrmi,
 @@ -953,6 +969,14 @@ pub const Inst = struct {
             r2: Register,
             payload: u32,
         },
+        /// Register, register, register, followed by Custom payload found in extra.
+        rrrx: struct {
+            fixes: Fixes = ._,
+            r1: Register,
+            r2: Register,
+            r3: Register,
+            payload: u32,
+        },
         /// Register, byte immediate, followed by Custom payload found in extra.
         rix: struct {
             fixes: Fixes = ._,

src/arch/x86_64/encodings.zig+68-1

...	@@ -1185,6 +1185,8 @@ pub const table = [_]Entry{	...	@@ -1185,6 +1185,8 @@ pub const table = [_]Entry{
1185		1185
1186	.{ .palignr, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 },	1186	.{ .palignr, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 },
1187		1187
		1188	.{ .pshufb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .none, .ssse3 },
		1189
1188	// SSE4.1	1190	// SSE4.1
1189	.{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },	1191	.{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },
1190		1192
...	@@ -1202,6 +1204,11 @@ pub const table = [_]Entry{	...	@@ -1202,6 +1204,11 @@ pub const table = [_]Entry{
1202		1204
1203	.{ .packusdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 },	1205	.{ .packusdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 },
1204		1206
		1207	.{ .pblendvb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 },
		1208	.{ .pblendvb, .rm, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 },
		1209
		1210	.{ .pblendw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .none, .sse4_1 },
		1211
1205	.{ .pcmpeqq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x29 }, 0, .none, .sse4_1 },	1212	.{ .pcmpeqq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x29 }, 0, .none, .sse4_1 },
1206		1213
1207	.{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },	1214	.{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
...	@@ -1228,6 +1235,20 @@ pub const table = [_]Entry{	...	@@ -1228,6 +1235,20 @@ pub const table = [_]Entry{
1228		1235
1229	.{ .pminud, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 },	1236	.{ .pminud, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 },
1230		1237
		1238	.{ .pmovsxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .none, .sse4_1 },
		1239	.{ .pmovsxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .none, .sse4_1 },
		1240	.{ .pmovsxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .none, .sse4_1 },
		1241	.{ .pmovsxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .none, .sse4_1 },
		1242	.{ .pmovsxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .none, .sse4_1 },
		1243	.{ .pmovsxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .none, .sse4_1 },
		1244
		1245	.{ .pmovzxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .none, .sse4_1 },
		1246	.{ .pmovzxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .none, .sse4_1 },
		1247	.{ .pmovzxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .none, .sse4_1 },
		1248	.{ .pmovzxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .none, .sse4_1 },
		1249	.{ .pmovzxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .none, .sse4_1 },
		1250	.{ .pmovzxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .none, .sse4_1 },
		1251
1231	.{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },	1252	.{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },
1232		1253
1233	.{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },	1254	.{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
...	@@ -1528,6 +1549,10 @@ pub const table = [_]Entry{	...	@@ -1528,6 +1549,10 @@ pub const table = [_]Entry{
1528		1549
1529	.{ .vpandn, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx },	1550	.{ .vpandn, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx },
1530		1551
		1552	.{ .vpblendvb, .rvmr, &.{ .xmm, .xmm, .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_128_w0, .avx },
		1553
		1554	.{ .vpblendw, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_128_wig, .avx },
		1555
1531	.{ .vpclmulqdq, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_128_wig, .@"pclmul avx" },	1556	.{ .vpclmulqdq, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_128_wig, .@"pclmul avx" },
1532		1557
1533	.{ .vpcmpeqb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_128_wig, .avx },	1558	.{ .vpcmpeqb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_128_wig, .avx },
...	@@ -1576,6 +1601,20 @@ pub const table = [_]Entry{	...	@@ -1576,6 +1601,20 @@ pub const table = [_]Entry{
1576	.{ .vpmovmskb, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },	1601	.{ .vpmovmskb, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },
1577	.{ .vpmovmskb, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },	1602	.{ .vpmovmskb, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },
1578		1603
		1604	.{ .vpmovsxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_128_wig, .avx },
		1605	.{ .vpmovsxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_128_wig, .avx },
		1606	.{ .vpmovsxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_128_wig, .avx },
		1607	.{ .vpmovsxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_128_wig, .avx },
		1608	.{ .vpmovsxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_128_wig, .avx },
		1609	.{ .vpmovsxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_128_wig, .avx },
		1610
		1611	.{ .vpmovzxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_128_wig, .avx },
		1612	.{ .vpmovzxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_128_wig, .avx },
		1613	.{ .vpmovzxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_128_wig, .avx },
		1614	.{ .vpmovzxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_128_wig, .avx },
		1615	.{ .vpmovzxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_128_wig, .avx },
		1616	.{ .vpmovzxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_128_wig, .avx },
		1617
1579	.{ .vpmulhw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx },	1618	.{ .vpmulhw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx },
1580		1619
1581	.{ .vpmulld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx },	1620	.{ .vpmulld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx },
...	@@ -1584,6 +1623,8 @@ pub const table = [_]Entry{	...	@@ -1584,6 +1623,8 @@ pub const table = [_]Entry{
1584		1623
1585	.{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },	1624	.{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },
1586		1625
		1626	.{ .vpshufb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx },
		1627
1587	.{ .vpshufd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },	1628	.{ .vpshufd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
1588		1629
1589	.{ .vpshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },	1630	.{ .vpshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
...	@@ -1728,6 +1769,10 @@ pub const table = [_]Entry{	...	@@ -1728,6 +1769,10 @@ pub const table = [_]Entry{
1728	.{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },	1769	.{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },
1729	.{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },	1770	.{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },
1730		1771
		1772	.{ .vextracti128, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x39 }, 0, .vex_256_w0, .avx2 },
		1773
		1774	.{ .vinserti128, .rvmi, &.{ .ymm, .ymm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x38 }, 0, .vex_256_w0, .avx2 },
		1775
1731	.{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 },	1776	.{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 },
1732	.{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 },	1777	.{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 },
1733	.{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 },	1778	.{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 },
...	@@ -1756,6 +1801,13 @@ pub const table = [_]Entry{	...	@@ -1756,6 +1801,13 @@ pub const table = [_]Entry{
1756		1801
1757	.{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },	1802	.{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
1758		1803
		1804	.{ .vpblendd, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_128_w0, .avx2 },
		1805	.{ .vpblendd, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_256_w0, .avx2 },
		1806
		1807	.{ .vpblendvb, .rvmr, &.{ .ymm, .ymm, .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_256_w0, .avx2 },
		1808
		1809	.{ .vpblendw, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_256_wig, .avx2 },
		1810
1759	.{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },	1811	.{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },
1760	.{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },	1812	.{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },
1761	.{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },	1813	.{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },
...	@@ -1764,7 +1816,7 @@ pub const table = [_]Entry{	...	@@ -1764,7 +1816,7 @@ pub const table = [_]Entry{
1764	.{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },	1816	.{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
1765	.{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },	1817	.{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
1766	.{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },	1818	.{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
1767	.{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },	1819	.{ .vbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
1768		1820
1769	.{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },	1821	.{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
1770	.{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },	1822	.{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },
...	@@ -1799,6 +1851,20 @@ pub const table = [_]Entry{	...	@@ -1799,6 +1851,20 @@ pub const table = [_]Entry{
1799	.{ .vpmovmskb, .rm, &.{ .r32, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },	1851	.{ .vpmovmskb, .rm, &.{ .r32, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },
1800	.{ .vpmovmskb, .rm, &.{ .r64, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },	1852	.{ .vpmovmskb, .rm, &.{ .r64, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },
1801		1853
		1854	.{ .vpmovsxbw, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_256_wig, .avx2 },
		1855	.{ .vpmovsxbd, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_256_wig, .avx2 },
		1856	.{ .vpmovsxbq, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_256_wig, .avx2 },
		1857	.{ .vpmovsxwd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_256_wig, .avx2 },
		1858	.{ .vpmovsxwq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_256_wig, .avx2 },
		1859	.{ .vpmovsxdq, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_256_wig, .avx2 },
		1860
		1861	.{ .vpmovzxbw, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_256_wig, .avx2 },
		1862	.{ .vpmovzxbd, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_256_wig, .avx2 },
		1863	.{ .vpmovzxbq, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_256_wig, .avx2 },
		1864	.{ .vpmovzxwd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_256_wig, .avx2 },
		1865	.{ .vpmovzxwq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_256_wig, .avx2 },
		1866	.{ .vpmovzxdq, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_256_wig, .avx2 },
		1867
1802	.{ .vpmulhw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 },	1868	.{ .vpmulhw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 },
1803		1869
1804	.{ .vpmulld, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 },	1870	.{ .vpmulld, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 },
...	@@ -1807,6 +1873,7 @@ pub const table = [_]Entry{	...	@@ -1807,6 +1873,7 @@ pub const table = [_]Entry{
1807		1873
1808	.{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },	1874	.{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },
1809		1875
		1876	.{ .vpshufb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 },
1810	.{ .vpshufd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },	1877	.{ .vpshufd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },
1811		1878
1812	.{ .vpshufhw, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },	1879	.{ .vpshufhw, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },

src/codegen.zig+27-30

...	@@ -405,7 +405,7 @@ pub fn generateSymbol(	...	@@ -405,7 +405,7 @@ pub fn generateSymbol(
405	.vector_type => \|vector_type\| {	405	.vector_type => \|vector_type\| {
406	const abi_size = math.cast(usize, typed_value.ty.abiSize(mod)) orelse	406	const abi_size = math.cast(usize, typed_value.ty.abiSize(mod)) orelse
407	return error.Overflow;	407	return error.Overflow;
408	if (Type.fromInterned(vector_type.child).bitSize(mod) == 1) {	408	if (vector_type.child == .bool_type) {
409	const bytes = try code.addManyAsSlice(abi_size);	409	const bytes = try code.addManyAsSlice(abi_size);
410	@memset(bytes, 0xaa);	410	@memset(bytes, 0xaa);
411	var index: usize = 0;	411	var index: usize = 0;
...	@@ -443,37 +443,34 @@ pub fn generateSymbol(	...	@@ -443,37 +443,34 @@ pub fn generateSymbol(
443	},	443	},
444	}) byte.* \|= mask else byte.* &= ~mask;	444	}) byte.* \|= mask else byte.* &= ~mask;
445	}	445	}
446	} else switch (aggregate.storage) {	446	} else {
447	.bytes => \|bytes\| try code.appendSlice(bytes),	447	switch (aggregate.storage) {
448	.elems, .repeated_elem => {	448	.bytes => \|bytes\| try code.appendSlice(bytes),
449	var index: u64 = 0;	449	.elems, .repeated_elem => {
450	while (index < vector_type.len) : (index += 1) {	450	var index: u64 = 0;
451	switch (try generateSymbol(bin_file, src_loc, .{	451	while (index < vector_type.len) : (index += 1) {
452	.ty = Type.fromInterned(vector_type.child),	452	switch (try generateSymbol(bin_file, src_loc, .{
453	.val = Value.fromInterned(switch (aggregate.storage) {	453	.ty = Type.fromInterned(vector_type.child),
454	.bytes => unreachable,	454	.val = Value.fromInterned(switch (aggregate.storage) {
455	.elems => \|elems\| elems[	455	.bytes => unreachable,
456	math.cast(usize, index) orelse return error.Overflow	456	.elems => \|elems\| elems[
457	],	457	math.cast(usize, index) orelse return error.Overflow
458	.repeated_elem => \|elem\| elem,	458	],
459	}),	459	.repeated_elem => \|elem\| elem,
460	}, code, debug_output, reloc_info)) {	460	}),
461	.ok => {},	461	}, code, debug_output, reloc_info)) {
462	.fail => \|em\| return .{ .fail = em },	462	.ok => {},
		463	.fail => \|em\| return .{ .fail = em },
		464	}
463	}	465	}
464	}	466	},
465	},	467	}
466	}
467		468
468	const padding = abi_size - (math.cast(usize, math.divCeil(	469	const padding = abi_size -
469	u64,	470	(math.cast(usize, Type.fromInterned(vector_type.child).abiSize(mod) * vector_type.len) orelse
470	Type.fromInterned(vector_type.child).bitSize(mod) * vector_type.len,	471	return error.Overflow);
471	8,	472	if (padding > 0) try code.appendNTimes(0, padding);
472	) catch \|err\| switch (err) {	473	}
473	error.DivisionByZero => unreachable,
474	else => \|e\| return e,
475	}) orelse return error.Overflow);
476	if (padding > 0) try code.appendNTimes(0, padding);
477	},	474	},
478	.anon_struct_type => \|tuple\| {	475	.anon_struct_type => \|tuple\| {
479	const struct_begin = code.items.len;	476	const struct_begin = code.items.len;

src/codegen/c.zig+45-53

...	@@ -4140,9 +4140,7 @@ fn airCmpOp(	...	@@ -4140,9 +4140,7 @@ fn airCmpOp(
4140	if (need_cast) try writer.writeAll("(void*)");	4140	if (need_cast) try writer.writeAll("(void*)");
4141	try f.writeCValue(writer, lhs, .Other);	4141	try f.writeCValue(writer, lhs, .Other);
4142	try v.elem(f, writer);	4142	try v.elem(f, writer);
4143	try writer.writeByte(' ');
4144	try writer.writeAll(compareOperatorC(operator));	4143	try writer.writeAll(compareOperatorC(operator));
4145	try writer.writeByte(' ');
4146	if (need_cast) try writer.writeAll("(void*)");	4144	if (need_cast) try writer.writeAll("(void*)");
4147	try f.writeCValue(writer, rhs, .Other);	4145	try f.writeCValue(writer, rhs, .Other);
4148	try v.elem(f, writer);	4146	try v.elem(f, writer);
...	@@ -4181,41 +4179,28 @@ fn airEquality(	...	@@ -4181,41 +4179,28 @@ fn airEquality(
4181	const writer = f.object.writer();	4179	const writer = f.object.writer();
4182	const inst_ty = f.typeOfIndex(inst);	4180	const inst_ty = f.typeOfIndex(inst);
4183	const local = try f.allocLocal(inst, inst_ty);	4181	const local = try f.allocLocal(inst, inst_ty);
		4182	const a = try Assignment.start(f, writer, inst_ty);
4184	try f.writeCValue(writer, local, .Other);	4183	try f.writeCValue(writer, local, .Other);
4185	try writer.writeAll(" = ");	4184	try a.assign(f, writer);
4186		4185
4187	if (operand_ty.zigTypeTag(mod) == .Optional and !operand_ty.optionalReprIsPayload(mod)) {	4186	if (operand_ty.zigTypeTag(mod) == .Optional and !operand_ty.optionalReprIsPayload(mod)) {
4188	// (A && B) \|\| (C && (A == B))	4187	try f.writeCValueMember(writer, lhs, .{ .identifier = "is_null" });
4189	// A = lhs.is_null ; B = rhs.is_null ; C = rhs.payload == lhs.payload	4188	try writer.writeAll(" \|\| ");
4190		4189	try f.writeCValueMember(writer, rhs, .{ .identifier = "is_null" });
4191	switch (operator) {	4190	try writer.writeAll(" ? ");
4192	.eq => {},	4191	try f.writeCValueMember(writer, lhs, .{ .identifier = "is_null" });
4193	.neq => try writer.writeByte('!'),	4192	try writer.writeAll(compareOperatorC(operator));
4194	else => unreachable,	4193	try f.writeCValueMember(writer, rhs, .{ .identifier = "is_null" });
4195	}	4194	try writer.writeAll(" : ");
4196	try writer.writeAll("((");	4195	try f.writeCValueMember(writer, lhs, .{ .identifier = "payload" });
4197	try f.writeCValue(writer, lhs, .Other);	4196	try writer.writeAll(compareOperatorC(operator));
4198	try writer.writeAll(".is_null && ");	4197	try f.writeCValueMember(writer, rhs, .{ .identifier = "payload" });
4199	try f.writeCValue(writer, rhs, .Other);	4198	} else {
4200	try writer.writeAll(".is_null) \|\| (");
4201	try f.writeCValue(writer, lhs, .Other);
4202	try writer.writeAll(".payload == ");
4203	try f.writeCValue(writer, rhs, .Other);
4204	try writer.writeAll(".payload && ");
4205	try f.writeCValue(writer, lhs, .Other);	4199	try f.writeCValue(writer, lhs, .Other);
4206	try writer.writeAll(".is_null == ");	4200	try writer.writeAll(compareOperatorC(operator));
4207	try f.writeCValue(writer, rhs, .Other);	4201	try f.writeCValue(writer, rhs, .Other);
4208	try writer.writeAll(".is_null));\n");
4209
4210	return local;
4211	}	4202	}
4212		4203	try a.end(f, writer);
4213	try f.writeCValue(writer, lhs, .Other);
4214	try writer.writeByte(' ');
4215	try writer.writeAll(compareOperatorC(operator));
4216	try writer.writeByte(' ');
4217	try f.writeCValue(writer, rhs, .Other);
4218	try writer.writeAll(";\n");
4219		4204
4220	return local;	4205	return local;
4221	}	4206	}
...	@@ -6109,41 +6094,48 @@ fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue {	...	@@ -6109,41 +6094,48 @@ fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue {
6109	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;	6094	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6110		6095
6111	const inst_ty = f.typeOfIndex(inst);	6096	const inst_ty = f.typeOfIndex(inst);
		6097	const inst_scalar_ty = inst_ty.scalarType(mod);
6112	const operand = try f.resolveInst(ty_op.operand);	6098	const operand = try f.resolveInst(ty_op.operand);
6113	try reap(f, inst, &.{ty_op.operand});	6099	try reap(f, inst, &.{ty_op.operand});
6114	const operand_ty = f.typeOf(ty_op.operand);	6100	const operand_ty = f.typeOf(ty_op.operand);
		6101	const scalar_ty = operand_ty.scalarType(mod);
6115	const target = f.object.dg.module.getTarget();	6102	const target = f.object.dg.module.getTarget();
6116	const operation = if (inst_ty.isRuntimeFloat() and operand_ty.isRuntimeFloat())	6103	const operation = if (inst_scalar_ty.isRuntimeFloat() and scalar_ty.isRuntimeFloat())
6117	if (inst_ty.floatBits(target) < operand_ty.floatBits(target)) "trunc" else "extend"	6104	if (inst_scalar_ty.floatBits(target) < scalar_ty.floatBits(target)) "trunc" else "extend"
6118	else if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat())	6105	else if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat())
6119	if (inst_ty.isSignedInt(mod)) "fix" else "fixuns"	6106	if (inst_scalar_ty.isSignedInt(mod)) "fix" else "fixuns"
6120	else if (inst_ty.isRuntimeFloat() and operand_ty.isInt(mod))	6107	else if (inst_scalar_ty.isRuntimeFloat() and scalar_ty.isInt(mod))
6121	if (operand_ty.isSignedInt(mod)) "float" else "floatun"	6108	if (scalar_ty.isSignedInt(mod)) "float" else "floatun"
6122	else	6109	else
6123	unreachable;	6110	unreachable;
6124		6111
6125	const writer = f.object.writer();	6112	const writer = f.object.writer();
6126	const local = try f.allocLocal(inst, inst_ty);	6113	const local = try f.allocLocal(inst, inst_ty);
		6114	const v = try Vectorize.start(f, inst, writer, operand_ty);
		6115	const a = try Assignment.start(f, writer, scalar_ty);
6127	try f.writeCValue(writer, local, .Other);	6116	try f.writeCValue(writer, local, .Other);
6128		6117	try v.elem(f, writer);
6129	try writer.writeAll(" = ");	6118	try a.assign(f, writer);
6130	if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat()) {	6119	if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat()) {
6131	try writer.writeAll("zig_wrap_");	6120	try writer.writeAll("zig_wrap_");
6132	try f.object.dg.renderTypeForBuiltinFnName(writer, inst_ty);	6121	try f.object.dg.renderTypeForBuiltinFnName(writer, inst_scalar_ty);
6133	try writer.writeByte('(');	6122	try writer.writeByte('(');
6134	}	6123	}
6135	try writer.writeAll("zig_");	6124	try writer.writeAll("zig_");
6136	try writer.writeAll(operation);	6125	try writer.writeAll(operation);
6137	try writer.writeAll(compilerRtAbbrev(operand_ty, mod));	6126	try writer.writeAll(compilerRtAbbrev(scalar_ty, mod));
6138	try writer.writeAll(compilerRtAbbrev(inst_ty, mod));	6127	try writer.writeAll(compilerRtAbbrev(inst_scalar_ty, mod));
6139	try writer.writeByte('(');	6128	try writer.writeByte('(');
6140	try f.writeCValue(writer, operand, .FunctionArgument);	6129	try f.writeCValue(writer, operand, .FunctionArgument);
		6130	try v.elem(f, writer);
6141	try writer.writeByte(')');	6131	try writer.writeByte(')');
6142	if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat()) {	6132	if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat()) {
6143	try f.object.dg.renderBuiltinInfo(writer, inst_ty, .bits);	6133	try f.object.dg.renderBuiltinInfo(writer, inst_scalar_ty, .bits);
6144	try writer.writeByte(')');	6134	try writer.writeByte(')');
6145	}	6135	}
6146	try writer.writeAll(";\n");	6136	try a.end(f, writer);
		6137	try v.end(f, inst, writer);
		6138
6147	return local;	6139	return local;
6148	}	6140	}
6149		6141
...	@@ -6315,7 +6307,7 @@ fn airCmpBuiltinCall(	...	@@ -6315,7 +6307,7 @@ fn airCmpBuiltinCall(
6315	try v.elem(f, writer);	6307	try v.elem(f, writer);
6316	try f.object.dg.renderBuiltinInfo(writer, scalar_ty, info);	6308	try f.object.dg.renderBuiltinInfo(writer, scalar_ty, info);
6317	try writer.writeByte(')');	6309	try writer.writeByte(')');
6318	if (!ref_ret) try writer.print(" {s} {}", .{	6310	if (!ref_ret) try writer.print("{s}{}", .{
6319	compareOperatorC(operator),	6311	compareOperatorC(operator),
6320	try f.fmtIntLiteral(Type.i32, try mod.intValue(Type.i32, 0)),	6312	try f.fmtIntLiteral(Type.i32, try mod.intValue(Type.i32, 0)),
6321	});	6313	});
...	@@ -7661,12 +7653,12 @@ fn compareOperatorAbbrev(operator: std.math.CompareOperator) []const u8 {	...	@@ -7661,12 +7653,12 @@ fn compareOperatorAbbrev(operator: std.math.CompareOperator) []const u8 {
7661		7653
7662	fn compareOperatorC(operator: std.math.CompareOperator) []const u8 {	7654	fn compareOperatorC(operator: std.math.CompareOperator) []const u8 {
7663	return switch (operator) {	7655	return switch (operator) {
7664	.lt => "<",	7656	.lt => " < ",
7665	.lte => "<=",	7657	.lte => " <= ",
7666	.eq => "==",	7658	.eq => " == ",
7667	.gte => ">=",	7659	.gte => " >= ",
7668	.gt => ">",	7660	.gt => " > ",
7669	.neq => "!=",	7661	.neq => " != ",
7670	};	7662	};
7671	}	7663	}
7672		7664

src/codegen/llvm.zig+8-2

 @@ -8646,8 +8646,6 @@ pub const FuncGen = struct {
         const operand_ty = self.typeOf(ty_op.operand);
         const dest_ty = self.typeOfIndex(inst);
         const target = mod.getTarget();
-        const dest_bits = dest_ty.floatBits(target);
-        const src_bits = operand_ty.floatBits(target);
         if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
             return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");
 @@ -8655,11 +8653,19 @@ pub const FuncGen = struct {
             const operand_llvm_ty = try o.lowerType(operand_ty);
             const dest_llvm_ty = try o.lowerType(dest_ty);
+            const dest_bits = dest_ty.scalarType(mod).floatBits(target);
+            const src_bits = operand_ty.scalarType(mod).floatBits(target);
             const fn_name = try o.builder.fmt("__extend{s}f{s}f2", .{
                 compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
             });
             const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
+            if (dest_ty.isVector(mod)) return self.buildElementwiseCall(
+                libc_fn,
+                &.{operand},
+                try o.builder.poisonValue(dest_llvm_ty),
+                dest_ty.vectorLen(mod),
+            );
             return self.wip.call(
                 .normal,
                 .ccc,

src/type.zig+40-9

...	@@ -905,11 +905,32 @@ pub const Type = struct {	...	@@ -905,11 +905,32 @@ pub const Type = struct {
905	return Type.fromInterned(array_type.child).abiAlignmentAdvanced(mod, strat);	905	return Type.fromInterned(array_type.child).abiAlignmentAdvanced(mod, strat);
906	},	906	},
907	.vector_type => \|vector_type\| {	907	.vector_type => \|vector_type\| {
908	const bits_u64 = try bitSizeAdvanced(Type.fromInterned(vector_type.child), mod, opt_sema);	908	if (vector_type.len == 0) return .{ .scalar = .@"1" };
909	const bits: u32 = @intCast(bits_u64);	909	switch (mod.comp.getZigBackend()) {
910	const bytes = ((bits * vector_type.len) + 7) / 8;	910	else => {
911	const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);	911	const elem_bits: u32 = @intCast(try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema));
912	return .{ .scalar = Alignment.fromByteUnits(alignment) };	912	if (elem_bits == 0) return .{ .scalar = .@"1" };
		913	const bytes = ((elem_bits * vector_type.len) + 7) / 8;
		914	const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);
		915	return .{ .scalar = Alignment.fromByteUnits(alignment) };
		916	},
		917	.stage2_x86_64 => {
		918	if (vector_type.child == .bool_type) {
		919	if (vector_type.len > 256 and std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return .{ .scalar = .@"64" };
		920	if (vector_type.len > 128 and std.Target.x86.featureSetHas(target.cpu.features, .avx2)) return .{ .scalar = .@"32" };
		921	if (vector_type.len > 64) return .{ .scalar = .@"16" };
		922	const bytes = std.math.divCeil(u32, vector_type.len, 8) catch unreachable;
		923	const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);
		924	return .{ .scalar = Alignment.fromByteUnits(alignment) };
		925	}
		926	const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);
		927	if (elem_bytes == 0) return .{ .scalar = .@"1" };
		928	const bytes = elem_bytes * vector_type.len;
		929	if (bytes > 32 and std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return .{ .scalar = .@"64" };
		930	if (bytes > 16 and std.Target.x86.featureSetHas(target.cpu.features, .avx)) return .{ .scalar = .@"32" };
		931	return .{ .scalar = .@"16" };
		932	},
		933	}
913	},	934	},
914		935
915	.opt_type => return abiAlignmentAdvancedOptional(ty, mod, strat),	936	.opt_type => return abiAlignmentAdvancedOptional(ty, mod, strat),
...	@@ -1237,9 +1258,6 @@ pub const Type = struct {	...	@@ -1237,9 +1258,6 @@ pub const Type = struct {
1237	.storage = .{ .lazy_size = ty.toIntern() },	1258	.storage = .{ .lazy_size = ty.toIntern() },
1238	} }))) },	1259	} }))) },
1239	};	1260	};
1240	const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);
1241	const total_bits = elem_bits * vector_type.len;
1242	const total_bytes = (total_bits + 7) / 8;
1243	const alignment = switch (try ty.abiAlignmentAdvanced(mod, strat)) {	1261	const alignment = switch (try ty.abiAlignmentAdvanced(mod, strat)) {
1244	.scalar => \|x\| x,	1262	.scalar => \|x\| x,
1245	.val => return .{ .val = Value.fromInterned((try mod.intern(.{ .int = .{	1263	.val => return .{ .val = Value.fromInterned((try mod.intern(.{ .int = .{
...	@@ -1247,6 +1265,18 @@ pub const Type = struct {	...	@@ -1247,6 +1265,18 @@ pub const Type = struct {
1247	.storage = .{ .lazy_size = ty.toIntern() },	1265	.storage = .{ .lazy_size = ty.toIntern() },
1248	} }))) },	1266	} }))) },
1249	};	1267	};
		1268	const total_bytes = switch (mod.comp.getZigBackend()) {
		1269	else => total_bytes: {
		1270	const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);
		1271	const total_bits = elem_bits * vector_type.len;
		1272	break :total_bytes (total_bits + 7) / 8;
		1273	},
		1274	.stage2_x86_64 => total_bytes: {
		1275	if (vector_type.child == .bool_type) break :total_bytes std.math.divCeil(u32, vector_type.len, 8) catch unreachable;
		1276	const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);
		1277	break :total_bytes elem_bytes * vector_type.len;
		1278	},
		1279	};
1250	return AbiSizeAdvanced{ .scalar = alignment.forward(total_bytes) };	1280	return AbiSizeAdvanced{ .scalar = alignment.forward(total_bytes) };
1251	},	1281	},
1252		1282
...	@@ -2108,7 +2138,8 @@ pub const Type = struct {	...	@@ -2108,7 +2138,8 @@ pub const Type = struct {
2108		2138
2109	/// Returns true if and only if the type is a fixed-width integer.	2139	/// Returns true if and only if the type is a fixed-width integer.
2110	pub fn isInt(self: Type, mod: *const Module) bool {	2140	pub fn isInt(self: Type, mod: *const Module) bool {
2111	return self.isSignedInt(mod) or self.isUnsignedInt(mod);	2141	return self.toIntern() != .comptime_int_type and
		2142	mod.intern_pool.isIntegerType(self.toIntern());
2112	}	2143	}
2113		2144
2114	/// Returns true if and only if the type is a fixed-width, signed integer.	2145	/// Returns true if and only if the type is a fixed-width, signed integer.

test/behavior/bitcast.zig+1-1

...	@@ -336,7 +336,7 @@ test "comptime @bitCast packed struct to int and back" {	...	@@ -336,7 +336,7 @@ test "comptime @bitCast packed struct to int and back" {
336	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;	336	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
337	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	337	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
338	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;	338	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
339	if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;	339	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
340		340
341	if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {	341	if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
342	// https://github.com/ziglang/zig/issues/13782	342	// https://github.com/ziglang/zig/issues/13782

test/behavior/cast.zig+33-21

 @@ -601,25 +601,25 @@ test "cast *[1][*]const u8 to [*]const ?[*]const u8" {
 test "@intCast on vector" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     const S = struct {
         fn doTheTest() !void {
             // Upcast (implicit, equivalent to @intCast)
             var up0: @Vector(2, u8) = [_]u8{ 0x55, 0xaa };
             _ = &up0;
-            const up1 = @as(@Vector(2, u16), up0);
+            const up1: @Vector(2, u16) = up0;
-            const up2 = @as(@Vector(2, u32), up0);
+            const up2: @Vector(2, u32) = up0;
-            const up3 = @as(@Vector(2, u64), up0);
+            const up3: @Vector(2, u64) = up0;
             // Downcast (safety-checked)
             var down0 = up3;
             _ = &down0;
-            const down1 = @as(@Vector(2, u32), @intCast(down0));
+            const down1: @Vector(2, u32) = @intCast(down0);
-            const down2 = @as(@Vector(2, u16), @intCast(down0));
+            const down2: @Vector(2, u16) = @intCast(down0);
-            const down3 = @as(@Vector(2, u8), @intCast(down0));
+            const down3: @Vector(2, u8) = @intCast(down0);
             try expect(mem.eql(u16, &@as([2]u16, up1), &[2]u16{ 0x55, 0xaa }));
             try expect(mem.eql(u32, &@as([2]u32, up2), &[2]u32{ 0x55, 0xaa }));
 @@ -629,20 +629,10 @@ test "@intCast on vector" {
             try expect(mem.eql(u16, &@as([2]u16, down2), &[2]u16{ 0x55, 0xaa }));
             try expect(mem.eql(u8, &@as([2]u8, down3), &[2]u8{ 0x55, 0xaa }));
         }
-        fn doTheTestFloat() !void {
-            var vec: @Vector(2, f32) = @splat(1234.0);
-            _ = &vec;
-            const wider: @Vector(2, f64) = vec;
-            try expect(wider[0] == 1234.0);
-            try expect(wider[1] == 1234.0);
-        }
     };
     try S.doTheTest();
     try comptime S.doTheTest();
-    try S.doTheTestFloat();
-    try comptime S.doTheTestFloat();
 }
 test "@floatCast cast down" {
 @@ -2340,10 +2330,31 @@ test "@floatCast on vector" {
     const S = struct {
         fn doTheTest() !void {
-            var a: @Vector(3, f64) = .{ 1.5, 2.5, 3.5 };
+            {
-            _ = &a;
+                var a: @Vector(2, f64) = .{ 1.5, 2.5 };
-            const b: @Vector(3, f32) = @floatCast(a);
+                _ = &a;
-            try expectEqual(@Vector(3, f32){ 1.5, 2.5, 3.5 }, b);
+                const b: @Vector(2, f32) = @floatCast(a);
+                try expectEqual(@Vector(2, f32){ 1.5, 2.5 }, b);
+            }
+            {
+                var a: @Vector(2, f32) = .{ 3.25, 4.25 };
+                _ = &a;
+                const b: @Vector(2, f64) = @floatCast(a);
+                try expectEqual(@Vector(2, f64){ 3.25, 4.25 }, b);
+            }
+            {
+                var a: @Vector(2, f32) = .{ 5.75, 6.75 };
+                _ = &a;
+                const b: @Vector(2, f64) = a;
+                try expectEqual(@Vector(2, f64){ 5.75, 6.75 }, b);
+            }
+            {
+                var vec: @Vector(2, f32) = @splat(1234.0);
+                _ = &vec;
+                const wider: @Vector(2, f64) = vec;
+                try expect(wider[0] == 1234.0);
+                try expect(wider[1] == 1234.0);
+            }
         }
     };
 @@ -2441,6 +2452,7 @@ test "@intFromBool on vector" {
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     const S = struct {
         fn doTheTest() !void {

test/behavior/optional.zig+77-32

...	@@ -110,44 +110,89 @@ test "nested optional field in struct" {	...	@@ -110,44 +110,89 @@ test "nested optional field in struct" {
110	try expect(s.x.?.y == 127);	110	try expect(s.x.?.y == 127);
111	}	111	}
112		112
113	test "equality compare optional with non-optional" {	113	test "equality compare optionals and non-optionals" {
114	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;	114	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
115	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO	115	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
116	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO	116	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
117		117
118	try test_cmp_optional_non_optional();	118	const S = struct {
119	try comptime test_cmp_optional_non_optional();	119	fn doTheTest() !void {
		120	var five: isize = 5;
		121	var ten: isize = 10;
		122	var opt_null: ?isize = null;
		123	var opt_ten: ?isize = 10;
		124	_ = .{ &five, &ten, &opt_null, &opt_ten };
		125	try expect(opt_null != five);
		126	try expect(opt_null != ten);
		127	try expect(opt_ten != five);
		128	try expect(opt_ten == ten);
		129
		130	var opt_int: ?isize = null;
		131	try expect(opt_int != five);
		132	try expect(opt_int != ten);
		133	try expect(opt_int == opt_null);
		134	try expect(opt_int != opt_ten);
		135
		136	opt_int = 10;
		137	try expect(opt_int != five);
		138	try expect(opt_int == ten);
		139	try expect(opt_int != opt_null);
		140	try expect(opt_int == opt_ten);
		141
		142	opt_int = five;
		143	try expect(opt_int == five);
		144	try expect(opt_int != ten);
		145	try expect(opt_int != opt_null);
		146	try expect(opt_int != opt_ten);
		147
		148	// test evaluation is always lexical
		149	// ensure that the optional isn't always computed before the non-optional
		150	var mutable_state: i32 = 0;
		151	_ = blk1: {
		152	mutable_state += 1;
		153	break :blk1 @as(?f64, 10.0);
		154	} != blk2: {
		155	try expect(mutable_state == 1);
		156	break :blk2 @as(f64, 5.0);
		157	};
		158	_ = blk1: {
		159	mutable_state += 1;
		160	break :blk1 @as(f64, 10.0);
		161	} != blk2: {
		162	try expect(mutable_state == 2);
		163	break :blk2 @as(?f64, 5.0);
		164	};
		165	}
		166	};
		167
		168	try S.doTheTest();
		169	try comptime S.doTheTest();
120	}	170	}
121		171
122	fn test_cmp_optional_non_optional() !void {	172	test "compare optionals with modified payloads" {
123	var ten: i32 = 10;	173	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
124	var opt_ten: ?i32 = 10;	174
125	var five: i32 = 5;	175	var lhs: ?bool = false;
126	var int_n: ?i32 = null;	176	const lhs_payload = &lhs.?;
127		177	var rhs: ?bool = true;
128	_ = .{ &ten, &opt_ten, &five, &int_n };	178	const rhs_payload = &rhs.?;
129		179	try expect(lhs != rhs and !(lhs == rhs));
130	try expect(int_n != ten);	180
131	try expect(opt_ten == ten);	181	lhs = null;
132	try expect(opt_ten != five);	182	lhs_payload.* = false;
133		183	rhs = false;
134	// test evaluation is always lexical	184	try expect(lhs != rhs and !(lhs == rhs));
135	// ensure that the optional isn't always computed before the non-optional	185
136	var mutable_state: i32 = 0;	186	lhs = true;
137	_ = blk1: {	187	rhs = null;
138	mutable_state += 1;	188	rhs_payload.* = true;
139	break :blk1 @as(?f64, 10.0);	189	try expect(lhs != rhs and !(lhs == rhs));
140	} != blk2: {	190
141	try expect(mutable_state == 1);	191	lhs = null;
142	break :blk2 @as(f64, 5.0);	192	lhs_payload.* = false;
143	};	193	rhs = null;
144	_ = blk1: {	194	rhs_payload.* = true;
145	mutable_state += 1;	195	try expect(lhs == rhs and !(lhs != rhs));
146	break :blk1 @as(f64, 10.0);
147	} != blk2: {
148	try expect(mutable_state == 2);
149	break :blk2 @as(?f64, 5.0);
150	};
151	}	196	}
152		197
153	test "unwrap function call with optional pointer return value" {	198	test "unwrap function call with optional pointer return value" {

test/behavior/select.zig+2-2

 @@ -5,7 +5,6 @@ const expect = std.testing.expect;
 test "@select vectors" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
 @@ -36,11 +35,12 @@ fn selectVectors() !void {
 test "@select arrays" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx2)) return error.SkipZigTest;
     try comptime selectArrays();
     try selectArrays();

test/behavior/shuffle.zig+2-1

 @@ -4,10 +4,11 @@ const mem = std.mem;
 const expect = std.testing.expect;
 test "@shuffle int" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;
     const S = struct {
         fn doTheTest() !void {

test/behavior/vector.zig+29-13

 @@ -29,7 +29,7 @@ test "vector wrap operators" {
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_x86_64 and
-        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest;
     const S = struct {
         fn doTheTest() !void {
 @@ -906,22 +906,26 @@ test "vector @reduce comptime" {
 }
 test "mask parameter of @shuffle is comptime scope" {
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64 and
+        !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;
     const __v4hi = @Vector(4, i16);
-    var v4_a = __v4hi{ 0, 0, 0, 0 };
+    var v4_a = __v4hi{ 1, 2, 3, 4 };
-    var v4_b = __v4hi{ 0, 0, 0, 0 };
+    var v4_b = __v4hi{ 5, 6, 7, 8 };
     _ = .{ &v4_a, &v4_b };
     const shuffled: __v4hi = @shuffle(i16, v4_a, v4_b, @Vector(4, i32){
         std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
-        std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
+        std.zig.c_translation.shuffleVectorIndex(2, @typeInfo(@TypeOf(v4_a)).Vector.len),
-        std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
+        std.zig.c_translation.shuffleVectorIndex(4, @typeInfo(@TypeOf(v4_a)).Vector.len),
-        std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
+        std.zig.c_translation.shuffleVectorIndex(6, @typeInfo(@TypeOf(v4_a)).Vector.len),
     });
-    _ = shuffled;
+    try expect(shuffled[0] == 1);
+    try expect(shuffled[1] == 3);
+    try expect(shuffled[2] == 5);
+    try expect(shuffled[3] == 7);
 }
 test "saturating add" {
 @@ -1177,10 +1181,22 @@ test "@shlWithOverflow" {
 }
 test "alignment of vectors" {
-    try expect(@alignOf(@Vector(2, u8)) == 2);
+    try expect(@alignOf(@Vector(2, u8)) == switch (builtin.zig_backend) {
-    try expect(@alignOf(@Vector(2, u1)) == 1);
+        else => 2,
-    try expect(@alignOf(@Vector(1, u1)) == 1);
+        .stage2_x86_64 => 16,
-    try expect(@alignOf(@Vector(2, u16)) == 4);
+    });
+    try expect(@alignOf(@Vector(2, u1)) == switch (builtin.zig_backend) {
+        else => 1,
+        .stage2_x86_64 => 16,
+    });
+    try expect(@alignOf(@Vector(1, u1)) == switch (builtin.zig_backend) {
+        else => 1,
+        .stage2_x86_64 => 16,
+    });
+    try expect(@alignOf(@Vector(2, u16)) == switch (builtin.zig_backend) {
+        else => 4,
+        .stage2_x86_64 => 16,
+    });
 }
 test "loading the second vector from a slice of vectors" {
 @@ -1316,10 +1332,10 @@ test "modRem with zero divisor" {
 test "array operands to shuffle are coerced to vectors" {
     if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
-    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
+    if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
     const mask = [5]i32{ -1, 0, 1, 2, 3 };