Commit 91fb211faa3f37d08da55b0c8df92a6475624316 | zig

author	Andrew Kelley <andrew@ziglang.org> 2024-02-25 21:43:20-08:00
committer	GitHub <noreply@github.com> 2024-02-25 21:43:20-08:00
log	91fb211faa3f37d08da55b0c8df92a6475624316
tree	ad824c8fcdd386e378669c21a3e65923b52d0133
parent	d656c2a7abe90d00ef6dbc3731b82bd26180038a
parent	4fcc750ba58f51606c49310bdd7c81c156d48cfc
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #18906 from jacobly0/x86_64-tests

x86_64: pass more tests

24 files changed, 2239 insertions(+), 666 deletions(-)

lib
- std
  - crypto
    aes.zig
    blake3.zig
    salsa20.zig
    sha2.zig
  - meta.zig
  - unicode.zig
  - zig
    c_translation.zig
src
- InternPool.zig
- Sema.zig
- arch
  - x86_64
    CodeGen.zig
    Encoding.zig
    Lower.zig
    Mir.zig
    encodings.zig
- codegen.zig
- codegen
  - c.zig
  - llvm.zig
- type.zig
test
- behavior

lib/std/crypto/aes.zig+1-1

...	...	@@ -6,7 +6,7 @@ const has_aesni = std.Target.x86.featureSetHas(builtin.cpu.features, .aes);
6	6	const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);
7	7	const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);
8	8	// C backend doesn't currently support passing vectors to inline asm.
9		const impl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and has_aesni and has_avx) impl: {
	9	const impl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_c and has_aesni and has_avx) impl: {
10	10	break :impl @import("aes/aesni.zig");
11	11	} else if (builtin.cpu.arch == .aarch64 and builtin.zig_backend != .stage2_c and has_armaes)
12	12	impl: {

lib/std/crypto/blake3.zig+1-1

...	...	@@ -200,7 +200,7 @@ const CompressGeneric = struct {
200	200	}
201	201	};
202	202
203		const compress = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64)
	203	const compress = if (builtin.cpu.arch == .x86_64)
204	204	CompressVectorized.compress
205	205	else
206	206	CompressGeneric.compress;

lib/std/crypto/salsa20.zig+4-1

...	...	@@ -302,7 +302,10 @@ fn SalsaNonVecImpl(comptime rounds: comptime_int) type {
302	302	};
303	303	}
304	304
305		const SalsaImpl = if (builtin.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_x86_64) SalsaVecImpl else SalsaNonVecImpl;
	305	const SalsaImpl = if (builtin.cpu.arch == .x86_64)
	306	SalsaVecImpl
	307	else
	308	SalsaNonVecImpl;
306	309
307	310	fn keyToWords(key: [32]u8) [8]u32 {
308	311	var k: [8]u32 = undefined;

lib/std/crypto/sha2.zig+1-1

...	...	@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
238	238	return;
239	239	},
240	240	// C backend doesn't currently support passing vectors to inline asm.
241		.x86_64 => if (builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
	241	.x86_64 => if (builtin.zig_backend != .stage2_c and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
242	242	var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
243	243	var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
244	244	const s_v = @as(*[16]v4u32, @ptrCast(&s));

lib/std/meta.zig+2-1

...	...	@@ -1286,5 +1286,6 @@ test "hasUniqueRepresentation" {
1286	1286	try testing.expect(!hasUniqueRepresentation([]u8));
1287	1287	try testing.expect(!hasUniqueRepresentation([]const u8));
1288	1288
1289		try testing.expect(hasUniqueRepresentation(@Vector(4, u16)));
	1289	try testing.expect(hasUniqueRepresentation(@Vector(std.simd.suggestVectorLength(u8) orelse 1, u8)));
	1290	try testing.expect(@sizeOf(@Vector(3, u8)) == 3 or !hasUniqueRepresentation(@Vector(3, u8)));
1290	1291	}

lib/std/unicode.zig+147-152

...	...	@@ -239,18 +239,19 @@ pub fn utf8ValidateSlice(input: []const u8) bool {
239	239	fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {
240	240	var remaining = input;
241	241
242		const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
243		const Chunk = @Vector(chunk_len, u8);
244
245		// Fast path. Check for and skip ASCII characters at the start of the input.
246		while (remaining.len >= chunk_len) {
247		const chunk: Chunk = remaining[0..chunk_len].*;
248		const mask: Chunk = @splat(0x80);
249		if (@reduce(.Or, chunk & mask == mask)) {
250		// found a non ASCII byte
251		break;
	242	if (std.simd.suggestVectorLength(u8)) \|chunk_len\| {
	243	const Chunk = @Vector(chunk_len, u8);
	244
	245	// Fast path. Check for and skip ASCII characters at the start of the input.
	246	while (remaining.len >= chunk_len) {
	247	const chunk: Chunk = remaining[0..chunk_len].*;
	248	const mask: Chunk = @splat(0x80);
	249	if (@reduce(.Or, chunk & mask == mask)) {
	250	// found a non ASCII byte
	251	break;
	252	}
	253	remaining = remaining[chunk_len..];
252	254	}
253		remaining = remaining[chunk_len..];
254	255	}
255	256
256	257	// default lowest and highest continuation byte
...	...	@@ -601,9 +602,9 @@ fn testUtf8IteratorOnAscii() !void {
601	602	const s = Utf8View.initComptime("abc");
602	603
603	604	var it1 = s.iterator();
604		try testing.expect(std.mem.eql(u8, "a", it1.nextCodepointSlice().?));
605		try testing.expect(std.mem.eql(u8, "b", it1.nextCodepointSlice().?));
606		try testing.expect(std.mem.eql(u8, "c", it1.nextCodepointSlice().?));
	605	try testing.expect(mem.eql(u8, "a", it1.nextCodepointSlice().?));
	606	try testing.expect(mem.eql(u8, "b", it1.nextCodepointSlice().?));
	607	try testing.expect(mem.eql(u8, "c", it1.nextCodepointSlice().?));
607	608	try testing.expect(it1.nextCodepointSlice() == null);
608	609
609	610	var it2 = s.iterator();
...	...	@@ -631,9 +632,9 @@ fn testUtf8ViewOk() !void {
631	632	const s = Utf8View.initComptime("東京市");
632	633
633	634	var it1 = s.iterator();
634		try testing.expect(std.mem.eql(u8, "東", it1.nextCodepointSlice().?));
635		try testing.expect(std.mem.eql(u8, "京", it1.nextCodepointSlice().?));
636		try testing.expect(std.mem.eql(u8, "市", it1.nextCodepointSlice().?));
	635	try testing.expect(mem.eql(u8, "東", it1.nextCodepointSlice().?));
	636	try testing.expect(mem.eql(u8, "京", it1.nextCodepointSlice().?));
	637	try testing.expect(mem.eql(u8, "市", it1.nextCodepointSlice().?));
637	638	try testing.expect(it1.nextCodepointSlice() == null);
638	639
639	640	var it2 = s.iterator();
...	...	@@ -771,20 +772,20 @@ fn testUtf8Peeking() !void {
771	772	const s = Utf8View.initComptime("noël");
772	773	var it = s.iterator();
773	774
774		try testing.expect(std.mem.eql(u8, "n", it.nextCodepointSlice().?));
	775	try testing.expect(mem.eql(u8, "n", it.nextCodepointSlice().?));
775	776
776		try testing.expect(std.mem.eql(u8, "o", it.peek(1)));
777		try testing.expect(std.mem.eql(u8, "oë", it.peek(2)));
778		try testing.expect(std.mem.eql(u8, "oël", it.peek(3)));
779		try testing.expect(std.mem.eql(u8, "oël", it.peek(4)));
780		try testing.expect(std.mem.eql(u8, "oël", it.peek(10)));
	777	try testing.expect(mem.eql(u8, "o", it.peek(1)));
	778	try testing.expect(mem.eql(u8, "oë", it.peek(2)));
	779	try testing.expect(mem.eql(u8, "oël", it.peek(3)));
	780	try testing.expect(mem.eql(u8, "oël", it.peek(4)));
	781	try testing.expect(mem.eql(u8, "oël", it.peek(10)));
781	782
782		try testing.expect(std.mem.eql(u8, "o", it.nextCodepointSlice().?));
783		try testing.expect(std.mem.eql(u8, "ë", it.nextCodepointSlice().?));
784		try testing.expect(std.mem.eql(u8, "l", it.nextCodepointSlice().?));
	783	try testing.expect(mem.eql(u8, "o", it.nextCodepointSlice().?));
	784	try testing.expect(mem.eql(u8, "ë", it.nextCodepointSlice().?));
	785	try testing.expect(mem.eql(u8, "l", it.nextCodepointSlice().?));
785	786	try testing.expect(it.nextCodepointSlice() == null);
786	787
787		try testing.expect(std.mem.eql(u8, &[_]u8{}, it.peek(1)));
	788	try testing.expect(mem.eql(u8, &[_]u8{}, it.peek(1)));
788	789	}
789	790
790	791	fn testError(bytes: []const u8, expected_err: anyerror) !void {
...	...	@@ -926,59 +927,50 @@ test "fmtUtf8" {
926	927	}
927	928
928	929	fn utf16LeToUtf8ArrayListImpl(
929		array_list: *std.ArrayList(u8),
	930	result: *std.ArrayList(u8),
930	931	utf16le: []const u16,
931	932	comptime surrogates: Surrogates,
932	933	) (switch (surrogates) {
933	934	.cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
934	935	.can_encode_surrogate_half => mem.Allocator.Error,
935	936	})!void {
936		// optimistically guess that it will all be ascii.
937		try array_list.ensureTotalCapacityPrecise(utf16le.len);
	937	assert(result.capacity >= utf16le.len);
938	938
939	939	var remaining = utf16le;
940		if (builtin.zig_backend != .stage2_x86_64) {
941		const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;
	940	vectorized: {
	941	const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
942	942	const Chunk = @Vector(chunk_len, u16);
943	943
944	944	// Fast path. Check for and encode ASCII characters at the start of the input.
945	945	while (remaining.len >= chunk_len) {
946	946	const chunk: Chunk = remaining[0..chunk_len].*;
947		const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));
	947	const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
948	948	if (@reduce(.Or, chunk \| mask != mask)) {
949	949	// found a non ASCII code unit
950	950	break;
951	951	}
952		const chunk_byte_len = chunk_len * 2;
953		const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
954		const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
955		const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
	952	const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
956	953	// We allocated enough space to encode every UTF-16 code unit
957	954	// as ASCII, so if the entire string is ASCII then we are
958	955	// guaranteed to have enough space allocated
959		array_list.appendSliceAssumeCapacity(&ascii_bytes);
	956	result.addManyAsArrayAssumeCapacity(chunk_len).* = ascii_chunk;
960	957	remaining = remaining[chunk_len..];
961	958	}
962	959	}
963	960
964		var out_index: usize = array_list.items.len;
965	961	switch (surrogates) {
966	962	.cannot_encode_surrogate_half => {
967	963	var it = Utf16LeIterator.init(remaining);
968	964	while (try it.nextCodepoint()) \|codepoint\| {
969	965	const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
970		try array_list.resize(array_list.items.len + utf8_len);
971		assert((utf8Encode(codepoint, array_list.items[out_index..]) catch unreachable) == utf8_len);
972		out_index += utf8_len;
	966	assert((utf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
973	967	}
974	968	},
975	969	.can_encode_surrogate_half => {
976	970	var it = Wtf16LeIterator.init(remaining);
977	971	while (it.nextCodepoint()) \|codepoint\| {
978	972	const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
979		try array_list.resize(array_list.items.len + utf8_len);
980		assert((wtf8Encode(codepoint, array_list.items[out_index..]) catch unreachable) == utf8_len);
981		out_index += utf8_len;
	973	assert((wtf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
982	974	}
983	975	},
984	976	}
...	...	@@ -986,8 +978,9 @@ fn utf16LeToUtf8ArrayListImpl(
986	978
987	979	pub const Utf16LeToUtf8AllocError = mem.Allocator.Error \|\| Utf16LeToUtf8Error;
988	980
989		pub fn utf16LeToUtf8ArrayList(array_list: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
990		return utf16LeToUtf8ArrayListImpl(array_list, utf16le, .cannot_encode_surrogate_half);
	981	pub fn utf16LeToUtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
	982	try result.ensureTotalCapacityPrecise(utf16le.len);
	983	return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
991	984	}
992	985
993	986	/// Deprecated; renamed to utf16LeToUtf8Alloc
...	...	@@ -999,8 +992,7 @@ pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) Utf16L
999	992	var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);
1000	993	errdefer result.deinit();
1001	994
1002		try utf16LeToUtf8ArrayList(&result, utf16le);
1003
	995	try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
1004	996	return result.toOwnedSlice();
1005	997	}
1006	998
...	...	@@ -1013,8 +1005,7 @@ pub fn utf16LeToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) Utf16
1013	1005	var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);
1014	1006	errdefer result.deinit();
1015	1007
1016		try utf16LeToUtf8ArrayList(&result, utf16le);
1017
	1008	try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
1018	1009	return result.toOwnedSliceSentinel(0);
1019	1010	}
1020	1011
...	...	@@ -1026,27 +1017,24 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1026	1017	.cannot_encode_surrogate_half => Utf16LeToUtf8Error,
1027	1018	.can_encode_surrogate_half => error{},
1028	1019	})!usize {
1029		var end_index: usize = 0;
	1020	var dest_index: usize = 0;
1030	1021
1031	1022	var remaining = utf16le;
1032		if (builtin.zig_backend != .stage2_x86_64) {
1033		const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;
	1023	vectorized: {
	1024	const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1034	1025	const Chunk = @Vector(chunk_len, u16);
1035	1026
1036	1027	// Fast path. Check for and encode ASCII characters at the start of the input.
1037	1028	while (remaining.len >= chunk_len) {
1038	1029	const chunk: Chunk = remaining[0..chunk_len].*;
1039		const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));
	1030	const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
1040	1031	if (@reduce(.Or, chunk \| mask != mask)) {
1041	1032	// found a non ASCII code unit
1042	1033	break;
1043	1034	}
1044		const chunk_byte_len = chunk_len * 2;
1045		const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
1046		const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
1047		const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
1048		@memcpy(utf8[end_index .. end_index + chunk_len], &ascii_bytes);
1049		end_index += chunk_len;
	1035	const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
	1036	utf8[dest_index..][0..chunk_len].* = ascii_chunk;
	1037	dest_index += chunk_len;
1050	1038	remaining = remaining[chunk_len..];
1051	1039	}
1052	1040	}
...	...	@@ -1055,7 +1043,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1055	1043	.cannot_encode_surrogate_half => {
1056	1044	var it = Utf16LeIterator.init(remaining);
1057	1045	while (try it.nextCodepoint()) \|codepoint\| {
1058		end_index += utf8Encode(codepoint, utf8[end_index..]) catch \|err\| switch (err) {
	1046	dest_index += utf8Encode(codepoint, utf8[dest_index..]) catch \|err\| switch (err) {
1059	1047	// The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
1060	1048	// which is within the valid codepoint range.
1061	1049	error.CodepointTooLarge => unreachable,
...	...	@@ -1068,7 +1056,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1068	1056	.can_encode_surrogate_half => {
1069	1057	var it = Wtf16LeIterator.init(remaining);
1070	1058	while (it.nextCodepoint()) \|codepoint\| {
1071		end_index += wtf8Encode(codepoint, utf8[end_index..]) catch \|err\| switch (err) {
	1059	dest_index += wtf8Encode(codepoint, utf8[dest_index..]) catch \|err\| switch (err) {
1072	1060	// The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
1073	1061	// which is within the valid codepoint range.
1074	1062	error.CodepointTooLarge => unreachable,
...	...	@@ -1076,7 +1064,7 @@ fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surr
1076	1064	}
1077	1065	},
1078	1066	}
1079		return end_index;
	1067	return dest_index;
1080	1068	}
1081	1069
1082	1070	/// Deprecated; renamed to utf16LeToUtf8
...	...	@@ -1149,14 +1137,12 @@ test utf16LeToUtf8 {
1149	1137	}
1150	1138	}
1151	1139
1152		fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
1153		// optimistically guess that it will not require surrogate pairs
1154		try array_list.ensureTotalCapacityPrecise(utf8.len);
	1140	fn utf8ToUtf16LeArrayListImpl(result: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
	1141	assert(result.capacity >= utf8.len);
1155	1142
1156	1143	var remaining = utf8;
1157		// Need support for std.simd.interlace
1158		if (builtin.zig_backend != .stage2_x86_64 and comptime !builtin.cpu.arch.isMIPS()) {
1159		const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
	1144	vectorized: {
	1145	const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1160	1146	const Chunk = @Vector(chunk_len, u8);
1161	1147
1162	1148	// Fast path. Check for and encode ASCII characters at the start of the input.
...	...	@@ -1167,9 +1153,8 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,
1167	1153	// found a non ASCII code unit
1168	1154	break;
1169	1155	}
1170		const zeroes: Chunk = @splat(0);
1171		const utf16_chunk: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });
1172		array_list.appendSliceAssumeCapacity(std.mem.bytesAsSlice(u16, &utf16_chunk));
	1156	const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
	1157	result.addManyAsArrayAssumeCapacity(chunk_len).* = utf16_chunk;
1173	1158	remaining = remaining[chunk_len..];
1174	1159	}
1175	1160	}
...	...	@@ -1181,21 +1166,18 @@ fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8,
1181	1166	var it = view.iterator();
1182	1167	while (it.nextCodepoint()) \|codepoint\| {
1183	1168	if (codepoint < 0x10000) {
1184		const short = @as(u16, @intCast(codepoint));
1185		try array_list.append(mem.nativeToLittle(u16, short));
	1169	try result.append(mem.nativeToLittle(u16, @intCast(codepoint)));
1186	1170	} else {
1187	1171	const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
1188	1172	const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
1189		var out: [2]u16 = undefined;
1190		out[0] = mem.nativeToLittle(u16, high);
1191		out[1] = mem.nativeToLittle(u16, low);
1192		try array_list.appendSlice(out[0..]);
	1173	try result.appendSlice(&.{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) });
1193	1174	}
1194	1175	}
1195	1176	}
1196	1177
1197		pub fn utf8ToUtf16LeArrayList(array_list: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
1198		return utf8ToUtf16LeArrayListImpl(array_list, utf8, .cannot_encode_surrogate_half);
	1178	pub fn utf8ToUtf16LeArrayList(result: *std.ArrayList(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
	1179	try result.ensureTotalCapacityPrecise(utf8.len);
	1180	return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
1199	1181	}
1200	1182
1201	1183	pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {
...	...	@@ -1204,7 +1186,6 @@ pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) error{ Inv
1204	1186	errdefer result.deinit();
1205	1187
1206	1188	try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
1207
1208	1189	return result.toOwnedSlice();
1209	1190	}
1210	1191
...	...	@@ -1217,7 +1198,6 @@ pub fn utf8ToUtf16LeAllocZ(allocator: mem.Allocator, utf8: []const u8) error{ In
1217	1198	errdefer result.deinit();
1218	1199
1219	1200	try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
1220
1221	1201	return result.toOwnedSliceSentinel(0);
1222	1202	}
1223	1203
...	...	@@ -1228,12 +1208,11 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) error{InvalidUtf8}!usize
1228	1208	}
1229	1209
1230	1210	pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates: Surrogates) !usize {
1231		var dest_i: usize = 0;
	1211	var dest_index: usize = 0;
1232	1212
1233	1213	var remaining = utf8;
1234		// Need support for std.simd.interlace
1235		if (builtin.zig_backend != .stage2_x86_64 and comptime !builtin.cpu.arch.isMIPS()) {
1236		const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
	1214	vectorized: {
	1215	const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
1237	1216	const Chunk = @Vector(chunk_len, u8);
1238	1217
1239	1218	// Fast path. Check for and encode ASCII characters at the start of the input.
...	...	@@ -1244,57 +1223,60 @@ pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates:
1244	1223	// found a non ASCII code unit
1245	1224	break;
1246	1225	}
1247		const zeroes: Chunk = @splat(0);
1248		const utf16_bytes: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });
1249		@memcpy(utf16le[dest_i..][0..chunk_len], std.mem.bytesAsSlice(u16, &utf16_bytes));
1250		dest_i += chunk_len;
	1226	const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
	1227	utf16le[dest_index..][0..chunk_len].* = utf16_chunk;
	1228	dest_index += chunk_len;
1251	1229	remaining = remaining[chunk_len..];
1252	1230	}
1253	1231	}
1254	1232
1255		var src_i: usize = 0;
1256		while (src_i < remaining.len) {
1257		const n = utf8ByteSequenceLength(remaining[src_i]) catch return switch (surrogates) {
1258		.cannot_encode_surrogate_half => error.InvalidUtf8,
1259		.can_encode_surrogate_half => error.InvalidWtf8,
1260		};
1261		const next_src_i = src_i + n;
1262		const codepoint = switch (surrogates) {
1263		.cannot_encode_surrogate_half => utf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidUtf8,
1264		.can_encode_surrogate_half => wtf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidWtf8,
1265		};
	1233	const view = switch (surrogates) {
	1234	.cannot_encode_surrogate_half => try Utf8View.init(remaining),
	1235	.can_encode_surrogate_half => try Wtf8View.init(remaining),
	1236	};
	1237	var it = view.iterator();
	1238	while (it.nextCodepoint()) \|codepoint\| {
1266	1239	if (codepoint < 0x10000) {
1267		const short = @as(u16, @intCast(codepoint));
1268		utf16le[dest_i] = mem.nativeToLittle(u16, short);
1269		dest_i += 1;
	1240	utf16le[dest_index] = mem.nativeToLittle(u16, @intCast(codepoint));
	1241	dest_index += 1;
1270	1242	} else {
1271	1243	const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
1272	1244	const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
1273		utf16le[dest_i] = mem.nativeToLittle(u16, high);
1274		utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);
1275		dest_i += 2;
	1245	utf16le[dest_index..][0..2].* = .{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) };
	1246	dest_index += 2;
1276	1247	}
1277		src_i = next_src_i;
1278	1248	}
1279		return dest_i;
	1249	return dest_index;
1280	1250	}
1281	1251
1282	1252	test "utf8ToUtf16Le" {
1283		var utf16le: [2]u16 = [_]u16{0} ** 2;
	1253	var utf16le: [128]u16 = undefined;
1284	1254	{
1285	1255	const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
1286		try testing.expectEqual(@as(usize, 2), length);
1287		try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..]));
	1256	try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..length]));
1288	1257	}
1289	1258	{
1290	1259	const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
1291		try testing.expectEqual(@as(usize, 2), length);
1292		try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..]));
	1260	try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..length]));
1293	1261	}
1294	1262	{
1295	1263	const result = utf8ToUtf16Le(utf16le[0..], "\xf4\x90\x80\x80");
1296	1264	try testing.expectError(error.InvalidUtf8, result);
1297	1265	}
	1266	{
	1267	const length = try utf8ToUtf16Le(utf16le[0..], "This string has been designed to test the vectorized implementat" ++
	1268	"ion by beginning with one hundred twenty-seven ASCII characters¡");
	1269	try testing.expectEqualSlices(u8, &.{
	1270	'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
	1271	'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
	1272	' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
	1273	'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
	1274	'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
	1275	' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
	1276	'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
	1277	'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0, '¡', 0,
	1278	}, mem.sliceAsBytes(utf16le[0..length]));
	1279	}
1298	1280	}
1299	1281
1300	1282	test utf8ToUtf16LeArrayList {
...	...	@@ -1339,25 +1321,40 @@ test utf8ToUtf16LeAllocZ {
1339	1321	{
1340	1322	const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "𐐷");
1341	1323	defer testing.allocator.free(utf16);
1342		try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));
	1324	try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16));
1343	1325	try testing.expect(utf16[2] == 0);
1344	1326	}
1345	1327	{
1346	1328	const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "\u{10FFFF}");
1347	1329	defer testing.allocator.free(utf16);
1348		try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));
	1330	try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16));
1349	1331	try testing.expect(utf16[2] == 0);
1350	1332	}
1351	1333	{
1352	1334	const result = utf8ToUtf16LeAllocZ(testing.allocator, "\xf4\x90\x80\x80");
1353	1335	try testing.expectError(error.InvalidUtf8, result);
1354	1336	}
	1337	{
	1338	const utf16 = try utf8ToUtf16LeWithNull(testing.allocator, "This string has been designed to test the vectorized implementat" ++
	1339	"ion by beginning with one hundred twenty-seven ASCII characters¡");
	1340	defer testing.allocator.free(utf16);
	1341	try testing.expectEqualSlices(u8, &.{
	1342	'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
	1343	'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
	1344	' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
	1345	'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
	1346	'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
	1347	' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
	1348	'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
	1349	'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0, '¡', 0,
	1350	}, mem.sliceAsBytes(utf16));
	1351	}
1355	1352	}
1356	1353
1357	1354	/// Converts a UTF-8 string literal into a UTF-16LE string literal.
1358		pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch unreachable:0]u16 {
	1355	pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch \|err\| @compileError(err):0]u16 {
1359	1356	return comptime blk: {
1360		const len: usize = calcUtf16LeLen(utf8) catch \|err\| @compileError(err);
	1357	const len: usize = calcUtf16LeLen(utf8) catch unreachable;
1361	1358	var utf16le: [len:0]u16 = [_:0]u16{0} ** len;
1362	1359	const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch \|err\| @compileError(err);
1363	1360	assert(len == utf16le_len);
...	...	@@ -1438,12 +1435,12 @@ test "fmtUtf16Le" {
1438	1435	try expectFmt("", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral(""))});
1439	1436	try expectFmt("foo", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("foo"))});
1440	1437	try expectFmt("𐐷", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("𐐷"))});
1441		try expectFmt("퟿", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xd7", native_endian)})});
1442		try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xd8", native_endian)})});
1443		try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xdb", native_endian)})});
1444		try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xdc", native_endian)})});
1445		try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xdf", native_endian)})});
1446		try expectFmt("", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xe0", native_endian)})});
	1438	try expectFmt("퟿", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xd7", native_endian)})});
	1439	try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xd8", native_endian)})});
	1440	try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdb", native_endian)})});
	1441	try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xdc", native_endian)})});
	1442	try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdf", native_endian)})});
	1443	try expectFmt("", "{}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xe0", native_endian)})});
1447	1444	}
1448	1445
1449	1446	test "utf8ToUtf16LeStringLiteral" {
...	...	@@ -1686,8 +1683,9 @@ pub const Wtf8Iterator = struct {
1686	1683	}
1687	1684	};
1688	1685
1689		pub fn wtf16LeToWtf8ArrayList(array_list: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {
1690		return utf16LeToUtf8ArrayListImpl(array_list, utf16le, .can_encode_surrogate_half);
	1686	pub fn wtf16LeToWtf8ArrayList(result: *std.ArrayList(u8), utf16le: []const u16) mem.Allocator.Error!void {
	1687	try result.ensureTotalCapacityPrecise(utf16le.len);
	1688	return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
1691	1689	}
1692	1690
1693	1691	/// Caller must free returned memory.
...	...	@@ -1696,8 +1694,7 @@ pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) mem.Al
1696	1694	var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len);
1697	1695	errdefer result.deinit();
1698	1696
1699		try wtf16LeToWtf8ArrayList(&result, wtf16le);
1700
	1697	try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
1701	1698	return result.toOwnedSlice();
1702	1699	}
1703	1700
...	...	@@ -1707,8 +1704,7 @@ pub fn wtf16LeToWtf8AllocZ(allocator: mem.Allocator, wtf16le: []const u16) mem.A
1707	1704	var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len + 1);
1708	1705	errdefer result.deinit();
1709	1706
1710		try wtf16LeToWtf8ArrayList(&result, wtf16le);
1711
	1707	try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
1712	1708	return result.toOwnedSliceSentinel(0);
1713	1709	}
1714	1710
...	...	@@ -1716,8 +1712,9 @@ pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {
1716	1712	return utf16LeToUtf8Impl(wtf8, wtf16le, .can_encode_surrogate_half) catch \|err\| switch (err) {};
1717	1713	}
1718	1714
1719		pub fn wtf8ToWtf16LeArrayList(array_list: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
1720		return utf8ToUtf16LeArrayListImpl(array_list, wtf8, .can_encode_surrogate_half);
	1715	pub fn wtf8ToWtf16LeArrayList(result: *std.ArrayList(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
	1716	try result.ensureTotalCapacityPrecise(wtf8.len);
	1717	return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
1721	1718	}
1722	1719
1723	1720	pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {
...	...	@@ -1726,7 +1723,6 @@ pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) error{ Inv
1726	1723	errdefer result.deinit();
1727	1724
1728	1725	try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
1729
1730	1726	return result.toOwnedSlice();
1731	1727	}
1732	1728
...	...	@@ -1736,7 +1732,6 @@ pub fn wtf8ToWtf16LeAllocZ(allocator: mem.Allocator, wtf8: []const u8) error{ In
1736	1732	errdefer result.deinit();
1737	1733
1738	1734	try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
1739
1740	1735	return result.toOwnedSliceSentinel(0);
1741	1736	}
1742	1737
...	...	@@ -1895,7 +1890,7 @@ pub const Wtf16LeIterator = struct {
1895	1890
1896	1891	pub fn init(s: []const u16) Wtf16LeIterator {
1897	1892	return Wtf16LeIterator{
1898		.bytes = std.mem.sliceAsBytes(s),
	1893	.bytes = mem.sliceAsBytes(s),
1899	1894	.i = 0,
1900	1895	};
1901	1896	}
...	...	@@ -1908,12 +1903,12 @@ pub const Wtf16LeIterator = struct {
1908	1903	assert(it.i <= it.bytes.len);
1909	1904	if (it.i == it.bytes.len) return null;
1910	1905	var code_units: [2]u16 = undefined;
1911		code_units[0] = std.mem.readInt(u16, it.bytes[it.i..][0..2], .little);
	1906	code_units[0] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
1912	1907	it.i += 2;
1913	1908	surrogate_pair: {
1914	1909	if (utf16IsHighSurrogate(code_units[0])) {
1915	1910	if (it.i >= it.bytes.len) break :surrogate_pair;
1916		code_units[1] = std.mem.readInt(u16, it.bytes[it.i..][0..2], .little);
	1911	code_units[1] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
1917	1912	const codepoint = utf16DecodeSurrogatePair(&code_units) catch break :surrogate_pair;
1918	1913	it.i += 2;
1919	1914	return codepoint;
...	...	@@ -2030,31 +2025,31 @@ fn testRoundtripWtf16(wtf16le: []const u16) !void {
2030	2025
2031	2026	test "well-formed WTF-16 roundtrips" {
2032	2027	try testRoundtripWtf16(&[_]u16{
2033		std.mem.nativeToLittle(u16, 0xD83D), // high surrogate
2034		std.mem.nativeToLittle(u16, 0xDCA9), // low surrogate
	2028	mem.nativeToLittle(u16, 0xD83D), // high surrogate
	2029	mem.nativeToLittle(u16, 0xDCA9), // low surrogate
2035	2030	});
2036	2031	try testRoundtripWtf16(&[_]u16{
2037		std.mem.nativeToLittle(u16, 0xD83D), // high surrogate
2038		std.mem.nativeToLittle(u16, ' '), // not surrogate
2039		std.mem.nativeToLittle(u16, 0xDCA9), // low surrogate
	2032	mem.nativeToLittle(u16, 0xD83D), // high surrogate
	2033	mem.nativeToLittle(u16, ' '), // not surrogate
	2034	mem.nativeToLittle(u16, 0xDCA9), // low surrogate
2040	2035	});
2041	2036	try testRoundtripWtf16(&[_]u16{
2042		std.mem.nativeToLittle(u16, 0xD800), // high surrogate
2043		std.mem.nativeToLittle(u16, 0xDBFF), // high surrogate
	2037	mem.nativeToLittle(u16, 0xD800), // high surrogate
	2038	mem.nativeToLittle(u16, 0xDBFF), // high surrogate
2044	2039	});
2045	2040	try testRoundtripWtf16(&[_]u16{
2046		std.mem.nativeToLittle(u16, 0xD800), // high surrogate
2047		std.mem.nativeToLittle(u16, 0xE000), // not surrogate
	2041	mem.nativeToLittle(u16, 0xD800), // high surrogate
	2042	mem.nativeToLittle(u16, 0xE000), // not surrogate
2048	2043	});
2049	2044	try testRoundtripWtf16(&[_]u16{
2050		std.mem.nativeToLittle(u16, 0xD7FF), // not surrogate
2051		std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
	2045	mem.nativeToLittle(u16, 0xD7FF), // not surrogate
	2046	mem.nativeToLittle(u16, 0xDC00), // low surrogate
2052	2047	});
2053	2048	try testRoundtripWtf16(&[_]u16{
2054		std.mem.nativeToLittle(u16, 0x61), // not surrogate
2055		std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
	2049	mem.nativeToLittle(u16, 0x61), // not surrogate
	2050	mem.nativeToLittle(u16, 0xDC00), // low surrogate
2056	2051	});
2057	2052	try testRoundtripWtf16(&[_]u16{
2058		std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
	2053	mem.nativeToLittle(u16, 0xDC00), // low surrogate
2059	2054	});
2060	2055	}

lib/std/zig/c_translation.zig+3-5

...	...	@@ -308,14 +308,12 @@ test "promoteIntLiteral" {
308	308
309	309	/// Convert from clang __builtin_shufflevector index to Zig @shuffle index
310	310	/// clang requires __builtin_shufflevector index arguments to be integer constants.
311		/// negative values for `this_index` indicate "don't care" so we arbitrarily choose 0
	311	/// negative values for `this_index` indicate "don't care".
312	312	/// clang enforces that `this_index` is less than the total number of vector elements
313	313	/// See https://ziglang.org/documentation/master/#shuffle
314	314	/// See https://clang.llvm.org/docs/LanguageExtensions.html#langext-builtin-shufflevector
315	315	pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len: usize) i32 {
316		if (this_index <= 0) return 0;
317
318		const positive_index = @as(usize, @intCast(this_index));
	316	const positive_index = std.math.cast(usize, this_index) orelse return undefined;
319	317	if (positive_index < source_vector_len) return @as(i32, @intCast(this_index));
320	318	const b_index = positive_index - source_vector_len;
321	319	return ~@as(i32, @intCast(b_index));
...	...	@@ -324,7 +322,7 @@ pub fn shuffleVectorIndex(comptime this_index: c_int, comptime source_vector_len
324	322	test "shuffleVectorIndex" {
325	323	const vector_len: usize = 4;
326	324
327		try testing.expect(shuffleVectorIndex(-1, vector_len) == 0);
	325	_ = shuffleVectorIndex(-1, vector_len);
328	326
329	327	try testing.expect(shuffleVectorIndex(0, vector_len) == 0);
330	328	try testing.expect(shuffleVectorIndex(1, vector_len) == 1);

src/InternPool.zig+7-2

...	...	@@ -3587,6 +3587,7 @@ pub const Alignment = enum(u6) {
3587	3587	@"8" = 3,
3588	3588	@"16" = 4,
3589	3589	@"32" = 5,
	3590	@"64" = 6,
3590	3591	none = std.math.maxInt(u6),
3591	3592	_,
3592	3593
...	...	@@ -7403,10 +7404,14 @@ pub fn isIntegerType(ip: *const InternPool, ty: Index) bool {
7403	7404	.c_ulong_type,
7404	7405	.c_longlong_type,
7405	7406	.c_ulonglong_type,
7406		.c_longdouble_type,
7407	7407	.comptime_int_type,
7408	7408	=> true,
7409		else => ip.indexToKey(ty) == .int_type,
	7409	else => switch (ip.items.items(.tag)[@intFromEnum(ty)]) {
	7410	.type_int_signed,
	7411	.type_int_unsigned,
	7412	=> true,
	7413	else => false,
	7414	},
7410	7415	};
7411	7416	}
7412	7417

src/Sema.zig+31-2

...	...	@@ -23315,7 +23315,8 @@ fn checkVectorElemType(
23315	23315	const mod = sema.mod;
23316	23316	switch (ty.zigTypeTag(mod)) {
23317	23317	.Int, .Float, .Bool => return,
23318		else => if (ty.isPtrAtRuntime(mod)) return,
	23318	.Optional, .Pointer => if (ty.isPtrAtRuntime(mod)) return,
	23319	else => {},
23319	23320	}
23320	23321	return sema.fail(block, ty_src, "expected integer, float, bool, or pointer for the vector element type; found '{}'", .{ty.fmt(mod)});
23321	23322	}
...	...	@@ -28442,7 +28443,7 @@ const CoerceOpts = struct {
28442	28443	report_err: bool = true,
28443	28444	/// Ignored if `report_err == false`.
28444	28445	is_ret: bool = false,
28445		/// Should coercion to comptime_int ermit an error message.
	28446	/// Should coercion to comptime_int emit an error message.
28446	28447	no_cast_to_comptime_int: bool = false,
28447	28448
28448	28449	param_src: struct {
...	...	@@ -31845,6 +31846,34 @@ fn coerceArrayLike(
31845	31846	}
31846	31847
31847	31848	const dest_elem_ty = dest_ty.childType(mod);
	31849	if (dest_ty.isVector(mod) and inst_ty.isVector(mod) and (try sema.resolveValue(inst)) == null) {
	31850	const inst_elem_ty = inst_ty.childType(mod);
	31851	switch (dest_elem_ty.zigTypeTag(mod)) {
	31852	.Int => if (inst_elem_ty.isInt(mod)) {
	31853	// integer widening
	31854	const dst_info = dest_elem_ty.intInfo(mod);
	31855	const src_info = inst_elem_ty.intInfo(mod);
	31856	if ((src_info.signedness == dst_info.signedness and dst_info.bits >= src_info.bits) or
	31857	// small enough unsigned ints can get casted to large enough signed ints
	31858	(dst_info.signedness == .signed and dst_info.bits > src_info.bits))
	31859	{
	31860	try sema.requireRuntimeBlock(block, inst_src, null);
	31861	return block.addTyOp(.intcast, dest_ty, inst);
	31862	}
	31863	},
	31864	.Float => if (inst_elem_ty.isRuntimeFloat()) {
	31865	// float widening
	31866	const src_bits = inst_elem_ty.floatBits(target);
	31867	const dst_bits = dest_elem_ty.floatBits(target);
	31868	if (dst_bits >= src_bits) {
	31869	try sema.requireRuntimeBlock(block, inst_src, null);
	31870	return block.addTyOp(.fpext, dest_ty, inst);
	31871	}
	31872	},
	31873	else => {},
	31874	}
	31875	}
	31876
31848	31877	const element_vals = try sema.arena.alloc(InternPool.Index, dest_len);
31849	31878	const element_refs = try sema.arena.alloc(Air.Inst.Ref, dest_len);
31850	31879	var runtime_src: ?LazySrcLoc = null;

src/arch/x86_64/CodeGen.zig+1659-327

...	...	@@ -1547,6 +1547,27 @@ fn asmRegisterRegisterMemory(
1547	1547	});
1548	1548	}
1549	1549
	1550	fn asmRegisterRegisterMemoryRegister(
	1551	self: *Self,
	1552	tag: Mir.Inst.FixedTag,
	1553	reg1: Register,
	1554	reg2: Register,
	1555	m: Memory,
	1556	reg3: Register,
	1557	) !void {
	1558	_ = try self.addInst(.{
	1559	.tag = tag[1],
	1560	.ops = .rrmr,
	1561	.data = .{ .rrrx = .{
	1562	.fixes = tag[0],
	1563	.r1 = reg1,
	1564	.r2 = reg2,
	1565	.r3 = reg3,
	1566	.payload = try self.addExtra(Mir.Memory.encode(m)),
	1567	} },
	1568	});
	1569	}
	1570
1550	1571	fn asmMemory(self: *Self, tag: Mir.Inst.FixedTag, m: Memory) !void {
1551	1572	_ = try self.addInst(.{
1552	1573	.tag = tag[1],
...	...	@@ -1570,6 +1591,25 @@ fn asmRegisterMemory(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, m: Memo
1570	1591	});
1571	1592	}
1572	1593
	1594	fn asmRegisterMemoryRegister(
	1595	self: *Self,
	1596	tag: Mir.Inst.FixedTag,
	1597	reg1: Register,
	1598	m: Memory,
	1599	reg2: Register,
	1600	) !void {
	1601	_ = try self.addInst(.{
	1602	.tag = tag[1],
	1603	.ops = .rmr,
	1604	.data = .{ .rrx = .{
	1605	.fixes = tag[0],
	1606	.r1 = reg1,
	1607	.r2 = reg2,
	1608	.payload = try self.addExtra(Mir.Memory.encode(m)),
	1609	} },
	1610	});
	1611	}
	1612
1573	1613	fn asmRegisterMemoryImmediate(
1574	1614	self: *Self,
1575	1615	tag: Mir.Inst.FixedTag,
...	...	@@ -2570,7 +2610,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
2570	2610
2571	2611	const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
2572	2612	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
2573		if (opts.update_tracking) ({}) else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
	2613	if (opts.update_tracking)
	2614	{} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
2574	2615
2575	2616	var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
2576	2617	stack.get(),
...	...	@@ -2812,11 +2853,14 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2812	2853	}
2813	2854
2814	2855	fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
	2856	const mod = self.bin_file.comp.module.?;
2815	2857	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2816	2858	const dst_ty = self.typeOfIndex(inst);
2817		const dst_bits = dst_ty.floatBits(self.target.*);
	2859	const dst_scalar_ty = dst_ty.scalarType(mod);
	2860	const dst_bits = dst_scalar_ty.floatBits(self.target.*);
2818	2861	const src_ty = self.typeOf(ty_op.operand);
2819		const src_bits = src_ty.floatBits(self.target.*);
	2862	const src_scalar_ty = src_ty.scalarType(mod);
	2863	const src_bits = src_scalar_ty.floatBits(self.target.*);
2820	2864
2821	2865	const result = result: {
2822	2866	if (switch (src_bits) {
...	...	@@ -2840,94 +2884,290 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2840	2884	},
2841	2885	else => unreachable,
2842	2886	}) {
	2887	if (dst_ty.isVector(mod)) break :result null;
2843	2888	var callee_buf: ["__extend?f?f2".len]u8 = undefined;
2844	2889	break :result try self.genCall(.{ .lib = .{
2845		.return_type = self.floatCompilerRtAbiType(dst_ty, src_ty).toIntern(),
2846		.param_types = &.{self.floatCompilerRtAbiType(src_ty, dst_ty).toIntern()},
	2890	.return_type = self.floatCompilerRtAbiType(dst_scalar_ty, src_scalar_ty).toIntern(),
	2891	.param_types = &.{self.floatCompilerRtAbiType(src_scalar_ty, dst_scalar_ty).toIntern()},
2847	2892	.callee = std.fmt.bufPrint(&callee_buf, "__extend{c}f{c}f2", .{
2848	2893	floatCompilerRtAbiName(src_bits),
2849	2894	floatCompilerRtAbiName(dst_bits),
2850	2895	}) catch unreachable,
2851		} }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
	2896	} }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }});
2852	2897	}
2853	2898
	2899	const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
2854	2900	const src_mcv = try self.resolveInst(ty_op.operand);
2855	2901	const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2856	2902	src_mcv
2857	2903	else
2858	2904	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
2859		const dst_reg = dst_mcv.getReg().?.to128();
	2905	const dst_reg = dst_mcv.getReg().?;
	2906	const dst_alias = registerAlias(dst_reg, @intCast(@max(dst_ty.abiSize(mod), 16)));
2860	2907	const dst_lock = self.register_manager.lockReg(dst_reg);
2861	2908	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
2862	2909
	2910	const vec_len = if (dst_ty.isVector(mod)) dst_ty.vectorLen(mod) else 1;
2863	2911	if (src_bits == 16) {
2864	2912	assert(self.hasFeature(.f16c));
2865	2913	const mat_src_reg = if (src_mcv.isRegister())
2866	2914	src_mcv.getReg().?
2867	2915	else
2868	2916	try self.copyToTmpRegister(src_ty, src_mcv);
2869		try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128());
	2917	try self.asmRegisterRegister(
	2918	.{ .v_ps, .cvtph2 },
	2919	dst_alias,
	2920	registerAlias(mat_src_reg, src_abi_size),
	2921	);
2870	2922	switch (dst_bits) {
2871	2923	32 => {},
2872	2924	64 => try self.asmRegisterRegisterRegister(
2873	2925	.{ .v_sd, .cvtss2 },
2874		dst_reg,
2875		dst_reg,
2876		dst_reg,
	2926	dst_alias,
	2927	dst_alias,
	2928	dst_alias,
2877	2929	),
2878	2930	else => unreachable,
2879	2931	}
2880	2932	} else {
2881	2933	assert(src_bits == 32 and dst_bits == 64);
2882		if (self.hasFeature(.avx)) if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
2883		.{ .v_sd, .cvtss2 },
2884		dst_reg,
2885		dst_reg,
2886		try src_mcv.mem(self, .dword),
2887		) else try self.asmRegisterRegisterRegister(
2888		.{ .v_sd, .cvtss2 },
2889		dst_reg,
2890		dst_reg,
2891		(if (src_mcv.isRegister())
2892		src_mcv.getReg().?
2893		else
2894		try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
2895		) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2896		.{ ._sd, .cvtss2 },
2897		dst_reg,
2898		try src_mcv.mem(self, .dword),
	2934	if (self.hasFeature(.avx)) switch (vec_len) {
	2935	1 => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	2936	.{ .v_sd, .cvtss2 },
	2937	dst_alias,
	2938	dst_alias,
	2939	try src_mcv.mem(self, self.memSize(src_ty)),
	2940	) else try self.asmRegisterRegisterRegister(
	2941	.{ .v_sd, .cvtss2 },
	2942	dst_alias,
	2943	dst_alias,
	2944	registerAlias(if (src_mcv.isRegister())
	2945	src_mcv.getReg().?
	2946	else
	2947	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
	2948	),
	2949	2...4 => if (src_mcv.isMemory()) try self.asmRegisterMemory(
	2950	.{ .v_pd, .cvtps2 },
	2951	dst_alias,
	2952	try src_mcv.mem(self, self.memSize(src_ty)),
	2953	) else try self.asmRegisterRegister(
	2954	.{ .v_pd, .cvtps2 },
	2955	dst_alias,
	2956	registerAlias(if (src_mcv.isRegister())
	2957	src_mcv.getReg().?
	2958	else
	2959	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
	2960	),
	2961	else => break :result null,
	2962	} else if (src_mcv.isMemory()) try self.asmRegisterMemory(
	2963	switch (vec_len) {
	2964	1 => .{ ._sd, .cvtss2 },
	2965	2 => .{ ._pd, .cvtps2 },
	2966	else => break :result null,
	2967	},
	2968	dst_alias,
	2969	try src_mcv.mem(self, self.memSize(src_ty)),
2899	2970	) else try self.asmRegisterRegister(
2900		.{ ._sd, .cvtss2 },
2901		dst_reg,
2902		(if (src_mcv.isRegister())
	2971	switch (vec_len) {
	2972	1 => .{ ._sd, .cvtss2 },
	2973	2 => .{ ._pd, .cvtps2 },
	2974	else => break :result null,
	2975	},
	2976	dst_alias,
	2977	registerAlias(if (src_mcv.isRegister())
2903	2978	src_mcv.getReg().?
2904	2979	else
2905		try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
	2980	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
2906	2981	);
2907	2982	}
2908	2983	break :result dst_mcv;
2909		};
	2984	} orelse return self.fail("TODO implement airFpext from {} to {}", .{
	2985	src_ty.fmt(mod), dst_ty.fmt(mod),
	2986	});
2910	2987	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2911	2988	}
2912	2989
2913	2990	fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2914	2991	const mod = self.bin_file.comp.module.?;
2915	2992	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2916		const result: MCValue = result: {
2917		const src_ty = self.typeOf(ty_op.operand);
2918		const src_int_info = src_ty.intInfo(mod);
	2993	const src_ty = self.typeOf(ty_op.operand);
	2994	const dst_ty = self.typeOfIndex(inst);
2919	2995
2920		const dst_ty = self.typeOfIndex(inst);
2921		const dst_int_info = dst_ty.intInfo(mod);
2922		const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
	2996	const result = @as(?MCValue, result: {
	2997	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
2923	2998
2924		const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
	2999	const src_int_info = src_ty.intInfo(mod);
	3000	const dst_int_info = dst_ty.intInfo(mod);
2925	3001	const extend = switch (src_int_info.signedness) {
2926	3002	.signed => dst_int_info,
2927	3003	.unsigned => src_int_info,
2928	3004	}.signedness;
2929	3005
2930	3006	const src_mcv = try self.resolveInst(ty_op.operand);
	3007	if (dst_ty.isVector(mod)) {
	3008	const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
	3009	const max_abi_size = @max(dst_abi_size, src_abi_size);
	3010	if (max_abi_size > @as(u32, if (self.hasFeature(.avx2)) 32 else 16)) break :result null;
	3011	const has_avx = self.hasFeature(.avx);
	3012
	3013	const dst_elem_abi_size = dst_ty.childType(mod).abiSize(mod);
	3014	const src_elem_abi_size = src_ty.childType(mod).abiSize(mod);
	3015	switch (math.order(dst_elem_abi_size, src_elem_abi_size)) {
	3016	.lt => {
	3017	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
	3018	else => break :result null,
	3019	1 => switch (src_elem_abi_size) {
	3020	else => break :result null,
	3021	2 => switch (dst_int_info.signedness) {
	3022	.signed => if (has_avx) .{ .vp_b, .ackssw } else .{ .p_b, .ackssw },
	3023	.unsigned => if (has_avx) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
	3024	},
	3025	},
	3026	2 => switch (src_elem_abi_size) {
	3027	else => break :result null,
	3028	4 => switch (dst_int_info.signedness) {
	3029	.signed => if (has_avx) .{ .vp_w, .ackssd } else .{ .p_w, .ackssd },
	3030	.unsigned => if (has_avx)
	3031	.{ .vp_w, .ackusd }
	3032	else if (self.hasFeature(.sse4_1))
	3033	.{ .p_w, .ackusd }
	3034	else
	3035	break :result null,
	3036	},
	3037	},
	3038	};
	3039
	3040	const dst_mcv: MCValue = if (src_mcv.isRegister() and
	3041	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	3042	src_mcv
	3043	else if (has_avx and src_mcv.isRegister())
	3044	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	3045	else
	3046	try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv);
	3047	const dst_reg = dst_mcv.getReg().?;
	3048	const dst_alias = registerAlias(dst_reg, dst_abi_size);
	3049
	3050	if (has_avx) try self.asmRegisterRegisterRegister(
	3051	mir_tag,
	3052	dst_alias,
	3053	registerAlias(if (src_mcv.isRegister())
	3054	src_mcv.getReg().?
	3055	else
	3056	dst_reg, src_abi_size),
	3057	dst_alias,
	3058	) else try self.asmRegisterRegister(
	3059	mir_tag,
	3060	dst_alias,
	3061	dst_alias,
	3062	);
	3063	break :result dst_mcv;
	3064	},
	3065	.eq => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	3066	break :result src_mcv
	3067	else {
	3068	const dst_mcv = try self.allocRegOrMem(inst, true);
	3069	try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
	3070	break :result dst_mcv;
	3071	},
	3072	.gt => if (self.hasFeature(.sse4_1)) {
	3073	const mir_tag: Mir.Inst.FixedTag = .{ switch (dst_elem_abi_size) {
	3074	else => break :result null,
	3075	2 => if (has_avx) .vp_w else .p_w,
	3076	4 => if (has_avx) .vp_d else .p_d,
	3077	8 => if (has_avx) .vp_q else .p_q,
	3078	}, switch (src_elem_abi_size) {
	3079	else => break :result null,
	3080	1 => switch (extend) {
	3081	.signed => .movsxb,
	3082	.unsigned => .movzxb,
	3083	},
	3084	2 => switch (extend) {
	3085	.signed => .movsxw,
	3086	.unsigned => .movzxw,
	3087	},
	3088	4 => switch (extend) {
	3089	.signed => .movsxd,
	3090	.unsigned => .movzxd,
	3091	},
	3092	} };
	3093
	3094	const dst_mcv: MCValue = if (src_mcv.isRegister() and
	3095	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	3096	src_mcv
	3097	else
	3098	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) };
	3099	const dst_reg = dst_mcv.getReg().?;
	3100	const dst_alias = registerAlias(dst_reg, dst_abi_size);
	3101
	3102	if (src_mcv.isMemory()) try self.asmRegisterMemory(
	3103	mir_tag,
	3104	dst_alias,
	3105	try src_mcv.mem(self, self.memSize(src_ty)),
	3106	) else try self.asmRegisterRegister(
	3107	mir_tag,
	3108	dst_alias,
	3109	registerAlias(if (src_mcv.isRegister())
	3110	src_mcv.getReg().?
	3111	else
	3112	try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
	3113	);
	3114	break :result dst_mcv;
	3115	} else {
	3116	const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
	3117	else => break :result null,
	3118	2 => switch (src_elem_abi_size) {
	3119	else => break :result null,
	3120	1 => .{ .p_, .unpcklbw },
	3121	},
	3122	4 => switch (src_elem_abi_size) {
	3123	else => break :result null,
	3124	2 => .{ .p_, .unpcklwd },
	3125	},
	3126	8 => switch (src_elem_abi_size) {
	3127	else => break :result null,
	3128	2 => .{ .p_, .unpckldq },
	3129	},
	3130	};
	3131
	3132	const dst_mcv: MCValue = if (src_mcv.isRegister() and
	3133	self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
	3134	src_mcv
	3135	else
	3136	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
	3137	const dst_reg = dst_mcv.getReg().?;
	3138
	3139	const ext_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
	3140	const ext_alias = registerAlias(ext_reg, src_abi_size);
	3141	const ext_lock = self.register_manager.lockRegAssumeUnused(ext_reg);
	3142	defer self.register_manager.unlockReg(ext_lock);
	3143
	3144	try self.asmRegisterRegister(.{ .p_, .xor }, ext_alias, ext_alias);
	3145	switch (extend) {
	3146	.signed => try self.asmRegisterRegister(
	3147	.{ switch (src_elem_abi_size) {
	3148	else => unreachable,
	3149	1 => .p_b,
	3150	2 => .p_w,
	3151	4 => .p_d,
	3152	}, .cmpgt },
	3153	ext_alias,
	3154	registerAlias(dst_reg, src_abi_size),
	3155	),
	3156	.unsigned => {},
	3157	}
	3158	try self.asmRegisterRegister(
	3159	mir_tag,
	3160	registerAlias(dst_reg, dst_abi_size),
	3161	registerAlias(ext_reg, dst_abi_size),
	3162	);
	3163	break :result dst_mcv;
	3164	},
	3165	}
	3166	@compileError("unreachable");
	3167	}
	3168
	3169	const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
	3170
2931	3171	const src_storage_bits: u16 = switch (src_mcv) {
2932	3172	.register, .register_offset => 64,
2933	3173	.register_pair => 128,
...	...	@@ -2945,13 +3185,13 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2945	3185	};
2946	3186
2947	3187	if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister())
2948		.{ .register = registerAlias(dst_mcv.getReg().?, abi_size) }
	3188	.{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) }
2949	3189	else
2950	3190	dst_mcv;
2951	3191
2952	3192	if (dst_mcv.isRegister()) {
2953	3193	try self.truncateRegister(src_ty, dst_mcv.getReg().?);
2954		break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };
	3194	break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
2955	3195	}
2956	3196
2957	3197	const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
...	...	@@ -2999,7 +3239,9 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2999	3239	);
3000	3240
3001	3241	break :result dst_mcv;
3002		};
	3242	}) orelse return self.fail("TODO implement airIntCast from {} to {}", .{
	3243	src_ty.fmt(mod), dst_ty.fmt(mod),
	3244	});
3003	3245	return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3004	3246	}
3005	3247
...	...	@@ -3022,7 +3264,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3022	3264	src_mcv
3023	3265	else if (dst_abi_size <= 8)
3024	3266	try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)
3025		else if (dst_abi_size <= 16) dst: {
	3267	else if (dst_abi_size <= 16 and !dst_ty.isVector(mod)) dst: {
3026	3268	const dst_regs =
3027	3269	try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);
3028	3270	const dst_mcv: MCValue = .{ .register_pair = dst_regs };
...	...	@@ -3032,26 +3274,29 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3032	3274	try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
3033	3275	break :dst dst_mcv;
3034	3276	} else dst: {
3035		const dst_mcv = try self.allocRegOrMem(inst, true);
3036		try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
	3277	const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true);
	3278	try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
3037	3279	break :dst dst_mcv;
3038	3280	};
3039	3281
3040	3282	if (dst_ty.zigTypeTag(mod) == .Vector) {
3041	3283	assert(src_ty.zigTypeTag(mod) == .Vector and dst_ty.vectorLen(mod) == src_ty.vectorLen(mod));
3042		const dst_info = dst_ty.childType(mod).intInfo(mod);
3043		const src_info = src_ty.childType(mod).intInfo(mod);
3044		const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_info.bits) {
3045		8 => switch (src_info.bits) {
3046		16 => switch (dst_ty.vectorLen(mod)) {
	3284	const dst_elem_ty = dst_ty.childType(mod);
	3285	const dst_elem_abi_size: u32 = @intCast(dst_elem_ty.abiSize(mod));
	3286	const src_elem_ty = src_ty.childType(mod);
	3287	const src_elem_abi_size: u32 = @intCast(src_elem_ty.abiSize(mod));
	3288
	3289	const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_elem_abi_size) {
	3290	1 => switch (src_elem_abi_size) {
	3291	2 => switch (dst_ty.vectorLen(mod)) {
3047	3292	1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
3048	3293	9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null,
3049	3294	else => null,
3050	3295	},
3051	3296	else => null,
3052	3297	},
3053		16 => switch (src_info.bits) {
3054		32 => switch (dst_ty.vectorLen(mod)) {
	3298	2 => switch (src_elem_abi_size) {
	3299	4 => switch (dst_ty.vectorLen(mod)) {
3055	3300	1...4 => if (self.hasFeature(.avx))
3056	3301	.{ .vp_w, .ackusd }
3057	3302	else if (self.hasFeature(.sse4_1))
...	...	@@ -3066,12 +3311,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3066	3311	else => null,
3067	3312	}) orelse return self.fail("TODO implement airTrunc for {}", .{dst_ty.fmt(mod)});
3068	3313
3069		const elem_ty = src_ty.childType(mod);
3070		const mask_val = try mod.intValue(elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
	3314	const dst_info = dst_elem_ty.intInfo(mod);
	3315	const src_info = src_elem_ty.intInfo(mod);
	3316
	3317	const mask_val = try mod.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
3071	3318
3072	3319	const splat_ty = try mod.vectorType(.{
3073	3320	.len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
3074		.child = elem_ty.ip_index,
	3321	.child = src_elem_ty.ip_index,
3075	3322	});
3076	3323	const splat_abi_size: u32 = @intCast(splat_ty.abiSize(mod));
3077	3324
...	...	@@ -3086,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
3086	3333	else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
3087	3334	};
3088	3335
3089		const dst_reg = registerAlias(dst_mcv.getReg().?, src_abi_size);
	3336	const dst_reg = dst_mcv.getReg().?;
	3337	const dst_alias = registerAlias(dst_reg, src_abi_size);
3090	3338	if (self.hasFeature(.avx)) {
3091	3339	try self.asmRegisterRegisterMemory(
3092	3340	.{ .vp_, .@"and" },
3093		dst_reg,
3094		dst_reg,
	3341	dst_alias,
	3342	dst_alias,
3095	3343	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3096	3344	);
3097		try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
	3345	if (src_abi_size > 16) {
	3346	const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
	3347	const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
	3348	defer self.register_manager.unlockReg(temp_lock);
	3349
	3350	try self.asmRegisterRegisterImmediate(
	3351	.{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
	3352	registerAlias(temp_reg, dst_abi_size),
	3353	dst_alias,
	3354	Immediate.u(1),
	3355	);
	3356	try self.asmRegisterRegisterRegister(
	3357	mir_tag,
	3358	registerAlias(dst_reg, dst_abi_size),
	3359	registerAlias(dst_reg, dst_abi_size),
	3360	registerAlias(temp_reg, dst_abi_size),
	3361	);
	3362	} else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias);
3098	3363	} else {
3099	3364	try self.asmRegisterMemory(
3100	3365	.{ .p_, .@"and" },
3101		dst_reg,
	3366	dst_alias,
3102	3367	try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3103	3368	);
3104		try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
	3369	try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
3105	3370	}
3106	3371	break :result dst_mcv;
3107	3372	}
...	...	@@ -4045,7 +4310,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4045	4310	if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
4046	4311	const slow_inc = self.hasFeature(.slow_incdec);
4047	4312	const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
4048		const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
	4313	const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable;
4049	4314
4050	4315	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
4051	4316	const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
...	...	@@ -4534,7 +4799,7 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
4534	4799	switch (lhs_ty.zigTypeTag(mod)) {
4535	4800	.Int => {
4536	4801	try self.spillRegisters(&.{.rcx});
4537		try self.register_manager.getReg(.rcx, null);
	4802	try self.register_manager.getKnownReg(.rcx, null);
4538	4803	const lhs_mcv = try self.resolveInst(bin_op.lhs);
4539	4804	const rhs_mcv = try self.resolveInst(bin_op.rhs);
4540	4805
...	...	@@ -6560,7 +6825,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
6560	6825
6561	6826	const dst_mcv: MCValue = .{ .register = .st0 };
6562	6827	if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))
6563		try self.register_manager.getReg(.st0, inst);
	6828	try self.register_manager.getKnownReg(.st0, inst);
6564	6829
6565	6830	try self.genCopy(ty, dst_mcv, src_mcv, .{});
6566	6831	switch (tag) {
...	...	@@ -6894,7 +7159,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6894	7159	},
6895	7160	else => {
6896	7161	const abi_size: u31 = @intCast(ty.abiSize(mod));
6897		const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
	7162	const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable;
6898	7163
6899	7164	const tmp_regs =
6900	7165	try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp);
...	...	@@ -8181,7 +8446,7 @@ fn genShiftBinOpMir(
8181	8446	try self.asmRegisterImmediate(
8182	8447	.{ ._, .@"and" },
8183	8448	.cl,
8184		Immediate.u(std.math.maxInt(u6)),
	8449	Immediate.u(math.maxInt(u6)),
8185	8450	);
8186	8451	try self.asmRegisterImmediate(
8187	8452	.{ ._r, .sh },
...	...	@@ -8218,7 +8483,7 @@ fn genShiftBinOpMir(
8218	8483	try self.asmRegisterImmediate(
8219	8484	.{ ._, .@"and" },
8220	8485	.cl,
8221		Immediate.u(std.math.maxInt(u6)),
	8486	Immediate.u(math.maxInt(u6)),
8222	8487	);
8223	8488	try self.asmRegisterImmediate(
8224	8489	.{ ._r, .sh },
...	...	@@ -8283,7 +8548,7 @@ fn genShiftBinOpMir(
8283	8548	}, .sh },
8284	8549	temp_regs[2].to64(),
8285	8550	temp_regs[3].to64(),
8286		Immediate.u(shift_imm & std.math.maxInt(u6)),
	8551	Immediate.u(shift_imm & math.maxInt(u6)),
8287	8552	),
8288	8553	else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) {
8289	8554	._l => ._ld,
...	...	@@ -8338,7 +8603,7 @@ fn genShiftBinOpMir(
8338	8603	.immediate => \|shift_imm\| try self.asmRegisterImmediate(
8339	8604	tag,
8340	8605	temp_regs[2].to64(),
8341		Immediate.u(shift_imm & std.math.maxInt(u6)),
	8606	Immediate.u(shift_imm & math.maxInt(u6)),
8342	8607	),
8343	8608	else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl),
8344	8609	}
...	...	@@ -8794,7 +9059,7 @@ fn genShiftBinOp(
8794	9059	lhs_ty.fmt(mod),
8795	9060	});
8796	9061
8797		try self.register_manager.getReg(.rcx, null);
	9062	try self.register_manager.getKnownReg(.rcx, null);
8798	9063	const rcx_lock = self.register_manager.lockReg(.rcx);
8799	9064	defer if (rcx_lock) \|lock\| self.register_manager.unlockReg(lock);
8800	9065
...	...	@@ -8933,7 +9198,7 @@ fn genMulDivBinOp(
8933	9198	switch (tag) {
8934	9199	.mul, .mul_wrap => {
8935	9200	const slow_inc = self.hasFeature(.slow_incdec);
8936		const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable;
	9201	const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable;
8937	9202
8938	9203	try self.spillRegisters(&.{ .rax, .rcx, .rdx });
8939	9204	const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx });
...	...	@@ -9117,8 +9382,8 @@ fn genMulDivBinOp(
9117	9382	.rem => maybe_inst,
9118	9383	else => null,
9119	9384	};
9120		try self.register_manager.getReg(.rax, track_inst_rax);
9121		try self.register_manager.getReg(.rdx, track_inst_rdx);
	9385	try self.register_manager.getKnownReg(.rax, track_inst_rax);
	9386	try self.register_manager.getKnownReg(.rdx, track_inst_rdx);
9122	9387
9123	9388	try self.genIntMulDivOpMir(switch (signedness) {
9124	9389	.signed => switch (tag) {
...	...	@@ -9158,8 +9423,11 @@ fn genMulDivBinOp(
9158	9423	},
9159	9424
9160	9425	.mod => {
9161		try self.register_manager.getReg(.rax, null);
9162		try self.register_manager.getReg(.rdx, if (signedness == .unsigned) maybe_inst else null);
	9426	try self.register_manager.getKnownReg(.rax, null);
	9427	try self.register_manager.getKnownReg(
	9428	.rdx,
	9429	if (signedness == .unsigned) maybe_inst else null,
	9430	);
9163	9431
9164	9432	switch (signedness) {
9165	9433	.signed => {
...	...	@@ -9200,8 +9468,11 @@ fn genMulDivBinOp(
9200	9468	},
9201	9469
9202	9470	.div_floor => {
9203		try self.register_manager.getReg(.rax, if (signedness == .unsigned) maybe_inst else null);
9204		try self.register_manager.getReg(.rdx, null);
	9471	try self.register_manager.getKnownReg(
	9472	.rax,
	9473	if (signedness == .unsigned) maybe_inst else null,
	9474	);
	9475	try self.register_manager.getKnownReg(.rdx, null);
9205	9476
9206	9477	const lhs_lock: ?RegisterLock = switch (lhs_mcv) {
9207	9478	.register => \|reg\| self.register_manager.lockRegAssumeUnused(reg),
...	...	@@ -9445,7 +9716,7 @@ fn genBinOp(
9445	9716	.rem, .mod => unreachable,
9446	9717	.max, .min => if (lhs_ty.scalarType(mod).isRuntimeFloat()) registerAlias(
9447	9718	if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {
9448		try self.register_manager.getReg(.xmm0, null);
	9719	try self.register_manager.getKnownReg(.xmm0, null);
9449	9720	break :mask .xmm0;
9450	9721	} else try self.register_manager.allocReg(null, abi.RegisterClass.sse),
9451	9722	abi_size,
...	...	@@ -10820,96 +11091,35 @@ fn genBinOp(
10820	11091	lhs_copy_reg.?,
10821	11092	mask_reg,
10822	11093	) else {
10823		try self.asmRegisterRegister(
10824		@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
10825		.Float => switch (lhs_ty.floatBits(self.target.*)) {
10826		32 => .{ ._ps, .@"and" },
10827		64 => .{ ._pd, .@"and" },
10828		16, 80, 128 => null,
10829		else => unreachable,
10830		},
10831		.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10832		.Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10833		32 => switch (lhs_ty.vectorLen(mod)) {
10834		1...4 => .{ ._ps, .@"and" },
10835		else => null,
10836		},
10837		64 => switch (lhs_ty.vectorLen(mod)) {
10838		1...2 => .{ ._pd, .@"and" },
10839		else => null,
10840		},
10841		16, 80, 128 => null,
10842		else => unreachable,
10843		},
10844		else => unreachable,
10845		},
	11094	const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(mod)) {
	11095	.Float => switch (lhs_ty.floatBits(self.target.*)) {
	11096	32 => ._ps,
	11097	64 => ._pd,
	11098	16, 80, 128 => null,
10846	11099	else => unreachable,
10847		}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
10848		@tagName(air_tag), lhs_ty.fmt(mod),
10849		}),
10850		dst_reg,
10851		mask_reg,
10852		);
10853		try self.asmRegisterRegister(
10854		@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
10855		.Float => switch (lhs_ty.floatBits(self.target.*)) {
10856		32 => .{ ._ps, .andn },
10857		64 => .{ ._pd, .andn },
10858		16, 80, 128 => null,
10859		else => unreachable,
10860		},
10861		.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10862		.Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10863		32 => switch (lhs_ty.vectorLen(mod)) {
10864		1...4 => .{ ._ps, .andn },
10865		else => null,
10866		},
10867		64 => switch (lhs_ty.vectorLen(mod)) {
10868		1...2 => .{ ._pd, .andn },
10869		else => null,
10870		},
10871		16, 80, 128 => null,
10872		else => unreachable,
	11100	},
	11101	.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
	11102	.Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
	11103	32 => switch (lhs_ty.vectorLen(mod)) {
	11104	1...4 => ._ps,
	11105	else => null,
10873	11106	},
10874		else => unreachable,
10875		},
10876		else => unreachable,
10877		}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
10878		@tagName(air_tag), lhs_ty.fmt(mod),
10879		}),
10880		mask_reg,
10881		lhs_copy_reg.?,
10882		);
10883		try self.asmRegisterRegister(
10884		@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) {
10885		.Float => switch (lhs_ty.floatBits(self.target.*)) {
10886		32 => .{ ._ps, .@"or" },
10887		64 => .{ ._pd, .@"or" },
10888		16, 80, 128 => null,
10889		else => unreachable,
10890		},
10891		.Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
10892		.Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) {
10893		32 => switch (lhs_ty.vectorLen(mod)) {
10894		1...4 => .{ ._ps, .@"or" },
10895		else => null,
10896		},
10897		64 => switch (lhs_ty.vectorLen(mod)) {
10898		1...2 => .{ ._pd, .@"or" },
10899		else => null,
10900		},
10901		16, 80, 128 => null,
10902		else => unreachable,
	11107	64 => switch (lhs_ty.vectorLen(mod)) {
	11108	1...2 => ._pd,
	11109	else => null,
10903	11110	},
	11111	16, 80, 128 => null,
10904	11112	else => unreachable,
10905	11113	},
10906	11114	else => unreachable,
10907		}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
10908		@tagName(air_tag), lhs_ty.fmt(mod),
10909		}),
10910		dst_reg,
10911		mask_reg,
10912		);
	11115	},
	11116	else => unreachable,
	11117	}) orelse return self.fail("TODO implement genBinOp for {s} {}", .{
	11118	@tagName(air_tag), lhs_ty.fmt(mod),
	11119	});
	11120	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg);
	11121	try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?);
	11122	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg);
10913	11123	}
10914	11124	},
10915	11125	.cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
...	...	@@ -12192,48 +12402,10 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
12192	12402	fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12193	12403	const mod = self.bin_file.comp.module.?;
12194	12404	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
12195		const ty = self.typeOf(bin_op.lhs);
	12405	var ty = self.typeOf(bin_op.lhs);
	12406	var null_compare: ?Mir.Inst.Index = null;
12196	12407
12197	12408	const result: Condition = result: {
12198		switch (ty.zigTypeTag(mod)) {
12199		.Float => {
12200		const float_bits = ty.floatBits(self.target.*);
12201		if (switch (float_bits) {
12202		16 => !self.hasFeature(.f16c),
12203		32, 64 => false,
12204		80, 128 => true,
12205		else => unreachable,
12206		}) {
12207		var callee_buf: ["__???f2".len]u8 = undefined;
12208		const ret = try self.genCall(.{ .lib = .{
12209		.return_type = .i32_type,
12210		.param_types = &.{ ty.toIntern(), ty.toIntern() },
12211		.callee = std.fmt.bufPrint(&callee_buf, "__{s}{c}f2", .{
12212		switch (op) {
12213		.eq => "eq",
12214		.neq => "ne",
12215		.lt => "lt",
12216		.lte => "le",
12217		.gt => "gt",
12218		.gte => "ge",
12219		},
12220		floatCompilerRtAbiName(float_bits),
12221		}) catch unreachable,
12222		} }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });
12223		try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);
12224		break :result switch (op) {
12225		.eq => .e,
12226		.neq => .ne,
12227		.lt => .l,
12228		.lte => .le,
12229		.gt => .g,
12230		.gte => .ge,
12231		};
12232		}
12233		},
12234		else => {},
12235		}
12236
12237	12409	try self.spillEflagsIfOccupied();
12238	12410
12239	12411	const lhs_mcv = try self.resolveInst(bin_op.lhs);
...	...	@@ -12260,6 +12432,103 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12260	12432	};
12261	12433	defer for (rhs_locks) \|rhs_lock\| if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
12262	12434
	12435	switch (ty.zigTypeTag(mod)) {
	12436	.Float => {
	12437	const float_bits = ty.floatBits(self.target.*);
	12438	if (switch (float_bits) {
	12439	16 => !self.hasFeature(.f16c),
	12440	32, 64 => false,
	12441	80, 128 => true,
	12442	else => unreachable,
	12443	}) {
	12444	var callee_buf: ["__???f2".len]u8 = undefined;
	12445	const ret = try self.genCall(.{ .lib = .{
	12446	.return_type = .i32_type,
	12447	.param_types = &.{ ty.toIntern(), ty.toIntern() },
	12448	.callee = std.fmt.bufPrint(&callee_buf, "__{s}{c}f2", .{
	12449	switch (op) {
	12450	.eq => "eq",
	12451	.neq => "ne",
	12452	.lt => "lt",
	12453	.lte => "le",
	12454	.gt => "gt",
	12455	.gte => "ge",
	12456	},
	12457	floatCompilerRtAbiName(float_bits),
	12458	}) catch unreachable,
	12459	} }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } });
	12460	try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret);
	12461	break :result switch (op) {
	12462	.eq => .e,
	12463	.neq => .ne,
	12464	.lt => .l,
	12465	.lte => .le,
	12466	.gt => .g,
	12467	.gte => .ge,
	12468	};
	12469	}
	12470	},
	12471	.Optional => if (!ty.optionalReprIsPayload(mod)) {
	12472	const opt_ty = ty;
	12473	const opt_abi_size: u31 = @intCast(opt_ty.abiSize(mod));
	12474	ty = opt_ty.optionalChild(mod);
	12475	const payload_abi_size: u31 = @intCast(ty.abiSize(mod));
	12476
	12477	const temp_lhs_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
	12478	const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg);
	12479	defer self.register_manager.unlockReg(temp_lhs_lock);
	12480
	12481	if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
	12482	.{ ._, .mov },
	12483	temp_lhs_reg.to8(),
	12484	try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte),
	12485	) else {
	12486	try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
	12487	try self.asmRegisterImmediate(
	12488	.{ ._r, .sh },
	12489	registerAlias(temp_lhs_reg, opt_abi_size),
	12490	Immediate.u(payload_abi_size * 8),
	12491	);
	12492	}
	12493
	12494	const payload_compare = payload_compare: {
	12495	if (rhs_mcv.isMemory()) {
	12496	const rhs_mem =
	12497	try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .byte);
	12498	try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
	12499	const payload_compare = try self.asmJccReloc(.nz, undefined);
	12500	try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
	12501	break :payload_compare payload_compare;
	12502	}
	12503
	12504	const temp_rhs_reg = try self.copyToTmpRegister(opt_ty, rhs_mcv);
	12505	const temp_rhs_lock = self.register_manager.lockRegAssumeUnused(temp_rhs_reg);
	12506	defer self.register_manager.unlockReg(temp_rhs_lock);
	12507
	12508	try self.asmRegisterImmediate(
	12509	.{ ._r, .sh },
	12510	registerAlias(temp_rhs_reg, opt_abi_size),
	12511	Immediate.u(payload_abi_size * 8),
	12512	);
	12513	try self.asmRegisterRegister(
	12514	.{ ._, .@"test" },
	12515	temp_lhs_reg.to8(),
	12516	temp_rhs_reg.to8(),
	12517	);
	12518	const payload_compare = try self.asmJccReloc(.nz, undefined);
	12519	try self.asmRegisterRegister(
	12520	.{ ._, .cmp },
	12521	temp_lhs_reg.to8(),
	12522	temp_rhs_reg.to8(),
	12523	);
	12524	break :payload_compare payload_compare;
	12525	};
	12526	null_compare = try self.asmJmpReloc(undefined);
	12527	self.performReloc(payload_compare);
	12528	},
	12529	else => {},
	12530	}
	12531
12263	12532	switch (ty.zigTypeTag(mod)) {
12264	12533	else => {
12265	12534	const abi_size: u16 = @intCast(ty.abiSize(mod));
...	...	@@ -12571,6 +12840,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
12571	12840	}
12572	12841	};
12573	12842
	12843	if (null_compare) \|reloc\| self.performReloc(reloc);
12574	12844	self.eflags_inst = inst;
12575	12845	return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
12576	12846	}
...	...	@@ -13521,6 +13791,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
13521	13791	} else if (constraint.len == 1 and std.ascii.isDigit(constraint[0])) arg: {
13522	13792	const index = std.fmt.charToDigit(constraint[0], 10) catch unreachable;
13523	13793	if (index >= args.items.len) return self.fail("constraint out of bounds: '{s}'", .{constraint});
	13794	try self.genCopy(ty, args.items[index], input_mcv, .{});
13524	13795	break :arg args.items[index];
13525	13796	} else return self.fail("invalid constraint: '{s}'", .{constraint});
13526	13797	if (arg_mcv.getReg()) \|reg\| if (RegisterManager.indexOfRegIntoTracked(reg)) \|_\| {
...	...	@@ -13619,25 +13890,26 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
13619	13890	label_gop.value_ptr.target = @intCast(self.mir_instructions.len);
13620	13891	} else continue;
13621	13892
13622		var mnem_size: ?Memory.Size = null;
13623		const mnem_tag = mnem: {
13624		mnem_size = if (mem.endsWith(u8, mnem_str, "b"))
13625		.byte
13626		else if (mem.endsWith(u8, mnem_str, "w"))
13627		.word
13628		else if (mem.endsWith(u8, mnem_str, "l"))
13629		.dword
13630		else if (mem.endsWith(u8, mnem_str, "q"))
13631		.qword
13632		else if (mem.endsWith(u8, mnem_str, "t"))
13633		.tbyte
13634		else
13635		break :mnem null;
13636		break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str[0 .. mnem_str.len - 1]);
13637		} orelse mnem: {
	13893	var mnem_size: ?Memory.Size = if (mem.endsWith(u8, mnem_str, "b"))
	13894	.byte
	13895	else if (mem.endsWith(u8, mnem_str, "w"))
	13896	.word
	13897	else if (mem.endsWith(u8, mnem_str, "l"))
	13898	.dword
	13899	else if (mem.endsWith(u8, mnem_str, "q") and
	13900	(std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq")))
	13901	.qword
	13902	else if (mem.endsWith(u8, mnem_str, "t"))
	13903	.tbyte
	13904	else
	13905	null;
	13906	const mnem_tag = while (true) break std.meta.stringToEnum(
	13907	Instruction.Mnemonic,
	13908	mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)],
	13909	) orelse if (mnem_size) \|_\| {
13638	13910	mnem_size = null;
13639		break :mnem std.meta.stringToEnum(Instruction.Mnemonic, mnem_str);
13640		} orelse return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
	13911	continue;
	13912	} else return self.fail("invalid mnemonic: '{s}'", .{mnem_str});
13641	13913	if (@as(?Memory.Size, switch (mnem_tag) {
13642	13914	.clflush => .byte,
13643	13915	.fldenv, .fnstenv, .fstenv => .none,
...	...	@@ -14135,30 +14407,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14135	14407	else => {},
14136	14408	},
14137	14409	.Int => switch (ty.childType(mod).intInfo(mod).bits) {
14138		8 => switch (ty.vectorLen(mod)) {
14139		1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
14140		.insert = .{ .vp_b, .insr },
14141		.extract = .{ .vp_b, .extr },
14142		} } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
14143		.insert = .{ .p_b, .insr },
14144		.extract = .{ .p_b, .extr },
14145		} },
14146		2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
14147		.insert = .{ .vp_w, .insr },
14148		.extract = .{ .vp_w, .extr },
14149		} } else .{ .insert_extract = .{
14150		.insert = .{ .p_w, .insr },
14151		.extract = .{ .p_w, .extr },
14152		} },
14153		3...4 => return .{ .move = if (self.hasFeature(.avx))
14154		.{ .v_d, .mov }
14155		else
14156		.{ ._d, .mov } },
14157		5...8 => return .{ .move = if (self.hasFeature(.avx))
14158		.{ .v_q, .mov }
14159		else
14160		.{ ._q, .mov } },
14161		9...16 => return .{ .move = if (self.hasFeature(.avx))
	14410	1...8 => switch (ty.vectorLen(mod)) {
	14411	1...16 => return .{ .move = if (self.hasFeature(.avx))
14162	14412	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14163	14413	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14164	14414	17...32 => if (self.hasFeature(.avx))
...	...	@@ -14168,23 +14418,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14168	14418	.{ .v_, .movdqu } },
14169	14419	else => {},
14170	14420	},
14171		16 => switch (ty.vectorLen(mod)) {
14172		1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
14173		.insert = .{ .vp_w, .insr },
14174		.extract = .{ .vp_w, .extr },
14175		} } else .{ .insert_extract = .{
14176		.insert = .{ .p_w, .insr },
14177		.extract = .{ .p_w, .extr },
14178		} },
14179		2 => return .{ .move = if (self.hasFeature(.avx))
14180		.{ .v_d, .mov }
14181		else
14182		.{ ._d, .mov } },
14183		3...4 => return .{ .move = if (self.hasFeature(.avx))
14184		.{ .v_q, .mov }
14185		else
14186		.{ ._q, .mov } },
14187		5...8 => return .{ .move = if (self.hasFeature(.avx))
	14421	9...16 => switch (ty.vectorLen(mod)) {
	14422	1...8 => return .{ .move = if (self.hasFeature(.avx))
14188	14423	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14189	14424	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14190	14425	9...16 => if (self.hasFeature(.avx))
...	...	@@ -14194,16 +14429,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14194	14429	.{ .v_, .movdqu } },
14195	14430	else => {},
14196	14431	},
14197		32 => switch (ty.vectorLen(mod)) {
14198		1 => return .{ .move = if (self.hasFeature(.avx))
14199		.{ .v_d, .mov }
14200		else
14201		.{ ._d, .mov } },
14202		2 => return .{ .move = if (self.hasFeature(.avx))
14203		.{ .v_q, .mov }
14204		else
14205		.{ ._q, .mov } },
14206		3...4 => return .{ .move = if (self.hasFeature(.avx))
	14432	17...32 => switch (ty.vectorLen(mod)) {
	14433	1...4 => return .{ .move = if (self.hasFeature(.avx))
14207	14434	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14208	14435	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14209	14436	5...8 => if (self.hasFeature(.avx))
...	...	@@ -14213,12 +14440,8 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14213	14440	.{ .v_, .movdqu } },
14214	14441	else => {},
14215	14442	},
14216		64 => switch (ty.vectorLen(mod)) {
14217		1 => return .{ .move = if (self.hasFeature(.avx))
14218		.{ .v_q, .mov }
14219		else
14220		.{ ._q, .mov } },
14221		2 => return .{ .move = if (self.hasFeature(.avx))
	14443	33...64 => switch (ty.vectorLen(mod)) {
	14444	1...2 => return .{ .move = if (self.hasFeature(.avx))
14222	14445	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14223	14446	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14224	14447	3...4 => if (self.hasFeature(.avx))
...	...	@@ -14228,7 +14451,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14228	14451	.{ .v_, .movdqu } },
14229	14452	else => {},
14230	14453	},
14231		128 => switch (ty.vectorLen(mod)) {
	14454	65...128 => switch (ty.vectorLen(mod)) {
14232	14455	1 => return .{ .move = if (self.hasFeature(.avx))
14233	14456	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14234	14457	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
...	...	@@ -14239,7 +14462,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14239	14462	.{ .v_, .movdqu } },
14240	14463	else => {},
14241	14464	},
14242		256 => switch (ty.vectorLen(mod)) {
	14465	129...256 => switch (ty.vectorLen(mod)) {
14243	14466	1 => if (self.hasFeature(.avx))
14244	14467	return .{ .move = if (aligned)
14245	14468	.{ .v_, .movdqa }
...	...	@@ -14251,11 +14474,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14251	14474	},
14252	14475	.Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod))
14253	14476	switch (ty.vectorLen(mod)) {
14254		1 => return .{ .move = if (self.hasFeature(.avx))
14255		.{ .v_q, .mov }
14256		else
14257		.{ ._q, .mov } },
14258		2 => return .{ .move = if (self.hasFeature(.avx))
	14477	1...2 => return .{ .move = if (self.hasFeature(.avx))
14259	14478	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14260	14479	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14261	14480	3...4 => if (self.hasFeature(.avx))
...	...	@@ -14269,22 +14488,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14269	14488	unreachable,
14270	14489	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
14271	14490	16 => switch (ty.vectorLen(mod)) {
14272		1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
14273		.insert = .{ .vp_w, .insr },
14274		.extract = .{ .vp_w, .extr },
14275		} } else .{ .insert_extract = .{
14276		.insert = .{ .p_w, .insr },
14277		.extract = .{ .p_w, .extr },
14278		} },
14279		2 => return .{ .move = if (self.hasFeature(.avx))
14280		.{ .v_d, .mov }
14281		else
14282		.{ ._d, .mov } },
14283		3...4 => return .{ .move = if (self.hasFeature(.avx))
14284		.{ .v_q, .mov }
14285		else
14286		.{ ._q, .mov } },
14287		5...8 => return .{ .move = if (self.hasFeature(.avx))
	14491	1...8 => return .{ .move = if (self.hasFeature(.avx))
14288	14492	if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
14289	14493	else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
14290	14494	9...16 => if (self.hasFeature(.avx))
...	...	@@ -14295,15 +14499,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14295	14499	else => {},
14296	14500	},
14297	14501	32 => switch (ty.vectorLen(mod)) {
14298		1 => return .{ .move = if (self.hasFeature(.avx))
14299		.{ .v_ss, .mov }
14300		else
14301		.{ ._ss, .mov } },
14302		2 => return .{ .move = if (self.hasFeature(.avx))
14303		.{ .v_sd, .mov }
14304		else
14305		.{ ._sd, .mov } },
14306		3...4 => return .{ .move = if (self.hasFeature(.avx))
	14502	1...4 => return .{ .move = if (self.hasFeature(.avx))
14307	14503	if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
14308	14504	else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
14309	14505	5...8 => if (self.hasFeature(.avx))
...	...	@@ -14314,11 +14510,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
14314	14510	else => {},
14315	14511	},
14316	14512	64 => switch (ty.vectorLen(mod)) {
14317		1 => return .{ .move = if (self.hasFeature(.avx))
14318		.{ .v_sd, .mov }
14319		else
14320		.{ ._sd, .mov } },
14321		2 => return .{ .move = if (self.hasFeature(.avx))
	14513	1...2 => return .{ .move = if (self.hasFeature(.avx))
14322	14514	if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
14323	14515	else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
14324	14516	3...4 => if (self.hasFeature(.avx))
...	...	@@ -14633,7 +14825,7 @@ fn genSetReg(
14633	14825	ty,
14634	14826	dst_reg.class(),
14635	14827	self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units(
14636		std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
	14828	math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)),
14637	14829	)),
14638	14830	),
14639	14831	.lea_frame => .{ .move = .{ ._, .lea } },
...	...	@@ -16296,7 +16488,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16296	16488	},
16297	16489	65...128 => switch (vector_len) {
16298	16490	else => null,
16299		1...2 => .{ .vp_i128, .broadcast },
	16491	1...2 => .{ .v_i128, .broadcast },
16300	16492	},
16301	16493	}) orelse break :avx2;
16302	16494
...	...	@@ -16310,7 +16502,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16310	16502	registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
16311	16503	try src_mcv.mem(self, self.memSize(scalar_ty)),
16312	16504	) else {
16313		if (mir_tag[0] == .vp_i128) break :avx2;
	16505	if (mir_tag[0] == .v_i128) break :avx2;
16314	16506	try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
16315	16507	try self.asmRegisterRegister(
16316	16508	mir_tag,
...	...	@@ -16352,7 +16544,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16352	16544	.{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
16353	16545	dst_alias,
16354	16546	dst_alias,
16355		Immediate.u(0),
	16547	Immediate.u(0b00_00_00_00),
16356	16548	);
16357	16549	if (switch (scalar_bits) {
16358	16550	1...8 => vector_len > 4,
...	...	@@ -16563,18 +16755,1158 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
16563	16755	}
16564	16756
16565	16757	fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
	16758	const mod = self.bin_file.comp.module.?;
16566	16759	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
16567	16760	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
16568		_ = extra;
16569		return self.fail("TODO implement airSelect for x86_64", .{});
16570		//return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
	16761	const ty = self.typeOfIndex(inst);
	16762	const vec_len = ty.vectorLen(mod);
	16763	const elem_ty = ty.childType(mod);
	16764	const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));
	16765	const abi_size: u32 = @intCast(ty.abiSize(mod));
	16766	const pred_ty = self.typeOf(pl_op.operand);
	16767
	16768	const result = result: {
	16769	const has_blend = self.hasFeature(.sse4_1);
	16770	const has_avx = self.hasFeature(.avx);
	16771	const need_xmm0 = has_blend and !has_avx;
	16772	const pred_mcv = try self.resolveInst(pl_op.operand);
	16773	const mask_reg = mask: {
	16774	switch (pred_mcv) {
	16775	.register => \|pred_reg\| switch (pred_reg.class()) {
	16776	.general_purpose => {},
	16777	.sse => if (need_xmm0 and pred_reg.id() != comptime Register.xmm0.id()) {
	16778	try self.register_manager.getKnownReg(.xmm0, null);
	16779	try self.genSetReg(.xmm0, pred_ty, pred_mcv, .{});
	16780	break :mask .xmm0;
	16781	} else break :mask if (has_blend)
	16782	pred_reg
	16783	else
	16784	try self.copyToTmpRegister(pred_ty, pred_mcv),
	16785	else => unreachable,
	16786	},
	16787	else => {},
	16788	}
	16789	const mask_reg: Register = if (need_xmm0) mask_reg: {
	16790	try self.register_manager.getKnownReg(.xmm0, null);
	16791	break :mask_reg .xmm0;
	16792	} else try self.register_manager.allocReg(null, abi.RegisterClass.sse);
	16793	const mask_alias = registerAlias(mask_reg, abi_size);
	16794	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
	16795	defer self.register_manager.unlockReg(mask_lock);
	16796
	16797	const pred_fits_in_elem = vec_len <= elem_abi_size;
	16798	if (self.hasFeature(.avx2) and abi_size <= 32) {
	16799	if (pred_mcv.isRegister()) broadcast: {
	16800	try self.asmRegisterRegister(
	16801	.{ .v_d, .mov },
	16802	mask_reg.to128(),
	16803	pred_mcv.getReg().?.to32(),
	16804	);
	16805	if (pred_fits_in_elem and vec_len > 1) try self.asmRegisterRegister(
	16806	.{ switch (elem_abi_size) {
	16807	1 => .vp_b,
	16808	2 => .vp_w,
	16809	3...4 => .vp_d,
	16810	5...8 => .vp_q,
	16811	9...16 => {
	16812	try self.asmRegisterRegisterRegisterImmediate(
	16813	.{ .v_f128, .insert },
	16814	mask_alias,
	16815	mask_alias,
	16816	mask_reg.to128(),
	16817	Immediate.u(1),
	16818	);
	16819	break :broadcast;
	16820	},
	16821	17...32 => break :broadcast,
	16822	else => unreachable,
	16823	}, .broadcast },
	16824	mask_alias,
	16825	mask_reg.to128(),
	16826	);
	16827	} else try self.asmRegisterMemory(
	16828	.{ switch (vec_len) {
	16829	1...8 => .vp_b,
	16830	9...16 => .vp_w,
	16831	17...32 => .vp_d,
	16832	else => unreachable,
	16833	}, .broadcast },
	16834	mask_alias,
	16835	if (pred_mcv.isMemory()) try pred_mcv.mem(self, .byte) else .{
	16836	.base = .{ .reg = (try self.copyToTmpRegister(
	16837	Type.usize,
	16838	pred_mcv.address(),
	16839	)).to64() },
	16840	.mod = .{ .rm = .{ .size = .byte } },
	16841	},
	16842	);
	16843	} else if (abi_size <= 16) broadcast: {
	16844	try self.asmRegisterRegister(
	16845	.{ if (has_avx) .v_d else ._d, .mov },
	16846	mask_alias,
	16847	(if (pred_mcv.isRegister())
	16848	pred_mcv.getReg().?
	16849	else
	16850	try self.copyToTmpRegister(pred_ty, pred_mcv.address())).to32(),
	16851	);
	16852	if (!pred_fits_in_elem or vec_len == 1) break :broadcast;
	16853	if (elem_abi_size <= 1) {
	16854	if (has_avx) try self.asmRegisterRegisterRegister(
	16855	.{ .vp_, .unpcklbw },
	16856	mask_alias,
	16857	mask_alias,
	16858	mask_alias,
	16859	) else try self.asmRegisterRegister(
	16860	.{ .p_, .unpcklbw },
	16861	mask_alias,
	16862	mask_alias,
	16863	);
	16864	if (abi_size <= 2) break :broadcast;
	16865	}
	16866	if (elem_abi_size <= 2) {
	16867	try self.asmRegisterRegisterImmediate(
	16868	.{ if (has_avx) .vp_w else .p_w, .shufl },
	16869	mask_alias,
	16870	mask_alias,
	16871	Immediate.u(0b00_00_00_00),
	16872	);
	16873	if (abi_size <= 8) break :broadcast;
	16874	}
	16875	try self.asmRegisterRegisterImmediate(
	16876	.{ if (has_avx) .vp_d else .p_d, .shuf },
	16877	mask_alias,
	16878	mask_alias,
	16879	Immediate.u(switch (elem_abi_size) {
	16880	1...2, 5...8 => 0b01_00_01_00,
	16881	3...4 => 0b00_00_00_00,
	16882	else => unreachable,
	16883	}),
	16884	);
	16885	} else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
	16886	const elem_bits: u16 = @intCast(elem_abi_size * 8);
	16887	const mask_elem_ty = try mod.intType(.unsigned, elem_bits);
	16888	const mask_ty = try mod.vectorType(.{ .len = vec_len, .child = mask_elem_ty.toIntern() });
	16889	if (!pred_fits_in_elem) if (self.hasFeature(.ssse3)) {
	16890	var mask_elems: [32]InternPool.Index = undefined;
	16891	for (mask_elems[0..vec_len], 0..) \|elem, bit\| elem. = try mod.intern(.{ .int = .{
	16892	.ty = mask_elem_ty.toIntern(),
	16893	.storage = .{ .u64 = bit / elem_bits },
	16894	} });
	16895	const mask_mcv = try self.genTypedValue(.{
	16896	.ty = mask_ty,
	16897	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
	16898	.ty = mask_ty.toIntern(),
	16899	.storage = .{ .elems = mask_elems[0..vec_len] },
	16900	} })),
	16901	});
	16902	const mask_mem: Memory = .{
	16903	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
	16904	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
	16905	};
	16906	if (has_avx) try self.asmRegisterRegisterMemory(
	16907	.{ .vp_b, .shuf },
	16908	mask_alias,
	16909	mask_alias,
	16910	mask_mem,
	16911	) else try self.asmRegisterMemory(
	16912	.{ .p_b, .shuf },
	16913	mask_alias,
	16914	mask_mem,
	16915	);
	16916	} else return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
	16917	{
	16918	var mask_elems: [32]InternPool.Index = undefined;
	16919	for (mask_elems[0..vec_len], 0..) \|elem, bit\| elem. = try mod.intern(.{ .int = .{
	16920	.ty = mask_elem_ty.toIntern(),
	16921	.storage = .{ .u64 = @as(u32, 1) << @intCast(bit & (elem_bits - 1)) },
	16922	} });
	16923	const mask_mcv = try self.genTypedValue(.{
	16924	.ty = mask_ty,
	16925	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
	16926	.ty = mask_ty.toIntern(),
	16927	.storage = .{ .elems = mask_elems[0..vec_len] },
	16928	} })),
	16929	});
	16930	const mask_mem: Memory = .{
	16931	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, mask_mcv.address()) },
	16932	.mod = .{ .rm = .{ .size = self.memSize(ty) } },
	16933	};
	16934	if (has_avx) {
	16935	try self.asmRegisterRegisterMemory(
	16936	.{ .vp_, .@"and" },
	16937	mask_alias,
	16938	mask_alias,
	16939	mask_mem,
	16940	);
	16941	try self.asmRegisterRegisterMemory(
	16942	.{ .vp_d, .cmpeq },
	16943	mask_alias,
	16944	mask_alias,
	16945	mask_mem,
	16946	);
	16947	} else {
	16948	try self.asmRegisterMemory(
	16949	.{ .p_, .@"and" },
	16950	mask_alias,
	16951	mask_mem,
	16952	);
	16953	try self.asmRegisterMemory(
	16954	.{ .p_d, .cmpeq },
	16955	mask_alias,
	16956	mask_mem,
	16957	);
	16958	}
	16959	}
	16960	break :mask mask_reg;
	16961	};
	16962	const mask_alias = registerAlias(mask_reg, abi_size);
	16963	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
	16964	defer self.register_manager.unlockReg(mask_lock);
	16965
	16966	const lhs_mcv = try self.resolveInst(extra.lhs);
	16967	const lhs_lock = switch (lhs_mcv) {
	16968	.register => \|lhs_reg\| self.register_manager.lockRegAssumeUnused(lhs_reg),
	16969	else => null,
	16970	};
	16971	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	16972
	16973	const rhs_mcv = try self.resolveInst(extra.rhs);
	16974	const rhs_lock = switch (rhs_mcv) {
	16975	.register => \|rhs_reg\| self.register_manager.lockReg(rhs_reg),
	16976	else => null,
	16977	};
	16978	defer if (rhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	16979
	16980	const reuse_mcv = if (has_blend) rhs_mcv else lhs_mcv;
	16981	const dst_mcv: MCValue = if (reuse_mcv.isRegister() and self.reuseOperand(
	16982	inst,
	16983	if (has_blend) extra.rhs else extra.lhs,
	16984	@intFromBool(has_blend),
	16985	reuse_mcv,
	16986	)) reuse_mcv else if (has_avx)
	16987	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	16988	else
	16989	try self.copyToRegisterWithInstTracking(inst, ty, reuse_mcv);
	16990	const dst_reg = dst_mcv.getReg().?;
	16991	const dst_alias = registerAlias(dst_reg, abi_size);
	16992	const dst_lock = self.register_manager.lockReg(dst_reg);
	16993	defer if (dst_lock) \|lock\| self.register_manager.unlockReg(lock);
	16994
	16995	const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.childType(mod).zigTypeTag(mod)) {
	16996	else => null,
	16997	.Int => switch (abi_size) {
	16998	0 => unreachable,
	16999	1...16 => if (has_avx)
	17000	.{ .vp_b, .blendv }
	17001	else if (has_blend)
	17002	.{ .p_b, .blendv }
	17003	else
	17004	.{ .p_, undefined },
	17005	17...32 => if (self.hasFeature(.avx2))
	17006	.{ .vp_b, .blendv }
	17007	else
	17008	null,
	17009	else => null,
	17010	},
	17011	.Float => switch (ty.childType(mod).floatBits(self.target.*)) {
	17012	else => unreachable,
	17013	16, 80, 128 => null,
	17014	32 => switch (vec_len) {
	17015	0 => unreachable,
	17016	1...4 => if (has_avx) .{ .v_ps, .blendv } else .{ ._ps, .blendv },
	17017	5...8 => if (has_avx) .{ .v_ps, .blendv } else null,
	17018	else => null,
	17019	},
	17020	64 => switch (vec_len) {
	17021	0 => unreachable,
	17022	1...2 => if (has_avx) .{ .v_pd, .blendv } else .{ ._pd, .blendv },
	17023	3...4 => if (has_avx) .{ .v_pd, .blendv } else null,
	17024	else => null,
	17025	},
	17026	},
	17027	}) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
	17028	if (has_avx) {
	17029	const rhs_alias = if (rhs_mcv.isRegister())
	17030	registerAlias(rhs_mcv.getReg().?, abi_size)
	17031	else rhs: {
	17032	try self.genSetReg(dst_reg, ty, rhs_mcv, .{});
	17033	break :rhs dst_alias;
	17034	};
	17035	if (lhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
	17036	mir_tag,
	17037	dst_alias,
	17038	rhs_alias,
	17039	try lhs_mcv.mem(self, self.memSize(ty)),
	17040	mask_alias,
	17041	) else try self.asmRegisterRegisterRegisterRegister(
	17042	mir_tag,
	17043	dst_alias,
	17044	rhs_alias,
	17045	registerAlias(if (lhs_mcv.isRegister())
	17046	lhs_mcv.getReg().?
	17047	else
	17048	try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
	17049	mask_alias,
	17050	);
	17051	} else if (has_blend) if (lhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
	17052	mir_tag,
	17053	dst_alias,
	17054	try lhs_mcv.mem(self, self.memSize(ty)),
	17055	mask_alias,
	17056	) else try self.asmRegisterRegisterRegister(
	17057	mir_tag,
	17058	dst_alias,
	17059	registerAlias(if (lhs_mcv.isRegister())
	17060	lhs_mcv.getReg().?
	17061	else
	17062	try self.copyToTmpRegister(ty, lhs_mcv), abi_size),
	17063	mask_alias,
	17064	) else {
	17065	const mir_fixes = @as(?Mir.Inst.Fixes, switch (elem_ty.zigTypeTag(mod)) {
	17066	else => null,
	17067	.Int => .p_,
	17068	.Float => switch (elem_ty.floatBits(self.target.*)) {
	17069	32 => ._ps,
	17070	64 => ._pd,
	17071	16, 80, 128 => null,
	17072	else => unreachable,
	17073	},
	17074	}) orelse return self.fail("TODO implement airSelect for {}", .{ty.fmt(mod)});
	17075	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
	17076	if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
	17077	.{ mir_fixes, .andn },
	17078	mask_alias,
	17079	try rhs_mcv.mem(self, Memory.Size.fromSize(abi_size)),
	17080	) else try self.asmRegisterRegister(
	17081	.{ mir_fixes, .andn },
	17082	mask_alias,
	17083	if (rhs_mcv.isRegister())
	17084	rhs_mcv.getReg().?
	17085	else
	17086	try self.copyToTmpRegister(ty, rhs_mcv),
	17087	);
	17088	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
	17089	}
	17090	break :result dst_mcv;
	17091	};
	17092	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
16571	17093	}
16572	17094
16573	17095	fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
	17096	const mod = self.bin_file.comp.module.?;
16574	17097	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
16575		_ = ty_pl;
16576		return self.fail("TODO implement airShuffle for x86_64", .{});
16577		//return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
	17098	const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
	17099
	17100	const dst_ty = self.typeOfIndex(inst);
	17101	const elem_ty = dst_ty.childType(mod);
	17102	const elem_abi_size: u16 = @intCast(elem_ty.abiSize(mod));
	17103	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));
	17104	const lhs_ty = self.typeOf(extra.a);
	17105	const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
	17106	const rhs_ty = self.typeOf(extra.b);
	17107	const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(mod));
	17108	const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size);
	17109
	17110	const ExpectedContents = [32]?i32;
	17111	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
	17112	std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
	17113	const allocator = stack.get();
	17114
	17115	const mask_elems = try allocator.alloc(?i32, extra.mask_len);
	17116	defer allocator.free(mask_elems);
	17117	for (mask_elems, 0..) \|*mask_elem, elem_index\| {
	17118	const mask_elem_val =
	17119	Value.fromInterned(extra.mask).elemValue(mod, elem_index) catch unreachable;
	17120	mask_elem.* = if (mask_elem_val.isUndef(mod))
	17121	null
	17122	else
	17123	@intCast(mask_elem_val.toSignedInt(mod));
	17124	}
	17125
	17126	const has_avx = self.hasFeature(.avx);
	17127	const result = @as(?MCValue, result: {
	17128	for (mask_elems) \|mask_elem\| {
	17129	if (mask_elem) \|_\| break;
	17130	} else break :result try self.allocRegOrMem(inst, true);
	17131
	17132	for (mask_elems, 0..) \|mask_elem, elem_index\| {
	17133	if (mask_elem orelse continue != elem_index) break;
	17134	} else {
	17135	const lhs_mcv = try self.resolveInst(extra.a);
	17136	if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv;
	17137	const dst_mcv = try self.allocRegOrMem(inst, true);
	17138	try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{});
	17139	break :result dst_mcv;
	17140	}
	17141
	17142	for (mask_elems, 0..) \|mask_elem, elem_index\| {
	17143	if (~(mask_elem orelse continue) != elem_index) break;
	17144	} else {
	17145	const rhs_mcv = try self.resolveInst(extra.b);
	17146	if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv;
	17147	const dst_mcv = try self.allocRegOrMem(inst, true);
	17148	try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{});
	17149	break :result dst_mcv;
	17150	}
	17151
	17152	for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) \|variant\| unpck: {
	17153	if (elem_abi_size > 8) break :unpck;
	17154	if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
	17155	has_avx
	17156	else
	17157	self.hasFeature(.avx2)) 32 else 16)) break :unpck;
	17158
	17159	var sources = [1]?u1{null} ** 2;
	17160	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
	17161	const mask_elem = maybe_mask_elem orelse continue;
	17162	const mask_elem_index =
	17163	math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
	17164	const elem_byte = (elem_index >> 1) * elem_abi_size;
	17165	if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {
	17166	.unpckl => 0b0000,
	17167	.unpckh => 0b1000,
	17168	else => unreachable,
	17169	}) \| (elem_byte << 1 & 0b10000)) break :unpck;
	17170
	17171	const source = @intFromBool(mask_elem < 0);
	17172	if (sources[elem_index & 0b00001]) \|prev_source\| {
	17173	if (source != prev_source) break :unpck;
	17174	} else sources[elem_index & 0b00001] = source;
	17175	}
	17176	if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck;
	17177
	17178	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
	17179	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
	17180	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
	17181	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
	17182
	17183	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
	17184	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
	17185	lhs_mcv
	17186	else if (has_avx and lhs_mcv.isRegister())
	17187	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	17188	else
	17189	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
	17190	const dst_reg = dst_mcv.getReg().?;
	17191	const dst_alias = registerAlias(dst_reg, max_abi_size);
	17192
	17193	const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
	17194	(dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) {
	17195	4 => if (has_avx) .v_ps else ._ps,
	17196	8 => if (has_avx) .v_pd else ._pd,
	17197	else => unreachable,
	17198	}, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) {
	17199	.unpckl => switch (elem_abi_size) {
	17200	1 => .unpcklbw,
	17201	2 => .unpcklwd,
	17202	4 => .unpckldq,
	17203	8 => .unpcklqdq,
	17204	else => unreachable,
	17205	},
	17206	.unpckh => switch (elem_abi_size) {
	17207	1 => .unpckhbw,
	17208	2 => .unpckhwd,
	17209	4 => .unpckhdq,
	17210	8 => .unpckhqdq,
	17211	else => unreachable,
	17212	},
	17213	else => unreachable,
	17214	} };
	17215	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemory(
	17216	mir_tag,
	17217	dst_alias,
	17218	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
	17219	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
	17220	) else try self.asmRegisterRegisterRegister(
	17221	mir_tag,
	17222	dst_alias,
	17223	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
	17224	registerAlias(if (rhs_mcv.isRegister())
	17225	rhs_mcv.getReg().?
	17226	else
	17227	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
	17228	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemory(
	17229	mir_tag,
	17230	dst_alias,
	17231	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
	17232	) else try self.asmRegisterRegister(
	17233	mir_tag,
	17234	dst_alias,
	17235	registerAlias(if (rhs_mcv.isRegister())
	17236	rhs_mcv.getReg().?
	17237	else
	17238	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
	17239	);
	17240	break :result dst_mcv;
	17241	}
	17242
	17243	pshufd: {
	17244	if (elem_abi_size != 4) break :pshufd;
	17245	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :pshufd;
	17246
	17247	var control: u8 = 0b00_00_00_00;
	17248	var sources = [1]?u1{null} ** 1;
	17249	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
	17250	const mask_elem = maybe_mask_elem orelse continue;
	17251	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
	17252	if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd;
	17253
	17254	const source = @intFromBool(mask_elem < 0);
	17255	if (sources[0]) \|prev_source\| {
	17256	if (source != prev_source) break :pshufd;
	17257	} else sources[(elem_index & 0b010) >> 1] = source;
	17258
	17259	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
	17260	const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
	17261	if (elem_index & 0b100 == 0)
	17262	control \|= select
	17263	else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd;
	17264	}
	17265
	17266	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
	17267	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
	17268	const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]);
	17269
	17270	const dst_reg = if (src_mcv.isRegister() and
	17271	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv))
	17272	src_mcv.getReg().?
	17273	else
	17274	try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
	17275	const dst_alias = registerAlias(dst_reg, max_abi_size);
	17276
	17277	if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	17278	.{ if (has_avx) .vp_d else .p_d, .shuf },
	17279	dst_alias,
	17280	try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
	17281	Immediate.u(control),
	17282	) else try self.asmRegisterRegisterImmediate(
	17283	.{ if (has_avx) .vp_d else .p_d, .shuf },
	17284	dst_alias,
	17285	registerAlias(if (src_mcv.isRegister())
	17286	src_mcv.getReg().?
	17287	else
	17288	try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
	17289	Immediate.u(control),
	17290	);
	17291	break :result .{ .register = dst_reg };
	17292	}
	17293
	17294	shufps: {
	17295	if (elem_abi_size != 4) break :shufps;
	17296	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufps;
	17297
	17298	var control: u8 = 0b00_00_00_00;
	17299	var sources = [1]?u1{null} ** 2;
	17300	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
	17301	const mask_elem = maybe_mask_elem orelse continue;
	17302	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
	17303	if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps;
	17304
	17305	const source = @intFromBool(mask_elem < 0);
	17306	if (sources[(elem_index & 0b010) >> 1]) \|prev_source\| {
	17307	if (source != prev_source) break :shufps;
	17308	} else sources[(elem_index & 0b010) >> 1] = source;
	17309
	17310	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
	17311	const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
	17312	if (elem_index & 0b100 == 0)
	17313	control \|= select
	17314	else if (control & @as(u8, 0b11) << select_bit != select) break :shufps;
	17315	}
	17316	if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
	17317
	17318	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
	17319	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
	17320	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
	17321	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
	17322
	17323	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
	17324	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
	17325	lhs_mcv
	17326	else if (has_avx and lhs_mcv.isRegister())
	17327	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	17328	else
	17329	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
	17330	const dst_reg = dst_mcv.getReg().?;
	17331	const dst_alias = registerAlias(dst_reg, max_abi_size);
	17332
	17333	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	17334	.{ .v_ps, .shuf },
	17335	dst_alias,
	17336	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
	17337	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
	17338	Immediate.u(control),
	17339	) else try self.asmRegisterRegisterRegisterImmediate(
	17340	.{ .v_ps, .shuf },
	17341	dst_alias,
	17342	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
	17343	registerAlias(if (rhs_mcv.isRegister())
	17344	rhs_mcv.getReg().?
	17345	else
	17346	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
	17347	Immediate.u(control),
	17348	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	17349	.{ ._ps, .shuf },
	17350	dst_alias,
	17351	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
	17352	Immediate.u(control),
	17353	) else try self.asmRegisterRegisterImmediate(
	17354	.{ ._ps, .shuf },
	17355	dst_alias,
	17356	registerAlias(if (rhs_mcv.isRegister())
	17357	rhs_mcv.getReg().?
	17358	else
	17359	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
	17360	Immediate.u(control),
	17361	);
	17362	break :result dst_mcv;
	17363	}
	17364
	17365	shufpd: {
	17366	if (elem_abi_size != 8) break :shufpd;
	17367	if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;
	17368
	17369	var control: u4 = 0b0_0_0_0;
	17370	var sources = [1]?u1{null} ** 2;
	17371	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
	17372	const mask_elem = maybe_mask_elem orelse continue;
	17373	const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
	17374	if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd;
	17375
	17376	const source = @intFromBool(mask_elem < 0);
	17377	if (sources[elem_index & 0b01]) \|prev_source\| {
	17378	if (source != prev_source) break :shufpd;
	17379	} else sources[elem_index & 0b01] = source;
	17380
	17381	control \|= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index);
	17382	}
	17383	if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd;
	17384
	17385	const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b };
	17386	const operand_tys: [2]Type = .{ lhs_ty, rhs_ty };
	17387	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
	17388	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
	17389
	17390	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
	17391	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
	17392	lhs_mcv
	17393	else if (has_avx and lhs_mcv.isRegister())
	17394	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	17395	else
	17396	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
	17397	const dst_reg = dst_mcv.getReg().?;
	17398	const dst_alias = registerAlias(dst_reg, max_abi_size);
	17399
	17400	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	17401	.{ .v_pd, .shuf },
	17402	dst_alias,
	17403	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
	17404	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
	17405	Immediate.u(control),
	17406	) else try self.asmRegisterRegisterRegisterImmediate(
	17407	.{ .v_pd, .shuf },
	17408	dst_alias,
	17409	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
	17410	registerAlias(if (rhs_mcv.isRegister())
	17411	rhs_mcv.getReg().?
	17412	else
	17413	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
	17414	Immediate.u(control),
	17415	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	17416	.{ ._pd, .shuf },
	17417	dst_alias,
	17418	try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),
	17419	Immediate.u(control),
	17420	) else try self.asmRegisterRegisterImmediate(
	17421	.{ ._pd, .shuf },
	17422	dst_alias,
	17423	registerAlias(if (rhs_mcv.isRegister())
	17424	rhs_mcv.getReg().?
	17425	else
	17426	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
	17427	Immediate.u(control),
	17428	);
	17429	break :result dst_mcv;
	17430	}
	17431
	17432	blend: {
	17433	if (elem_abi_size < 2) break :blend;
	17434	if (dst_abi_size > @as(u32, if (has_avx) 32 else 16)) break :blend;
	17435	if (!self.hasFeature(.sse4_1)) break :blend;
	17436
	17437	var control: u8 = 0b0_0_0_0_0_0_0_0;
	17438	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
	17439	const mask_elem = maybe_mask_elem orelse continue;
	17440	const mask_elem_index =
	17441	math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
	17442	if (mask_elem_index != elem_index) break :blend;
	17443
	17444	const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
	17445	if (elem_index & 0b1000 == 0)
	17446	control \|= select
	17447	else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend;
	17448	}
	17449
	17450	if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
	17451	const expanded_control = switch (elem_abi_size) {
	17452	4 => control,
	17453	8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) \|
	17454	@as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) \|
	17455	@as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) \|
	17456	@as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00),
	17457	else => break :vpblendd,
	17458	};
	17459
	17460	const lhs_mcv = try self.resolveInst(extra.a);
	17461	const lhs_reg = if (lhs_mcv.isRegister())
	17462	lhs_mcv.getReg().?
	17463	else
	17464	try self.copyToTmpRegister(dst_ty, lhs_mcv);
	17465	const lhs_lock = self.register_manager.lockReg(lhs_reg);
	17466	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
	17467
	17468	const rhs_mcv = try self.resolveInst(extra.b);
	17469	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
	17470	if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	17471	.{ .vp_d, .blend },
	17472	registerAlias(dst_reg, dst_abi_size),
	17473	registerAlias(lhs_reg, dst_abi_size),
	17474	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
	17475	Immediate.u(expanded_control),
	17476	) else try self.asmRegisterRegisterRegisterImmediate(
	17477	.{ .vp_d, .blend },
	17478	registerAlias(dst_reg, dst_abi_size),
	17479	registerAlias(lhs_reg, dst_abi_size),
	17480	registerAlias(if (rhs_mcv.isRegister())
	17481	rhs_mcv.getReg().?
	17482	else
	17483	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
	17484	Immediate.u(expanded_control),
	17485	);
	17486	break :result .{ .register = dst_reg };
	17487	}
	17488
	17489	if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: {
	17490	const expanded_control = switch (elem_abi_size) {
	17491	2 => control,
	17492	4 => if (dst_abi_size <= 16 or
	17493	@as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0)))
	17494	@as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) \|
	17495	@as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) \|
	17496	@as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) \|
	17497	@as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00)
	17498	else
	17499	break :pblendw,
	17500	8 => if (dst_abi_size <= 16 or
	17501	@as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0)))
	17502	@as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) \|
	17503	@as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000)
	17504	else
	17505	break :pblendw,
	17506	16 => break :pblendw,
	17507	else => unreachable,
	17508	};
	17509
	17510	const lhs_mcv = try self.resolveInst(extra.a);
	17511	const rhs_mcv = try self.resolveInst(extra.b);
	17512
	17513	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
	17514	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
	17515	lhs_mcv
	17516	else if (has_avx and lhs_mcv.isRegister())
	17517	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	17518	else
	17519	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
	17520	const dst_reg = dst_mcv.getReg().?;
	17521
	17522	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	17523	.{ .vp_w, .blend },
	17524	registerAlias(dst_reg, dst_abi_size),
	17525	registerAlias(if (lhs_mcv.isRegister())
	17526	lhs_mcv.getReg().?
	17527	else
	17528	dst_reg, dst_abi_size),
	17529	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
	17530	Immediate.u(expanded_control),
	17531	) else try self.asmRegisterRegisterRegisterImmediate(
	17532	.{ .vp_w, .blend },
	17533	registerAlias(dst_reg, dst_abi_size),
	17534	registerAlias(if (lhs_mcv.isRegister())
	17535	lhs_mcv.getReg().?
	17536	else
	17537	dst_reg, dst_abi_size),
	17538	registerAlias(if (rhs_mcv.isRegister())
	17539	rhs_mcv.getReg().?
	17540	else
	17541	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
	17542	Immediate.u(expanded_control),
	17543	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	17544	.{ .p_w, .blend },
	17545	registerAlias(dst_reg, dst_abi_size),
	17546	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
	17547	Immediate.u(expanded_control),
	17548	) else try self.asmRegisterRegisterImmediate(
	17549	.{ .p_w, .blend },
	17550	registerAlias(dst_reg, dst_abi_size),
	17551	registerAlias(if (rhs_mcv.isRegister())
	17552	rhs_mcv.getReg().?
	17553	else
	17554	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
	17555	Immediate.u(expanded_control),
	17556	);
	17557	break :result .{ .register = dst_reg };
	17558	}
	17559
	17560	const expanded_control = switch (elem_abi_size) {
	17561	4, 8 => control,
	17562	16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) \|
	17563	@as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
	17564	else => unreachable,
	17565	};
	17566
	17567	const lhs_mcv = try self.resolveInst(extra.a);
	17568	const rhs_mcv = try self.resolveInst(extra.b);
	17569
	17570	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
	17571	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
	17572	lhs_mcv
	17573	else if (has_avx and lhs_mcv.isRegister())
	17574	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	17575	else
	17576	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
	17577	const dst_reg = dst_mcv.getReg().?;
	17578
	17579	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
	17580	switch (elem_abi_size) {
	17581	4 => .{ .v_ps, .blend },
	17582	8, 16 => .{ .v_pd, .blend },
	17583	else => unreachable,
	17584	},
	17585	registerAlias(dst_reg, dst_abi_size),
	17586	registerAlias(if (lhs_mcv.isRegister())
	17587	lhs_mcv.getReg().?
	17588	else
	17589	dst_reg, dst_abi_size),
	17590	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
	17591	Immediate.u(expanded_control),
	17592	) else try self.asmRegisterRegisterRegisterImmediate(
	17593	switch (elem_abi_size) {
	17594	4 => .{ .v_ps, .blend },
	17595	8, 16 => .{ .v_pd, .blend },
	17596	else => unreachable,
	17597	},
	17598	registerAlias(dst_reg, dst_abi_size),
	17599	registerAlias(if (lhs_mcv.isRegister())
	17600	lhs_mcv.getReg().?
	17601	else
	17602	dst_reg, dst_abi_size),
	17603	registerAlias(if (rhs_mcv.isRegister())
	17604	rhs_mcv.getReg().?
	17605	else
	17606	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
	17607	Immediate.u(expanded_control),
	17608	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
	17609	switch (elem_abi_size) {
	17610	4 => .{ ._ps, .blend },
	17611	8, 16 => .{ ._pd, .blend },
	17612	else => unreachable,
	17613	},
	17614	registerAlias(dst_reg, dst_abi_size),
	17615	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
	17616	Immediate.u(expanded_control),
	17617	) else try self.asmRegisterRegisterImmediate(
	17618	switch (elem_abi_size) {
	17619	4 => .{ ._ps, .blend },
	17620	8, 16 => .{ ._pd, .blend },
	17621	else => unreachable,
	17622	},
	17623	registerAlias(dst_reg, dst_abi_size),
	17624	registerAlias(if (rhs_mcv.isRegister())
	17625	rhs_mcv.getReg().?
	17626	else
	17627	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
	17628	Immediate.u(expanded_control),
	17629	);
	17630	break :result .{ .register = dst_reg };
	17631	}
	17632
	17633	blendv: {
	17634	if (dst_abi_size > @as(u32, if (if (elem_abi_size >= 4)
	17635	has_avx
	17636	else
	17637	self.hasFeature(.avx2)) 32 else 16)) break :blendv;
	17638
	17639	const select_mask_elem_ty = try mod.intType(.unsigned, elem_abi_size * 8);
	17640	const select_mask_ty = try mod.vectorType(.{
	17641	.len = @intCast(mask_elems.len),
	17642	.child = select_mask_elem_ty.toIntern(),
	17643	});
	17644	var select_mask_elems: [32]InternPool.Index = undefined;
	17645	for (
	17646	select_mask_elems[0..mask_elems.len],
	17647	mask_elems,
	17648	0..,
	17649	) \|*select_mask_elem, maybe_mask_elem, elem_index\| {
	17650	const mask_elem = maybe_mask_elem orelse continue;
	17651	const mask_elem_index =
	17652	math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
	17653	if (mask_elem_index != elem_index) break :blendv;
	17654
	17655	select_mask_elem.* = (if (mask_elem < 0)
	17656	try select_mask_elem_ty.maxIntScalar(mod, select_mask_elem_ty)
	17657	else
	17658	try select_mask_elem_ty.minIntScalar(mod, select_mask_elem_ty)).toIntern();
	17659	}
	17660	const select_mask_mcv = try self.genTypedValue(.{
	17661	.ty = select_mask_ty,
	17662	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
	17663	.ty = select_mask_ty.toIntern(),
	17664	.storage = .{ .elems = select_mask_elems[0..mask_elems.len] },
	17665	} })),
	17666	});
	17667
	17668	if (self.hasFeature(.sse4_1)) {
	17669	const mir_tag: Mir.Inst.FixedTag = .{
	17670	if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
	17671	(dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
	17672	4 => if (has_avx) .v_ps else ._ps,
	17673	8 => if (has_avx) .v_pd else ._pd,
	17674	else => unreachable,
	17675	} else if (has_avx) .vp_b else .p_b,
	17676	.blendv,
	17677	};
	17678
	17679	const select_mask_reg = if (!has_avx) reg: {
	17680	try self.register_manager.getKnownReg(.xmm0, null);
	17681	try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
	17682	break :reg .xmm0;
	17683	} else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
	17684	const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
	17685	const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
	17686	defer self.register_manager.unlockReg(select_mask_lock);
	17687
	17688	const lhs_mcv = try self.resolveInst(extra.a);
	17689	const rhs_mcv = try self.resolveInst(extra.b);
	17690
	17691	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
	17692	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
	17693	lhs_mcv
	17694	else if (has_avx and lhs_mcv.isRegister())
	17695	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
	17696	else
	17697	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
	17698	const dst_reg = dst_mcv.getReg().?;
	17699	const dst_alias = registerAlias(dst_reg, dst_abi_size);
	17700
	17701	if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryRegister(
	17702	mir_tag,
	17703	dst_alias,
	17704	if (lhs_mcv.isRegister())
	17705	registerAlias(lhs_mcv.getReg().?, dst_abi_size)
	17706	else
	17707	dst_alias,
	17708	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
	17709	select_mask_alias,
	17710	) else try self.asmRegisterRegisterRegisterRegister(
	17711	mir_tag,
	17712	dst_alias,
	17713	if (lhs_mcv.isRegister())
	17714	registerAlias(lhs_mcv.getReg().?, dst_abi_size)
	17715	else
	17716	dst_alias,
	17717	registerAlias(if (rhs_mcv.isRegister())
	17718	rhs_mcv.getReg().?
	17719	else
	17720	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
	17721	select_mask_alias,
	17722	) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryRegister(
	17723	mir_tag,
	17724	dst_alias,
	17725	try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
	17726	select_mask_alias,
	17727	) else try self.asmRegisterRegisterRegister(
	17728	mir_tag,
	17729	dst_alias,
	17730	registerAlias(if (rhs_mcv.isRegister())
	17731	rhs_mcv.getReg().?
	17732	else
	17733	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
	17734	select_mask_alias,
	17735	);
	17736	break :result dst_mcv;
	17737	}
	17738
	17739	const lhs_mcv = try self.resolveInst(extra.a);
	17740	const rhs_mcv = try self.resolveInst(extra.b);
	17741
	17742	const dst_mcv: MCValue = if (rhs_mcv.isRegister() and
	17743	self.reuseOperand(inst, extra.b, 1, rhs_mcv))
	17744	rhs_mcv
	17745	else
	17746	try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);
	17747	const dst_reg = dst_mcv.getReg().?;
	17748	const dst_alias = registerAlias(dst_reg, dst_abi_size);
	17749
	17750	const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
	17751	const mask_alias = registerAlias(mask_reg, dst_abi_size);
	17752	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
	17753	defer self.register_manager.unlockReg(mask_lock);
	17754
	17755	const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())
	17756	switch (elem_ty.floatBits(self.target.*)) {
	17757	16, 80, 128 => .p_,
	17758	32 => ._ps,
	17759	64 => ._pd,
	17760	else => unreachable,
	17761	}
	17762	else
	17763	.p_;
	17764	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
	17765	if (lhs_mcv.isMemory()) try self.asmRegisterMemory(
	17766	.{ mir_fixes, .andn },
	17767	mask_alias,
	17768	try lhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)),
	17769	) else try self.asmRegisterRegister(
	17770	.{ mir_fixes, .andn },
	17771	mask_alias,
	17772	if (lhs_mcv.isRegister())
	17773	lhs_mcv.getReg().?
	17774	else
	17775	try self.copyToTmpRegister(dst_ty, lhs_mcv),
	17776	);
	17777	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
	17778	break :result dst_mcv;
	17779	}
	17780
	17781	pshufb: {
	17782	if (max_abi_size > 16) break :pshufb;
	17783	if (!self.hasFeature(.ssse3)) break :pshufb;
	17784
	17785	const temp_regs =
	17786	try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
	17787	const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
	17788	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
	17789
	17790	const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
	17791	try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
	17792
	17793	const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
	17794	try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
	17795
	17796	var lhs_mask_elems: [16]InternPool.Index = undefined;
	17797	for (lhs_mask_elems[0..max_abi_size], 0..) \|*lhs_mask_elem, byte_index\| {
	17798	const elem_index = byte_index / elem_abi_size;
	17799	lhs_mask_elem.* = try mod.intern(.{ .int = .{
	17800	.ty = .u8_type,
	17801	.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
	17802	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
	17803	if (mask_elem < 0) break :elem 0b1_00_00000;
	17804	const mask_elem_index: u31 = @intCast(mask_elem);
	17805	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
	17806	break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
	17807	} },
	17808	} });
	17809	}
	17810	const lhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
	17811	const lhs_mask_mcv = try self.genTypedValue(.{
	17812	.ty = lhs_mask_ty,
	17813	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
	17814	.ty = lhs_mask_ty.toIntern(),
	17815	.storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },
	17816	} })),
	17817	});
	17818	const lhs_mask_mem: Memory = .{
	17819	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },
	17820	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
	17821	};
	17822	if (has_avx) try self.asmRegisterRegisterMemory(
	17823	.{ .vp_b, .shuf },
	17824	lhs_temp_alias,
	17825	lhs_temp_alias,
	17826	lhs_mask_mem,
	17827	) else try self.asmRegisterMemory(
	17828	.{ .p_b, .shuf },
	17829	lhs_temp_alias,
	17830	lhs_mask_mem,
	17831	);
	17832
	17833	var rhs_mask_elems: [16]InternPool.Index = undefined;
	17834	for (rhs_mask_elems[0..max_abi_size], 0..) \|*rhs_mask_elem, byte_index\| {
	17835	const elem_index = byte_index / elem_abi_size;
	17836	rhs_mask_elem.* = try mod.intern(.{ .int = .{
	17837	.ty = .u8_type,
	17838	.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
	17839	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
	17840	if (mask_elem >= 0) break :elem 0b1_00_00000;
	17841	const mask_elem_index: u31 = @intCast(~mask_elem);
	17842	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
	17843	break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);
	17844	} },
	17845	} });
	17846	}
	17847	const rhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });
	17848	const rhs_mask_mcv = try self.genTypedValue(.{
	17849	.ty = rhs_mask_ty,
	17850	.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{
	17851	.ty = rhs_mask_ty.toIntern(),
	17852	.storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },
	17853	} })),
	17854	});
	17855	const rhs_mask_mem: Memory = .{
	17856	.base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },
	17857	.mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },
	17858	};
	17859	if (has_avx) try self.asmRegisterRegisterMemory(
	17860	.{ .vp_b, .shuf },
	17861	rhs_temp_alias,
	17862	rhs_temp_alias,
	17863	rhs_mask_mem,
	17864	) else try self.asmRegisterMemory(
	17865	.{ .p_b, .shuf },
	17866	rhs_temp_alias,
	17867	rhs_mask_mem,
	17868	);
	17869
	17870	if (has_avx) try self.asmRegisterRegisterRegister(
	17871	.{ switch (elem_ty.zigTypeTag(mod)) {
	17872	else => break :result null,
	17873	.Int => .vp_,
	17874	.Float => switch (elem_ty.floatBits(self.target.*)) {
	17875	32 => .v_ps,
	17876	64 => .v_pd,
	17877	16, 80, 128 => break :result null,
	17878	else => unreachable,
	17879	},
	17880	}, .@"or" },
	17881	lhs_temp_alias,
	17882	lhs_temp_alias,
	17883	rhs_temp_alias,
	17884	) else try self.asmRegisterRegister(
	17885	.{ switch (elem_ty.zigTypeTag(mod)) {
	17886	else => break :result null,
	17887	.Int => .p_,
	17888	.Float => switch (elem_ty.floatBits(self.target.*)) {
	17889	32 => ._ps,
	17890	64 => ._pd,
	17891	16, 80, 128 => break :result null,
	17892	else => unreachable,
	17893	},
	17894	}, .@"or" },
	17895	lhs_temp_alias,
	17896	rhs_temp_alias,
	17897	);
	17898	break :result .{ .register = temp_regs[0] };
	17899	}
	17900
	17901	break :result null;
	17902	}) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{
	17903	lhs_ty.fmt(mod), rhs_ty.fmt(mod), dst_ty.fmt(mod),
	17904	Value.fromInterned(extra.mask).fmtValue(
	17905	Type.fromInterned(mod.intern_pool.typeOf(extra.mask)),
	17906	mod,
	17907	),
	17908	});
	17909	return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
16578	17910	}
16579	17911
16580	17912	fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
...	...	@@ -16751,7 +18083,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
16751	18083	},
16752	18084	.Array, .Vector => {
16753	18085	const elem_ty = result_ty.childType(mod);
16754		if (result_ty.isVector(mod) and elem_ty.bitSize(mod) == 1) {
	18086	if (result_ty.isVector(mod) and elem_ty.toIntern() == .bool_type) {
16755	18087	const result_size: u32 = @intCast(result_ty.abiSize(mod));
16756	18088	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
16757	18089	try self.asmRegisterRegister(
...	...	@@ -17801,7 +19133,7 @@ fn splitType(self: *Self, ty: Type) ![2]Type {
17801	19133	else => unreachable,
17802	19134	},
17803	19135	.float => Type.f32,
17804		.float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),
	19136	.float_combine => try mod.arrayType(.{ .len = 2, .child = .f32_type }),
17805	19137	.sse => Type.f64,
17806	19138	else => break,
17807	19139	};

src/arch/x86_64/Encoding.zig+11-4

...	...	@@ -324,16 +324,19 @@ pub const Mnemonic = enum {
324	324	// SSE3
325	325	movddup, movshdup, movsldup,
326	326	// SSSE3
327		pabsb, pabsd, pabsw, palignr,
	327	pabsb, pabsd, pabsw, palignr, pshufb,
328	328	// SSE4.1
329	329	blendpd, blendps, blendvpd, blendvps,
330	330	extractps,
331	331	insertps,
332	332	packusdw,
	333	pblendvb, pblendw,
333	334	pcmpeqq,
334	335	pextrb, pextrd, pextrq,
335	336	pinsrb, pinsrd, pinsrq,
336	337	pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,
	338	pmovsxbd, pmovsxbq, pmovsxbw, pmovsxdq, pmovsxwd, pmovsxwq,
	339	pmovzxbd, pmovzxbq, pmovzxbw, pmovzxdq, pmovzxwd, pmovzxwq,
337	340	pmulld,
338	341	roundpd, roundps, roundsd, roundss,
339	342	// SSE4.2
...	...	@@ -377,7 +380,8 @@ pub const Mnemonic = enum {
377	380	vpabsb, vpabsd, vpabsw,
378	381	vpackssdw, vpacksswb, vpackusdw, vpackuswb,
379	382	vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,
380		vpalignr, vpand, vpandn, vpclmulqdq,
	383	vpalignr, vpand, vpandn,
	384	vpblendvb, vpblendw, vpclmulqdq,
381	385	vpcmpeqb, vpcmpeqd, vpcmpeqq, vpcmpeqw,
382	386	vpcmpgtb, vpcmpgtd, vpcmpgtq, vpcmpgtw,
383	387	vpextrb, vpextrd, vpextrq, vpextrw,
...	...	@@ -385,9 +389,11 @@ pub const Mnemonic = enum {
385	389	vpmaxsb, vpmaxsd, vpmaxsw, vpmaxub, vpmaxud, vpmaxuw,
386	390	vpminsb, vpminsd, vpminsw, vpminub, vpminud, vpminuw,
387	391	vpmovmskb,
	392	vpmovsxbd, vpmovsxbq, vpmovsxbw, vpmovsxdq, vpmovsxwd, vpmovsxwq,
	393	vpmovzxbd, vpmovzxbq, vpmovzxbw, vpmovzxdq, vpmovzxwd, vpmovzxwq,
388	394	vpmulhw, vpmulld, vpmullw,
389	395	vpor,
390		vpshufd, vpshufhw, vpshuflw,
	396	vpshufb, vpshufd, vpshufhw, vpshuflw,
391	397	vpslld, vpslldq, vpsllq, vpsllw,
392	398	vpsrad, vpsraq, vpsraw,
393	399	vpsrld, vpsrldq, vpsrlq, vpsrlw,
...	...	@@ -409,7 +415,8 @@ pub const Mnemonic = enum {
409	415	vfmadd132sd, vfmadd213sd, vfmadd231sd,
410	416	vfmadd132ss, vfmadd213ss, vfmadd231ss,
411	417	// AVX2
412		vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,
	418	vbroadcasti128, vpbroadcastb, vpbroadcastd, vpbroadcastq, vpbroadcastw,
	419	vextracti128, vinserti128, vpblendd,
413	420	// zig fmt: on
414	421	};
415	422

src/arch/x86_64/Lower.zig+13-1

...	...	@@ -477,8 +477,9 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
477	477	.rri_s, .rri_u => inst.data.rri.fixes,
478	478	.ri_s, .ri_u => inst.data.ri.fixes,
479	479	.ri64, .rm, .rmi_s, .mr => inst.data.rx.fixes,
480		.mrr, .rrm => inst.data.rrx.fixes,
	480	.mrr, .rrm, .rmr => inst.data.rrx.fixes,
481	481	.rmi, .mri => inst.data.rix.fixes,
	482	.rrmr => inst.data.rrrx.fixes,
482	483	.rrmi => inst.data.rrix.fixes,
483	484	.mi_u, .mi_s => inst.data.x.fixes,
484	485	.m => inst.data.x.fixes,
...	...	@@ -565,6 +566,11 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
565	566	.{ .reg = inst.data.rx.r1 },
566	567	.{ .mem = lower.mem(inst.data.rx.payload) },
567	568	},
	569	.rmr => &.{
	570	.{ .reg = inst.data.rrx.r1 },
	571	.{ .mem = lower.mem(inst.data.rrx.payload) },
	572	.{ .reg = inst.data.rrx.r2 },
	573	},
568	574	.rmi => &.{
569	575	.{ .reg = inst.data.rix.r1 },
570	576	.{ .mem = lower.mem(inst.data.rix.payload) },
...	...	@@ -597,6 +603,12 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
597	603	.{ .reg = inst.data.rrx.r2 },
598	604	.{ .mem = lower.mem(inst.data.rrx.payload) },
599	605	},
	606	.rrmr => &.{
	607	.{ .reg = inst.data.rrrx.r1 },
	608	.{ .reg = inst.data.rrrx.r2 },
	609	.{ .mem = lower.mem(inst.data.rrrx.payload) },
	610	.{ .reg = inst.data.rrrx.r3 },
	611	},
600	612	.rrmi => &.{
601	613	.{ .reg = inst.data.rrix.r1 },
602	614	.{ .reg = inst.data.rrix.r2 },

src/arch/x86_64/Mir.zig+27-3

...	...	@@ -230,6 +230,8 @@ pub const Inst = struct {
230	230	v_d,
231	231	/// VEX-Encoded ___ QuadWord
232	232	v_q,
	233	/// VEX-Encoded ___ Integer Data
	234	v_i128,
233	235	/// VEX-Encoded Packed ___
234	236	vp_,
235	237	/// VEX-Encoded Packed ___ Byte
...	...	@@ -242,8 +244,6 @@ pub const Inst = struct {
242	244	vp_q,
243	245	/// VEX-Encoded Packed ___ Double Quadword
244	246	vp_dq,
245		/// VEX-Encoded Packed ___ Integer Data
246		vp_i128,
247	247	/// VEX-Encoded ___ Scalar Single-Precision Values
248	248	v_ss,
249	249	/// VEX-Encoded ___ Packed Single-Precision Values
...	...	@@ -654,10 +654,19 @@ pub const Inst = struct {
654	654	/// Variable blend scalar double-precision floating-point values
655	655	blendv,
656	656	/// Extract packed floating-point values
	657	/// Extract packed integer values
657	658	extract,
658	659	/// Insert scalar single-precision floating-point value
659	660	/// Insert packed floating-point values
660	661	insert,
	662	/// Packed move with sign extend
	663	movsxb,
	664	movsxd,
	665	movsxw,
	666	/// Packed move with zero extend
	667	movzxb,
	668	movzxd,
	669	movzxw,
661	670	/// Round packed single-precision floating-point values
662	671	/// Round scalar single-precision floating-point value
663	672	/// Round packed double-precision floating-point values
...	...	@@ -688,6 +697,7 @@ pub const Inst = struct {
688	697	sha256rnds2,
689	698
690	699	/// Load with broadcast floating-point data
	700	/// Load integer and broadcast
691	701	broadcast,
692	702
693	703	/// Convert 16-bit floating-point values to single-precision floating-point values
...	...	@@ -762,8 +772,11 @@ pub const Inst = struct {
762	772	/// Uses `imm` payload.
763	773	rel,
764	774	/// Register, memory operands.
765		/// Uses `rx` payload.
	775	/// Uses `rx` payload with extra data of type `Memory`.
766	776	rm,
	777	/// Register, memory, register operands.
	778	/// Uses `rrx` payload with extra data of type `Memory`.
	779	rmr,
767	780	/// Register, memory, immediate (word) operands.
768	781	/// Uses `rix` payload with extra data of type `Memory`.
769	782	rmi,
...	...	@@ -776,6 +789,9 @@ pub const Inst = struct {
776	789	/// Register, register, memory.
777	790	/// Uses `rrix` payload with extra data of type `Memory`.
778	791	rrm,
	792	/// Register, register, memory, register.
	793	/// Uses `rrrx` payload with extra data of type `Memory`.
	794	rrmr,
779	795	/// Register, register, memory, immediate (byte) operands.
780	796	/// Uses `rrix` payload with extra data of type `Memory`.
781	797	rrmi,
...	...	@@ -953,6 +969,14 @@ pub const Inst = struct {
953	969	r2: Register,
954	970	payload: u32,
955	971	},
	972	/// Register, register, register, followed by Custom payload found in extra.
	973	rrrx: struct {
	974	fixes: Fixes = ._,
	975	r1: Register,
	976	r2: Register,
	977	r3: Register,
	978	payload: u32,
	979	},
956	980	/// Register, byte immediate, followed by Custom payload found in extra.
957	981	rix: struct {
958	982	fixes: Fixes = ._,

src/arch/x86_64/encodings.zig+68-1

...	...	@@ -1185,6 +1185,8 @@ pub const table = [_]Entry{
1185	1185
1186	1186	.{ .palignr, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0f }, 0, .none, .ssse3 },
1187	1187
	1188	.{ .pshufb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .none, .ssse3 },
	1189
1188	1190	// SSE4.1
1189	1191	.{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },
1190	1192
...	...	@@ -1202,6 +1204,11 @@ pub const table = [_]Entry{
1202	1204
1203	1205	.{ .packusdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 },
1204	1206
	1207	.{ .pblendvb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 },
	1208	.{ .pblendvb, .rm, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x66, 0x0f, 0x38, 0x10 }, 0, .none, .sse4_1 },
	1209
	1210	.{ .pblendw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .none, .sse4_1 },
	1211
1205	1212	.{ .pcmpeqq, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x29 }, 0, .none, .sse4_1 },
1206	1213
1207	1214	.{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
...	...	@@ -1228,6 +1235,20 @@ pub const table = [_]Entry{
1228	1235
1229	1236	.{ .pminud, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x3b }, 0, .none, .sse4_1 },
1230	1237
	1238	.{ .pmovsxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .none, .sse4_1 },
	1239	.{ .pmovsxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .none, .sse4_1 },
	1240	.{ .pmovsxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .none, .sse4_1 },
	1241	.{ .pmovsxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .none, .sse4_1 },
	1242	.{ .pmovsxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .none, .sse4_1 },
	1243	.{ .pmovsxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .none, .sse4_1 },
	1244
	1245	.{ .pmovzxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .none, .sse4_1 },
	1246	.{ .pmovzxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .none, .sse4_1 },
	1247	.{ .pmovzxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .none, .sse4_1 },
	1248	.{ .pmovzxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .none, .sse4_1 },
	1249	.{ .pmovzxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .none, .sse4_1 },
	1250	.{ .pmovzxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .none, .sse4_1 },
	1251
1231	1252	.{ .pmulld, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .none, .sse4_1 },
1232	1253
1233	1254	.{ .roundpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x09 }, 0, .none, .sse4_1 },
...	...	@@ -1528,6 +1549,10 @@ pub const table = [_]Entry{
1528	1549
1529	1550	.{ .vpandn, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_128_wig, .avx },
1530	1551
	1552	.{ .vpblendvb, .rvmr, &.{ .xmm, .xmm, .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_128_w0, .avx },
	1553
	1554	.{ .vpblendw, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_128_wig, .avx },
	1555
1531	1556	.{ .vpclmulqdq, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x44 }, 0, .vex_128_wig, .@"pclmul avx" },
1532	1557
1533	1558	.{ .vpcmpeqb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_128_wig, .avx },
...	...	@@ -1576,6 +1601,20 @@ pub const table = [_]Entry{
1576	1601	.{ .vpmovmskb, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },
1577	1602	.{ .vpmovmskb, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_128_wig, .avx },
1578	1603
	1604	.{ .vpmovsxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_128_wig, .avx },
	1605	.{ .vpmovsxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_128_wig, .avx },
	1606	.{ .vpmovsxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_128_wig, .avx },
	1607	.{ .vpmovsxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_128_wig, .avx },
	1608	.{ .vpmovsxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_128_wig, .avx },
	1609	.{ .vpmovsxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_128_wig, .avx },
	1610
	1611	.{ .vpmovzxbw, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_128_wig, .avx },
	1612	.{ .vpmovzxbd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_128_wig, .avx },
	1613	.{ .vpmovzxbq, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_128_wig, .avx },
	1614	.{ .vpmovzxwd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_128_wig, .avx },
	1615	.{ .vpmovzxwq, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_128_wig, .avx },
	1616	.{ .vpmovzxdq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_128_wig, .avx },
	1617
1579	1618	.{ .vpmulhw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_128_wig, .avx },
1580	1619
1581	1620	.{ .vpmulld, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_128_wig, .avx },
...	...	@@ -1584,6 +1623,8 @@ pub const table = [_]Entry{
1584	1623
1585	1624	.{ .vpor, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_128_wig, .avx },
1586	1625
	1626	.{ .vpshufb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx },
	1627
1587	1628	.{ .vpshufd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
1588	1629
1589	1630	.{ .vpshufhw, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_128_wig, .avx },
...	...	@@ -1728,6 +1769,10 @@ pub const table = [_]Entry{
1728	1769	.{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },
1729	1770	.{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },
1730	1771
	1772	.{ .vextracti128, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x39 }, 0, .vex_256_w0, .avx2 },
	1773
	1774	.{ .vinserti128, .rvmi, &.{ .ymm, .ymm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x38 }, 0, .vex_256_w0, .avx2 },
	1775
1731	1776	.{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 },
1732	1777	.{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 },
1733	1778	.{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 },
...	...	@@ -1756,6 +1801,13 @@ pub const table = [_]Entry{
1756	1801
1757	1802	.{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
1758	1803
	1804	.{ .vpblendd, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_128_w0, .avx2 },
	1805	.{ .vpblendd, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x02 }, 0, .vex_256_w0, .avx2 },
	1806
	1807	.{ .vpblendvb, .rvmr, &.{ .ymm, .ymm, .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x3a, 0x4c }, 0, .vex_256_w0, .avx2 },
	1808
	1809	.{ .vpblendw, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0e }, 0, .vex_256_wig, .avx2 },
	1810
1759	1811	.{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },
1760	1812	.{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },
1761	1813	.{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },
...	...	@@ -1764,7 +1816,7 @@ pub const table = [_]Entry{
1764	1816	.{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
1765	1817	.{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
1766	1818	.{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
1767		.{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
	1819	.{ .vbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
1768	1820
1769	1821	.{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
1770	1822	.{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },
...	...	@@ -1799,6 +1851,20 @@ pub const table = [_]Entry{
1799	1851	.{ .vpmovmskb, .rm, &.{ .r32, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },
1800	1852	.{ .vpmovmskb, .rm, &.{ .r64, .ymm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .vex_256_wig, .avx2 },
1801	1853
	1854	.{ .vpmovsxbw, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x20 }, 0, .vex_256_wig, .avx2 },
	1855	.{ .vpmovsxbd, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x21 }, 0, .vex_256_wig, .avx2 },
	1856	.{ .vpmovsxbq, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x22 }, 0, .vex_256_wig, .avx2 },
	1857	.{ .vpmovsxwd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x23 }, 0, .vex_256_wig, .avx2 },
	1858	.{ .vpmovsxwq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x24 }, 0, .vex_256_wig, .avx2 },
	1859	.{ .vpmovsxdq, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x25 }, 0, .vex_256_wig, .avx2 },
	1860
	1861	.{ .vpmovzxbw, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x30 }, 0, .vex_256_wig, .avx2 },
	1862	.{ .vpmovzxbd, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x31 }, 0, .vex_256_wig, .avx2 },
	1863	.{ .vpmovzxbq, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x32 }, 0, .vex_256_wig, .avx2 },
	1864	.{ .vpmovzxwd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x33 }, 0, .vex_256_wig, .avx2 },
	1865	.{ .vpmovzxwq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x34 }, 0, .vex_256_wig, .avx2 },
	1866	.{ .vpmovzxdq, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x35 }, 0, .vex_256_wig, .avx2 },
	1867
1802	1868	.{ .vpmulhw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xe5 }, 0, .vex_256_wig, .avx2 },
1803	1869
1804	1870	.{ .vpmulld, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x40 }, 0, .vex_256_wig, .avx2 },
...	...	@@ -1807,6 +1873,7 @@ pub const table = [_]Entry{
1807	1873
1808	1874	.{ .vpor, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xeb }, 0, .vex_256_wig, .avx2 },
1809	1875
	1876	.{ .vpshufb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 },
1810	1877	.{ .vpshufd, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },
1811	1878
1812	1879	.{ .vpshufhw, .rmi, &.{ .ymm, .ymm_m256, .imm8 }, &.{ 0xf3, 0x0f, 0x70 }, 0, .vex_256_wig, .avx2 },

src/codegen.zig+27-30

...	...	@@ -405,7 +405,7 @@ pub fn generateSymbol(
405	405	.vector_type => \|vector_type\| {
406	406	const abi_size = math.cast(usize, typed_value.ty.abiSize(mod)) orelse
407	407	return error.Overflow;
408		if (Type.fromInterned(vector_type.child).bitSize(mod) == 1) {
	408	if (vector_type.child == .bool_type) {
409	409	const bytes = try code.addManyAsSlice(abi_size);
410	410	@memset(bytes, 0xaa);
411	411	var index: usize = 0;
...	...	@@ -443,37 +443,34 @@ pub fn generateSymbol(
443	443	},
444	444	}) byte.* \|= mask else byte.* &= ~mask;
445	445	}
446		} else switch (aggregate.storage) {
447		.bytes => \|bytes\| try code.appendSlice(bytes),
448		.elems, .repeated_elem => {
449		var index: u64 = 0;
450		while (index < vector_type.len) : (index += 1) {
451		switch (try generateSymbol(bin_file, src_loc, .{
452		.ty = Type.fromInterned(vector_type.child),
453		.val = Value.fromInterned(switch (aggregate.storage) {
454		.bytes => unreachable,
455		.elems => \|elems\| elems[
456		math.cast(usize, index) orelse return error.Overflow
457		],
458		.repeated_elem => \|elem\| elem,
459		}),
460		}, code, debug_output, reloc_info)) {
461		.ok => {},
462		.fail => \|em\| return .{ .fail = em },
	446	} else {
	447	switch (aggregate.storage) {
	448	.bytes => \|bytes\| try code.appendSlice(bytes),
	449	.elems, .repeated_elem => {
	450	var index: u64 = 0;
	451	while (index < vector_type.len) : (index += 1) {
	452	switch (try generateSymbol(bin_file, src_loc, .{
	453	.ty = Type.fromInterned(vector_type.child),
	454	.val = Value.fromInterned(switch (aggregate.storage) {
	455	.bytes => unreachable,
	456	.elems => \|elems\| elems[
	457	math.cast(usize, index) orelse return error.Overflow
	458	],
	459	.repeated_elem => \|elem\| elem,
	460	}),
	461	}, code, debug_output, reloc_info)) {
	462	.ok => {},
	463	.fail => \|em\| return .{ .fail = em },
	464	}
463	465	}
464		}
465		},
466		}
	466	},
	467	}
467	468
468		const padding = abi_size - (math.cast(usize, math.divCeil(
469		u64,
470		Type.fromInterned(vector_type.child).bitSize(mod) * vector_type.len,
471		8,
472		) catch \|err\| switch (err) {
473		error.DivisionByZero => unreachable,
474		else => \|e\| return e,
475		}) orelse return error.Overflow);
476		if (padding > 0) try code.appendNTimes(0, padding);
	469	const padding = abi_size -
	470	(math.cast(usize, Type.fromInterned(vector_type.child).abiSize(mod) * vector_type.len) orelse
	471	return error.Overflow);
	472	if (padding > 0) try code.appendNTimes(0, padding);
	473	}
477	474	},
478	475	.anon_struct_type => \|tuple\| {
479	476	const struct_begin = code.items.len;

src/codegen/c.zig+45-53

...	...	@@ -4140,9 +4140,7 @@ fn airCmpOp(
4140	4140	if (need_cast) try writer.writeAll("(void*)");
4141	4141	try f.writeCValue(writer, lhs, .Other);
4142	4142	try v.elem(f, writer);
4143		try writer.writeByte(' ');
4144	4143	try writer.writeAll(compareOperatorC(operator));
4145		try writer.writeByte(' ');
4146	4144	if (need_cast) try writer.writeAll("(void*)");
4147	4145	try f.writeCValue(writer, rhs, .Other);
4148	4146	try v.elem(f, writer);
...	...	@@ -4181,41 +4179,28 @@ fn airEquality(
4181	4179	const writer = f.object.writer();
4182	4180	const inst_ty = f.typeOfIndex(inst);
4183	4181	const local = try f.allocLocal(inst, inst_ty);
	4182	const a = try Assignment.start(f, writer, inst_ty);
4184	4183	try f.writeCValue(writer, local, .Other);
4185		try writer.writeAll(" = ");
	4184	try a.assign(f, writer);
4186	4185
4187	4186	if (operand_ty.zigTypeTag(mod) == .Optional and !operand_ty.optionalReprIsPayload(mod)) {
4188		// (A && B) \|\| (C && (A == B))
4189		// A = lhs.is_null ; B = rhs.is_null ; C = rhs.payload == lhs.payload
4190
4191		switch (operator) {
4192		.eq => {},
4193		.neq => try writer.writeByte('!'),
4194		else => unreachable,
4195		}
4196		try writer.writeAll("((");
4197		try f.writeCValue(writer, lhs, .Other);
4198		try writer.writeAll(".is_null && ");
4199		try f.writeCValue(writer, rhs, .Other);
4200		try writer.writeAll(".is_null) \|\| (");
4201		try f.writeCValue(writer, lhs, .Other);
4202		try writer.writeAll(".payload == ");
4203		try f.writeCValue(writer, rhs, .Other);
4204		try writer.writeAll(".payload && ");
	4187	try f.writeCValueMember(writer, lhs, .{ .identifier = "is_null" });
	4188	try writer.writeAll(" \|\| ");
	4189	try f.writeCValueMember(writer, rhs, .{ .identifier = "is_null" });
	4190	try writer.writeAll(" ? ");
	4191	try f.writeCValueMember(writer, lhs, .{ .identifier = "is_null" });
	4192	try writer.writeAll(compareOperatorC(operator));
	4193	try f.writeCValueMember(writer, rhs, .{ .identifier = "is_null" });
	4194	try writer.writeAll(" : ");
	4195	try f.writeCValueMember(writer, lhs, .{ .identifier = "payload" });
	4196	try writer.writeAll(compareOperatorC(operator));
	4197	try f.writeCValueMember(writer, rhs, .{ .identifier = "payload" });
	4198	} else {
4205	4199	try f.writeCValue(writer, lhs, .Other);
4206		try writer.writeAll(".is_null == ");
	4200	try writer.writeAll(compareOperatorC(operator));
4207	4201	try f.writeCValue(writer, rhs, .Other);
4208		try writer.writeAll(".is_null));\n");
4209
4210		return local;
4211	4202	}
4212
4213		try f.writeCValue(writer, lhs, .Other);
4214		try writer.writeByte(' ');
4215		try writer.writeAll(compareOperatorC(operator));
4216		try writer.writeByte(' ');
4217		try f.writeCValue(writer, rhs, .Other);
4218		try writer.writeAll(";\n");
	4203	try a.end(f, writer);
4219	4204
4220	4205	return local;
4221	4206	}
...	...	@@ -6109,41 +6094,48 @@ fn airFloatCast(f: *Function, inst: Air.Inst.Index) !CValue {
6109	6094	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6110	6095
6111	6096	const inst_ty = f.typeOfIndex(inst);
	6097	const inst_scalar_ty = inst_ty.scalarType(mod);
6112	6098	const operand = try f.resolveInst(ty_op.operand);
6113	6099	try reap(f, inst, &.{ty_op.operand});
6114	6100	const operand_ty = f.typeOf(ty_op.operand);
	6101	const scalar_ty = operand_ty.scalarType(mod);
6115	6102	const target = f.object.dg.module.getTarget();
6116		const operation = if (inst_ty.isRuntimeFloat() and operand_ty.isRuntimeFloat())
6117		if (inst_ty.floatBits(target) < operand_ty.floatBits(target)) "trunc" else "extend"
6118		else if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat())
6119		if (inst_ty.isSignedInt(mod)) "fix" else "fixuns"
6120		else if (inst_ty.isRuntimeFloat() and operand_ty.isInt(mod))
6121		if (operand_ty.isSignedInt(mod)) "float" else "floatun"
	6103	const operation = if (inst_scalar_ty.isRuntimeFloat() and scalar_ty.isRuntimeFloat())
	6104	if (inst_scalar_ty.floatBits(target) < scalar_ty.floatBits(target)) "trunc" else "extend"
	6105	else if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat())
	6106	if (inst_scalar_ty.isSignedInt(mod)) "fix" else "fixuns"
	6107	else if (inst_scalar_ty.isRuntimeFloat() and scalar_ty.isInt(mod))
	6108	if (scalar_ty.isSignedInt(mod)) "float" else "floatun"
6122	6109	else
6123	6110	unreachable;
6124	6111
6125	6112	const writer = f.object.writer();
6126	6113	const local = try f.allocLocal(inst, inst_ty);
	6114	const v = try Vectorize.start(f, inst, writer, operand_ty);
	6115	const a = try Assignment.start(f, writer, scalar_ty);
6127	6116	try f.writeCValue(writer, local, .Other);
6128
6129		try writer.writeAll(" = ");
6130		if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat()) {
	6117	try v.elem(f, writer);
	6118	try a.assign(f, writer);
	6119	if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat()) {
6131	6120	try writer.writeAll("zig_wrap_");
6132		try f.object.dg.renderTypeForBuiltinFnName(writer, inst_ty);
	6121	try f.object.dg.renderTypeForBuiltinFnName(writer, inst_scalar_ty);
6133	6122	try writer.writeByte('(');
6134	6123	}
6135	6124	try writer.writeAll("zig_");
6136	6125	try writer.writeAll(operation);
6137		try writer.writeAll(compilerRtAbbrev(operand_ty, mod));
6138		try writer.writeAll(compilerRtAbbrev(inst_ty, mod));
	6126	try writer.writeAll(compilerRtAbbrev(scalar_ty, mod));
	6127	try writer.writeAll(compilerRtAbbrev(inst_scalar_ty, mod));
6139	6128	try writer.writeByte('(');
6140	6129	try f.writeCValue(writer, operand, .FunctionArgument);
	6130	try v.elem(f, writer);
6141	6131	try writer.writeByte(')');
6142		if (inst_ty.isInt(mod) and operand_ty.isRuntimeFloat()) {
6143		try f.object.dg.renderBuiltinInfo(writer, inst_ty, .bits);
	6132	if (inst_scalar_ty.isInt(mod) and scalar_ty.isRuntimeFloat()) {
	6133	try f.object.dg.renderBuiltinInfo(writer, inst_scalar_ty, .bits);
6144	6134	try writer.writeByte(')');
6145	6135	}
6146		try writer.writeAll(";\n");
	6136	try a.end(f, writer);
	6137	try v.end(f, inst, writer);
	6138
6147	6139	return local;
6148	6140	}
6149	6141
...	...	@@ -6315,7 +6307,7 @@ fn airCmpBuiltinCall(
6315	6307	try v.elem(f, writer);
6316	6308	try f.object.dg.renderBuiltinInfo(writer, scalar_ty, info);
6317	6309	try writer.writeByte(')');
6318		if (!ref_ret) try writer.print(" {s} {}", .{
	6310	if (!ref_ret) try writer.print("{s}{}", .{
6319	6311	compareOperatorC(operator),
6320	6312	try f.fmtIntLiteral(Type.i32, try mod.intValue(Type.i32, 0)),
6321	6313	});
...	...	@@ -7661,12 +7653,12 @@ fn compareOperatorAbbrev(operator: std.math.CompareOperator) []const u8 {
7661	7653
7662	7654	fn compareOperatorC(operator: std.math.CompareOperator) []const u8 {
7663	7655	return switch (operator) {
7664		.lt => "<",
7665		.lte => "<=",
7666		.eq => "==",
7667		.gte => ">=",
7668		.gt => ">",
7669		.neq => "!=",
	7656	.lt => " < ",
	7657	.lte => " <= ",
	7658	.eq => " == ",
	7659	.gte => " >= ",
	7660	.gt => " > ",
	7661	.neq => " != ",
7670	7662	};
7671	7663	}
7672	7664

src/codegen/llvm.zig+8-2

...	...	@@ -8646,8 +8646,6 @@ pub const FuncGen = struct {
8646	8646	const operand_ty = self.typeOf(ty_op.operand);
8647	8647	const dest_ty = self.typeOfIndex(inst);
8648	8648	const target = mod.getTarget();
8649		const dest_bits = dest_ty.floatBits(target);
8650		const src_bits = operand_ty.floatBits(target);
8651	8649
8652	8650	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
8653	8651	return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");
...	...	@@ -8655,11 +8653,19 @@ pub const FuncGen = struct {
8655	8653	const operand_llvm_ty = try o.lowerType(operand_ty);
8656	8654	const dest_llvm_ty = try o.lowerType(dest_ty);
8657	8655
	8656	const dest_bits = dest_ty.scalarType(mod).floatBits(target);
	8657	const src_bits = operand_ty.scalarType(mod).floatBits(target);
8658	8658	const fn_name = try o.builder.fmt("__extend{s}f{s}f2", .{
8659	8659	compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
8660	8660	});
8661	8661
8662	8662	const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
	8663	if (dest_ty.isVector(mod)) return self.buildElementwiseCall(
	8664	libc_fn,
	8665	&.{operand},
	8666	try o.builder.poisonValue(dest_llvm_ty),
	8667	dest_ty.vectorLen(mod),
	8668	);
8663	8669	return self.wip.call(
8664	8670	.normal,
8665	8671	.ccc,

src/type.zig+40-9

...	...	@@ -905,11 +905,32 @@ pub const Type = struct {
905	905	return Type.fromInterned(array_type.child).abiAlignmentAdvanced(mod, strat);
906	906	},
907	907	.vector_type => \|vector_type\| {
908		const bits_u64 = try bitSizeAdvanced(Type.fromInterned(vector_type.child), mod, opt_sema);
909		const bits: u32 = @intCast(bits_u64);
910		const bytes = ((bits * vector_type.len) + 7) / 8;
911		const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);
912		return .{ .scalar = Alignment.fromByteUnits(alignment) };
	908	if (vector_type.len == 0) return .{ .scalar = .@"1" };
	909	switch (mod.comp.getZigBackend()) {
	910	else => {
	911	const elem_bits: u32 = @intCast(try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema));
	912	if (elem_bits == 0) return .{ .scalar = .@"1" };
	913	const bytes = ((elem_bits * vector_type.len) + 7) / 8;
	914	const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);
	915	return .{ .scalar = Alignment.fromByteUnits(alignment) };
	916	},
	917	.stage2_x86_64 => {
	918	if (vector_type.child == .bool_type) {
	919	if (vector_type.len > 256 and std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return .{ .scalar = .@"64" };
	920	if (vector_type.len > 128 and std.Target.x86.featureSetHas(target.cpu.features, .avx2)) return .{ .scalar = .@"32" };
	921	if (vector_type.len > 64) return .{ .scalar = .@"16" };
	922	const bytes = std.math.divCeil(u32, vector_type.len, 8) catch unreachable;
	923	const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);
	924	return .{ .scalar = Alignment.fromByteUnits(alignment) };
	925	}
	926	const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);
	927	if (elem_bytes == 0) return .{ .scalar = .@"1" };
	928	const bytes = elem_bytes * vector_type.len;
	929	if (bytes > 32 and std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return .{ .scalar = .@"64" };
	930	if (bytes > 16 and std.Target.x86.featureSetHas(target.cpu.features, .avx)) return .{ .scalar = .@"32" };
	931	return .{ .scalar = .@"16" };
	932	},
	933	}
913	934	},
914	935
915	936	.opt_type => return abiAlignmentAdvancedOptional(ty, mod, strat),
...	...	@@ -1237,9 +1258,6 @@ pub const Type = struct {
1237	1258	.storage = .{ .lazy_size = ty.toIntern() },
1238	1259	} }))) },
1239	1260	};
1240		const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);
1241		const total_bits = elem_bits * vector_type.len;
1242		const total_bytes = (total_bits + 7) / 8;
1243	1261	const alignment = switch (try ty.abiAlignmentAdvanced(mod, strat)) {
1244	1262	.scalar => \|x\| x,
1245	1263	.val => return .{ .val = Value.fromInterned((try mod.intern(.{ .int = .{
...	...	@@ -1247,6 +1265,18 @@ pub const Type = struct {
1247	1265	.storage = .{ .lazy_size = ty.toIntern() },
1248	1266	} }))) },
1249	1267	};
	1268	const total_bytes = switch (mod.comp.getZigBackend()) {
	1269	else => total_bytes: {
	1270	const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);
	1271	const total_bits = elem_bits * vector_type.len;
	1272	break :total_bytes (total_bits + 7) / 8;
	1273	},
	1274	.stage2_x86_64 => total_bytes: {
	1275	if (vector_type.child == .bool_type) break :total_bytes std.math.divCeil(u32, vector_type.len, 8) catch unreachable;
	1276	const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);
	1277	break :total_bytes elem_bytes * vector_type.len;
	1278	},
	1279	};
1250	1280	return AbiSizeAdvanced{ .scalar = alignment.forward(total_bytes) };
1251	1281	},
1252	1282
...	...	@@ -2108,7 +2138,8 @@ pub const Type = struct {
2108	2138
2109	2139	/// Returns true if and only if the type is a fixed-width integer.
2110	2140	pub fn isInt(self: Type, mod: *const Module) bool {
2111		return self.isSignedInt(mod) or self.isUnsignedInt(mod);
	2141	return self.toIntern() != .comptime_int_type and
	2142	mod.intern_pool.isIntegerType(self.toIntern());
2112	2143	}
2113	2144
2114	2145	/// Returns true if and only if the type is a fixed-width, signed integer.

test/behavior/bitcast.zig+1-1

...	...	@@ -336,7 +336,7 @@ test "comptime @bitCast packed struct to int and back" {
336	336	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
337	337	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
338	338	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
339		if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
	339	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
340	340
341	341	if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
342	342	// https://github.com/ziglang/zig/issues/13782

test/behavior/cast.zig+33-21

...	...	@@ -601,25 +601,25 @@ test "cast [1][]const u8 to []const ?[]const u8" {
601	601
602	602	test "@intCast on vector" {
603	603	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
604		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
605	604	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
606	605	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
607	606	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
	607	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
608	608
609	609	const S = struct {
610	610	fn doTheTest() !void {
611	611	// Upcast (implicit, equivalent to @intCast)
612	612	var up0: @Vector(2, u8) = [_]u8{ 0x55, 0xaa };
613	613	_ = &up0;
614		const up1 = @as(@Vector(2, u16), up0);
615		const up2 = @as(@Vector(2, u32), up0);
616		const up3 = @as(@Vector(2, u64), up0);
	614	const up1: @Vector(2, u16) = up0;
	615	const up2: @Vector(2, u32) = up0;
	616	const up3: @Vector(2, u64) = up0;
617	617	// Downcast (safety-checked)
618	618	var down0 = up3;
619	619	_ = &down0;
620		const down1 = @as(@Vector(2, u32), @intCast(down0));
621		const down2 = @as(@Vector(2, u16), @intCast(down0));
622		const down3 = @as(@Vector(2, u8), @intCast(down0));
	620	const down1: @Vector(2, u32) = @intCast(down0);
	621	const down2: @Vector(2, u16) = @intCast(down0);
	622	const down3: @Vector(2, u8) = @intCast(down0);
623	623
624	624	try expect(mem.eql(u16, &@as([2]u16, up1), &[2]u16{ 0x55, 0xaa }));
625	625	try expect(mem.eql(u32, &@as([2]u32, up2), &[2]u32{ 0x55, 0xaa }));
...	...	@@ -629,20 +629,10 @@ test "@intCast on vector" {
629	629	try expect(mem.eql(u16, &@as([2]u16, down2), &[2]u16{ 0x55, 0xaa }));
630	630	try expect(mem.eql(u8, &@as([2]u8, down3), &[2]u8{ 0x55, 0xaa }));
631	631	}
632
633		fn doTheTestFloat() !void {
634		var vec: @Vector(2, f32) = @splat(1234.0);
635		_ = &vec;
636		const wider: @Vector(2, f64) = vec;
637		try expect(wider[0] == 1234.0);
638		try expect(wider[1] == 1234.0);
639		}
640	632	};
641	633
642	634	try S.doTheTest();
643	635	try comptime S.doTheTest();
644		try S.doTheTestFloat();
645		try comptime S.doTheTestFloat();
646	636	}
647	637
648	638	test "@floatCast cast down" {
...	...	@@ -2340,10 +2330,31 @@ test "@floatCast on vector" {
2340	2330
2341	2331	const S = struct {
2342	2332	fn doTheTest() !void {
2343		var a: @Vector(3, f64) = .{ 1.5, 2.5, 3.5 };
2344		_ = &a;
2345		const b: @Vector(3, f32) = @floatCast(a);
2346		try expectEqual(@Vector(3, f32){ 1.5, 2.5, 3.5 }, b);
	2333	{
	2334	var a: @Vector(2, f64) = .{ 1.5, 2.5 };
	2335	_ = &a;
	2336	const b: @Vector(2, f32) = @floatCast(a);
	2337	try expectEqual(@Vector(2, f32){ 1.5, 2.5 }, b);
	2338	}
	2339	{
	2340	var a: @Vector(2, f32) = .{ 3.25, 4.25 };
	2341	_ = &a;
	2342	const b: @Vector(2, f64) = @floatCast(a);
	2343	try expectEqual(@Vector(2, f64){ 3.25, 4.25 }, b);
	2344	}
	2345	{
	2346	var a: @Vector(2, f32) = .{ 5.75, 6.75 };
	2347	_ = &a;
	2348	const b: @Vector(2, f64) = a;
	2349	try expectEqual(@Vector(2, f64){ 5.75, 6.75 }, b);
	2350	}
	2351	{
	2352	var vec: @Vector(2, f32) = @splat(1234.0);
	2353	_ = &vec;
	2354	const wider: @Vector(2, f64) = vec;
	2355	try expect(wider[0] == 1234.0);
	2356	try expect(wider[1] == 1234.0);
	2357	}
2347	2358	}
2348	2359	};
2349	2360
...	...	@@ -2441,6 +2452,7 @@ test "@intFromBool on vector" {
2441	2452	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
2442	2453	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
2443	2454	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
	2455	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2444	2456
2445	2457	const S = struct {
2446	2458	fn doTheTest() !void {

test/behavior/optional.zig+77-32

...	...	@@ -110,44 +110,89 @@ test "nested optional field in struct" {
110	110	try expect(s.x.?.y == 127);
111	111	}
112	112
113		test "equality compare optional with non-optional" {
	113	test "equality compare optionals and non-optionals" {
114	114	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
115	115	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
116	116	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
117	117
118		try test_cmp_optional_non_optional();
119		try comptime test_cmp_optional_non_optional();
	118	const S = struct {
	119	fn doTheTest() !void {
	120	var five: isize = 5;
	121	var ten: isize = 10;
	122	var opt_null: ?isize = null;
	123	var opt_ten: ?isize = 10;
	124	_ = .{ &five, &ten, &opt_null, &opt_ten };
	125	try expect(opt_null != five);
	126	try expect(opt_null != ten);
	127	try expect(opt_ten != five);
	128	try expect(opt_ten == ten);
	129
	130	var opt_int: ?isize = null;
	131	try expect(opt_int != five);
	132	try expect(opt_int != ten);
	133	try expect(opt_int == opt_null);
	134	try expect(opt_int != opt_ten);
	135
	136	opt_int = 10;
	137	try expect(opt_int != five);
	138	try expect(opt_int == ten);
	139	try expect(opt_int != opt_null);
	140	try expect(opt_int == opt_ten);
	141
	142	opt_int = five;
	143	try expect(opt_int == five);
	144	try expect(opt_int != ten);
	145	try expect(opt_int != opt_null);
	146	try expect(opt_int != opt_ten);
	147
	148	// test evaluation is always lexical
	149	// ensure that the optional isn't always computed before the non-optional
	150	var mutable_state: i32 = 0;
	151	_ = blk1: {
	152	mutable_state += 1;
	153	break :blk1 @as(?f64, 10.0);
	154	} != blk2: {
	155	try expect(mutable_state == 1);
	156	break :blk2 @as(f64, 5.0);
	157	};
	158	_ = blk1: {
	159	mutable_state += 1;
	160	break :blk1 @as(f64, 10.0);
	161	} != blk2: {
	162	try expect(mutable_state == 2);
	163	break :blk2 @as(?f64, 5.0);
	164	};
	165	}
	166	};
	167
	168	try S.doTheTest();
	169	try comptime S.doTheTest();
120	170	}
121	171
122		fn test_cmp_optional_non_optional() !void {
123		var ten: i32 = 10;
124		var opt_ten: ?i32 = 10;
125		var five: i32 = 5;
126		var int_n: ?i32 = null;
127
128		_ = .{ &ten, &opt_ten, &five, &int_n };
129
130		try expect(int_n != ten);
131		try expect(opt_ten == ten);
132		try expect(opt_ten != five);
133
134		// test evaluation is always lexical
135		// ensure that the optional isn't always computed before the non-optional
136		var mutable_state: i32 = 0;
137		_ = blk1: {
138		mutable_state += 1;
139		break :blk1 @as(?f64, 10.0);
140		} != blk2: {
141		try expect(mutable_state == 1);
142		break :blk2 @as(f64, 5.0);
143		};
144		_ = blk1: {
145		mutable_state += 1;
146		break :blk1 @as(f64, 10.0);
147		} != blk2: {
148		try expect(mutable_state == 2);
149		break :blk2 @as(?f64, 5.0);
150		};
	172	test "compare optionals with modified payloads" {
	173	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
	174
	175	var lhs: ?bool = false;
	176	const lhs_payload = &lhs.?;
	177	var rhs: ?bool = true;
	178	const rhs_payload = &rhs.?;
	179	try expect(lhs != rhs and !(lhs == rhs));
	180
	181	lhs = null;
	182	lhs_payload.* = false;
	183	rhs = false;
	184	try expect(lhs != rhs and !(lhs == rhs));
	185
	186	lhs = true;
	187	rhs = null;
	188	rhs_payload.* = true;
	189	try expect(lhs != rhs and !(lhs == rhs));
	190
	191	lhs = null;
	192	lhs_payload.* = false;
	193	rhs = null;
	194	rhs_payload.* = true;
	195	try expect(lhs == rhs and !(lhs != rhs));
151	196	}
152	197
153	198	test "unwrap function call with optional pointer return value" {

test/behavior/select.zig+2-2

...	...	@@ -5,7 +5,6 @@ const expect = std.testing.expect;
5	5
6	6	test "@select vectors" {
7	7	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
8		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
9	8	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10	9	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
11	10	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...	...	@@ -36,11 +35,12 @@ fn selectVectors() !void {
36	35
37	36	test "@select arrays" {
38	37	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
39		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
40	38	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
41	39	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
42	40	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
43	41	if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
	42	if (builtin.zig_backend == .stage2_x86_64 and
	43	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx2)) return error.SkipZigTest;
44	44
45	45	try comptime selectArrays();
46	46	try selectArrays();

test/behavior/shuffle.zig+2-1

...	...	@@ -4,10 +4,11 @@ const mem = std.mem;
4	4	const expect = std.testing.expect;
5	5
6	6	test "@shuffle int" {
7		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8	7	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
9	8	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
10	9	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
	10	if (builtin.zig_backend == .stage2_x86_64 and
	11	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;
11	12
12	13	const S = struct {
13	14	fn doTheTest() !void {

test/behavior/vector.zig+29-13

...	...	@@ -29,7 +29,7 @@ test "vector wrap operators" {
29	29	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
30	30	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
31	31	if (builtin.zig_backend == .stage2_x86_64 and
32		!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
	32	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest;
33	33
34	34	const S = struct {
35	35	fn doTheTest() !void {
...	...	@@ -906,22 +906,26 @@ test "vector @reduce comptime" {
906	906	}
907	907
908	908	test "mask parameter of @shuffle is comptime scope" {
909		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
910	909	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
911	910	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
912	911	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
	912	if (builtin.zig_backend == .stage2_x86_64 and
	913	!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;
913	914
914	915	const __v4hi = @Vector(4, i16);
915		var v4_a = __v4hi{ 0, 0, 0, 0 };
916		var v4_b = __v4hi{ 0, 0, 0, 0 };
	916	var v4_a = __v4hi{ 1, 2, 3, 4 };
	917	var v4_b = __v4hi{ 5, 6, 7, 8 };
917	918	_ = .{ &v4_a, &v4_b };
918	919	const shuffled: __v4hi = @shuffle(i16, v4_a, v4_b, @Vector(4, i32){
919	920	std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
920		std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
921		std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
922		std.zig.c_translation.shuffleVectorIndex(0, @typeInfo(@TypeOf(v4_a)).Vector.len),
	921	std.zig.c_translation.shuffleVectorIndex(2, @typeInfo(@TypeOf(v4_a)).Vector.len),
	922	std.zig.c_translation.shuffleVectorIndex(4, @typeInfo(@TypeOf(v4_a)).Vector.len),
	923	std.zig.c_translation.shuffleVectorIndex(6, @typeInfo(@TypeOf(v4_a)).Vector.len),
923	924	});
924		_ = shuffled;
	925	try expect(shuffled[0] == 1);
	926	try expect(shuffled[1] == 3);
	927	try expect(shuffled[2] == 5);
	928	try expect(shuffled[3] == 7);
925	929	}
926	930
927	931	test "saturating add" {
...	...	@@ -1177,10 +1181,22 @@ test "@shlWithOverflow" {
1177	1181	}
1178	1182
1179	1183	test "alignment of vectors" {
1180		try expect(@alignOf(@Vector(2, u8)) == 2);
1181		try expect(@alignOf(@Vector(2, u1)) == 1);
1182		try expect(@alignOf(@Vector(1, u1)) == 1);
1183		try expect(@alignOf(@Vector(2, u16)) == 4);
	1184	try expect(@alignOf(@Vector(2, u8)) == switch (builtin.zig_backend) {
	1185	else => 2,
	1186	.stage2_x86_64 => 16,
	1187	});
	1188	try expect(@alignOf(@Vector(2, u1)) == switch (builtin.zig_backend) {
	1189	else => 1,
	1190	.stage2_x86_64 => 16,
	1191	});
	1192	try expect(@alignOf(@Vector(1, u1)) == switch (builtin.zig_backend) {
	1193	else => 1,
	1194	.stage2_x86_64 => 16,
	1195	});
	1196	try expect(@alignOf(@Vector(2, u16)) == switch (builtin.zig_backend) {
	1197	else => 4,
	1198	.stage2_x86_64 => 16,
	1199	});
1184	1200	}
1185	1201
1186	1202	test "loading the second vector from a slice of vectors" {
...	...	@@ -1316,10 +1332,10 @@ test "modRem with zero divisor" {
1316	1332
1317	1333	test "array operands to shuffle are coerced to vectors" {
1318	1334	if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1319		if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1320	1335	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1321	1336	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1322	1337	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
	1338	if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1323	1339
1324	1340	const mask = [5]i32{ -1, 0, 1, 2, 3 };
1325	1341