authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-03 23:18:21-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-03 23:18:21-04:00
log509be7cf1f10c5d329d2b0524f2af6bfcabd52de
tree655825e5800a1c79e8ccc0abe5011dd47655a96a
parentf6de3ec963e3a7d96cd4f6c72b0f076f0437c45d

x86_64: fix std test failures

45 files changed, 1413 insertions(+), 842 deletions(-)

lib/std/Thread/Condition.zig-10
...@@ -324,8 +324,6 @@ test "Condition - wait and signal" {...@@ -324,8 +324,6 @@ test "Condition - wait and signal" {
324 return error.SkipZigTest;324 return error.SkipZigTest;
325 }325 }
326326
327 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
328
329 const num_threads = 4;327 const num_threads = 4;
330328
331 const MultiWait = struct {329 const MultiWait = struct {
...@@ -371,8 +369,6 @@ test "Condition - signal" {...@@ -371,8 +369,6 @@ test "Condition - signal" {
371 return error.SkipZigTest;369 return error.SkipZigTest;
372 }370 }
373371
374 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
375
376 const num_threads = 4;372 const num_threads = 4;
377373
378 const SignalTest = struct {374 const SignalTest = struct {
...@@ -440,8 +436,6 @@ test "Condition - multi signal" {...@@ -440,8 +436,6 @@ test "Condition - multi signal" {
440 return error.SkipZigTest;436 return error.SkipZigTest;
441 }437 }
442438
443 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
444
445 const num_threads = 4;439 const num_threads = 4;
446 const num_iterations = 4;440 const num_iterations = 4;
447441
...@@ -504,8 +498,6 @@ test "Condition - broadcasting" {...@@ -504,8 +498,6 @@ test "Condition - broadcasting" {
504 return error.SkipZigTest;498 return error.SkipZigTest;
505 }499 }
506500
507 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
508
509 const num_threads = 10;501 const num_threads = 10;
510502
511 const BroadcastTest = struct {503 const BroadcastTest = struct {
...@@ -573,8 +565,6 @@ test "Condition - broadcasting - wake all threads" {...@@ -573,8 +565,6 @@ test "Condition - broadcasting - wake all threads" {
573 return error.SkipZigTest;565 return error.SkipZigTest;
574 }566 }
575567
576 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
577
578 var num_runs: usize = 1;568 var num_runs: usize = 1;
579 const num_threads = 10;569 const num_threads = 10;
580570
lib/std/Thread/Mutex.zig-2
...@@ -289,8 +289,6 @@ test "Mutex - many contended" {...@@ -289,8 +289,6 @@ test "Mutex - many contended" {
289 return error.SkipZigTest;289 return error.SkipZigTest;
290 }290 }
291291
292 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
293
294 const num_threads = 4;292 const num_threads = 4;
295 const num_increments = 1000;293 const num_increments = 1000;
296294
lib/std/Thread/RwLock.zig-2
...@@ -297,8 +297,6 @@ test "RwLock - concurrent access" {...@@ -297,8 +297,6 @@ test "RwLock - concurrent access" {
297 if (builtin.single_threaded)297 if (builtin.single_threaded)
298 return;298 return;
299299
300 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
301
302 const num_writers: usize = 2;300 const num_writers: usize = 2;
303 const num_readers: usize = 4;301 const num_readers: usize = 4;
304 const num_writes: usize = 10000;302 const num_writes: usize = 10000;
lib/std/Thread/Semaphore.zig-2
...@@ -39,8 +39,6 @@ test "Thread.Semaphore" {...@@ -39,8 +39,6 @@ test "Thread.Semaphore" {
39 return error.SkipZigTest;39 return error.SkipZigTest;
40 }40 }
4141
42 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
43
44 const TestContext = struct {42 const TestContext = struct {
45 sem: *Semaphore,43 sem: *Semaphore,
46 n: *i32,44 n: *i32,
lib/std/atomic/Atomic.zig-2
...@@ -467,8 +467,6 @@ test "Atomic.fetchSub" {...@@ -467,8 +467,6 @@ test "Atomic.fetchSub" {
467}467}
468468
469test "Atomic.fetchMin" {469test "Atomic.fetchMin" {
470 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
471
472 inline for (atomicIntTypes()) |Int| {470 inline for (atomicIntTypes()) |Int| {
473 inline for (atomic_rmw_orderings) |ordering| {471 inline for (atomic_rmw_orderings) |ordering| {
474 var x = Atomic(Int).init(5);472 var x = Atomic(Int).init(5);
lib/std/atomic/queue.zig-2
...@@ -175,8 +175,6 @@ const puts_per_thread = 500;...@@ -175,8 +175,6 @@ const puts_per_thread = 500;
175const put_thread_count = 3;175const put_thread_count = 3;
176176
177test "std.atomic.Queue" {177test "std.atomic.Queue" {
178 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
179
180 var plenty_of_memory = try std.heap.page_allocator.alloc(u8, 300 * 1024);178 var plenty_of_memory = try std.heap.page_allocator.alloc(u8, 300 * 1024);
181 defer std.heap.page_allocator.free(plenty_of_memory);179 defer std.heap.page_allocator.free(plenty_of_memory);
182180
lib/std/base64.zig-4
...@@ -355,8 +355,6 @@ pub const Base64DecoderWithIgnore = struct {...@@ -355,8 +355,6 @@ pub const Base64DecoderWithIgnore = struct {
355};355};
356356
357test "base64" {357test "base64" {
358 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
359
360 @setEvalBranchQuota(8000);358 @setEvalBranchQuota(8000);
361 try testBase64();359 try testBase64();
362 try comptime testAllApis(standard, "comptime", "Y29tcHRpbWU=");360 try comptime testAllApis(standard, "comptime", "Y29tcHRpbWU=");
...@@ -377,8 +375,6 @@ test "base64 padding dest overflow" {...@@ -377,8 +375,6 @@ test "base64 padding dest overflow" {
377}375}
378376
379test "base64 url_safe_no_pad" {377test "base64 url_safe_no_pad" {
380 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
381
382 @setEvalBranchQuota(8000);378 @setEvalBranchQuota(8000);
383 try testBase64UrlSafeNoPad();379 try testBase64UrlSafeNoPad();
384 try comptime testAllApis(url_safe_no_pad, "comptime", "Y29tcHRpbWU");380 try comptime testAllApis(url_safe_no_pad, "comptime", "Y29tcHRpbWU");
lib/std/bit_set.zig-3
...@@ -1638,7 +1638,6 @@ fn testStaticBitSet(comptime Set: type) !void {...@@ -1638,7 +1638,6 @@ fn testStaticBitSet(comptime Set: type) !void {
16381638
1639test "IntegerBitSet" {1639test "IntegerBitSet" {
1640 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;1640 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1641 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
16421641
1643 try testStaticBitSet(IntegerBitSet(0));1642 try testStaticBitSet(IntegerBitSet(0));
1644 try testStaticBitSet(IntegerBitSet(1));1643 try testStaticBitSet(IntegerBitSet(1));
...@@ -1651,8 +1650,6 @@ test "IntegerBitSet" {...@@ -1651,8 +1650,6 @@ test "IntegerBitSet" {
1651}1650}
16521651
1653test "ArrayBitSet" {1652test "ArrayBitSet" {
1654 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1655
1656 inline for (.{ 0, 1, 2, 31, 32, 33, 63, 64, 65, 254, 500, 3000 }) |size| {1653 inline for (.{ 0, 1, 2, 31, 32, 33, 63, 64, 65, 254, 500, 3000 }) |size| {
1657 try testStaticBitSet(ArrayBitSet(u8, size));1654 try testStaticBitSet(ArrayBitSet(u8, size));
1658 try testStaticBitSet(ArrayBitSet(u16, size));1655 try testStaticBitSet(ArrayBitSet(u16, size));
lib/std/compress/zstandard.zig-2
...@@ -264,8 +264,6 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {...@@ -264,8 +264,6 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {
264}264}
265265
266test "zstandard decompression" {266test "zstandard decompression" {
267 if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
268
269 const uncompressed = @embedFile("testdata/rfc8478.txt");267 const uncompressed = @embedFile("testdata/rfc8478.txt");
270 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");268 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
271 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");269 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
lib/std/crypto/25519/ed25519.zig+2-13
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const builtin = @import("builtin");
3const crypto = std.crypto;2const crypto = std.crypto;
4const debug = std.debug;3const debug = std.debug;
5const fmt = std.fmt;4const fmt = std.fmt;
...@@ -276,8 +275,8 @@ pub const Ed25519 = struct {...@@ -276,8 +275,8 @@ pub const Ed25519 = struct {
276 pub fn fromSecretKey(secret_key: SecretKey) (NonCanonicalError || EncodingError || IdentityElementError)!KeyPair {275 pub fn fromSecretKey(secret_key: SecretKey) (NonCanonicalError || EncodingError || IdentityElementError)!KeyPair {
277 // It is critical for EdDSA to use the correct public key.276 // It is critical for EdDSA to use the correct public key.
278 // In order to enforce this, a SecretKey implicitly includes a copy of the public key.277 // In order to enforce this, a SecretKey implicitly includes a copy of the public key.
279 // In Debug mode, we can still afford checking that the public key is correct for extra safety.278 // With runtime safety, we can still afford checking that the public key is correct.
280 if (builtin.mode == .Debug) {279 if (std.debug.runtime_safety) {
281 const pk_p = try Curve.fromBytes(secret_key.publicKeyBytes());280 const pk_p = try Curve.fromBytes(secret_key.publicKeyBytes());
282 const recomputed_kp = try create(secret_key.seed());281 const recomputed_kp = try create(secret_key.seed());
283 debug.assert(mem.eql(u8, &recomputed_kp.public_key.toBytes(), &pk_p.toBytes()));282 debug.assert(mem.eql(u8, &recomputed_kp.public_key.toBytes(), &pk_p.toBytes()));
...@@ -493,8 +492,6 @@ test "ed25519 key pair creation" {...@@ -493,8 +492,6 @@ test "ed25519 key pair creation" {
493}492}
494493
495test "ed25519 signature" {494test "ed25519 signature" {
496 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
497
498 var seed: [32]u8 = undefined;495 var seed: [32]u8 = undefined;
499 _ = try fmt.hexToBytes(seed[0..], "8052030376d47112be7f73ed7a019293dd12ad910b654455798b4667d73de166");496 _ = try fmt.hexToBytes(seed[0..], "8052030376d47112be7f73ed7a019293dd12ad910b654455798b4667d73de166");
500 const key_pair = try Ed25519.KeyPair.create(seed);497 const key_pair = try Ed25519.KeyPair.create(seed);
...@@ -507,8 +504,6 @@ test "ed25519 signature" {...@@ -507,8 +504,6 @@ test "ed25519 signature" {
507}504}
508505
509test "ed25519 batch verification" {506test "ed25519 batch verification" {
510 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
511
512 var i: usize = 0;507 var i: usize = 0;
513 while (i < 100) : (i += 1) {508 while (i < 100) : (i += 1) {
514 const key_pair = try Ed25519.KeyPair.create(null);509 const key_pair = try Ed25519.KeyPair.create(null);
...@@ -538,8 +533,6 @@ test "ed25519 batch verification" {...@@ -538,8 +533,6 @@ test "ed25519 batch verification" {
538}533}
539534
540test "ed25519 test vectors" {535test "ed25519 test vectors" {
541 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
542
543 const Vec = struct {536 const Vec = struct {
544 msg_hex: *const [64:0]u8,537 msg_hex: *const [64:0]u8,
545 public_key_hex: *const [64:0]u8,538 public_key_hex: *const [64:0]u8,
...@@ -642,8 +635,6 @@ test "ed25519 test vectors" {...@@ -642,8 +635,6 @@ test "ed25519 test vectors" {
642}635}
643636
644test "ed25519 with blind keys" {637test "ed25519 with blind keys" {
645 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
646
647 const BlindKeyPair = Ed25519.key_blinding.BlindKeyPair;638 const BlindKeyPair = Ed25519.key_blinding.BlindKeyPair;
648639
649 // Create a standard Ed25519 key pair640 // Create a standard Ed25519 key pair
...@@ -667,8 +658,6 @@ test "ed25519 with blind keys" {...@@ -667,8 +658,6 @@ test "ed25519 with blind keys" {
667}658}
668659
669test "ed25519 signatures with streaming" {660test "ed25519 signatures with streaming" {
670 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
671
672 const kp = try Ed25519.KeyPair.create(null);661 const kp = try Ed25519.KeyPair.create(null);
673662
674 var signer = try kp.signer(null);663 var signer = try kp.signer(null);
lib/std/crypto/Certificate.zig+13-13
...@@ -614,18 +614,18 @@ const Date = struct {...@@ -614,18 +614,18 @@ const Date = struct {
614};614};
615615
616pub fn parseTimeDigits(text: *const [2]u8, min: u8, max: u8) !u8 {616pub fn parseTimeDigits(text: *const [2]u8, min: u8, max: u8) !u8 {
617 const nn: @Vector(2, u16) = .{ text[0], text[1] };617 const result = if (use_vectors) result: {
618 const zero: @Vector(2, u16) = .{ '0', '0' };618 const nn: @Vector(2, u16) = .{ text[0], text[1] };
619 const mm: @Vector(2, u16) = .{ 10, 1 };619 const zero: @Vector(2, u16) = .{ '0', '0' };
620 const result = @reduce(.Add, (nn -% zero) *% mm);620 const mm: @Vector(2, u16) = .{ 10, 1 };
621 break :result @reduce(.Add, (nn -% zero) *% mm);
622 } else std.fmt.parseInt(u8, text, 10) catch return error.CertificateTimeInvalid;
621 if (result < min) return error.CertificateTimeInvalid;623 if (result < min) return error.CertificateTimeInvalid;
622 if (result > max) return error.CertificateTimeInvalid;624 if (result > max) return error.CertificateTimeInvalid;
623 return @truncate(result);625 return @truncate(result);
624}626}
625627
626test parseTimeDigits {628test parseTimeDigits {
627 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
628
629 const expectEqual = std.testing.expectEqual;629 const expectEqual = std.testing.expectEqual;
630 try expectEqual(@as(u8, 0), try parseTimeDigits("00", 0, 99));630 try expectEqual(@as(u8, 0), try parseTimeDigits("00", 0, 99));
631 try expectEqual(@as(u8, 99), try parseTimeDigits("99", 0, 99));631 try expectEqual(@as(u8, 99), try parseTimeDigits("99", 0, 99));
...@@ -638,17 +638,17 @@ test parseTimeDigits {...@@ -638,17 +638,17 @@ test parseTimeDigits {
638}638}
639639
640pub fn parseYear4(text: *const [4]u8) !u16 {640pub fn parseYear4(text: *const [4]u8) !u16 {
641 const nnnn: @Vector(4, u32) = .{ text[0], text[1], text[2], text[3] };641 const result = if (use_vectors) result: {
642 const zero: @Vector(4, u32) = .{ '0', '0', '0', '0' };642 const nnnn: @Vector(4, u32) = .{ text[0], text[1], text[2], text[3] };
643 const mmmm: @Vector(4, u32) = .{ 1000, 100, 10, 1 };643 const zero: @Vector(4, u32) = .{ '0', '0', '0', '0' };
644 const result = @reduce(.Add, (nnnn -% zero) *% mmmm);644 const mmmm: @Vector(4, u32) = .{ 1000, 100, 10, 1 };
645 break :result @reduce(.Add, (nnnn -% zero) *% mmmm);
646 } else std.fmt.parseInt(u16, text, 10) catch return error.CertificateTimeInvalid;
645 if (result > 9999) return error.CertificateTimeInvalid;647 if (result > 9999) return error.CertificateTimeInvalid;
646 return @truncate(result);648 return @truncate(result);
647}649}
648650
649test parseYear4 {651test parseYear4 {
650 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
651
652 const expectEqual = std.testing.expectEqual;652 const expectEqual = std.testing.expectEqual;
653 try expectEqual(@as(u16, 0), try parseYear4("0000"));653 try expectEqual(@as(u16, 0), try parseYear4("0000"));
654 try expectEqual(@as(u16, 9999), try parseYear4("9999"));654 try expectEqual(@as(u16, 9999), try parseYear4("9999"));
...@@ -1124,4 +1124,4 @@ pub const rsa = struct {...@@ -1124,4 +1124,4 @@ pub const rsa = struct {
1124 }1124 }
1125};1125};
11261126
1127const builtin = @import("builtin");1127const use_vectors = @import("builtin").zig_backend != .stage2_x86_64;
lib/std/crypto/Certificate/Bundle.zig-2
...@@ -318,8 +318,6 @@ const MapContext = struct {...@@ -318,8 +318,6 @@ const MapContext = struct {
318test "scan for OS-provided certificates" {318test "scan for OS-provided certificates" {
319 if (builtin.os.tag == .wasi) return error.SkipZigTest;319 if (builtin.os.tag == .wasi) return error.SkipZigTest;
320320
321 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
322
323 var bundle: Bundle = .{};321 var bundle: Bundle = .{};
324 defer bundle.deinit(std.testing.allocator);322 defer bundle.deinit(std.testing.allocator);
325323
lib/std/crypto/aes.zig-2
...@@ -28,8 +28,6 @@ pub const Aes128 = impl.Aes128;...@@ -28,8 +28,6 @@ pub const Aes128 = impl.Aes128;
28pub const Aes256 = impl.Aes256;28pub const Aes256 = impl.Aes256;
2929
30test "ctr" {30test "ctr" {
31 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
32
33 // NIST SP 800-38A pp 55-5831 // NIST SP 800-38A pp 55-58
34 const ctr = @import("modes.zig").ctr;32 const ctr = @import("modes.zig").ctr;
3533
lib/std/crypto/aes_gcm.zig+2-11
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const builtin = @import("builtin");
3const assert = std.debug.assert;2const assert = std.debug.assert;
4const crypto = std.crypto;3const crypto = std.crypto;
5const debug = std.debug;4const debug = std.debug;
...@@ -42,7 +41,7 @@ fn AesGcm(comptime Aes: anytype) type {...@@ -42,7 +41,7 @@ fn AesGcm(comptime Aes: anytype) type {
42 mac.pad();41 mac.pad();
4342
44 mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);43 mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);
45 modes.ctr(@TypeOf(aes), aes, c, m, j, std.builtin.Endian.big);44 modes.ctr(@TypeOf(aes), aes, c, m, j, .big);
46 mac.update(c[0..m.len][0..]);45 mac.update(c[0..m.len][0..]);
47 mac.pad();46 mac.pad();
4847
...@@ -104,7 +103,7 @@ fn AesGcm(comptime Aes: anytype) type {...@@ -104,7 +103,7 @@ fn AesGcm(comptime Aes: anytype) type {
104 }103 }
105104
106 mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);105 mem.writeInt(u32, j[nonce_length..][0..4], 2, .big);
107 modes.ctr(@TypeOf(aes), aes, m, c, j, std.builtin.Endian.big);106 modes.ctr(@TypeOf(aes), aes, m, c, j, .big);
108 }107 }
109 };108 };
110}109}
...@@ -113,8 +112,6 @@ const htest = @import("test.zig");...@@ -113,8 +112,6 @@ const htest = @import("test.zig");
113const testing = std.testing;112const testing = std.testing;
114113
115test "Aes256Gcm - Empty message and no associated data" {114test "Aes256Gcm - Empty message and no associated data" {
116 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
117
118 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;115 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
119 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;116 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
120 const ad = "";117 const ad = "";
...@@ -127,8 +124,6 @@ test "Aes256Gcm - Empty message and no associated data" {...@@ -127,8 +124,6 @@ test "Aes256Gcm - Empty message and no associated data" {
127}124}
128125
129test "Aes256Gcm - Associated data only" {126test "Aes256Gcm - Associated data only" {
130 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
131
132 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;127 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
133 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;128 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
134 const m = "";129 const m = "";
...@@ -141,8 +136,6 @@ test "Aes256Gcm - Associated data only" {...@@ -141,8 +136,6 @@ test "Aes256Gcm - Associated data only" {
141}136}
142137
143test "Aes256Gcm - Message only" {138test "Aes256Gcm - Message only" {
144 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
145
146 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;139 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
147 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;140 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
148 const m = "Test with message only";141 const m = "Test with message only";
...@@ -160,8 +153,6 @@ test "Aes256Gcm - Message only" {...@@ -160,8 +153,6 @@ test "Aes256Gcm - Message only" {
160}153}
161154
162test "Aes256Gcm - Message and associated data" {155test "Aes256Gcm - Message and associated data" {
163 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
164
165 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;156 const key: [Aes256Gcm.key_length]u8 = [_]u8{0x69} ** Aes256Gcm.key_length;
166 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;157 const nonce: [Aes256Gcm.nonce_length]u8 = [_]u8{0x42} ** Aes256Gcm.nonce_length;
167 const m = "Test with message";158 const m = "Test with message";
lib/std/crypto/argon2.zig-4
...@@ -896,8 +896,6 @@ test "kdf" {...@@ -896,8 +896,6 @@ test "kdf" {
896}896}
897897
898test "phc format hasher" {898test "phc format hasher" {
899 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
900
901 const allocator = std.testing.allocator;899 const allocator = std.testing.allocator;
902 const password = "testpass";900 const password = "testpass";
903901
...@@ -913,8 +911,6 @@ test "phc format hasher" {...@@ -913,8 +911,6 @@ test "phc format hasher" {
913}911}
914912
915test "password hash and password verify" {913test "password hash and password verify" {
916 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
917
918 const allocator = std.testing.allocator;914 const allocator = std.testing.allocator;
919 const password = "testpass";915 const password = "testpass";
920916
lib/std/crypto/bcrypt.zig-7
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const builtin = @import("builtin");
3const base64 = std.base64;2const base64 = std.base64;
4const crypto = std.crypto;3const crypto = std.crypto;
5const debug = std.debug;4const debug = std.debug;
...@@ -754,8 +753,6 @@ pub fn strVerify(...@@ -754,8 +753,6 @@ pub fn strVerify(
754}753}
755754
756test "bcrypt codec" {755test "bcrypt codec" {
757 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
758
759 var salt: [salt_length]u8 = undefined;756 var salt: [salt_length]u8 = undefined;
760 crypto.random.bytes(&salt);757 crypto.random.bytes(&salt);
761 var salt_str: [salt_str_length]u8 = undefined;758 var salt_str: [salt_str_length]u8 = undefined;
...@@ -766,8 +763,6 @@ test "bcrypt codec" {...@@ -766,8 +763,6 @@ test "bcrypt codec" {
766}763}
767764
768test "bcrypt crypt format" {765test "bcrypt crypt format" {
769 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
770
771 var hash_options = HashOptions{766 var hash_options = HashOptions{
772 .params = .{ .rounds_log = 5 },767 .params = .{ .rounds_log = 5 },
773 .encoding = .crypt,768 .encoding = .crypt,
...@@ -808,8 +803,6 @@ test "bcrypt crypt format" {...@@ -808,8 +803,6 @@ test "bcrypt crypt format" {
808}803}
809804
810test "bcrypt phc format" {805test "bcrypt phc format" {
811 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
812
813 var hash_options = HashOptions{806 var hash_options = HashOptions{
814 .params = .{ .rounds_log = 5 },807 .params = .{ .rounds_log = 5 },
815 .encoding = .phc,808 .encoding = .phc,
lib/std/crypto/cmac.zig-9
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("std");1const std = @import("std");
2const builtin = @import("builtin");
3const crypto = std.crypto;2const crypto = std.crypto;
4const mem = std.mem;3const mem = std.mem;
54
...@@ -94,8 +93,6 @@ pub fn Cmac(comptime BlockCipher: type) type {...@@ -94,8 +93,6 @@ pub fn Cmac(comptime BlockCipher: type) type {
94const testing = std.testing;93const testing = std.testing;
9594
96test "CmacAes128 - Example 1: len = 0" {95test "CmacAes128 - Example 1: len = 0" {
97 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
98
99 const key = [_]u8{96 const key = [_]u8{
100 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,97 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
101 };98 };
...@@ -109,8 +106,6 @@ test "CmacAes128 - Example 1: len = 0" {...@@ -109,8 +106,6 @@ test "CmacAes128 - Example 1: len = 0" {
109}106}
110107
111test "CmacAes128 - Example 2: len = 16" {108test "CmacAes128 - Example 2: len = 16" {
112 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
113
114 const key = [_]u8{109 const key = [_]u8{
115 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,110 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
116 };111 };
...@@ -126,8 +121,6 @@ test "CmacAes128 - Example 2: len = 16" {...@@ -126,8 +121,6 @@ test "CmacAes128 - Example 2: len = 16" {
126}121}
127122
128test "CmacAes128 - Example 3: len = 40" {123test "CmacAes128 - Example 3: len = 40" {
129 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
130
131 const key = [_]u8{124 const key = [_]u8{
132 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,125 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
133 };126 };
...@@ -145,8 +138,6 @@ test "CmacAes128 - Example 3: len = 40" {...@@ -145,8 +138,6 @@ test "CmacAes128 - Example 3: len = 40" {
145}138}
146139
147test "CmacAes128 - Example 4: len = 64" {140test "CmacAes128 - Example 4: len = 64" {
148 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
149
150 const key = [_]u8{141 const key = [_]u8{
151 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,142 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c,
152 };143 };
lib/std/crypto/ecdsa.zig-5
...@@ -373,7 +373,6 @@ pub fn Ecdsa(comptime Curve: type, comptime Hash: type) type {...@@ -373,7 +373,6 @@ pub fn Ecdsa(comptime Curve: type, comptime Hash: type) type {
373373
374test "ECDSA - Basic operations over EcdsaP384Sha384" {374test "ECDSA - Basic operations over EcdsaP384Sha384" {
375 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;375 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
376 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
377376
378 const Scheme = EcdsaP384Sha384;377 const Scheme = EcdsaP384Sha384;
379 const kp = try Scheme.KeyPair.create(null);378 const kp = try Scheme.KeyPair.create(null);
...@@ -407,7 +406,6 @@ test "ECDSA - Basic operations over Secp256k1" {...@@ -407,7 +406,6 @@ test "ECDSA - Basic operations over Secp256k1" {
407406
408test "ECDSA - Basic operations over EcdsaP384Sha256" {407test "ECDSA - Basic operations over EcdsaP384Sha256" {
409 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;408 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
410 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
411409
412 const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);410 const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);
413 const kp = try Scheme.KeyPair.create(null);411 const kp = try Scheme.KeyPair.create(null);
...@@ -424,7 +422,6 @@ test "ECDSA - Basic operations over EcdsaP384Sha256" {...@@ -424,7 +422,6 @@ test "ECDSA - Basic operations over EcdsaP384Sha256" {
424422
425test "ECDSA - Verifying a existing signature with EcdsaP384Sha256" {423test "ECDSA - Verifying a existing signature with EcdsaP384Sha256" {
426 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;424 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
427 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
428425
429 const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);426 const Scheme = Ecdsa(crypto.ecc.P384, crypto.hash.sha2.Sha256);
430 // zig fmt: off427 // zig fmt: off
...@@ -469,7 +466,6 @@ const TestVector = struct {...@@ -469,7 +466,6 @@ const TestVector = struct {
469466
470test "ECDSA - Test vectors from Project Wycheproof" {467test "ECDSA - Test vectors from Project Wycheproof" {
471 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;468 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
472 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
473469
474 const vectors = [_]TestVector{470 const vectors = [_]TestVector{
475 .{ .key = "042927b10512bae3eddcfe467828128bad2903269919f7086069c8c4df6c732838c7787964eaac00e5921fb1498a60f4606766b3d9685001558d1a974e7341513e", .msg = "313233343030", .sig = "304402202ba3a8be6b94d5ec80a6d9d1190a436effe50d85a1eee859b8cc6af9bd5c2e1802204cd60b855d442f5b3c7b11eb6c4e0ae7525fe710fab9aa7c77a67f79e6fadd76", .result = .valid },471 .{ .key = "042927b10512bae3eddcfe467828128bad2903269919f7086069c8c4df6c732838c7787964eaac00e5921fb1498a60f4606766b3d9685001558d1a974e7341513e", .msg = "313233343030", .sig = "304402202ba3a8be6b94d5ec80a6d9d1190a436effe50d85a1eee859b8cc6af9bd5c2e1802204cd60b855d442f5b3c7b11eb6c4e0ae7525fe710fab9aa7c77a67f79e6fadd76", .result = .valid },
...@@ -884,7 +880,6 @@ fn tvTry(vector: TestVector) !void {...@@ -884,7 +880,6 @@ fn tvTry(vector: TestVector) !void {
884880
885test "ECDSA - Sec1 encoding/decoding" {881test "ECDSA - Sec1 encoding/decoding" {
886 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;882 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
887 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
888883
889 const Scheme = EcdsaP384Sha384;884 const Scheme = EcdsaP384Sha384;
890 const kp = try Scheme.KeyPair.create(null);885 const kp = try Scheme.KeyPair.create(null);
lib/std/crypto/ghash_polyval.zig-4
...@@ -422,8 +422,6 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {...@@ -422,8 +422,6 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {
422const htest = @import("test.zig");422const htest = @import("test.zig");
423423
424test "ghash" {424test "ghash" {
425 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
426
427 const key = [_]u8{0x42} ** 16;425 const key = [_]u8{0x42} ** 16;
428 const m = [_]u8{0x69} ** 256;426 const m = [_]u8{0x69} ** 256;
429427
...@@ -441,8 +439,6 @@ test "ghash" {...@@ -441,8 +439,6 @@ test "ghash" {
441}439}
442440
443test "ghash2" {441test "ghash2" {
444 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
445
446 var key: [16]u8 = undefined;442 var key: [16]u8 = undefined;
447 var i: usize = 0;443 var i: usize = 0;
448 while (i < key.len) : (i += 1) {444 while (i < key.len) : (i += 1) {
lib/std/crypto/pcurves/p256.zig-2
...@@ -478,7 +478,5 @@ pub const AffineCoordinates = struct {...@@ -478,7 +478,5 @@ pub const AffineCoordinates = struct {
478};478};
479479
480test {480test {
481 if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
482
483 _ = @import("tests/p256.zig");481 _ = @import("tests/p256.zig");
484}482}
lib/std/crypto/phc_encoding.zig-3
...@@ -1,7 +1,6 @@...@@ -1,7 +1,6 @@
1// https://github.com/P-H-C/phc-string-format1// https://github.com/P-H-C/phc-string-format
22
3const std = @import("std");3const std = @import("std");
4const builtin = @import("builtin");
5const fmt = std.fmt;4const fmt = std.fmt;
6const io = std.io;5const io = std.io;
7const mem = std.mem;6const mem = std.mem;
...@@ -264,8 +263,6 @@ fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {...@@ -264,8 +263,6 @@ fn kvSplit(str: []const u8) !struct { key: []const u8, value: []const u8 } {
264}263}
265264
266test "phc format - encoding/decoding" {265test "phc format - encoding/decoding" {
267 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
268
269 const Input = struct {266 const Input = struct {
270 str: []const u8,267 str: []const u8,
271 HashResult: type,268 HashResult: type,
lib/std/crypto/sha2.zig+1-1
...@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {...@@ -238,7 +238,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
238 return;238 return;
239 },239 },
240 // C backend doesn't currently support passing vectors to inline asm.240 // C backend doesn't currently support passing vectors to inline asm.
241 .x86_64 => if (builtin.zig_backend != .stage2_c and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {241 .x86_64 => if (builtin.zig_backend != .stage2_c and builtin.zig_backend != .stage2_x86_64 and comptime std.Target.x86.featureSetHasAll(builtin.cpu.features, .{ .sha, .avx2 })) {
242 var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };242 var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
243 var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };243 var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
244 const s_v = @as(*[16]v4u32, @ptrCast(&s));244 const s_v = @as(*[16]v4u32, @ptrCast(&s));
lib/std/fmt/parse_float.zig-2
...@@ -83,8 +83,6 @@ test "fmt.parseFloat #11169" {...@@ -83,8 +83,6 @@ test "fmt.parseFloat #11169" {
83}83}
8484
85test "fmt.parseFloat hex.special" {85test "fmt.parseFloat hex.special" {
86 if (@import("builtin").zig_backend == .stage2_x86_64) return error.SkipZigTest;
87
88 try testing.expect(math.isNan(try parseFloat(f32, "nAn")));86 try testing.expect(math.isNan(try parseFloat(f32, "nAn")));
89 try testing.expect(math.isPositiveInf(try parseFloat(f32, "iNf")));87 try testing.expect(math.isPositiveInf(try parseFloat(f32, "iNf")));
90 try testing.expect(math.isPositiveInf(try parseFloat(f32, "+Inf")));88 try testing.expect(math.isPositiveInf(try parseFloat(f32, "+Inf")));
lib/std/hash/xxhash.zig+2-1
...@@ -2,6 +2,7 @@ const std = @import("std");...@@ -2,6 +2,7 @@ const std = @import("std");
2const builtin = @import("builtin");2const builtin = @import("builtin");
3const mem = std.mem;3const mem = std.mem;
4const expectEqual = std.testing.expectEqual;4const expectEqual = std.testing.expectEqual;
5const native_endian = builtin.cpu.arch.endian();
56
6const rotl = std.math.rotl;7const rotl = std.math.rotl;
78
...@@ -472,7 +473,7 @@ pub const XxHash3 = struct {...@@ -472,7 +473,7 @@ pub const XxHash3 = struct {
472 }473 }
473474
474 inline fn swap(x: anytype) @TypeOf(x) {475 inline fn swap(x: anytype) @TypeOf(x) {
475 return if (builtin.cpu.arch.endian() == .big) @byteSwap(x) else x;476 return if (native_endian == .big) @byteSwap(x) else x;
476 }477 }
477478
478 inline fn disableAutoVectorization(x: anytype) void {479 inline fn disableAutoVectorization(x: anytype) void {
lib/std/http/Client.zig+16-10
...@@ -9,6 +9,7 @@ const net = std.net;...@@ -9,6 +9,7 @@ const net = std.net;
9const Uri = std.Uri;9const Uri = std.Uri;
10const Allocator = mem.Allocator;10const Allocator = mem.Allocator;
11const assert = std.debug.assert;11const assert = std.debug.assert;
12const use_vectors = builtin.zig_backend != .stage2_x86_64;
1213
13const Client = @This();14const Client = @This();
14const proto = @import("protocol.zig");15const proto = @import("protocol.zig");
...@@ -408,7 +409,7 @@ pub const Response = struct {...@@ -408,7 +409,7 @@ pub const Response = struct {
408 else => return error.HttpHeadersInvalid,409 else => return error.HttpHeadersInvalid,
409 };410 };
410 if (first_line[8] != ' ') return error.HttpHeadersInvalid;411 if (first_line[8] != ' ') return error.HttpHeadersInvalid;
411 const status = @as(http.Status, @enumFromInt(parseInt3(first_line[9..12].*)));412 const status: http.Status = @enumFromInt(parseInt3(first_line[9..12]));
412 const reason = mem.trimLeft(u8, first_line[12..], " ");413 const reason = mem.trimLeft(u8, first_line[12..], " ");
413414
414 res.version = version;415 res.version = version;
...@@ -481,20 +482,24 @@ pub const Response = struct {...@@ -481,20 +482,24 @@ pub const Response = struct {
481 }482 }
482483
483 inline fn int64(array: *const [8]u8) u64 {484 inline fn int64(array: *const [8]u8) u64 {
484 return @as(u64, @bitCast(array.*));485 return @bitCast(array.*);
485 }486 }
486487
487 fn parseInt3(nnn: @Vector(3, u8)) u10 {488 fn parseInt3(text: *const [3]u8) u10 {
488 const zero: @Vector(3, u8) = .{ '0', '0', '0' };489 if (use_vectors) {
489 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };490 const nnn: @Vector(3, u8) = text.*;
490 return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);491 const zero: @Vector(3, u8) = .{ '0', '0', '0' };
492 const mmm: @Vector(3, u10) = .{ 100, 10, 1 };
493 return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm);
494 }
495 return std.fmt.parseInt(u10, text, 10) catch unreachable;
491 }496 }
492497
493 test parseInt3 {498 test parseInt3 {
494 const expectEqual = testing.expectEqual;499 const expectEqual = testing.expectEqual;
495 try expectEqual(@as(u10, 0), parseInt3("000".*));500 try expectEqual(@as(u10, 0), parseInt3("000"));
496 try expectEqual(@as(u10, 418), parseInt3("418".*));501 try expectEqual(@as(u10, 418), parseInt3("418"));
497 try expectEqual(@as(u10, 999), parseInt3("999".*));502 try expectEqual(@as(u10, 999), parseInt3("999"));
498 }503 }
499504
500 version: http.Version,505 version: http.Version,
...@@ -1588,7 +1593,8 @@ test {...@@ -1588,7 +1593,8 @@ test {
15881593
1589 if (builtin.os.tag == .wasi) return error.SkipZigTest;1594 if (builtin.os.tag == .wasi) return error.SkipZigTest;
15901595
1591 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;1596 if (builtin.zig_backend == .stage2_x86_64 and
1597 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) return error.SkipZigTest;
15921598
1593 std.testing.refAllDecls(@This());1599 std.testing.refAllDecls(@This());
1594}1600}
lib/std/http/Server.zig-2
...@@ -736,8 +736,6 @@ test "HTTP server handles a chunked transfer coding request" {...@@ -736,8 +736,6 @@ test "HTTP server handles a chunked transfer coding request" {
736 return error.SkipZigTest;736 return error.SkipZigTest;
737 }737 }
738738
739 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
740
741 const native_endian = comptime builtin.cpu.arch.endian();739 const native_endian = comptime builtin.cpu.arch.endian();
742 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {740 if (builtin.zig_backend == .stage2_llvm and native_endian == .big) {
743 // https://github.com/ziglang/zig/issues/13782741 // https://github.com/ziglang/zig/issues/13782
lib/std/http/protocol.zig+23-12
...@@ -1,8 +1,10 @@...@@ -1,8 +1,10 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const builtin = @import("builtin");
2const testing = std.testing;3const testing = std.testing;
3const mem = std.mem;4const mem = std.mem;
45
5const assert = std.debug.assert;6const assert = std.debug.assert;
7const use_vectors = builtin.zig_backend != .stage2_x86_64;
68
7pub const State = enum {9pub const State = enum {
8 /// Begin header parsing states.10 /// Begin header parsing states.
...@@ -83,7 +85,7 @@ pub const HeadersParser = struct {...@@ -83,7 +85,7 @@ pub const HeadersParser = struct {
83 /// first byte of content is located at `bytes[result]`.85 /// first byte of content is located at `bytes[result]`.
84 pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {86 pub fn findHeadersEnd(r: *HeadersParser, bytes: []const u8) u32 {
85 const vector_len: comptime_int = @max(std.simd.suggestVectorSize(u8) orelse 1, 8);87 const vector_len: comptime_int = @max(std.simd.suggestVectorSize(u8) orelse 1, 8);
86 const len = @as(u32, @intCast(bytes.len));88 const len: u32 = @intCast(bytes.len);
87 var index: u32 = 0;89 var index: u32 = 0;
8890
89 while (true) {91 while (true) {
...@@ -175,18 +177,27 @@ pub const HeadersParser = struct {...@@ -175,18 +177,27 @@ pub const HeadersParser = struct {
175 continue;177 continue;
176 },178 },
177 else => {179 else => {
178 const Vector = @Vector(vector_len, u8);
179 // const BoolVector = @Vector(vector_len, bool);
180 const BitVector = @Vector(vector_len, u1);
181 const SizeVector = @Vector(vector_len, u8);
182
183 const chunk = bytes[index..][0..vector_len];180 const chunk = bytes[index..][0..vector_len];
184 const v: Vector = chunk.*;181 const matches = if (use_vectors) matches: {
185 const matches_r = @as(BitVector, @bitCast(v == @as(Vector, @splat('\r'))));182 const Vector = @Vector(vector_len, u8);
186 const matches_n = @as(BitVector, @bitCast(v == @as(Vector, @splat('\n'))));183 // const BoolVector = @Vector(vector_len, bool);
187 const matches_or: SizeVector = matches_r | matches_n;184 const BitVector = @Vector(vector_len, u1);
188185 const SizeVector = @Vector(vector_len, u8);
189 const matches = @reduce(.Add, matches_or);186
187 const v: Vector = chunk.*;
188 const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
189 const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
190 const matches_or: SizeVector = matches_r | matches_n;
191
192 break :matches @reduce(.Add, matches_or);
193 } else matches: {
194 var matches: u8 = 0;
195 for (chunk) |byte| switch (byte) {
196 '\r', '\n' => matches += 1,
197 else => {},
198 };
199 break :matches matches;
200 };
190 switch (matches) {201 switch (matches) {
191 0 => {},202 0 => {},
192 1 => switch (chunk[vector_len - 1]) {203 1 => switch (chunk[vector_len - 1]) {
lib/std/math.zig+2-21
...@@ -492,8 +492,6 @@ pub fn shl(comptime T: type, a: T, shift_amt: anytype) T {...@@ -492,8 +492,6 @@ pub fn shl(comptime T: type, a: T, shift_amt: anytype) T {
492}492}
493493
494test "shl" {494test "shl" {
495 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
496
497 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {495 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
498 // https://github.com/ziglang/zig/issues/12012496 // https://github.com/ziglang/zig/issues/12012
499 return error.SkipZigTest;497 return error.SkipZigTest;
...@@ -539,8 +537,6 @@ pub fn shr(comptime T: type, a: T, shift_amt: anytype) T {...@@ -539,8 +537,6 @@ pub fn shr(comptime T: type, a: T, shift_amt: anytype) T {
539}537}
540538
541test "shr" {539test "shr" {
542 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
543
544 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {540 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
545 // https://github.com/ziglang/zig/issues/12012541 // https://github.com/ziglang/zig/issues/12012
546 return error.SkipZigTest;542 return error.SkipZigTest;
...@@ -587,8 +583,6 @@ pub fn rotr(comptime T: type, x: T, r: anytype) T {...@@ -587,8 +583,6 @@ pub fn rotr(comptime T: type, x: T, r: anytype) T {
587}583}
588584
589test "rotr" {585test "rotr" {
590 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
591
592 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {586 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
593 // https://github.com/ziglang/zig/issues/12012587 // https://github.com/ziglang/zig/issues/12012
594 return error.SkipZigTest;588 return error.SkipZigTest;
...@@ -634,8 +628,6 @@ pub fn rotl(comptime T: type, x: T, r: anytype) T {...@@ -634,8 +628,6 @@ pub fn rotl(comptime T: type, x: T, r: anytype) T {
634}628}
635629
636test "rotl" {630test "rotl" {
637 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
638
639 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {631 if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) {
640 // https://github.com/ziglang/zig/issues/12012632 // https://github.com/ziglang/zig/issues/12012
641 return error.SkipZigTest;633 return error.SkipZigTest;
...@@ -764,8 +756,6 @@ pub fn divTrunc(comptime T: type, numerator: T, denominator: T) !T {...@@ -764,8 +756,6 @@ pub fn divTrunc(comptime T: type, numerator: T, denominator: T) !T {
764}756}
765757
766test "divTrunc" {758test "divTrunc" {
767 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
768
769 try testDivTrunc();759 try testDivTrunc();
770 try comptime testDivTrunc();760 try comptime testDivTrunc();
771}761}
...@@ -790,8 +780,6 @@ pub fn divFloor(comptime T: type, numerator: T, denominator: T) !T {...@@ -790,8 +780,6 @@ pub fn divFloor(comptime T: type, numerator: T, denominator: T) !T {
790}780}
791781
792test "divFloor" {782test "divFloor" {
793 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
794
795 try testDivFloor();783 try testDivFloor();
796 try comptime testDivFloor();784 try comptime testDivFloor();
797}785}
...@@ -829,8 +817,6 @@ pub fn divCeil(comptime T: type, numerator: T, denominator: T) !T {...@@ -829,8 +817,6 @@ pub fn divCeil(comptime T: type, numerator: T, denominator: T) !T {
829}817}
830818
831test "divCeil" {819test "divCeil" {
832 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
833
834 try testDivCeil();820 try testDivCeil();
835 try comptime testDivCeil();821 try comptime testDivCeil();
836}822}
...@@ -875,8 +861,6 @@ pub fn divExact(comptime T: type, numerator: T, denominator: T) !T {...@@ -875,8 +861,6 @@ pub fn divExact(comptime T: type, numerator: T, denominator: T) !T {
875}861}
876862
877test "divExact" {863test "divExact" {
878 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
879
880 try testDivExact();864 try testDivExact();
881 try comptime testDivExact();865 try comptime testDivExact();
882}866}
...@@ -903,8 +887,6 @@ pub fn mod(comptime T: type, numerator: T, denominator: T) !T {...@@ -903,8 +887,6 @@ pub fn mod(comptime T: type, numerator: T, denominator: T) !T {
903}887}
904888
905test "mod" {889test "mod" {
906 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
907
908 try testMod();890 try testMod();
909 try comptime testMod();891 try comptime testMod();
910}892}
...@@ -931,8 +913,6 @@ pub fn rem(comptime T: type, numerator: T, denominator: T) !T {...@@ -931,8 +913,6 @@ pub fn rem(comptime T: type, numerator: T, denominator: T) !T {
931}913}
932914
933test "rem" {915test "rem" {
934 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
935
936 try testRem();916 try testRem();
937 try comptime testRem();917 try comptime testRem();
938}918}
...@@ -1285,7 +1265,8 @@ pub fn lerp(a: anytype, b: anytype, t: anytype) @TypeOf(a, b, t) {...@@ -1285,7 +1265,8 @@ pub fn lerp(a: anytype, b: anytype, t: anytype) @TypeOf(a, b, t) {
1285}1265}
12861266
1287test "lerp" {1267test "lerp" {
1288 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;1268 if (builtin.zig_backend == .stage2_x86_64 and
1269 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .fma)) return error.SkipZigTest;
12891270
1290 try testing.expectEqual(@as(f64, 75), lerp(50, 100, 0.5));1271 try testing.expectEqual(@as(f64, 75), lerp(50, 100, 0.5));
1291 try testing.expectEqual(@as(f32, 43.75), lerp(50, 25, 0.25));1272 try testing.expectEqual(@as(f32, 43.75), lerp(50, 25, 0.25));
lib/std/math/big/int.zig+3-3
...@@ -1318,7 +1318,7 @@ pub const Mutable = struct {...@@ -1318,7 +1318,7 @@ pub const Mutable = struct {
1318 ///1318 ///
1319 /// `limbs_buffer` is used for temporary storage.1319 /// `limbs_buffer` is used for temporary storage.
1320 /// The amount required is given by `calcPowLimbsBufferLen`.1320 /// The amount required is given by `calcPowLimbsBufferLen`.
1321 pub fn pow(r: *Mutable, a: Const, b: u32, limbs_buffer: []Limb) !void {1321 pub fn pow(r: *Mutable, a: Const, b: u32, limbs_buffer: []Limb) void {
1322 assert(r.limbs.ptr != a.limbs.ptr); // illegal aliasing1322 assert(r.limbs.ptr != a.limbs.ptr); // illegal aliasing
13231323
1324 // Handle all the trivial cases first1324 // Handle all the trivial cases first
...@@ -3213,7 +3213,7 @@ pub const Managed = struct {...@@ -3213,7 +3213,7 @@ pub const Managed = struct {
3213 var m = try Managed.initCapacity(rma.allocator, needed_limbs);3213 var m = try Managed.initCapacity(rma.allocator, needed_limbs);
3214 errdefer m.deinit();3214 errdefer m.deinit();
3215 var m_mut = m.toMutable();3215 var m_mut = m.toMutable();
3216 try m_mut.pow(a.toConst(), b, limbs_buffer);3216 m_mut.pow(a.toConst(), b, limbs_buffer);
3217 m.setMetadata(m_mut.positive, m_mut.len);3217 m.setMetadata(m_mut.positive, m_mut.len);
32183218
3219 rma.deinit();3219 rma.deinit();
...@@ -3221,7 +3221,7 @@ pub const Managed = struct {...@@ -3221,7 +3221,7 @@ pub const Managed = struct {
3221 } else {3221 } else {
3222 try rma.ensureCapacity(needed_limbs);3222 try rma.ensureCapacity(needed_limbs);
3223 var rma_mut = rma.toMutable();3223 var rma_mut = rma.toMutable();
3224 try rma_mut.pow(a.toConst(), b, limbs_buffer);3224 rma_mut.pow(a.toConst(), b, limbs_buffer);
3225 rma.setMetadata(rma_mut.positive, rma_mut.len);3225 rma.setMetadata(rma_mut.positive, rma_mut.len);
3226 }3226 }
3227 }3227 }
lib/std/math/big/int_test.zig-6
...@@ -2568,8 +2568,6 @@ test "big.int const to managed" {...@@ -2568,8 +2568,6 @@ test "big.int const to managed" {
2568}2568}
25692569
2570test "big.int pow" {2570test "big.int pow" {
2571 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2572
2573 {2571 {
2574 var a = try Managed.initSet(testing.allocator, -3);2572 var a = try Managed.initSet(testing.allocator, -3);
2575 defer a.deinit();2573 defer a.deinit();
...@@ -2763,8 +2761,6 @@ fn popCountTest(val: *const Managed, bit_count: usize, expected: usize) !void {...@@ -2763,8 +2761,6 @@ fn popCountTest(val: *const Managed, bit_count: usize, expected: usize) !void {
2763}2761}
27642762
2765test "big int conversion read/write twos complement" {2763test "big int conversion read/write twos complement" {
2766 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2767
2768 var a = try Managed.initSet(testing.allocator, (1 << 493) - 1);2764 var a = try Managed.initSet(testing.allocator, (1 << 493) - 1);
2769 defer a.deinit();2765 defer a.deinit();
2770 var b = try Managed.initSet(testing.allocator, (1 << 493) - 1);2766 var b = try Managed.initSet(testing.allocator, (1 << 493) - 1);
...@@ -2863,8 +2859,6 @@ test "big int write twos complement +/- zero" {...@@ -2863,8 +2859,6 @@ test "big int write twos complement +/- zero" {
2863}2859}
28642860
2865test "big int conversion write twos complement with padding" {2861test "big int conversion write twos complement with padding" {
2866 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2867
2868 var a = try Managed.initSet(testing.allocator, 0x01_ffffffff_ffffffff_ffffffff);2862 var a = try Managed.initSet(testing.allocator, 0x01_ffffffff_ffffffff_ffffffff);
2869 defer a.deinit();2863 defer a.deinit();
28702864
lib/std/mem.zig-4
...@@ -315,8 +315,6 @@ pub fn zeroes(comptime T: type) T {...@@ -315,8 +315,6 @@ pub fn zeroes(comptime T: type) T {
315}315}
316316
317test "zeroes" {317test "zeroes" {
318 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
319
320 const C_struct = extern struct {318 const C_struct = extern struct {
321 x: u32,319 x: u32,
322 y: u32 align(128),320 y: u32 align(128),
...@@ -4342,8 +4340,6 @@ pub fn alignInSlice(slice: anytype, comptime new_alignment: usize) ?AlignedSlice...@@ -4342,8 +4340,6 @@ pub fn alignInSlice(slice: anytype, comptime new_alignment: usize) ?AlignedSlice
4342}4340}
43434341
4344test "read/write(Var)PackedInt" {4342test "read/write(Var)PackedInt" {
4345 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
4346
4347 switch (builtin.cpu.arch) {4343 switch (builtin.cpu.arch) {
4348 // This test generates too much code to execute on WASI.4344 // This test generates too much code to execute on WASI.
4349 // LLVM backend fails with "too many locals: locals exceed maximum"4345 // LLVM backend fails with "too many locals: locals exceed maximum"
lib/std/net/test.zig+1-1
...@@ -60,7 +60,7 @@ test "parse and render IPv6 addresses" {...@@ -60,7 +60,7 @@ test "parse and render IPv6 addresses" {
60}60}
6161
62test "invalid but parseable IPv6 scope ids" {62test "invalid but parseable IPv6 scope ids" {
63 if (builtin.os.tag != .linux or comptime !builtin.os.tag.isDarwin()) {63 if (builtin.os.tag != .linux and comptime !builtin.os.tag.isDarwin()) {
64 // Currently, resolveIp6 with alphanumerical scope IDs only works on Linux.64 // Currently, resolveIp6 with alphanumerical scope IDs only works on Linux.
65 // TODO Make this test pass on other operating systems.65 // TODO Make this test pass on other operating systems.
66 return error.SkipZigTest;66 return error.SkipZigTest;
lib/std/once.zig-2
...@@ -46,8 +46,6 @@ fn incr() void {...@@ -46,8 +46,6 @@ fn incr() void {
46}46}
4747
48test "Once executes its function just once" {48test "Once executes its function just once" {
49 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
50
51 if (builtin.single_threaded) {49 if (builtin.single_threaded) {
52 global_once.call();50 global_once.call();
53 global_once.call();51 global_once.call();
lib/std/os/test.zig-4
...@@ -375,8 +375,6 @@ fn testThreadIdFn(thread_id: *Thread.Id) void {...@@ -375,8 +375,6 @@ fn testThreadIdFn(thread_id: *Thread.Id) void {
375test "std.Thread.getCurrentId" {375test "std.Thread.getCurrentId" {
376 if (builtin.single_threaded) return error.SkipZigTest;376 if (builtin.single_threaded) return error.SkipZigTest;
377377
378 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
379
380 var thread_current_id: Thread.Id = undefined;378 var thread_current_id: Thread.Id = undefined;
381 const thread = try Thread.spawn(.{}, testThreadIdFn, .{&thread_current_id});379 const thread = try Thread.spawn(.{}, testThreadIdFn, .{&thread_current_id});
382 thread.join();380 thread.join();
...@@ -420,8 +418,6 @@ test "cpu count" {...@@ -420,8 +418,6 @@ test "cpu count" {
420test "thread local storage" {418test "thread local storage" {
421 if (builtin.single_threaded) return error.SkipZigTest;419 if (builtin.single_threaded) return error.SkipZigTest;
422420
423 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
424
425 const thread1 = try Thread.spawn(.{}, testTls, .{});421 const thread1 = try Thread.spawn(.{}, testTls, .{});
426 const thread2 = try Thread.spawn(.{}, testTls, .{});422 const thread2 = try Thread.spawn(.{}, testTls, .{});
427 try testTls();423 try testTls();
lib/std/rand/test.zig-7
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const builtin = @import("builtin");
3const math = std.math;2const math = std.math;
4const DefaultPrng = std.rand.DefaultPrng;3const DefaultPrng = std.rand.DefaultPrng;
5const Random = std.rand.Random;4const Random = std.rand.Random;
...@@ -200,8 +199,6 @@ fn testRandomIntLessThan() !void {...@@ -200,8 +199,6 @@ fn testRandomIntLessThan() !void {
200}199}
201200
202test "Random intAtMost" {201test "Random intAtMost" {
203 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
204
205 @setEvalBranchQuota(10000);202 @setEvalBranchQuota(10000);
206 try testRandomIntAtMost();203 try testRandomIntAtMost();
207 try comptime testRandomIntAtMost();204 try comptime testRandomIntAtMost();
...@@ -242,8 +239,6 @@ fn testRandomIntAtMost() !void {...@@ -242,8 +239,6 @@ fn testRandomIntAtMost() !void {
242}239}
243240
244test "Random Biased" {241test "Random Biased" {
245 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
246
247 var prng = DefaultPrng.init(0);242 var prng = DefaultPrng.init(0);
248 const random = prng.random();243 const random = prng.random();
249 // Not thoroughly checking the logic here.244 // Not thoroughly checking the logic here.
...@@ -452,8 +447,6 @@ test "CSPRNG" {...@@ -452,8 +447,6 @@ test "CSPRNG" {
452}447}
453448
454test "Random weightedIndex" {449test "Random weightedIndex" {
455 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
456
457 // Make sure weightedIndex works for various integers and floats450 // Make sure weightedIndex works for various integers and floats
458 inline for (.{ u64, i4, f32, f64 }) |T| {451 inline for (.{ u64, i4, f32, f64 }) |T| {
459 var prng = DefaultPrng.init(0);452 var prng = DefaultPrng.init(0);
lib/std/zig/tokenizer.zig-9
...@@ -1,5 +1,4 @@...@@ -1,5 +1,4 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const builtin = @import("builtin");
32
4pub const Token = struct {3pub const Token = struct {
5 tag: Tag,4 tag: Tag,
...@@ -1450,8 +1449,6 @@ test "chars" {...@@ -1450,8 +1449,6 @@ test "chars" {
1450}1449}
14511450
1452test "invalid token characters" {1451test "invalid token characters" {
1453 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1454
1455 try testTokenize("#", &.{.invalid});1452 try testTokenize("#", &.{.invalid});
1456 try testTokenize("`", &.{.invalid});1453 try testTokenize("`", &.{.invalid});
1457 try testTokenize("'c", &.{.invalid});1454 try testTokenize("'c", &.{.invalid});
...@@ -1571,8 +1568,6 @@ test "pipe and then invalid" {...@@ -1571,8 +1568,6 @@ test "pipe and then invalid" {
1571}1568}
15721569
1573test "line comment and doc comment" {1570test "line comment and doc comment" {
1574 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1575
1576 try testTokenize("//", &.{});1571 try testTokenize("//", &.{});
1577 try testTokenize("// a / b", &.{});1572 try testTokenize("// a / b", &.{});
1578 try testTokenize("// /", &.{});1573 try testTokenize("// /", &.{});
...@@ -1647,8 +1642,6 @@ test "range literals" {...@@ -1647,8 +1642,6 @@ test "range literals" {
1647}1642}
16481643
1649test "number literals decimal" {1644test "number literals decimal" {
1650 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1651
1652 try testTokenize("0", &.{.number_literal});1645 try testTokenize("0", &.{.number_literal});
1653 try testTokenize("1", &.{.number_literal});1646 try testTokenize("1", &.{.number_literal});
1654 try testTokenize("2", &.{.number_literal});1647 try testTokenize("2", &.{.number_literal});
...@@ -1897,8 +1890,6 @@ test "invalid token with unfinished escape right before eof" {...@@ -1897,8 +1890,6 @@ test "invalid token with unfinished escape right before eof" {
1897}1890}
18981891
1899test "saturating operators" {1892test "saturating operators" {
1900 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
1901
1902 try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});1893 try testTokenize("<<", &.{.angle_bracket_angle_bracket_left});
1903 try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe});1894 try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe});
1904 try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});1895 try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal});
src/Compilation.zig+3-1
...@@ -1121,7 +1121,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {...@@ -1121,7 +1121,9 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1121 const include_compiler_rt = options.want_compiler_rt orelse needs_c_symbols;1121 const include_compiler_rt = options.want_compiler_rt orelse needs_c_symbols;
11221122
1123 const must_single_thread = target_util.isSingleThreaded(options.target);1123 const must_single_thread = target_util.isSingleThreaded(options.target);
1124 const single_threaded = options.single_threaded orelse must_single_thread;1124 const single_threaded = options.single_threaded orelse must_single_thread or
1125 // x86_64 codegen doesn't support TLV for most object formats
1126 (!use_llvm and options.target.cpu.arch == .x86_64 and options.target.ofmt != .macho);
1125 if (must_single_thread and !single_threaded) {1127 if (must_single_thread and !single_threaded) {
1126 return error.TargetRequiresSingleThreaded;1128 return error.TargetRequiresSingleThreaded;
1127 }1129 }
src/arch/x86_64/CodeGen.zig+1317-641
...@@ -388,7 +388,7 @@ pub const MCValue = union(enum) {...@@ -388,7 +388,7 @@ pub const MCValue = union(enum) {
388 };388 };
389 }389 }
390390
391 fn mem(mcv: MCValue, size: Memory.Size) Memory {391 fn mem(mcv: MCValue, function: *Self, size: Memory.Size) !Memory {
392 return switch (mcv) {392 return switch (mcv) {
393 .none,393 .none,
394 .unreach,394 .unreach,
...@@ -409,7 +409,6 @@ pub const MCValue = union(enum) {...@@ -409,7 +409,6 @@ pub const MCValue = union(enum) {
409 .lea_frame,409 .lea_frame,
410 .reserved_frame,410 .reserved_frame,
411 .air_ref,411 .air_ref,
412 .load_symbol,
413 .lea_symbol,412 .lea_symbol,
414 => unreachable,413 => unreachable,
415 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{414 .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{
...@@ -433,6 +432,19 @@ pub const MCValue = union(enum) {...@@ -433,6 +432,19 @@ pub const MCValue = union(enum) {
433 .disp = frame_addr.off,432 .disp = frame_addr.off,
434 } },433 } },
435 },434 },
435 .load_symbol => |sym_off| {
436 assert(sym_off.off == 0);
437 return .{
438 .base = .{ .reloc = .{
439 .atom_index = try function.owner.getSymbolIndex(function),
440 .sym_index = sym_off.sym,
441 } },
442 .mod = .{ .rm = .{
443 .size = size,
444 .disp = sym_off.off,
445 } },
446 };
447 },
436 };448 };
437 }449 }
438450
...@@ -722,12 +734,14 @@ const InstTracking = struct {...@@ -722,12 +734,14 @@ const InstTracking = struct {
722734
723const FrameAlloc = struct {735const FrameAlloc = struct {
724 abi_size: u31,736 abi_size: u31,
737 spill_pad: u3,
725 abi_align: Alignment,738 abi_align: Alignment,
726 ref_count: u16,739 ref_count: u16,
727740
728 fn init(alloc_abi: struct { size: u64, alignment: Alignment }) FrameAlloc {741 fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc {
729 return .{742 return .{
730 .abi_size = @intCast(alloc_abi.size),743 .abi_size = @intCast(alloc_abi.size),
744 .spill_pad = alloc_abi.pad,
731 .abi_align = alloc_abi.alignment,745 .abi_align = alloc_abi.alignment,
732 .ref_count = 0,746 .ref_count = 0,
733 };747 };
...@@ -738,6 +752,20 @@ const FrameAlloc = struct {...@@ -738,6 +752,20 @@ const FrameAlloc = struct {
738 .alignment = ty.abiAlignment(mod),752 .alignment = ty.abiAlignment(mod),
739 });753 });
740 }754 }
755 fn initSpill(ty: Type, mod: *Module) FrameAlloc {
756 const abi_size = ty.abiSize(mod);
757 const spill_size = if (abi_size < 8)
758 math.ceilPowerOfTwoAssert(u64, abi_size)
759 else
760 std.mem.alignForward(u64, abi_size, 8);
761 return init(.{
762 .size = spill_size,
763 .pad = @intCast(spill_size - abi_size),
764 .alignment = ty.abiAlignment(mod).maxStrict(
765 Alignment.fromNonzeroByteUnits(@min(spill_size, 8)),
766 ),
767 });
768 }
741};769};
742770
743const StackAllocation = struct {771const StackAllocation = struct {
...@@ -1668,8 +1696,7 @@ fn gen(self: *Self) InnerError!void {...@@ -1668,8 +1696,7 @@ fn gen(self: *Self) InnerError!void {
1668 // The address where to store the return value for the caller is in a1696 // The address where to store the return value for the caller is in a
1669 // register which the callee is free to clobber. Therefore, we purposely1697 // register which the callee is free to clobber. Therefore, we purposely
1670 // spill it to stack immediately.1698 // spill it to stack immediately.
1671 const frame_index =1699 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(Type.usize, mod));
1672 try self.allocFrameIndex(FrameAlloc.initType(Type.usize, mod));
1673 try self.genSetMem(1700 try self.genSetMem(
1674 .{ .frame = frame_index },1701 .{ .frame = frame_index },
1675 0,1702 0,
...@@ -2434,7 +2461,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b...@@ -2434,7 +2461,7 @@ fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: b
2434 }2461 }
2435 }2462 }
24362463
2437 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ty, mod));2464 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ty, mod));
2438 return .{ .load_frame = .{ .index = frame_index } };2465 return .{ .load_frame = .{ .index = frame_index } };
2439}2466}
24402467
...@@ -2445,7 +2472,10 @@ fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {...@@ -2445,7 +2472,10 @@ fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet {
2445 80 => abi.RegisterClass.x87,2472 80 => abi.RegisterClass.x87,
2446 else => abi.RegisterClass.sse,2473 else => abi.RegisterClass.sse,
2447 },2474 },
2448 .Vector => abi.RegisterClass.sse,2475 .Vector => switch (ty.childType(mod).toIntern()) {
2476 .bool_type => abi.RegisterClass.gp,
2477 else => abi.RegisterClass.sse,
2478 },
2449 else => abi.RegisterClass.gp,2479 else => abi.RegisterClass.gp,
2450 };2480 };
2451}2481}
...@@ -2699,7 +2729,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2699,7 +2729,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2699 .{ .v_ss, .cvtsd2 },2729 .{ .v_ss, .cvtsd2 },
2700 dst_reg,2730 dst_reg,
2701 dst_reg,2731 dst_reg,
2702 src_mcv.mem(.qword),2732 try src_mcv.mem(self, .qword),
2703 ) else try self.asmRegisterRegisterRegister(2733 ) else try self.asmRegisterRegisterRegister(
2704 .{ .v_ss, .cvtsd2 },2734 .{ .v_ss, .cvtsd2 },
2705 dst_reg,2735 dst_reg,
...@@ -2711,7 +2741,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2711,7 +2741,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void {
2711 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(2741 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2712 .{ ._ss, .cvtsd2 },2742 .{ ._ss, .cvtsd2 },
2713 dst_reg,2743 dst_reg,
2714 src_mcv.mem(.qword),2744 try src_mcv.mem(self, .qword),
2715 ) else try self.asmRegisterRegister(2745 ) else try self.asmRegisterRegister(
2716 .{ ._ss, .cvtsd2 },2746 .{ ._ss, .cvtsd2 },
2717 dst_reg,2747 dst_reg,
...@@ -2798,7 +2828,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -2798,7 +2828,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2798 .{ .v_sd, .cvtss2 },2828 .{ .v_sd, .cvtss2 },
2799 dst_reg,2829 dst_reg,
2800 dst_reg,2830 dst_reg,
2801 src_mcv.mem(.dword),2831 try src_mcv.mem(self, .dword),
2802 ) else try self.asmRegisterRegisterRegister(2832 ) else try self.asmRegisterRegisterRegister(
2803 .{ .v_sd, .cvtss2 },2833 .{ .v_sd, .cvtss2 },
2804 dst_reg,2834 dst_reg,
...@@ -2810,7 +2840,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -2810,7 +2840,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
2810 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(2840 ) else if (src_mcv.isMemory()) try self.asmRegisterMemory(
2811 .{ ._sd, .cvtss2 },2841 .{ ._sd, .cvtss2 },
2812 dst_reg,2842 dst_reg,
2813 src_mcv.mem(.dword),2843 try src_mcv.mem(self, .dword),
2814 ) else try self.asmRegisterRegister(2844 ) else try self.asmRegisterRegister(
2815 .{ ._sd, .cvtss2 },2845 .{ ._sd, .cvtss2 },
2816 dst_reg,2846 dst_reg,
...@@ -2851,8 +2881,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2851,8 +2881,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2851 };2881 };
28522882
2853 const dst_mcv = if (dst_int_info.bits <= src_storage_bits and2883 const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
2854 std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==2884 math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
2855 std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and2885 math.divCeil(u32, src_storage_bits, 64) catch unreachable and
2856 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {2886 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
2857 const dst_mcv = try self.allocRegOrMem(inst, true);2887 const dst_mcv = try self.allocRegOrMem(inst, true);
2858 try self.genCopy(min_ty, dst_mcv, src_mcv);2888 try self.genCopy(min_ty, dst_mcv, src_mcv);
...@@ -2869,22 +2899,28 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -2869,22 +2899,28 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2869 break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };2899 break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };
2870 }2900 }
28712901
2872 const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable;2902 const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
2873 const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;2903 const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
28742904
2875 const high_mcv = dst_mcv.address().offset((src_limbs_len - 1) * 8).deref();2905 const high_mcv: MCValue = if (dst_mcv.isMemory())
2876 const high_reg = try self.copyToTmpRegister(switch (src_int_info.signedness) {2906 dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
2877 .signed => Type.isize,2907 else
2878 .unsigned => Type.usize,2908 .{ .register = dst_mcv.register_pair[1] };
2879 }, high_mcv);2909 const high_reg = if (high_mcv.isRegister())
2910 high_mcv.getReg().?
2911 else
2912 try self.copyToTmpRegister(switch (src_int_info.signedness) {
2913 .signed => Type.isize,
2914 .unsigned => Type.usize,
2915 }, high_mcv);
2880 const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);2916 const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
2881 defer self.register_manager.unlockReg(high_lock);2917 defer self.register_manager.unlockReg(high_lock);
28822918
2883 const high_bits = src_int_info.bits % 64;2919 const high_bits = src_int_info.bits % 64;
2884 if (high_bits > 0) {2920 if (high_bits > 0) {
2885 const high_ty = try mod.intType(extend, high_bits);2921 try self.truncateRegister(src_ty, high_reg);
2886 try self.truncateRegister(high_ty, high_reg);2922 const high_ty = if (dst_int_info.bits >= 64) Type.usize else dst_ty;
2887 try self.genCopy(Type.usize, high_mcv, .{ .register = high_reg });2923 try self.genCopy(high_ty, high_mcv, .{ .register = high_reg });
2888 }2924 }
28892925
2890 if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(2926 if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(
...@@ -2995,14 +3031,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {...@@ -2995,14 +3031,14 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2995 .{ .vp_, .@"and" },3031 .{ .vp_, .@"and" },
2996 dst_reg,3032 dst_reg,
2997 dst_reg,3033 dst_reg,
2998 splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),3034 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
2999 );3035 );
3000 try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);3036 try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
3001 } else {3037 } else {
3002 try self.asmRegisterMemory(3038 try self.asmRegisterMemory(
3003 .{ .p_, .@"and" },3039 .{ .p_, .@"and" },
3004 dst_reg,3040 dst_reg,
3005 splat_addr_mcv.deref().mem(Memory.Size.fromSize(splat_abi_size)),3041 try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)),
3006 );3042 );
3007 try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);3043 try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
3008 }3044 }
...@@ -3048,7 +3084,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -3048,7 +3084,7 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void {
3048 const len = try self.resolveInst(bin_op.rhs);3084 const len = try self.resolveInst(bin_op.rhs);
3049 const len_ty = self.typeOf(bin_op.rhs);3085 const len_ty = self.typeOf(bin_op.rhs);
30503086
3051 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, mod));3087 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
3052 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);3088 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
3053 try self.genSetMem(3089 try self.genSetMem(
3054 .{ .frame = frame_index },3090 .{ .frame = frame_index },
...@@ -3068,8 +3104,36 @@ fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {...@@ -3068,8 +3104,36 @@ fn airUnOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3068}3104}
30693105
3070fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {3106fn airBinOp(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
3107 const mod = self.bin_file.options.module.?;
3071 const bin_op = self.air.instructions.items(.data)[inst].bin_op;3108 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
3072 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);3109 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
3110
3111 const dst_ty = self.typeOfIndex(inst);
3112 if (dst_ty.isAbiInt(mod)) {
3113 const abi_size: u32 = @intCast(dst_ty.abiSize(mod));
3114 const bit_size: u32 = @intCast(dst_ty.bitSize(mod));
3115 if (abi_size * 8 > bit_size) {
3116 const dst_lock = switch (dst_mcv) {
3117 .register => |dst_reg| self.register_manager.lockRegAssumeUnused(dst_reg),
3118 else => null,
3119 };
3120 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
3121
3122 if (dst_mcv.isRegister()) {
3123 try self.truncateRegister(dst_ty, dst_mcv.getReg().?);
3124 } else {
3125 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
3126 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3127 defer self.register_manager.unlockReg(tmp_lock);
3128
3129 const hi_ty = try mod.intType(.unsigned, @intCast((dst_ty.bitSize(mod) - 1) % 64 + 1));
3130 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
3131 try self.genSetReg(tmp_reg, hi_ty, hi_mcv);
3132 try self.truncateRegister(dst_ty, tmp_reg);
3133 try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg });
3134 }
3135 }
3136 }
3073 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3137 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
3074}3138}
30753139
...@@ -3176,7 +3240,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -3176,7 +3240,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3176 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(3240 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3177 .{ ._, .mov },3241 .{ ._, .mov },
3178 tmp_reg,3242 tmp_reg,
3179 mat_lhs_mcv.address().offset(8).deref().mem(.qword),3243 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3180 ) else try self.asmRegisterRegister(3244 ) else try self.asmRegisterRegister(
3181 .{ ._, .mov },3245 .{ ._, .mov },
3182 tmp_reg,3246 tmp_reg,
...@@ -3200,7 +3264,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -3200,7 +3264,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3200 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(3264 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3201 .{ ._, .xor },3265 .{ ._, .xor },
3202 tmp_reg,3266 tmp_reg,
3203 mat_rhs_mcv.address().offset(8).deref().mem(.qword),3267 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3204 ) else try self.asmRegisterRegister(3268 ) else try self.asmRegisterRegister(
3205 .{ ._, .xor },3269 .{ ._, .xor },
3206 tmp_reg,3270 tmp_reg,
...@@ -3300,12 +3364,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -3300,12 +3364,12 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
3300 try self.asmRegisterMemory(3364 try self.asmRegisterMemory(
3301 .{ ._, .add },3365 .{ ._, .add },
3302 tmp_regs[0],3366 tmp_regs[0],
3303 mat_rhs_mcv.mem(.qword),3367 try mat_rhs_mcv.mem(self, .qword),
3304 );3368 );
3305 try self.asmRegisterMemory(3369 try self.asmRegisterMemory(
3306 .{ ._, .adc },3370 .{ ._, .adc },
3307 tmp_regs[1],3371 tmp_regs[1],
3308 mat_rhs_mcv.address().offset(8).deref().mem(.qword),3372 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3309 );3373 );
3310 } else for (3374 } else for (
3311 [_]Mir.Inst.Tag{ .add, .adc },3375 [_]Mir.Inst.Tag{ .add, .adc },
...@@ -3534,7 +3598,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -3534,7 +3598,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3534 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(3598 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3535 .{ ._, .mov },3599 .{ ._, .mov },
3536 tmp_reg,3600 tmp_reg,
3537 mat_lhs_mcv.address().offset(8).deref().mem(.qword),3601 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3538 ) else try self.asmRegisterRegister(3602 ) else try self.asmRegisterRegister(
3539 .{ ._, .mov },3603 .{ ._, .mov },
3540 tmp_reg,3604 tmp_reg,
...@@ -3558,7 +3622,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -3558,7 +3622,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
3558 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(3622 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3559 .{ ._, .xor },3623 .{ ._, .xor },
3560 tmp_reg,3624 tmp_reg,
3561 mat_rhs_mcv.address().offset(8).deref().mem(.qword),3625 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3562 ) else try self.asmRegisterRegister(3626 ) else try self.asmRegisterRegister(
3563 .{ ._, .xor },3627 .{ ._, .xor },
3564 tmp_reg,3628 tmp_reg,
...@@ -3567,7 +3631,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -3567,7 +3631,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
35673631
3568 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));3632 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63));
3569 try self.asmRegister(.{ ._, .not }, tmp_reg);3633 try self.asmRegister(.{ ._, .not }, tmp_reg);
3570 try self.asmMemoryImmediate(.{ ._, .cmp }, overflow.mem(.dword), Immediate.s(0));3634 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0));
3571 try self.freeValue(overflow);3635 try self.freeValue(overflow);
3572 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);3636 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
3573 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));3637 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63));
...@@ -3665,7 +3729,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3665,7 +3729,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3665 }3729 }
36663730
3667 const frame_index =3731 const frame_index =
3668 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));3732 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3669 try self.genSetMem(3733 try self.genSetMem(
3670 .{ .frame = frame_index },3734 .{ .frame = frame_index },
3671 @intCast(tuple_ty.structFieldOffset(1, mod)),3735 @intCast(tuple_ty.structFieldOffset(1, mod)),
...@@ -3682,7 +3746,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3682,7 +3746,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3682 }3746 }
36833747
3684 const frame_index =3748 const frame_index =
3685 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));3749 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3686 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);3750 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
3687 break :result .{ .load_frame = .{ .index = frame_index } };3751 break :result .{ .load_frame = .{ .index = frame_index } };
3688 },3752 },
...@@ -3738,7 +3802,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3738,7 +3802,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3738 }3802 }
37393803
3740 const frame_index =3804 const frame_index =
3741 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));3805 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3742 try self.genSetMem(3806 try self.genSetMem(
3743 .{ .frame = frame_index },3807 .{ .frame = frame_index },
3744 @intCast(tuple_ty.structFieldOffset(1, mod)),3808 @intCast(tuple_ty.structFieldOffset(1, mod)),
...@@ -3755,7 +3819,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3755,7 +3819,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3755 }3819 }
37563820
3757 const frame_index =3821 const frame_index =
3758 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));3822 try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
3759 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);3823 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
3760 break :result .{ .load_frame = .{ .index = frame_index } };3824 break :result .{ .load_frame = .{ .index = frame_index } };
3761 },3825 },
...@@ -3874,7 +3938,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3874,7 +3938,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3874 );3938 );
3875 try self.asmMemoryImmediate(3939 try self.asmMemoryImmediate(
3876 .{ ._, .cmp },3940 .{ ._, .cmp },
3877 overflow.mem(self.memSize(Type.c_int)),3941 try overflow.mem(self, self.memSize(Type.c_int)),
3878 Immediate.s(0),3942 Immediate.s(0),
3879 );3943 );
3880 try self.genSetMem(3944 try self.genSetMem(
...@@ -3926,14 +3990,19 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3926,14 +3990,19 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3926 };3990 };
3927 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);3991 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
39283992
3929 if (mat_lhs_mcv.isMemory())3993 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3930 try self.asmRegisterMemory(.{ ._, .mov }, .rax, mat_lhs_mcv.mem(.qword))3994 .{ ._, .mov },
3931 else3995 .rax,
3932 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);3996 try mat_lhs_mcv.mem(self, .qword),
3997 ) else try self.asmRegisterRegister(
3998 .{ ._, .mov },
3999 .rax,
4000 mat_lhs_mcv.register_pair[0],
4001 );
3933 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(4002 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3934 .{ ._, .mov },4003 .{ ._, .mov },
3935 tmp_regs[0],4004 tmp_regs[0],
3936 mat_rhs_mcv.address().offset(8).deref().mem(.qword),4005 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
3937 ) else try self.asmRegisterRegister(4006 ) else try self.asmRegisterRegister(
3938 .{ ._, .mov },4007 .{ ._, .mov },
3939 tmp_regs[0],4008 tmp_regs[0],
...@@ -3944,7 +4013,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3944,7 +4013,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3944 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);4013 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
3945 try self.asmSetccRegister(.o, tmp_regs[2].to8());4014 try self.asmSetccRegister(.o, tmp_regs[2].to8());
3946 if (mat_rhs_mcv.isMemory())4015 if (mat_rhs_mcv.isMemory())
3947 try self.asmMemory(.{ ._, .mul }, mat_rhs_mcv.mem(.qword))4016 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
3948 else4017 else
3949 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);4018 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
3950 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);4019 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...@@ -3953,7 +4022,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3953,7 +4022,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3953 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(4022 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
3954 .{ ._, .mov },4023 .{ ._, .mov },
3955 tmp_regs[0],4024 tmp_regs[0],
3956 mat_lhs_mcv.address().offset(8).deref().mem(.qword),4025 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
3957 ) else try self.asmRegisterRegister(4026 ) else try self.asmRegisterRegister(
3958 .{ ._, .mov },4027 .{ ._, .mov },
3959 tmp_regs[0],4028 tmp_regs[0],
...@@ -3967,14 +4036,15 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3967,14 +4036,15 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3967 tmp_regs[3].to8(),4036 tmp_regs[3].to8(),
3968 );4037 );
3969 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());4038 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
3970 if (mat_rhs_mcv.isMemory())4039 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
3971 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_regs[0], mat_rhs_mcv.mem(.qword))4040 .{ .i_, .mul },
3972 else4041 tmp_regs[0],
3973 try self.asmRegisterRegister(4042 try mat_rhs_mcv.mem(self, .qword),
3974 .{ .i_, .mul },4043 ) else try self.asmRegisterRegister(
3975 tmp_regs[0],4044 .{ .i_, .mul },
3976 mat_rhs_mcv.register_pair[0],4045 tmp_regs[0],
3977 );4046 mat_rhs_mcv.register_pair[0],
4047 );
3978 try self.asmSetccRegister(.o, tmp_regs[2].to8());4048 try self.asmSetccRegister(.o, tmp_regs[2].to8());
3979 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());4049 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
3980 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);4050 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
...@@ -4020,8 +4090,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4020,8 +4090,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4020 self.eflags_inst = inst;4090 self.eflags_inst = inst;
4021 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };4091 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
4022 } else {4092 } else {
4023 const frame_index =4093 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
4024 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
4025 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);4094 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
4026 break :result .{ .load_frame = .{ .index = frame_index } };4095 break :result .{ .load_frame = .{ .index = frame_index } };
4027 },4096 },
...@@ -4032,8 +4101,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -4032,8 +4101,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
4032 src_ty.fmt(mod), dst_ty.fmt(mod),4101 src_ty.fmt(mod), dst_ty.fmt(mod),
4033 });4102 });
40344103
4035 const frame_index =4104 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(tuple_ty, mod));
4036 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, mod));
4037 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {4105 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
4038 try self.genSetMem(4106 try self.genSetMem(
4039 .{ .frame = frame_index },4107 .{ .frame = frame_index },
...@@ -4106,7 +4174,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue...@@ -4106,7 +4174,7 @@ fn genIntMulDivOpMir(self: *Self, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue
4106 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),4174 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
4107 .memory, .indirect, .load_frame => try self.asmMemory(4175 .memory, .indirect, .load_frame => try self.asmMemory(
4108 tag,4176 tag,
4109 mat_rhs.mem(Memory.Size.fromSize(abi_size)),4177 try mat_rhs.mem(self, Memory.Size.fromSize(abi_size)),
4110 ),4178 ),
4111 else => unreachable,4179 else => unreachable,
4112 }4180 }
...@@ -4160,8 +4228,8 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa...@@ -4160,8 +4228,8 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa
4160 );4228 );
4161 try self.asmCmovccRegisterRegister(4229 try self.asmCmovccRegisterRegister(
4162 .z,4230 .z,
4163 registerAlias(divisor, abi_size),4231 registerAlias(divisor, @max(abi_size, 2)),
4164 registerAlias(.rdx, abi_size),4232 registerAlias(.rdx, @max(abi_size, 2)),
4165 );4233 );
4166 try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });4234 try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
4167 return MCValue{ .register = divisor };4235 return MCValue{ .register = divisor };
...@@ -4171,47 +4239,268 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -4171,47 +4239,268 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void {
4171 const mod = self.bin_file.options.module.?;4239 const mod = self.bin_file.options.module.?;
4172 const bin_op = self.air.instructions.items(.data)[inst].bin_op;4240 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
41734241
4174 try self.spillRegisters(&.{.rcx});4242 const air_tags = self.air.instructions.items(.tag);
41754243 const tag = air_tags[inst];
4176 const tag = self.air.instructions.items(.tag)[inst];
4177 try self.register_manager.getReg(.rcx, null);
4178 const lhs = try self.resolveInst(bin_op.lhs);
4179 const rhs = try self.resolveInst(bin_op.rhs);
4180 const lhs_ty = self.typeOf(bin_op.lhs);4244 const lhs_ty = self.typeOf(bin_op.lhs);
4181 const rhs_ty = self.typeOf(bin_op.rhs);4245 const rhs_ty = self.typeOf(bin_op.rhs);
4246 const result: MCValue = result: {
4247 switch (lhs_ty.zigTypeTag(mod)) {
4248 .Int => {
4249 try self.spillRegisters(&.{.rcx});
4250 try self.register_manager.getReg(.rcx, null);
4251 const lhs_mcv = try self.resolveInst(bin_op.lhs);
4252 const rhs_mcv = try self.resolveInst(bin_op.rhs);
41824253
4183 const dst_mcv = try self.genShiftBinOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty);4254 const dst_mcv = try self.genShiftBinOp(tag, inst, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty);
4184 switch (tag) {4255 switch (tag) {
4185 .shr, .shr_exact, .shl_exact => {},4256 .shr, .shr_exact, .shl_exact => {},
4186 .shl => switch (dst_mcv) {4257 .shl => switch (dst_mcv) {
4187 .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg),4258 .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg),
4188 .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]),4259 .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]),
4189 .load_frame => |frame_addr| {4260 .load_frame => |frame_addr| {
4190 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);4261 const tmp_reg =
4191 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);4262 try self.register_manager.allocReg(null, abi.RegisterClass.gp);
4192 defer self.register_manager.unlockReg(tmp_lock);4263 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
4264 defer self.register_manager.unlockReg(tmp_lock);
41934265
4194 const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));4266 const lhs_bits: u31 = @intCast(lhs_ty.bitSize(mod));
4195 const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;4267 const tmp_ty = if (lhs_bits > 64) Type.usize else lhs_ty;
4196 const off = frame_addr.off + lhs_bits / 64 * 8;4268 const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;
4197 try self.genSetReg(4269 try self.genSetReg(
4198 tmp_reg,4270 tmp_reg,
4199 tmp_ty,4271 tmp_ty,
4200 .{ .load_frame = .{ .index = frame_addr.index, .off = off } },4272 .{ .load_frame = .{ .index = frame_addr.index, .off = off } },
4201 );4273 );
4202 try self.truncateRegister(lhs_ty, tmp_reg);4274 try self.truncateRegister(lhs_ty, tmp_reg);
4203 try self.genSetMem(4275 try self.genSetMem(
4204 .{ .frame = frame_addr.index },4276 .{ .frame = frame_addr.index },
4205 off,4277 off,
4206 tmp_ty,4278 tmp_ty,
4207 .{ .register = tmp_reg },4279 .{ .register = tmp_reg },
4208 );4280 );
4281 },
4282 else => {},
4283 },
4284 else => unreachable,
4285 }
4286 break :result dst_mcv;
4287 },
4288 .Vector => switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
4289 .Int => if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.childType(mod).intInfo(mod).bits) {
4290 else => null,
4291 16 => switch (lhs_ty.vectorLen(mod)) {
4292 else => null,
4293 1...8 => switch (tag) {
4294 else => unreachable,
4295 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4296 .signed => if (self.hasFeature(.avx))
4297 .{ .vp_w, .sra }
4298 else
4299 .{ .p_w, .sra },
4300 .unsigned => if (self.hasFeature(.avx))
4301 .{ .vp_w, .srl }
4302 else
4303 .{ .p_w, .srl },
4304 },
4305 .shl, .shl_exact => if (self.hasFeature(.avx))
4306 .{ .vp_w, .sll }
4307 else
4308 .{ .p_w, .sll },
4309 },
4310 9...16 => switch (tag) {
4311 else => unreachable,
4312 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4313 .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .sra } else null,
4314 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .srl } else null,
4315 },
4316 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_w, .sll } else null,
4317 },
4318 },
4319 32 => switch (lhs_ty.vectorLen(mod)) {
4320 else => null,
4321 1...4 => switch (tag) {
4322 else => unreachable,
4323 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4324 .signed => if (self.hasFeature(.avx))
4325 .{ .vp_d, .sra }
4326 else
4327 .{ .p_d, .sra },
4328 .unsigned => if (self.hasFeature(.avx))
4329 .{ .vp_d, .srl }
4330 else
4331 .{ .p_d, .srl },
4332 },
4333 .shl, .shl_exact => if (self.hasFeature(.avx))
4334 .{ .vp_d, .sll }
4335 else
4336 .{ .p_d, .sll },
4337 },
4338 5...8 => switch (tag) {
4339 else => unreachable,
4340 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4341 .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .sra } else null,
4342 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .srl } else null,
4343 },
4344 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_d, .sll } else null,
4345 },
4346 },
4347 64 => switch (lhs_ty.vectorLen(mod)) {
4348 else => null,
4349 1...2 => switch (tag) {
4350 else => unreachable,
4351 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4352 .signed => if (self.hasFeature(.avx))
4353 .{ .vp_q, .sra }
4354 else
4355 .{ .p_q, .sra },
4356 .unsigned => if (self.hasFeature(.avx))
4357 .{ .vp_q, .srl }
4358 else
4359 .{ .p_q, .srl },
4360 },
4361 .shl, .shl_exact => if (self.hasFeature(.avx))
4362 .{ .vp_q, .sll }
4363 else
4364 .{ .p_q, .sll },
4365 },
4366 3...4 => switch (tag) {
4367 else => unreachable,
4368 .shr, .shr_exact => switch (lhs_ty.childType(mod).intInfo(mod).signedness) {
4369 .signed => if (self.hasFeature(.avx2)) .{ .vp_q, .sra } else null,
4370 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_q, .srl } else null,
4371 },
4372 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_q, .sll } else null,
4373 },
4374 },
4375 })) |mir_tag| if (try self.air.value(bin_op.rhs, mod)) |rhs_val| {
4376 switch (mod.intern_pool.indexToKey(rhs_val.toIntern())) {
4377 .aggregate => |rhs_aggregate| switch (rhs_aggregate.storage) {
4378 .repeated_elem => |rhs_elem| {
4379 const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
4380
4381 const lhs_mcv = try self.resolveInst(bin_op.lhs);
4382 const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
4383 self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
4384 .{lhs_mcv.getReg().?} ** 2
4385 else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
4386 try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
4387 lhs_mcv.getReg().?,
4388 } else .{(try self.copyToRegisterWithInstTracking(
4389 inst,
4390 lhs_ty,
4391 lhs_mcv,
4392 )).register} ** 2;
4393 const reg_locks =
4394 self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
4395 defer for (reg_locks) |reg_lock| if (reg_lock) |lock|
4396 self.register_manager.unlockReg(lock);
4397
4398 const shift_imm =
4399 Immediate.u(@intCast(rhs_elem.toValue().toUnsignedInt(mod)));
4400 if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
4401 mir_tag,
4402 registerAlias(dst_reg, abi_size),
4403 registerAlias(lhs_reg, abi_size),
4404 shift_imm,
4405 ) else {
4406 assert(dst_reg.id() == lhs_reg.id());
4407 try self.asmRegisterImmediate(
4408 mir_tag,
4409 registerAlias(dst_reg, abi_size),
4410 shift_imm,
4411 );
4412 }
4413 break :result .{ .register = dst_reg };
4414 },
4415 else => {},
4416 },
4417 else => {},
4418 }
4419 } else if (Air.refToIndex(bin_op.rhs)) |rhs_inst| switch (air_tags[rhs_inst]) {
4420 .splat => {
4421 const abi_size: u32 = @intCast(lhs_ty.abiSize(mod));
4422
4423 const lhs_mcv = try self.resolveInst(bin_op.lhs);
4424 const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
4425 self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
4426 .{lhs_mcv.getReg().?} ** 2
4427 else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
4428 try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
4429 lhs_mcv.getReg().?,
4430 } else .{(try self.copyToRegisterWithInstTracking(
4431 inst,
4432 lhs_ty,
4433 lhs_mcv,
4434 )).register} ** 2;
4435 const reg_locks = self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
4436 defer for (reg_locks) |reg_lock| if (reg_lock) |lock|
4437 self.register_manager.unlockReg(lock);
4438
4439 const shift_reg =
4440 try self.copyToTmpRegister(rhs_ty, .{ .air_ref = bin_op.rhs });
4441 const shift_lock = self.register_manager.lockRegAssumeUnused(shift_reg);
4442 defer self.register_manager.unlockReg(shift_lock);
4443
4444 const mask_ty = try mod.vectorType(.{ .len = 16, .child = .u8_type });
4445 const mask_mcv = try self.genTypedValue(.{
4446 .ty = mask_ty,
4447 .val = (try mod.intern(.{ .aggregate = .{
4448 .ty = mask_ty.toIntern(),
4449 .storage = .{ .elems = &([1]InternPool.Index{
4450 (try rhs_ty.childType(mod).maxIntScalar(mod, Type.u8)).toIntern(),
4451 } ++ [1]InternPool.Index{
4452 (try mod.intValue(Type.u8, 0)).toIntern(),
4453 } ** 15) },
4454 } })).toValue(),
4455 });
4456 const mask_addr_reg =
4457 try self.copyToTmpRegister(Type.usize, mask_mcv.address());
4458 const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);
4459 defer self.register_manager.unlockReg(mask_addr_lock);
4460
4461 if (self.hasFeature(.avx)) {
4462 try self.asmRegisterRegisterMemory(
4463 .{ .vp_, .@"and" },
4464 shift_reg.to128(),
4465 shift_reg.to128(),
4466 .{
4467 .base = .{ .reg = mask_addr_reg },
4468 .mod = .{ .rm = .{ .size = .xword } },
4469 },
4470 );
4471 try self.asmRegisterRegisterRegister(
4472 mir_tag,
4473 registerAlias(dst_reg, abi_size),
4474 registerAlias(lhs_reg, abi_size),
4475 shift_reg.to128(),
4476 );
4477 } else {
4478 try self.asmRegisterMemory(
4479 .{ .p_, .@"and" },
4480 shift_reg.to128(),
4481 .{
4482 .base = .{ .reg = mask_addr_reg },
4483 .mod = .{ .rm = .{ .size = .xword } },
4484 },
4485 );
4486 assert(dst_reg.id() == lhs_reg.id());
4487 try self.asmRegisterRegister(
4488 mir_tag,
4489 registerAlias(dst_reg, abi_size),
4490 shift_reg.to128(),
4491 );
4492 }
4493 break :result .{ .register = dst_reg };
4494 },
4495 else => {},
4496 },
4497 else => {},
4209 },4498 },
4210 else => {},4499 else => {},
4211 },4500 }
4212 else => unreachable,4501 return self.fail("TODO implement airShlShrBinOp for {}", .{lhs_ty.fmt(mod)});
4213 }4502 };
4214 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });4503 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4215}4504}
42164505
4217fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {4506fn airShlSat(self: *Self, inst: Air.Inst.Index) !void {
...@@ -4230,12 +4519,18 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -4230,12 +4519,18 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void {
42304519
4231 const opt_mcv = try self.resolveInst(ty_op.operand);4520 const opt_mcv = try self.resolveInst(ty_op.operand);
4232 if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {4521 if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
4233 switch (opt_mcv) {4522 const pl_mcv: MCValue = switch (opt_mcv) {
4234 .register => |reg| try self.truncateRegister(pl_ty, reg),4523 .register_overflow => |ro| pl: {
4235 .register_overflow => |ro| try self.truncateRegister(pl_ty, ro.reg),4524 self.eflags_inst = null; // actually stop tracking the overflow part
4525 break :pl .{ .register = ro.reg };
4526 },
4527 else => opt_mcv,
4528 };
4529 switch (pl_mcv) {
4530 .register => |pl_reg| try self.truncateRegister(pl_ty, pl_reg),
4236 else => {},4531 else => {},
4237 }4532 }
4238 break :result opt_mcv;4533 break :result pl_mcv;
4239 }4534 }
42404535
4241 const pl_mcv = try self.allocRegOrMem(inst, true);4536 const pl_mcv = try self.allocRegOrMem(inst, true);
...@@ -4472,8 +4767,9 @@ fn genUnwrapErrUnionPayloadMir(...@@ -4472,8 +4767,9 @@ fn genUnwrapErrUnionPayloadMir(
4472 const eu_lock = self.register_manager.lockReg(reg);4767 const eu_lock = self.register_manager.lockReg(reg);
4473 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);4768 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
44744769
4475 const result_mcv: MCValue = if (maybe_inst) |inst|4770 const payload_in_gp = self.regClassForType(payload_ty).supersetOf(abi.RegisterClass.gp);
4476 try self.copyToRegisterWithInstTracking(inst, err_union_ty, err_union)4771 const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
4772 try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
4477 else4773 else
4478 .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };4774 .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
4479 if (payload_off > 0) try self.genShiftBinOpMir(4775 if (payload_off > 0) try self.genShiftBinOpMir(
...@@ -4482,7 +4778,12 @@ fn genUnwrapErrUnionPayloadMir(...@@ -4482,7 +4778,12 @@ fn genUnwrapErrUnionPayloadMir(
4482 result_mcv,4778 result_mcv,
4483 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },4779 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },
4484 ) else try self.truncateRegister(payload_ty, result_mcv.register);4780 ) else try self.truncateRegister(payload_ty, result_mcv.register);
4485 break :result result_mcv;4781 break :result if (payload_in_gp)
4782 result_mcv
4783 else if (maybe_inst) |inst|
4784 try self.copyToRegisterWithInstTracking(inst, payload_ty, result_mcv)
4785 else
4786 .{ .register = try self.copyToTmpRegister(payload_ty, result_mcv) };
4486 },4787 },
4487 else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),4788 else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {}", .{err_union}),
4488 }4789 }
...@@ -4593,7 +4894,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {...@@ -4593,7 +4894,7 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
4593 const result: MCValue = result: {4894 const result: MCValue = result: {
4594 if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .{ .immediate = 0 };4895 if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .{ .immediate = 0 };
45954896
4596 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));4897 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
4597 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));4898 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
4598 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));4899 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
4599 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);4900 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand);
...@@ -4615,7 +4916,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {...@@ -4615,7 +4916,7 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
4615 const result: MCValue = result: {4916 const result: MCValue = result: {
4616 if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);4917 if (!pl_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result try self.resolveInst(ty_op.operand);
46174918
4618 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(eu_ty, mod));4919 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, mod));
4619 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));4920 const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, mod));
4620 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));4921 const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, mod));
4621 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);4922 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef);
...@@ -4770,14 +5071,19 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {...@@ -4770,14 +5071,19 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
4770fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {5071fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void {
4771 const mod = self.bin_file.options.module.?;5072 const mod = self.bin_file.options.module.?;
4772 const bin_op = self.air.instructions.items(.data)[inst].bin_op;5073 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
4773 const slice_ty = self.typeOf(bin_op.lhs);
47745074
4775 const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);5075 const result: MCValue = result: {
4776 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);5076 const elem_ty = self.typeOfIndex(inst);
4777 const dst_mcv = try self.allocRegOrMem(inst, false);5077 if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod)) break :result .none;
4778 try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
47795078
4780 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });5079 const slice_ty = self.typeOf(bin_op.lhs);
5080 const slice_ptr_field_type = slice_ty.slicePtrFieldType(mod);
5081 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
5082 const dst_mcv = try self.allocRegOrMem(inst, false);
5083 try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
5084 break :result dst_mcv;
5085 };
5086 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
4781}5087}
47825088
4783fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {5089fn airSliceElemPtr(self: *Self, inst: Air.Inst.Index) !void {
...@@ -4810,11 +5116,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4810,11 +5116,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4810 };5116 };
4811 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);5117 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
48125118
4813 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
4814 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
4815 defer self.register_manager.unlockReg(offset_reg_lock);
4816
4817 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);5119 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5120 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
5121 defer self.register_manager.unlockReg(addr_lock);
5122
4818 switch (array) {5123 switch (array) {
4819 .register => {5124 .register => {
4820 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod));5125 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod));
...@@ -4843,6 +5148,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -4843,6 +5148,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void {
4843 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),5148 else => return self.fail("TODO implement array_elem_val when array is {}", .{array}),
4844 }5149 }
48455150
5151 const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size);
5152 const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
5153 defer self.register_manager.unlockReg(offset_lock);
5154
4846 // TODO we could allocate register here, but need to expect addr register and potentially5155 // TODO we could allocate register here, but need to expect addr register and potentially
4847 // offset register.5156 // offset register.
4848 try self.spillEflagsIfOccupied();5157 try self.spillEflagsIfOccupied();
...@@ -5093,7 +5402,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5093,7 +5402,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5093 .{ ._, .sub },5402 .{ ._, .sub },
5094 dst_ty,5403 dst_ty,
5095 dst_mcv,5404 dst_mcv,
5096 .{ .immediate = 8 + self.regExtraBits(src_ty) },5405 .{ .immediate = 32 - src_bits },
5097 );5406 );
5098 } else if (src_bits <= 64) {5407 } else if (src_bits <= 64) {
5099 try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);5408 try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
...@@ -5361,7 +5670,9 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {...@@ -5361,7 +5670,9 @@ fn airPopCount(self: *Self, inst: Air.Inst.Index) !void {
5361 mat_src_mcv5670 mat_src_mcv
5362 else5671 else
5363 .{ .register = mat_src_mcv.register_pair[0] }, false);5672 .{ .register = mat_src_mcv.register_pair[0] }, false);
5364 try self.genPopCount(tmp_regs[1], Type.usize, if (mat_src_mcv.isMemory())5673 const src_info = src_ty.intInfo(mod);
5674 const hi_ty = try mod.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1);
5675 try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isMemory())
5365 mat_src_mcv.address().offset(8).deref()5676 mat_src_mcv.address().offset(8).deref()
5366 else5677 else
5367 .{ .register = mat_src_mcv.register_pair[1] }, false);5678 .{ .register = mat_src_mcv.register_pair[1] }, false);
...@@ -5383,9 +5694,13 @@ fn genPopCount(...@@ -5383,9 +5694,13 @@ fn genPopCount(
5383 const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));5694 const src_abi_size: u32 = @intCast(src_ty.abiSize(mod));
5384 if (self.hasFeature(.popcnt)) return self.genBinOpMir(5695 if (self.hasFeature(.popcnt)) return self.genBinOpMir(
5385 .{ ._, .popcnt },5696 .{ ._, .popcnt },
5386 if (src_abi_size > 1) src_ty else Type.u16,5697 if (src_abi_size > 1) src_ty else Type.u32,
5387 .{ .register = dst_reg },5698 .{ .register = dst_reg },
5388 src_mcv,5699 if (src_abi_size > 1) src_mcv else src: {
5700 if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv);
5701 try self.truncateRegister(try src_ty.toUnsigned(mod), dst_reg);
5702 break :src .{ .register = dst_reg };
5703 },
5389 );5704 );
53905705
5391 const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);5706 const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
...@@ -5517,9 +5832,9 @@ fn genByteSwap(...@@ -5517,9 +5832,9 @@ fn genByteSwap(
5517 try self.asmRegisterMemory(5832 try self.asmRegisterMemory(
5518 .{ ._, .movbe },5833 .{ ._, .movbe },
5519 dst_regs[0],5834 dst_regs[0],
5520 src_mcv.address().offset(8).deref().mem(.qword),5835 try src_mcv.address().offset(8).deref().mem(self, .qword),
5521 );5836 );
5522 try self.asmRegisterMemory(.{ ._, .movbe }, dst_regs[1], src_mcv.mem(.qword));5837 try self.asmRegisterMemory(.{ ._, .movbe }, dst_regs[1], try src_mcv.mem(self, .qword));
5523 } else for (dst_regs, src_mcv.register_pair) |dst_reg, src_reg| {5838 } else for (dst_regs, src_mcv.register_pair) |dst_reg, src_reg| {
5524 try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to64(), src_reg.to64());5839 try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to64(), src_reg.to64());
5525 try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());5840 try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
...@@ -5762,7 +6077,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)...@@ -5762,7 +6077,7 @@ fn floatSign(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, ty: Type)
5762 else => unreachable,6077 else => unreachable,
5763 } });6078 } });
5764 const sign_mem: Memory = if (sign_mcv.isMemory())6079 const sign_mem: Memory = if (sign_mcv.isMemory())
5765 sign_mcv.mem(Memory.Size.fromSize(abi_size))6080 try sign_mcv.mem(self, Memory.Size.fromSize(abi_size))
5766 else6081 else
5767 .{6082 .{
5768 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },6083 .base = .{ .reg = try self.copyToTmpRegister(Type.usize, sign_mcv.address()) },
...@@ -5945,7 +6260,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -5945,7 +6260,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5945 mir_tag,6260 mir_tag,
5946 dst_alias,6261 dst_alias,
5947 dst_alias,6262 dst_alias,
5948 src_mcv.mem(Memory.Size.fromSize(abi_size)),6263 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
5949 Immediate.u(@as(u5, @bitCast(mode))),6264 Immediate.u(@as(u5, @bitCast(mode))),
5950 ) else try self.asmRegisterRegisterRegisterImmediate(6265 ) else try self.asmRegisterRegisterRegisterImmediate(
5951 mir_tag,6266 mir_tag,
...@@ -5960,7 +6275,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro...@@ -5960,7 +6275,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro
5960 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(6275 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
5961 mir_tag,6276 mir_tag,
5962 dst_alias,6277 dst_alias,
5963 src_mcv.mem(Memory.Size.fromSize(abi_size)),6278 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
5964 Immediate.u(@as(u5, @bitCast(mode))),6279 Immediate.u(@as(u5, @bitCast(mode))),
5965 ) else try self.asmRegisterRegisterImmediate(6280 ) else try self.asmRegisterRegisterImmediate(
5966 mir_tag,6281 mir_tag,
...@@ -6000,7 +6315,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -6000,7 +6315,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6000 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(6315 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
6001 .l,6316 .l,
6002 registerAlias(dst_mcv.register, cmov_abi_size),6317 registerAlias(dst_mcv.register, cmov_abi_size),
6003 src_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),6318 try src_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
6004 ),6319 ),
6005 else => {6320 else => {
6006 const val_reg = try self.copyToTmpRegister(ty, src_mcv);6321 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
...@@ -6100,7 +6415,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {...@@ -6100,7 +6415,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void {
6100 if (src_mcv.isMemory()) try self.asmRegisterMemory(6415 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6101 mir_tag,6416 mir_tag,
6102 dst_alias,6417 dst_alias,
6103 src_mcv.mem(self.memSize(ty)),6418 try src_mcv.mem(self, self.memSize(ty)),
6104 ) else try self.asmRegisterRegister(6419 ) else try self.asmRegisterRegister(
6105 mir_tag,6420 mir_tag,
6106 dst_alias,6421 dst_alias,
...@@ -6206,7 +6521,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -6206,7 +6521,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6206 if (src_mcv.isMemory()) try self.asmRegisterMemory(6521 if (src_mcv.isMemory()) try self.asmRegisterMemory(
6207 .{ .v_ps, .cvtph2 },6522 .{ .v_ps, .cvtph2 },
6208 wide_reg,6523 wide_reg,
6209 src_mcv.mem(Memory.Size.fromSize(6524 try src_mcv.mem(self, Memory.Size.fromSize(
6210 @intCast(@divExact(wide_reg.bitSize(), 16)),6525 @intCast(@divExact(wide_reg.bitSize(), 16)),
6211 )),6526 )),
6212 ) else try self.asmRegisterRegister(6527 ) else try self.asmRegisterRegister(
...@@ -6254,7 +6569,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -6254,7 +6569,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6254 mir_tag,6569 mir_tag,
6255 dst_reg,6570 dst_reg,
6256 dst_reg,6571 dst_reg,
6257 src_mcv.mem(Memory.Size.fromSize(abi_size)),6572 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
6258 ) else try self.asmRegisterRegisterRegister(6573 ) else try self.asmRegisterRegisterRegister(
6259 mir_tag,6574 mir_tag,
6260 dst_reg,6575 dst_reg,
...@@ -6267,7 +6582,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {...@@ -6267,7 +6582,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
6267 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(6582 else => if (src_mcv.isMemory()) try self.asmRegisterMemory(
6268 mir_tag,6583 mir_tag,
6269 dst_reg,6584 dst_reg,
6270 src_mcv.mem(Memory.Size.fromSize(abi_size)),6585 try src_mcv.mem(self, Memory.Size.fromSize(abi_size)),
6271 ) else try self.asmRegisterRegister(6586 ) else try self.asmRegisterRegister(
6272 mir_tag,6587 mir_tag,
6273 dst_reg,6588 dst_reg,
...@@ -6332,7 +6647,7 @@ fn reuseOperandAdvanced(...@@ -6332,7 +6647,7 @@ fn reuseOperandAdvanced(
6332 return false;6647 return false;
63336648
6334 switch (mcv) {6649 switch (mcv) {
6335 .register, .register_pair => for (mcv.getRegs()) |reg| {6650 .register, .register_pair, .register_overflow => for (mcv.getRegs()) |reg| {
6336 // If it's in the registers table, need to associate the register(s) with the6651 // If it's in the registers table, need to associate the register(s) with the
6337 // new instruction.6652 // new instruction.
6338 if (maybe_tracked_inst) |tracked_inst| {6653 if (maybe_tracked_inst) |tracked_inst| {
...@@ -6346,6 +6661,10 @@ fn reuseOperandAdvanced(...@@ -6346,6 +6661,10 @@ fn reuseOperandAdvanced(
6346 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,6661 .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false,
6347 else => return false,6662 else => return false,
6348 }6663 }
6664 switch (mcv) {
6665 .eflags, .register_overflow => self.eflags_inst = maybe_tracked_inst,
6666 else => {},
6667 }
63496668
6350 // Prevent the operand deaths processing code from deallocating it.6669 // Prevent the operand deaths processing code from deallocating it.
6351 self.liveness.clearOperandDeath(inst, op_index);6670 self.liveness.clearOperandDeath(inst, op_index);
...@@ -6363,11 +6682,36 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn...@@ -6363,11 +6682,36 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
6363 if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return;6682 if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return;
6364 const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));6683 const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
63656684
6685 if (ptr_info.packed_offset.bit_offset % 8 == 0) {
6686 try self.load(
6687 dst_mcv,
6688 ptr_ty,
6689 ptr_mcv.offset(@intCast(@divExact(ptr_info.packed_offset.bit_offset, 8))),
6690 );
6691 const val_bit_size: u32 = @intCast(val_ty.bitSize(mod));
6692 if (val_abi_size * 8 > val_bit_size) {
6693 if (dst_mcv.isRegister()) {
6694 try self.truncateRegister(val_ty, dst_mcv.getReg().?);
6695 } else {
6696 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6697 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6698 defer self.register_manager.unlockReg(tmp_lock);
6699
6700 const hi_mcv = dst_mcv.address().offset(@intCast(val_bit_size / 64 * 8)).deref();
6701 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
6702 try self.truncateRegister(val_ty, tmp_reg);
6703 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
6704 }
6705 }
6706 return;
6707 }
6708
6366 if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{val_ty.fmt(mod)});6709 if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{val_ty.fmt(mod)});
63676710
6368 const limb_abi_size: u32 = @min(val_abi_size, 8);6711 const limb_abi_size: u32 = @min(val_abi_size, 8);
6369 const limb_abi_bits = limb_abi_size * 8;6712 const limb_abi_bits = limb_abi_size * 8;
6370 const val_byte_off: i32 = @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);6713 const val_byte_off: i32 =
6714 @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size);
6371 const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;6715 const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits;
6372 const val_extra_bits = self.regExtraBits(val_ty);6716 const val_extra_bits = self.regExtraBits(val_ty);
63736717
...@@ -6530,7 +6874,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -6530,7 +6874,7 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6530 .base = .{ .reg = ptr_reg },6874 .base = .{ .reg = ptr_reg },
6531 .mod = .{ .rm = .{6875 .mod = .{ .rm = .{
6532 .size = Memory.Size.fromSize(limb_abi_size),6876 .size = Memory.Size.fromSize(limb_abi_size),
6533 .disp = src_byte_off + limb_i * limb_abi_bits,6877 .disp = src_byte_off + limb_i * limb_abi_size,
6534 } },6878 } },
6535 };6879 };
65366880
...@@ -6575,6 +6919,22 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In...@@ -6575,6 +6919,22 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In
6575 limb_mem,6919 limb_mem,
6576 registerAlias(tmp_reg, limb_abi_size),6920 registerAlias(tmp_reg, limb_abi_size),
6577 );6921 );
6922 } else if (src_bit_size <= 128 and src_bit_off == 0) {
6923 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6924 const tmp_mcv = MCValue{ .register = tmp_reg };
6925 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6926 defer self.register_manager.unlockReg(tmp_lock);
6927
6928 try self.genSetReg(tmp_reg, limb_ty, switch (limb_i) {
6929 0 => src_mcv,
6930 else => src_mcv.address().offset(limb_i * limb_abi_size).deref(),
6931 });
6932 try self.genBinOpMir(.{ ._, .@"and" }, limb_ty, tmp_mcv, .{ .immediate = part_mask });
6933 try self.asmMemoryRegister(
6934 .{ ._, .@"or" },
6935 limb_mem,
6936 registerAlias(tmp_reg, limb_abi_size),
6937 );
6578 } else return self.fail("TODO: implement packed store of {}", .{src_ty.fmt(mod)});6938 } else return self.fail("TODO: implement packed store of {}", .{src_ty.fmt(mod)});
6579 }6939 }
6580}6940}
...@@ -6808,17 +7168,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6808,17 +7168,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6808 .register_overflow => |ro| {7168 .register_overflow => |ro| {
6809 switch (index) {7169 switch (index) {
6810 // Get wrapped value for overflow operation.7170 // Get wrapped value for overflow operation.
6811 0 => break :result if (self.liveness.operandDies(inst, 0))7171 0 => if (self.reuseOperand(inst, extra.struct_operand, 0, src_mcv)) {
6812 .{ .register = ro.reg }7172 self.eflags_inst = null; // actually stop tracking the overflow part
6813 else7173 break :result .{ .register = ro.reg };
6814 try self.copyToRegisterWithInstTracking(7174 } else break :result try self.copyToRegisterWithInstTracking(
6815 inst,7175 inst,
6816 Type.usize,7176 Type.usize,
6817 .{ .register = ro.reg },7177 .{ .register = ro.reg },
6818 ),7178 ),
6819 // Get overflow bit.7179 // Get overflow bit.
6820 1 => if (self.liveness.operandDies(inst, 0)) {7180 1 => if (self.reuseOperandAdvanced(inst, extra.struct_operand, 0, src_mcv, null)) {
6821 self.eflags_inst = inst;7181 self.eflags_inst = inst; // actually keep tracking the overflow part
6822 break :result .{ .eflags = ro.eflags };7182 break :result .{ .eflags = ro.eflags };
6823 } else {7183 } else {
6824 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);7184 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
...@@ -6833,11 +7193,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6833,11 +7193,12 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6833 if (field_off % 8 == 0) {7193 if (field_off % 8 == 0) {
6834 const off_mcv =7194 const off_mcv =
6835 src_mcv.address().offset(@intCast(@divExact(field_off, 8))).deref();7195 src_mcv.address().offset(@intCast(@divExact(field_off, 8))).deref();
7196 const field_bit_size = field_ty.bitSize(mod);
68367197
6837 if (field_abi_size <= 8) {7198 if (field_abi_size <= 8) {
6838 const int_ty = try mod.intType(7199 const int_ty = try mod.intType(
6839 if (field_ty.isAbiInt(mod)) field_ty.intInfo(mod).signedness else .unsigned,7200 if (field_ty.isAbiInt(mod)) field_ty.intInfo(mod).signedness else .unsigned,
6840 @intCast(field_ty.bitSize(mod)),7201 @intCast(field_bit_size),
6841 );7202 );
68427203
6843 const dst_reg = try self.register_manager.allocReg(7204 const dst_reg = try self.register_manager.allocReg(
...@@ -6856,10 +7217,24 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -6856,10 +7217,24 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
6856 try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);7217 try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv);
6857 }7218 }
68587219
6859 if (self.reuseOperand(inst, operand, 0, src_mcv)) break :result off_mcv;7220 const dst_mcv = if (self.reuseOperand(inst, operand, 0, src_mcv))
68607221 off_mcv
6861 const dst_mcv = try self.allocRegOrMem(inst, true);7222 else dst: {
6862 try self.genCopy(field_ty, dst_mcv, off_mcv);7223 const dst_mcv = try self.allocRegOrMem(inst, true);
7224 try self.genCopy(field_ty, dst_mcv, off_mcv);
7225 break :dst dst_mcv;
7226 };
7227 if (field_abi_size * 8 > field_bit_size and dst_mcv.isMemory()) {
7228 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
7229 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7230 defer self.register_manager.unlockReg(tmp_lock);
7231
7232 const hi_mcv =
7233 dst_mcv.address().offset(@intCast(field_bit_size / 64 * 8)).deref();
7234 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
7235 try self.truncateRegister(field_ty, tmp_reg);
7236 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
7237 }
6863 break :result dst_mcv;7238 break :result dst_mcv;
6864 }7239 }
68657240
...@@ -7013,7 +7388,25 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:...@@ -7013,7 +7388,25 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air:
7013 } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);7388 } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
7014 }7389 }
7015 },7390 },
7016 .neg => try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv),7391 .neg => {
7392 try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv);
7393 const abi_size: u16 = @intCast(src_ty.abiSize(mod));
7394 const bit_size = src_ty.intInfo(mod).bits;
7395 if (abi_size * 8 > bit_size) {
7396 if (dst_mcv.isRegister()) {
7397 try self.truncateRegister(src_ty, dst_mcv.getReg().?);
7398 } else {
7399 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
7400 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7401 defer self.register_manager.unlockReg(tmp_lock);
7402
7403 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
7404 try self.genSetReg(tmp_reg, Type.usize, hi_mcv);
7405 try self.truncateRegister(src_ty, tmp_reg);
7406 try self.genCopy(Type.usize, hi_mcv, .{ .register = tmp_reg });
7407 }
7408 }
7409 },
7017 else => unreachable,7410 else => unreachable,
7018 }7411 }
7019 return dst_mcv;7412 return dst_mcv;
...@@ -7054,7 +7447,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC...@@ -7054,7 +7447,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC
7054 },7447 },
7055 .indirect, .load_frame => try self.asmMemory(7448 .indirect, .load_frame => try self.asmMemory(
7056 mir_tag,7449 mir_tag,
7057 dst_mcv.mem(Memory.Size.fromSize(abi_size)),7450 try dst_mcv.mem(self, Memory.Size.fromSize(abi_size)),
7058 ),7451 ),
7059 }7452 }
7060}7453}
...@@ -7552,27 +7945,27 @@ fn genMulDivBinOp(...@@ -7552,27 +7945,27 @@ fn genMulDivBinOp(
7552 defer self.register_manager.unlockReg(tmp_lock);7945 defer self.register_manager.unlockReg(tmp_lock);
75537946
7554 if (mat_lhs_mcv.isMemory())7947 if (mat_lhs_mcv.isMemory())
7555 try self.asmRegisterMemory(.{ ._, .mov }, .rax, mat_lhs_mcv.mem(.qword))7948 try self.asmRegisterMemory(.{ ._, .mov }, .rax, try mat_lhs_mcv.mem(self, .qword))
7556 else7949 else
7557 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);7950 try self.asmRegisterRegister(.{ ._, .mov }, .rax, mat_lhs_mcv.register_pair[0]);
7558 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(7951 if (mat_rhs_mcv.isMemory()) try self.asmRegisterMemory(
7559 .{ ._, .mov },7952 .{ ._, .mov },
7560 tmp_reg,7953 tmp_reg,
7561 mat_rhs_mcv.address().offset(8).deref().mem(.qword),7954 try mat_rhs_mcv.address().offset(8).deref().mem(self, .qword),
7562 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);7955 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_rhs_mcv.register_pair[1]);
7563 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);7956 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, .rax);
7564 if (mat_rhs_mcv.isMemory())7957 if (mat_rhs_mcv.isMemory())
7565 try self.asmMemory(.{ ._, .mul }, mat_rhs_mcv.mem(.qword))7958 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .qword))
7566 else7959 else
7567 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);7960 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
7568 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);7961 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
7569 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(7962 if (mat_lhs_mcv.isMemory()) try self.asmRegisterMemory(
7570 .{ ._, .mov },7963 .{ ._, .mov },
7571 tmp_reg,7964 tmp_reg,
7572 mat_lhs_mcv.address().offset(8).deref().mem(.qword),7965 try mat_lhs_mcv.address().offset(8).deref().mem(self, .qword),
7573 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);7966 ) else try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, mat_lhs_mcv.register_pair[1]);
7574 if (mat_rhs_mcv.isMemory())7967 if (mat_rhs_mcv.isMemory())
7575 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.mem(.qword))7968 try self.asmRegisterMemory(.{ .i_, .mul }, tmp_reg, try mat_rhs_mcv.mem(self, .qword))
7576 else7969 else
7577 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);7970 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_reg, mat_rhs_mcv.register_pair[0]);
7578 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);7971 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_reg);
...@@ -7833,7 +8226,7 @@ fn genBinOp(...@@ -7833,7 +8226,7 @@ fn genBinOp(
7833 .{ .vp_w, .insr },8226 .{ .vp_w, .insr },
7834 dst_reg,8227 dst_reg,
7835 dst_reg,8228 dst_reg,
7836 rhs_mcv.mem(.word),8229 try rhs_mcv.mem(self, .word),
7837 Immediate.u(1),8230 Immediate.u(1),
7838 ) else try self.asmRegisterRegisterRegister(8231 ) else try self.asmRegisterRegisterRegister(
7839 .{ .vp_, .unpcklwd },8232 .{ .vp_, .unpcklwd },
...@@ -7858,7 +8251,7 @@ fn genBinOp(...@@ -7858,7 +8251,7 @@ fn genBinOp(
7858 mir_tag,8251 mir_tag,
7859 dst_reg,8252 dst_reg,
7860 dst_reg,8253 dst_reg,
7861 src_mcv.mem(Memory.Size.fromBitSize(float_bits)),8254 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
7862 ) else try self.asmRegisterRegisterRegister(8255 ) else try self.asmRegisterRegisterRegister(
7863 mir_tag,8256 mir_tag,
7864 dst_reg,8257 dst_reg,
...@@ -7877,7 +8270,7 @@ fn genBinOp(...@@ -7877,7 +8270,7 @@ fn genBinOp(
7877 if (src_mcv.isMemory()) try self.asmRegisterMemory(8270 if (src_mcv.isMemory()) try self.asmRegisterMemory(
7878 mir_tag,8271 mir_tag,
7879 dst_reg,8272 dst_reg,
7880 src_mcv.mem(Memory.Size.fromBitSize(float_bits)),8273 try src_mcv.mem(self, Memory.Size.fromBitSize(float_bits)),
7881 ) else try self.asmRegisterRegister(8274 ) else try self.asmRegisterRegister(
7882 mir_tag,8275 mir_tag,
7883 dst_reg,8276 dst_reg,
...@@ -7919,12 +8312,18 @@ fn genBinOp(...@@ -7919,12 +8312,18 @@ fn genBinOp(
7919 };8312 };
7920 }8313 }
79218314
7922 if ((lhs_ty.scalarType(mod).isRuntimeFloat() and8315 const sse_op = switch (lhs_ty.zigTypeTag(mod)) {
8316 else => false,
8317 .Float => true,
8318 .Vector => switch (lhs_ty.childType(mod).toIntern()) {
8319 .bool_type => false,
8320 else => true,
8321 },
8322 };
8323 if (sse_op and ((lhs_ty.scalarType(mod).isRuntimeFloat() and
7923 lhs_ty.scalarType(mod).floatBits(self.target.*) == 80) or8324 lhs_ty.scalarType(mod).floatBits(self.target.*) == 80) or
7924 lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16))8325 lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16)))
7925 return self.fail("TODO implement genBinOp for {s} {}", .{8326 return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(air_tag), lhs_ty.fmt(mod) });
7926 @tagName(air_tag), lhs_ty.fmt(mod),
7927 });
79288327
7929 const maybe_mask_reg = switch (air_tag) {8328 const maybe_mask_reg = switch (air_tag) {
7930 else => null,8329 else => null,
...@@ -7941,10 +8340,16 @@ fn genBinOp(...@@ -7941,10 +8340,16 @@ fn genBinOp(
7941 if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null;8340 if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null;
7942 defer if (mask_lock) |lock| self.register_manager.unlockReg(lock);8341 defer if (mask_lock) |lock| self.register_manager.unlockReg(lock);
79438342
7944 const ordered_air = if (lhs_ty.isVector(mod) and lhs_ty.childType(mod).isAbiInt(mod) and8343 const ordered_air = if (lhs_ty.isVector(mod) and switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
7945 switch (air_tag) {8344 .Int => switch (air_tag) {
7946 .cmp_lt, .cmp_gte => true,8345 .cmp_lt, .cmp_gte => true,
7947 else => false,8346 else => false,
8347 },
8348 .Float => switch (air_tag) {
8349 .cmp_gte, .cmp_gt => true,
8350 else => false,
8351 },
8352 else => unreachable,
7948 }) .{ .lhs = rhs_air, .rhs = lhs_air } else .{ .lhs = lhs_air, .rhs = rhs_air };8353 }) .{ .lhs = rhs_air, .rhs = lhs_air } else .{ .lhs = lhs_air, .rhs = rhs_air };
79498354
7950 const lhs_mcv = try self.resolveInst(ordered_air.lhs);8355 const lhs_mcv = try self.resolveInst(ordered_air.lhs);
...@@ -7971,14 +8376,12 @@ fn genBinOp(...@@ -7971,14 +8376,12 @@ fn genBinOp(
7971 .xor,8376 .xor,
7972 .min,8377 .min,
7973 .max,8378 .max,
8379 .cmp_eq,
8380 .cmp_neq,
7974 => true,8381 => true,
79758382
7976 else => false,8383 else => false,
7977 };8384 };
7978 const vec_op = switch (lhs_ty.zigTypeTag(mod)) {
7979 else => false,
7980 .Float, .Vector => true,
7981 };
79828385
7983 const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {8386 const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
7984 .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },8387 .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
...@@ -8000,23 +8403,23 @@ fn genBinOp(...@@ -8000,23 +8403,23 @@ fn genBinOp(
8000 var flipped = false;8403 var flipped = false;
8001 var copied_to_dst = true;8404 var copied_to_dst = true;
8002 const dst_mcv: MCValue = dst: {8405 const dst_mcv: MCValue = dst: {
8406 const tracked_inst = switch (air_tag) {
8407 else => maybe_inst,
8408 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
8409 };
8003 if (maybe_inst) |inst| {8410 if (maybe_inst) |inst| {
8004 const tracked_inst = switch (air_tag) {8411 if ((!sse_op or lhs_mcv.isRegister()) and
8005 else => inst,
8006 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
8007 };
8008 if ((!vec_op or lhs_mcv.isRegister()) and
8009 self.reuseOperandAdvanced(inst, ordered_air.lhs, 0, lhs_mcv, tracked_inst))8412 self.reuseOperandAdvanced(inst, ordered_air.lhs, 0, lhs_mcv, tracked_inst))
8010 break :dst lhs_mcv;8413 break :dst lhs_mcv;
8011 if (is_commutative and (!vec_op or rhs_mcv.isRegister()) and8414 if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and
8012 self.reuseOperandAdvanced(inst, ordered_air.rhs, 1, rhs_mcv, tracked_inst))8415 self.reuseOperandAdvanced(inst, ordered_air.rhs, 1, rhs_mcv, tracked_inst))
8013 {8416 {
8014 flipped = true;8417 flipped = true;
8015 break :dst rhs_mcv;8418 break :dst rhs_mcv;
8016 }8419 }
8017 }8420 }
8018 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, maybe_inst, true);8421 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, tracked_inst, true);
8019 if (vec_op and lhs_mcv.isRegister() and self.hasFeature(.avx))8422 if (sse_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
8020 copied_to_dst = false8423 copied_to_dst = false
8021 else8424 else
8022 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);8425 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv);
...@@ -8046,7 +8449,7 @@ fn genBinOp(...@@ -8046,7 +8449,7 @@ fn genBinOp(
8046 };8449 };
8047 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);8450 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);
80488451
8049 if (!vec_op) {8452 if (!sse_op) {
8050 switch (air_tag) {8453 switch (air_tag) {
8051 .add,8454 .add,
8052 .add_wrap,8455 .add_wrap,
...@@ -8130,17 +8533,25 @@ fn genBinOp(...@@ -8130,17 +8533,25 @@ fn genBinOp(
81308533
8131 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);8534 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
8132 if (src_mcv.isMemory()) {8535 if (src_mcv.isMemory()) {
8133 try self.asmRegisterMemory(.{ ._, .cmp }, dst_regs[0], src_mcv.mem(.qword));8536 try self.asmRegisterMemory(
8537 .{ ._, .cmp },
8538 dst_regs[0],
8539 try src_mcv.mem(self, .qword),
8540 );
8134 try self.asmRegisterMemory(8541 try self.asmRegisterMemory(
8135 .{ ._, .sbb },8542 .{ ._, .sbb },
8136 tmp_reg,8543 tmp_reg,
8137 src_mcv.address().offset(8).deref().mem(.qword),8544 try src_mcv.address().offset(8).deref().mem(self, .qword),
8545 );
8546 try self.asmCmovccRegisterMemory(
8547 cc,
8548 dst_regs[0],
8549 try src_mcv.mem(self, .qword),
8138 );8550 );
8139 try self.asmCmovccRegisterMemory(cc, dst_regs[0], src_mcv.mem(.qword));
8140 try self.asmCmovccRegisterMemory(8551 try self.asmCmovccRegisterMemory(
8141 cc,8552 cc,
8142 dst_regs[1],8553 dst_regs[1],
8143 src_mcv.address().offset(8).deref().mem(.qword),8554 try src_mcv.address().offset(8).deref().mem(self, .qword),
8144 );8555 );
8145 } else {8556 } else {
8146 try self.asmRegisterRegister(8557 try self.asmRegisterRegister(
...@@ -8292,7 +8703,7 @@ fn genBinOp(...@@ -8292,7 +8703,7 @@ fn genBinOp(
8292 .{ .vp_w, .insr },8703 .{ .vp_w, .insr },
8293 dst_reg,8704 dst_reg,
8294 dst_reg,8705 dst_reg,
8295 src_mcv.mem(.word),8706 try src_mcv.mem(self, .word),
8296 Immediate.u(1),8707 Immediate.u(1),
8297 ) else try self.asmRegisterRegisterRegister(8708 ) else try self.asmRegisterRegisterRegister(
8298 .{ .vp_, .unpcklwd },8709 .{ .vp_, .unpcklwd },
...@@ -8738,7 +9149,7 @@ fn genBinOp(...@@ -8738,7 +9149,7 @@ fn genBinOp(
8738 .{ .vp_w, .insr },9149 .{ .vp_w, .insr },
8739 dst_reg,9150 dst_reg,
8740 dst_reg,9151 dst_reg,
8741 src_mcv.mem(.word),9152 try src_mcv.mem(self, .word),
8742 Immediate.u(1),9153 Immediate.u(1),
8743 ) else try self.asmRegisterRegisterRegister(9154 ) else try self.asmRegisterRegisterRegister(
8744 .{ .vp_, .unpcklwd },9155 .{ .vp_, .unpcklwd },
...@@ -8784,7 +9195,7 @@ fn genBinOp(...@@ -8784,7 +9195,7 @@ fn genBinOp(
8784 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(9195 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
8785 .{ .vp_d, .insr },9196 .{ .vp_d, .insr },
8786 dst_reg,9197 dst_reg,
8787 src_mcv.mem(.dword),9198 try src_mcv.mem(self, .dword),
8788 Immediate.u(1),9199 Immediate.u(1),
8789 ) else try self.asmRegisterRegisterRegister(9200 ) else try self.asmRegisterRegisterRegister(
8790 .{ .v_ps, .unpckl },9201 .{ .v_ps, .unpckl },
...@@ -8836,7 +9247,7 @@ fn genBinOp(...@@ -8836,7 +9247,7 @@ fn genBinOp(
8836 if (src_mcv.isMemory()) try self.asmRegisterMemory(9247 if (src_mcv.isMemory()) try self.asmRegisterMemory(
8837 .{ .v_ps, .cvtph2 },9248 .{ .v_ps, .cvtph2 },
8838 tmp_reg,9249 tmp_reg,
8839 src_mcv.mem(.qword),9250 try src_mcv.mem(self, .qword),
8840 ) else try self.asmRegisterRegister(9251 ) else try self.asmRegisterRegister(
8841 .{ .v_ps, .cvtph2 },9252 .{ .v_ps, .cvtph2 },
8842 tmp_reg,9253 tmp_reg,
...@@ -8879,7 +9290,7 @@ fn genBinOp(...@@ -8879,7 +9290,7 @@ fn genBinOp(
8879 if (src_mcv.isMemory()) try self.asmRegisterMemory(9290 if (src_mcv.isMemory()) try self.asmRegisterMemory(
8880 .{ .v_ps, .cvtph2 },9291 .{ .v_ps, .cvtph2 },
8881 tmp_reg,9292 tmp_reg,
8882 src_mcv.mem(.xword),9293 try src_mcv.mem(self, .xword),
8883 ) else try self.asmRegisterRegister(9294 ) else try self.asmRegisterRegister(
8884 .{ .v_ps, .cvtph2 },9295 .{ .v_ps, .cvtph2 },
8885 tmp_reg,9296 tmp_reg,
...@@ -8925,6 +9336,13 @@ fn genBinOp(...@@ -8925,6 +9336,13 @@ fn genBinOp(
8925 => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },9336 => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
8926 .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },9337 .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
8927 .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },9338 .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
9339 .cmp_lt,
9340 .cmp_lte,
9341 .cmp_eq,
9342 .cmp_gte,
9343 .cmp_gt,
9344 .cmp_neq,
9345 => if (self.hasFeature(.avx)) .{ .v_ss, .cmp } else .{ ._ss, .cmp },
8928 else => unreachable,9346 else => unreachable,
8929 },9347 },
8930 2...4 => switch (air_tag) {9348 2...4 => switch (air_tag) {
...@@ -8938,7 +9356,14 @@ fn genBinOp(...@@ -8938,7 +9356,14 @@ fn genBinOp(
8938 => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },9356 => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
8939 .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },9357 .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
8940 .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },9358 .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
8941 else => unreachable,9359 .cmp_lt,
9360 .cmp_lte,
9361 .cmp_eq,
9362 .cmp_gte,
9363 .cmp_gt,
9364 .cmp_neq,
9365 => if (self.hasFeature(.avx)) .{ .v_ps, .cmp } else .{ ._ps, .cmp },
9366 else => unreachable,
8942 },9367 },
8943 5...8 => if (self.hasFeature(.avx)) switch (air_tag) {9368 5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
8944 .add => .{ .v_ps, .add },9369 .add => .{ .v_ps, .add },
...@@ -8947,6 +9372,7 @@ fn genBinOp(...@@ -8947,6 +9372,7 @@ fn genBinOp(
8947 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },9372 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
8948 .max => .{ .v_ps, .max },9373 .max => .{ .v_ps, .max },
8949 .min => .{ .v_ps, .min },9374 .min => .{ .v_ps, .min },
9375 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_ps, .cmp },
8950 else => unreachable,9376 else => unreachable,
8951 } else null,9377 } else null,
8952 else => null,9378 else => null,
...@@ -8963,6 +9389,13 @@ fn genBinOp(...@@ -8963,6 +9389,13 @@ fn genBinOp(
8963 => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },9389 => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
8964 .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },9390 .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
8965 .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },9391 .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
9392 .cmp_lt,
9393 .cmp_lte,
9394 .cmp_eq,
9395 .cmp_gte,
9396 .cmp_gt,
9397 .cmp_neq,
9398 => if (self.hasFeature(.avx)) .{ .v_sd, .cmp } else .{ ._sd, .cmp },
8966 else => unreachable,9399 else => unreachable,
8967 },9400 },
8968 2 => switch (air_tag) {9401 2 => switch (air_tag) {
...@@ -8976,6 +9409,13 @@ fn genBinOp(...@@ -8976,6 +9409,13 @@ fn genBinOp(
8976 => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },9409 => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
8977 .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },9410 .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
8978 .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },9411 .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
9412 .cmp_lt,
9413 .cmp_lte,
9414 .cmp_eq,
9415 .cmp_gte,
9416 .cmp_gt,
9417 .cmp_neq,
9418 => if (self.hasFeature(.avx)) .{ .v_pd, .cmp } else .{ ._pd, .cmp },
8979 else => unreachable,9419 else => unreachable,
8980 },9420 },
8981 3...4 => if (self.hasFeature(.avx)) switch (air_tag) {9421 3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
...@@ -8984,6 +9424,7 @@ fn genBinOp(...@@ -8984,6 +9424,7 @@ fn genBinOp(
8984 .mul => .{ .v_pd, .mul },9424 .mul => .{ .v_pd, .mul },
8985 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },9425 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
8986 .max => .{ .v_pd, .max },9426 .max => .{ .v_pd, .max },
9427 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_pd, .cmp },
8987 .min => .{ .v_pd, .min },9428 .min => .{ .v_pd, .min },
8988 else => unreachable,9429 else => unreachable,
8989 } else null,9430 } else null,
...@@ -9004,43 +9445,96 @@ fn genBinOp(...@@ -9004,43 +9445,96 @@ fn genBinOp(
9004 const lhs_copy_lock = if (lhs_copy_reg) |reg| self.register_manager.lockReg(reg) else null;9445 const lhs_copy_lock = if (lhs_copy_reg) |reg| self.register_manager.lockReg(reg) else null;
9005 defer if (lhs_copy_lock) |lock| self.register_manager.unlockReg(lock);9446 defer if (lhs_copy_lock) |lock| self.register_manager.unlockReg(lock);
90069447
9007 if (self.hasFeature(.avx)) {9448 switch (mir_tag[1]) {
9008 const lhs_reg =9449 else => if (self.hasFeature(.avx)) {
9009 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);9450 const lhs_reg =
9010 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(9451 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
9011 mir_tag,9452 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
9012 dst_reg,9453 mir_tag,
9013 lhs_reg,9454 dst_reg,
9014 src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {9455 lhs_reg,
9015 else => Memory.Size.fromSize(abi_size),9456 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
9016 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),9457 else => Memory.Size.fromSize(abi_size),
9017 }),9458 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9018 ) else try self.asmRegisterRegisterRegister(9459 }),
9019 mir_tag,9460 ) else try self.asmRegisterRegisterRegister(
9020 dst_reg,9461 mir_tag,
9021 lhs_reg,9462 dst_reg,
9022 registerAlias(if (src_mcv.isRegister())9463 lhs_reg,
9023 src_mcv.getReg().?9464 registerAlias(if (src_mcv.isRegister())
9024 else9465 src_mcv.getReg().?
9025 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),9466 else
9026 );9467 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9027 } else {9468 );
9028 assert(copied_to_dst);9469 } else {
9029 if (src_mcv.isMemory()) try self.asmRegisterMemory(9470 assert(copied_to_dst);
9030 mir_tag,9471 if (src_mcv.isMemory()) try self.asmRegisterMemory(
9031 dst_reg,9472 mir_tag,
9032 src_mcv.mem(switch (lhs_ty.zigTypeTag(mod)) {9473 dst_reg,
9033 else => Memory.Size.fromSize(abi_size),9474 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
9034 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),9475 else => Memory.Size.fromSize(abi_size),
9035 }),9476 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9036 ) else try self.asmRegisterRegister(9477 }),
9037 mir_tag,9478 ) else try self.asmRegisterRegister(
9038 dst_reg,9479 mir_tag,
9039 registerAlias(if (src_mcv.isRegister())9480 dst_reg,
9040 src_mcv.getReg().?9481 registerAlias(if (src_mcv.isRegister())
9041 else9482 src_mcv.getReg().?
9042 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),9483 else
9043 );9484 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9485 );
9486 },
9487 .cmp => {
9488 const imm = Immediate.u(switch (air_tag) {
9489 .cmp_eq => 0,
9490 .cmp_lt, .cmp_gt => 1,
9491 .cmp_lte, .cmp_gte => 2,
9492 .cmp_neq => 4,
9493 else => unreachable,
9494 });
9495 if (self.hasFeature(.avx)) {
9496 const lhs_reg =
9497 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
9498 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
9499 mir_tag,
9500 dst_reg,
9501 lhs_reg,
9502 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
9503 else => Memory.Size.fromSize(abi_size),
9504 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9505 }),
9506 imm,
9507 ) else try self.asmRegisterRegisterRegisterImmediate(
9508 mir_tag,
9509 dst_reg,
9510 lhs_reg,
9511 registerAlias(if (src_mcv.isRegister())
9512 src_mcv.getReg().?
9513 else
9514 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9515 imm,
9516 );
9517 } else {
9518 assert(copied_to_dst);
9519 if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
9520 mir_tag,
9521 dst_reg,
9522 try src_mcv.mem(self, switch (lhs_ty.zigTypeTag(mod)) {
9523 else => Memory.Size.fromSize(abi_size),
9524 .Vector => Memory.Size.fromBitSize(dst_reg.bitSize()),
9525 }),
9526 imm,
9527 ) else try self.asmRegisterRegisterImmediate(
9528 mir_tag,
9529 dst_reg,
9530 registerAlias(if (src_mcv.isRegister())
9531 src_mcv.getReg().?
9532 else
9533 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
9534 imm,
9535 );
9536 }
9537 },
9044 }9538 }
90459539
9046 switch (air_tag) {9540 switch (air_tag) {
...@@ -9281,48 +9775,46 @@ fn genBinOp(...@@ -9281,48 +9775,46 @@ fn genBinOp(
9281 );9775 );
9282 }9776 }
9283 },9777 },
9284 .cmp_lt,9778 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
9285 .cmp_lte,9779 switch (lhs_ty.childType(mod).zigTypeTag(mod)) {
9286 .cmp_eq,9780 .Int => switch (air_tag) {
9287 .cmp_gte,9781 .cmp_lt,
9288 .cmp_gt,9782 .cmp_eq,
9289 .cmp_neq,9783 .cmp_gt,
9290 => {9784 => {},
9291 switch (air_tag) {9785 .cmp_lte,
9292 .cmp_lt,9786 .cmp_gte,
9293 .cmp_eq,9787 .cmp_neq,
9294 .cmp_gt,9788 => {
9295 => {},9789 const unsigned_ty = try lhs_ty.toUnsigned(mod);
9296 .cmp_lte,9790 const not_mcv = try self.genTypedValue(.{
9297 .cmp_gte,9791 .ty = lhs_ty,
9298 .cmp_neq,9792 .val = try unsigned_ty.maxInt(mod, unsigned_ty),
9299 => {9793 });
9300 const unsigned_ty = try lhs_ty.toUnsigned(mod);9794 const not_mem: Memory = if (not_mcv.isMemory())
9301 const not_mcv = try self.genTypedValue(.{9795 try not_mcv.mem(self, Memory.Size.fromSize(abi_size))
9302 .ty = lhs_ty,9796 else
9303 .val = try unsigned_ty.maxInt(mod, unsigned_ty),9797 .{ .base = .{
9304 });9798 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),
9305 const not_mem: Memory = if (not_mcv.isMemory())9799 }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };
9306 not_mcv.mem(Memory.Size.fromSize(abi_size))9800 switch (mir_tag[0]) {
9307 else9801 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
9308 .{ .base = .{9802 .{ .vp_, .xor },
9309 .reg = try self.copyToTmpRegister(Type.usize, not_mcv.address()),9803 dst_reg,
9310 }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size) } } };9804 dst_reg,
9311 switch (mir_tag[0]) {9805 not_mem,
9312 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(9806 ),
9313 .{ .vp_, .xor },9807 .p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(
9314 dst_reg,9808 .{ .p_, .xor },
9315 dst_reg,9809 dst_reg,
9316 not_mem,9810 not_mem,
9317 ),9811 ),
9318 .p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(9812 else => unreachable,
9319 .{ .p_, .xor },9813 }
9320 dst_reg,9814 },
9321 not_mem,9815 else => unreachable,
9322 ),
9323 else => unreachable,
9324 }
9325 },9816 },
9817 .Float => {},
9326 else => unreachable,9818 else => unreachable,
9327 }9819 }
93289820
...@@ -9331,8 +9823,12 @@ fn genBinOp(...@@ -9331,8 +9823,12 @@ fn genBinOp(
9331 defer self.register_manager.unlockReg(gp_lock);9823 defer self.register_manager.unlockReg(gp_lock);
93329824
9333 try self.asmRegisterRegister(switch (mir_tag[0]) {9825 try self.asmRegisterRegister(switch (mir_tag[0]) {
9334 .vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk },9826 ._pd, ._sd => .{ ._pd, .movmsk },
9827 ._ps, ._ss => .{ ._ps, .movmsk },
9335 .p_b, .p_d, .p_q, .p_w => .{ .p_b, .movmsk },9828 .p_b, .p_d, .p_q, .p_w => .{ .p_b, .movmsk },
9829 .v_pd, .v_sd => .{ .v_pd, .movmsk },
9830 .v_ps, .v_ss => .{ .v_ps, .movmsk },
9831 .vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk },
9336 else => unreachable,9832 else => unreachable,
9337 }, gp_reg.to32(), dst_reg);9833 }, gp_reg.to32(), dst_reg);
9338 return .{ .register = gp_reg };9834 return .{ .register = gp_reg };
...@@ -9459,13 +9955,13 @@ fn genBinOpMir(...@@ -9459,13 +9955,13 @@ fn genBinOpMir(
9459 .load_frame,9955 .load_frame,
9460 .lea_frame,9956 .lea_frame,
9461 => {9957 => {
9462 blk: {9958 direct: {
9463 return self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {9959 try self.asmRegisterMemory(mir_limb_tag, dst_alias, switch (src_mcv) {
9464 .memory => |addr| .{9960 .memory => |addr| .{
9465 .base = .{ .reg = .ds },9961 .base = .{ .reg = .ds },
9466 .mod = .{ .rm = .{9962 .mod = .{ .rm = .{
9467 .size = Memory.Size.fromSize(limb_abi_size),9963 .size = Memory.Size.fromSize(limb_abi_size),
9468 .disp = math.cast(i32, addr + off) orelse break :blk,9964 .disp = math.cast(i32, addr + off) orelse break :direct,
9469 } },9965 } },
9470 },9966 },
9471 .indirect => |reg_off| .{9967 .indirect => |reg_off| .{
...@@ -9482,8 +9978,9 @@ fn genBinOpMir(...@@ -9482,8 +9978,9 @@ fn genBinOpMir(
9482 .disp = frame_addr.off + off,9978 .disp = frame_addr.off + off,
9483 } },9979 } },
9484 },9980 },
9485 else => break :blk,9981 else => break :direct,
9486 });9982 });
9983 continue;
9487 }9984 }
94889985
9489 switch (src_mcv) {9986 switch (src_mcv) {
...@@ -10180,7 +10677,7 @@ fn genCall(self: *Self, info: union(enum) {...@@ -10180,7 +10677,7 @@ fn genCall(self: *Self, info: union(enum) {
10180 .none, .unreach => {},10677 .none, .unreach => {},
10181 .indirect => |reg_off| {10678 .indirect => |reg_off| {
10182 const ret_ty = fn_info.return_type.toType();10679 const ret_ty = fn_info.return_type.toType();
10183 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(ret_ty, mod));10680 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(ret_ty, mod));
10184 try self.genSetReg(reg_off.reg, Type.usize, .{10681 try self.genSetReg(reg_off.reg, Type.usize, .{
10185 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },10682 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },
10186 });10683 });
...@@ -10306,19 +10803,20 @@ fn genCall(self: *Self, info: union(enum) {...@@ -10306,19 +10803,20 @@ fn genCall(self: *Self, info: union(enum) {
10306fn airRet(self: *Self, inst: Air.Inst.Index) !void {10803fn airRet(self: *Self, inst: Air.Inst.Index) !void {
10307 const mod = self.bin_file.options.module.?;10804 const mod = self.bin_file.options.module.?;
10308 const un_op = self.air.instructions.items(.data)[inst].un_op;10805 const un_op = self.air.instructions.items(.data)[inst].un_op;
10309 const operand = try self.resolveInst(un_op);
1031010806
10311 const ret_ty = self.fn_type.fnReturnType(mod);10807 const ret_ty = self.fn_type.fnReturnType(mod);
10312 switch (self.ret_mcv.short) {10808 switch (self.ret_mcv.short) {
10313 .none => {},10809 .none => {},
10314 .register, .register_pair => try self.genCopy(ret_ty, self.ret_mcv.short, operand),10810 .register,
10811 .register_pair,
10812 => try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }),
10315 .indirect => |reg_off| {10813 .indirect => |reg_off| {
10316 try self.register_manager.getReg(reg_off.reg, null);10814 try self.register_manager.getReg(reg_off.reg, null);
10317 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);10815 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);
10318 defer self.register_manager.unlockReg(lock);10816 defer self.register_manager.unlockReg(lock);
1031910817
10320 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);10818 try self.genSetReg(reg_off.reg, Type.usize, self.ret_mcv.long);
10321 try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, operand);10819 try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ret_ty, .{ .air_ref = un_op });
10322 },10820 },
10323 else => unreachable,10821 else => unreachable,
10324 }10822 }
...@@ -10593,7 +11091,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -10593,7 +11091,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
10593 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);11091 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
10594 defer for (locks) |lock| self.register_manager.unlockReg(lock);11092 defer for (locks) |lock| self.register_manager.unlockReg(lock);
1059511093
10596 const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;11094 const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable;
10597 var limb_i: u16 = 0;11095 var limb_i: u16 = 0;
10598 while (limb_i < limbs_len) : (limb_i += 1) {11096 while (limb_i < limbs_len) : (limb_i += 1) {
10599 const off = limb_i * 8;11097 const off = limb_i * 8;
...@@ -10688,7 +11186,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -10688,7 +11186,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
10688 .{ .vp_w, .insr },11186 .{ .vp_w, .insr },
10689 tmp1_reg,11187 tmp1_reg,
10690 dst_reg.to128(),11188 dst_reg.to128(),
10691 src_mcv.mem(.word),11189 try src_mcv.mem(self, .word),
10692 Immediate.u(1),11190 Immediate.u(1),
10693 ) else try self.asmRegisterRegisterRegister(11191 ) else try self.asmRegisterRegisterRegister(
10694 .{ .vp_, .unpcklwd },11192 .{ .vp_, .unpcklwd },
...@@ -10892,8 +11390,8 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {...@@ -10892,8 +11390,8 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index {
10892 },11390 },
10893 .register => |reg| {11391 .register => |reg| {
10894 try self.spillEflagsIfOccupied();11392 try self.spillEflagsIfOccupied();
10895 try self.asmRegisterImmediate(.{ ._, .@"test" }, reg, Immediate.u(1));11393 try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1));
10896 return self.asmJccReloc(.e, undefined);11394 return self.asmJccReloc(.z, undefined);
10897 },11395 },
10898 .immediate,11396 .immediate,
10899 .load_frame,11397 .load_frame,
...@@ -11433,12 +11931,12 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {...@@ -11433,12 +11931,12 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void {
11433 if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {11931 if (self.reuseOperandAdvanced(inst, br.operand, 0, src_mcv, br.block_inst)) {
11434 if (first_br) break :result src_mcv;11932 if (first_br) break :result src_mcv;
1143511933
11436 if (block_tracking.getReg()) |block_reg|11934 for (block_tracking.getRegs()) |block_reg|
11437 try self.register_manager.getReg(block_reg, br.block_inst);11935 try self.register_manager.getReg(block_reg, br.block_inst);
11438 // .long = .none to avoid merging operand and block result stack frames.11936 // .long = .none to avoid merging operand and block result stack frames.
11439 var current_tracking = InstTracking{ .long = .none, .short = src_mcv };11937 var current_tracking = InstTracking{ .long = .none, .short = src_mcv };
11440 try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);11938 try current_tracking.materializeUnsafe(self, br.block_inst, block_tracking.*);
11441 if (src_mcv.getReg()) |src_reg| self.register_manager.freeReg(src_reg);11939 for (src_mcv.getRegs()) |src_reg| self.register_manager.freeReg(src_reg);
11442 break :result block_tracking.short;11940 break :result block_tracking.short;
11443 }11941 }
1144411942
...@@ -12177,16 +12675,87 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -12177,16 +12675,87 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
12177 .general_purpose, .segment => return .{ .move = .{ ._, .mov } },12675 .general_purpose, .segment => return .{ .move = .{ ._, .mov } },
12178 .x87 => return .x87_load_store,12676 .x87 => return .x87_load_store,
12179 .mmx => {},12677 .mmx => {},
12180 .sse => {12678 .sse => switch (ty.zigTypeTag(mod)) {
12181 switch (ty.zigTypeTag(mod)) {12679 else => {
12182 else => {12680 const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
12183 const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);12681 assert(std.mem.indexOfNone(abi.Class, classes, &.{
12184 assert(std.mem.indexOfNone(abi.Class, classes, &.{12682 .integer, .sse, .float, .float_combine,
12185 .integer, .sse, .float, .float_combine,12683 }) == null);
12186 }) == null);12684 const abi_size = ty.abiSize(mod);
12187 const abi_size = ty.abiSize(mod);12685 if (abi_size < 4 or
12188 if (abi_size < 4 or12686 std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) {
12189 std.mem.indexOfScalar(abi.Class, classes, .integer) != null) switch (abi_size) {12687 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12688 .insert = .{ .vp_b, .insr },
12689 .extract = .{ .vp_b, .extr },
12690 } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
12691 .insert = .{ .p_b, .insr },
12692 .extract = .{ .p_b, .extr },
12693 } },
12694 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12695 .insert = .{ .vp_w, .insr },
12696 .extract = .{ .vp_w, .extr },
12697 } } else .{ .insert_extract = .{
12698 .insert = .{ .p_w, .insr },
12699 .extract = .{ .p_w, .extr },
12700 } },
12701 3...4 => return .{ .move = if (self.hasFeature(.avx))
12702 .{ .v_d, .mov }
12703 else
12704 .{ ._d, .mov } },
12705 5...8 => return .{ .move = if (self.hasFeature(.avx))
12706 .{ .v_q, .mov }
12707 else
12708 .{ ._q, .mov } },
12709 9...16 => return .{ .move = if (self.hasFeature(.avx))
12710 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12711 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12712 17...32 => if (self.hasFeature(.avx))
12713 return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },
12714 else => {},
12715 } else switch (abi_size) {
12716 4 => return .{ .move = if (self.hasFeature(.avx))
12717 .{ .v_ss, .mov }
12718 else
12719 .{ ._ss, .mov } },
12720 5...8 => return .{ .move = if (self.hasFeature(.avx))
12721 .{ .v_sd, .mov }
12722 else
12723 .{ ._sd, .mov } },
12724 9...16 => return .{ .move = if (self.hasFeature(.avx))
12725 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12726 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12727 17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
12728 .{ .v_pd, .mova }
12729 else
12730 .{ .v_pd, .movu } },
12731 else => {},
12732 }
12733 },
12734 .Float => switch (ty.floatBits(self.target.*)) {
12735 16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12736 .insert = .{ .vp_w, .insr },
12737 .extract = .{ .vp_w, .extr },
12738 } } else .{ .insert_extract = .{
12739 .insert = .{ .p_w, .insr },
12740 .extract = .{ .p_w, .extr },
12741 } },
12742 32 => return .{ .move = if (self.hasFeature(.avx))
12743 .{ .v_ss, .mov }
12744 else
12745 .{ ._ss, .mov } },
12746 64 => return .{ .move = if (self.hasFeature(.avx))
12747 .{ .v_sd, .mov }
12748 else
12749 .{ ._sd, .mov } },
12750 128 => return .{ .move = if (self.hasFeature(.avx))
12751 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12752 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12753 else => {},
12754 },
12755 .Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12756 .Bool => {},
12757 .Int => switch (ty.childType(mod).intInfo(mod).bits) {
12758 8 => switch (ty.vectorLen(mod)) {
12190 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{12759 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12191 .insert = .{ .vp_b, .insr },12760 .insert = .{ .vp_b, .insr },
12192 .extract = .{ .vp_b, .extr },12761 .extract = .{ .vp_b, .extr },
...@@ -12213,242 +12782,169 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo...@@ -12213,242 +12782,169 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo
12213 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }12782 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12214 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },12783 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12215 17...32 => if (self.hasFeature(.avx))12784 17...32 => if (self.hasFeature(.avx))
12216 return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } },12785 return .{ .move = if (aligned)
12786 .{ .v_, .movdqa }
12787 else
12788 .{ .v_, .movdqu } },
12217 else => {},12789 else => {},
12218 } else switch (abi_size) {12790 },
12219 4 => return .{ .move = if (self.hasFeature(.avx))12791 16 => switch (ty.vectorLen(mod)) {
12220 .{ .v_ss, .mov }12792 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12793 .insert = .{ .vp_w, .insr },
12794 .extract = .{ .vp_w, .extr },
12795 } } else .{ .insert_extract = .{
12796 .insert = .{ .p_w, .insr },
12797 .extract = .{ .p_w, .extr },
12798 } },
12799 2 => return .{ .move = if (self.hasFeature(.avx))
12800 .{ .v_d, .mov }
12221 else12801 else
12222 .{ ._ss, .mov } },12802 .{ ._d, .mov } },
12803 3...4 => return .{ .move = if (self.hasFeature(.avx))
12804 .{ .v_q, .mov }
12805 else
12806 .{ ._q, .mov } },
12223 5...8 => return .{ .move = if (self.hasFeature(.avx))12807 5...8 => return .{ .move = if (self.hasFeature(.avx))
12224 .{ .v_sd, .mov }12808 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12809 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12810 9...16 => if (self.hasFeature(.avx))
12811 return .{ .move = if (aligned)
12812 .{ .v_, .movdqa }
12813 else
12814 .{ .v_, .movdqu } },
12815 else => {},
12816 },
12817 32 => switch (ty.vectorLen(mod)) {
12818 1 => return .{ .move = if (self.hasFeature(.avx))
12819 .{ .v_d, .mov }
12225 else12820 else
12226 .{ ._sd, .mov } },12821 .{ ._d, .mov } },
12227 9...16 => return .{ .move = if (self.hasFeature(.avx))12822 2 => return .{ .move = if (self.hasFeature(.avx))
12228 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }12823 .{ .v_q, .mov }
12229 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12230 17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned)
12231 .{ .v_pd, .mova }
12232 else12824 else
12233 .{ .v_pd, .movu } },12825 .{ ._q, .mov } },
12234 else => {},12826 3...4 => return .{ .move = if (self.hasFeature(.avx))
12235 }12827 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12236 },12828 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12237 .Float => switch (ty.floatBits(self.target.*)) {12829 5...8 => if (self.hasFeature(.avx))
12238 16 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{12830 return .{ .move = if (aligned)
12239 .insert = .{ .vp_w, .insr },12831 .{ .v_, .movdqa }
12240 .extract = .{ .vp_w, .extr },
12241 } } else .{ .insert_extract = .{
12242 .insert = .{ .p_w, .insr },
12243 .extract = .{ .p_w, .extr },
12244 } },
12245 32 => return .{ .move = if (self.hasFeature(.avx))
12246 .{ .v_ss, .mov }
12247 else
12248 .{ ._ss, .mov } },
12249 64 => return .{ .move = if (self.hasFeature(.avx))
12250 .{ .v_sd, .mov }
12251 else
12252 .{ ._sd, .mov } },
12253 128 => return .{ .move = if (self.hasFeature(.avx))
12254 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12255 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12256 else => {},
12257 },
12258 .Vector => switch (ty.childType(mod).zigTypeTag(mod)) {
12259 .Bool => return .{ .move = .{ ._, .mov } },
12260 .Int => switch (ty.childType(mod).intInfo(mod).bits) {
12261 8 => switch (ty.vectorLen(mod)) {
12262 1 => if (self.hasFeature(.avx)) return .{ .vex_insert_extract = .{
12263 .insert = .{ .vp_b, .insr },
12264 .extract = .{ .vp_b, .extr },
12265 } } else if (self.hasFeature(.sse4_2)) return .{ .insert_extract = .{
12266 .insert = .{ .p_b, .insr },
12267 .extract = .{ .p_b, .extr },
12268 } },
12269 2 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12270 .insert = .{ .vp_w, .insr },
12271 .extract = .{ .vp_w, .extr },
12272 } } else .{ .insert_extract = .{
12273 .insert = .{ .p_w, .insr },
12274 .extract = .{ .p_w, .extr },
12275 } },
12276 3...4 => return .{ .move = if (self.hasFeature(.avx))
12277 .{ .v_d, .mov }
12278 else
12279 .{ ._d, .mov } },
12280 5...8 => return .{ .move = if (self.hasFeature(.avx))
12281 .{ .v_q, .mov }
12282 else
12283 .{ ._q, .mov } },
12284 9...16 => return .{ .move = if (self.hasFeature(.avx))
12285 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12286 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12287 17...32 => if (self.hasFeature(.avx))
12288 return .{ .move = if (aligned)
12289 .{ .v_, .movdqa }
12290 else
12291 .{ .v_, .movdqu } },
12292 else => {},
12293 },
12294 16 => switch (ty.vectorLen(mod)) {
12295 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12296 .insert = .{ .vp_w, .insr },
12297 .extract = .{ .vp_w, .extr },
12298 } } else .{ .insert_extract = .{
12299 .insert = .{ .p_w, .insr },
12300 .extract = .{ .p_w, .extr },
12301 } },
12302 2 => return .{ .move = if (self.hasFeature(.avx))
12303 .{ .v_d, .mov }
12304 else
12305 .{ ._d, .mov } },
12306 3...4 => return .{ .move = if (self.hasFeature(.avx))
12307 .{ .v_q, .mov }
12308 else
12309 .{ ._q, .mov } },
12310 5...8 => return .{ .move = if (self.hasFeature(.avx))
12311 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12312 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12313 9...16 => if (self.hasFeature(.avx))
12314 return .{ .move = if (aligned)
12315 .{ .v_, .movdqa }
12316 else
12317 .{ .v_, .movdqu } },
12318 else => {},
12319 },
12320 32 => switch (ty.vectorLen(mod)) {
12321 1 => return .{ .move = if (self.hasFeature(.avx))
12322 .{ .v_d, .mov }
12323 else12832 else
12324 .{ ._d, .mov } },12833 .{ .v_, .movdqu } },
12325 2 => return .{ .move = if (self.hasFeature(.avx))12834 else => {},
12326 .{ .v_q, .mov }12835 },
12836 64 => switch (ty.vectorLen(mod)) {
12837 1 => return .{ .move = if (self.hasFeature(.avx))
12838 .{ .v_q, .mov }
12839 else
12840 .{ ._q, .mov } },
12841 2 => return .{ .move = if (self.hasFeature(.avx))
12842 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12843 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12844 3...4 => if (self.hasFeature(.avx))
12845 return .{ .move = if (aligned)
12846 .{ .v_, .movdqa }
12327 else12847 else
12328 .{ ._q, .mov } },12848 .{ .v_, .movdqu } },
12329 3...4 => return .{ .move = if (self.hasFeature(.avx))12849 else => {},
12330 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }12850 },
12331 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },12851 128 => switch (ty.vectorLen(mod)) {
12332 5...8 => if (self.hasFeature(.avx))12852 1 => return .{ .move = if (self.hasFeature(.avx))
12333 return .{ .move = if (aligned)12853 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12334 .{ .v_, .movdqa }12854 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12335 else12855 2 => if (self.hasFeature(.avx))
12336 .{ .v_, .movdqu } },12856 return .{ .move = if (aligned)
12337 else => {},12857 .{ .v_, .movdqa }
12338 },
12339 64 => switch (ty.vectorLen(mod)) {
12340 1 => return .{ .move = if (self.hasFeature(.avx))
12341 .{ .v_q, .mov }
12342 else12858 else
12343 .{ ._q, .mov } },12859 .{ .v_, .movdqu } },
12344 2 => return .{ .move = if (self.hasFeature(.avx))
12345 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12346 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12347 3...4 => if (self.hasFeature(.avx))
12348 return .{ .move = if (aligned)
12349 .{ .v_, .movdqa }
12350 else
12351 .{ .v_, .movdqu } },
12352 else => {},
12353 },
12354 128 => switch (ty.vectorLen(mod)) {
12355 1 => return .{ .move = if (self.hasFeature(.avx))
12356 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12357 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12358 2 => if (self.hasFeature(.avx))
12359 return .{ .move = if (aligned)
12360 .{ .v_, .movdqa }
12361 else
12362 .{ .v_, .movdqu } },
12363 else => {},
12364 },
12365 256 => switch (ty.vectorLen(mod)) {
12366 1 => if (self.hasFeature(.avx))
12367 return .{ .move = if (aligned)
12368 .{ .v_, .movdqa }
12369 else
12370 .{ .v_, .movdqu } },
12371 else => {},
12372 },
12373 else => {},12860 else => {},
12374 },12861 },
12375 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {12862 256 => switch (ty.vectorLen(mod)) {
12376 16 => switch (ty.vectorLen(mod)) {12863 1 => if (self.hasFeature(.avx))
12377 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{12864 return .{ .move = if (aligned)
12378 .insert = .{ .vp_w, .insr },12865 .{ .v_, .movdqa }
12379 .extract = .{ .vp_w, .extr },
12380 } } else .{ .insert_extract = .{
12381 .insert = .{ .p_w, .insr },
12382 .extract = .{ .p_w, .extr },
12383 } },
12384 2 => return .{ .move = if (self.hasFeature(.avx))
12385 .{ .v_d, .mov }
12386 else12866 else
12387 .{ ._d, .mov } },12867 .{ .v_, .movdqu } },
12388 3...4 => return .{ .move = if (self.hasFeature(.avx))12868 else => {},
12389 .{ .v_q, .mov }12869 },
12870 else => {},
12871 },
12872 .Float => switch (ty.childType(mod).floatBits(self.target.*)) {
12873 16 => switch (ty.vectorLen(mod)) {
12874 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{
12875 .insert = .{ .vp_w, .insr },
12876 .extract = .{ .vp_w, .extr },
12877 } } else .{ .insert_extract = .{
12878 .insert = .{ .p_w, .insr },
12879 .extract = .{ .p_w, .extr },
12880 } },
12881 2 => return .{ .move = if (self.hasFeature(.avx))
12882 .{ .v_d, .mov }
12883 else
12884 .{ ._d, .mov } },
12885 3...4 => return .{ .move = if (self.hasFeature(.avx))
12886 .{ .v_q, .mov }
12887 else
12888 .{ ._q, .mov } },
12889 5...8 => return .{ .move = if (self.hasFeature(.avx))
12890 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12891 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12892 9...16 => if (self.hasFeature(.avx))
12893 return .{ .move = if (aligned)
12894 .{ .v_, .movdqa }
12390 else12895 else
12391 .{ ._q, .mov } },12896 .{ .v_, .movdqu } },
12392 5...8 => return .{ .move = if (self.hasFeature(.avx))12897 else => {},
12393 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }12898 },
12394 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },12899 32 => switch (ty.vectorLen(mod)) {
12395 9...16 => if (self.hasFeature(.avx))12900 1 => return .{ .move = if (self.hasFeature(.avx))
12396 return .{ .move = if (aligned)12901 .{ .v_ss, .mov }
12397 .{ .v_, .movdqa }12902 else
12398 else12903 .{ ._ss, .mov } },
12399 .{ .v_, .movdqu } },12904 2 => return .{ .move = if (self.hasFeature(.avx))
12400 else => {},12905 .{ .v_sd, .mov }
12401 },12906 else
12402 32 => switch (ty.vectorLen(mod)) {12907 .{ ._sd, .mov } },
12403 1 => return .{ .move = if (self.hasFeature(.avx))12908 3...4 => return .{ .move = if (self.hasFeature(.avx))
12404 .{ .v_ss, .mov }12909 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }
12910 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },
12911 5...8 => if (self.hasFeature(.avx))
12912 return .{ .move = if (aligned)
12913 .{ .v_ps, .mova }
12405 else12914 else
12406 .{ ._ss, .mov } },12915 .{ .v_ps, .movu } },
12407 2 => return .{ .move = if (self.hasFeature(.avx))12916 else => {},
12408 .{ .v_sd, .mov }12917 },
12918 64 => switch (ty.vectorLen(mod)) {
12919 1 => return .{ .move = if (self.hasFeature(.avx))
12920 .{ .v_sd, .mov }
12921 else
12922 .{ ._sd, .mov } },
12923 2 => return .{ .move = if (self.hasFeature(.avx))
12924 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12925 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12926 3...4 => if (self.hasFeature(.avx))
12927 return .{ .move = if (aligned)
12928 .{ .v_pd, .mova }
12409 else12929 else
12410 .{ ._sd, .mov } },12930 .{ .v_pd, .movu } },
12411 3...4 => return .{ .move = if (self.hasFeature(.avx))12931 else => {},
12412 if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu }12932 },
12413 else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } },12933 128 => switch (ty.vectorLen(mod)) {
12414 5...8 => if (self.hasFeature(.avx))12934 1 => return .{ .move = if (self.hasFeature(.avx))
12415 return .{ .move = if (aligned)12935 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12416 .{ .v_ps, .mova }12936 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12417 else12937 2 => if (self.hasFeature(.avx))
12418 .{ .v_ps, .movu } },12938 return .{ .move = if (aligned)
12419 else => {},12939 .{ .v_, .movdqa }
12420 },
12421 64 => switch (ty.vectorLen(mod)) {
12422 1 => return .{ .move = if (self.hasFeature(.avx))
12423 .{ .v_sd, .mov }
12424 else12940 else
12425 .{ ._sd, .mov } },12941 .{ .v_, .movdqu } },
12426 2 => return .{ .move = if (self.hasFeature(.avx))
12427 if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu }
12428 else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } },
12429 3...4 => if (self.hasFeature(.avx))
12430 return .{ .move = if (aligned)
12431 .{ .v_pd, .mova }
12432 else
12433 .{ .v_pd, .movu } },
12434 else => {},
12435 },
12436 128 => switch (ty.vectorLen(mod)) {
12437 1 => return .{ .move = if (self.hasFeature(.avx))
12438 if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu }
12439 else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } },
12440 2 => if (self.hasFeature(.avx))
12441 return .{ .move = if (aligned)
12442 .{ .v_, .movdqa }
12443 else
12444 .{ .v_, .movdqu } },
12445 else => {},
12446 },
12447 else => {},12942 else => {},
12448 },12943 },
12449 else => {},12944 else => {},
12450 },12945 },
12451 }12946 else => {},
12947 },
12452 },12948 },
12453 }12949 }
12454 return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)});12950 return self.fail("TODO moveStrategy for {}", .{ty.fmt(mod)});
...@@ -12514,32 +13010,18 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError...@@ -12514,32 +13010,18 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError
12514 };13010 };
12515 defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock);13011 defer if (src_info) |info| self.register_manager.unlockReg(info.addr_lock);
1251613012
12517 const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);13013 var part_disp: i32 = 0;
12518 for (dst_regs, classes, 0..) |dst_reg, class, dst_reg_i| {13014 for (dst_regs, try self.splitType(ty), 0..) |dst_reg, dst_ty, part_i| {
12519 const class_ty = switch (class) {13015 try self.genSetReg(dst_reg, dst_ty, switch (src_mcv) {
12520 .integer => Type.usize,13016 .register_pair => |src_regs| .{ .register = src_regs[part_i] },
12521 .sse, .float, .float_combine => Type.f64,13017 .memory, .indirect, .load_frame => src_mcv.address().offset(part_disp).deref(),
12522 else => unreachable,13018 .load_symbol, .load_direct, .load_got, .load_tlv => .{ .indirect = .{
12523 };13019 .reg = src_info.?.addr_reg,
12524 const off: i32 = @intCast(dst_reg_i * 8);13020 .off = part_disp,
12525 switch (src_mcv) {13021 } },
12526 .register_pair => |src_regs| try self.genSetReg(
12527 dst_reg,
12528 class_ty,
12529 .{ .register = src_regs[dst_reg_i] },
12530 ),
12531 .memory, .indirect, .load_frame => try self.genSetReg(
12532 dst_reg,
12533 class_ty,
12534 src_mcv.address().offset(off).deref(),
12535 ),
12536 .load_symbol, .load_direct, .load_got, .load_tlv => try self.genSetReg(
12537 dst_reg,
12538 class_ty,
12539 .{ .indirect = .{ .reg = src_info.?.addr_reg, .off = off } },
12540 ),
12541 else => unreachable,13022 else => unreachable,
12542 }13023 });
13024 part_disp += @intCast(dst_ty.abiSize(mod));
12543 }13025 }
12544 },13026 },
12545 .indirect => |reg_off| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),13027 .indirect => |reg_off| try self.genSetMem(.{ .reg = reg_off.reg }, reg_off.off, ty, src_mcv),
...@@ -12584,6 +13066,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -12584,6 +13066,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
12584 if (imm == 0) {13066 if (imm == 0) {
12585 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit13067 // 32-bit moves zero-extend to 64-bit, so xoring the 32-bit
12586 // register is the fastest way to zero a register.13068 // register is the fastest way to zero a register.
13069 try self.spillEflagsIfOccupied();
12587 try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());13070 try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32());
12588 } else if (abi_size > 4 and math.cast(u32, imm) != null) {13071 } else if (abi_size > 4 and math.cast(u32, imm) != null) {
12589 // 32-bit moves zero-extend to 64-bit.13072 // 32-bit moves zero-extend to 64-bit.
...@@ -12933,44 +13416,65 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal...@@ -12933,44 +13416,65 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
12933 .eflags => |cc| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{13416 .eflags => |cc| try self.asmSetccMemory(cc, .{ .base = base, .mod = .{
12934 .rm = .{ .size = .byte, .disp = disp },13417 .rm = .{ .size = .byte, .disp = disp },
12935 } }),13418 } }),
12936 .register => |src_reg| try (try self.moveStrategy(ty, src_reg.class(), switch (base) {13419 .register => |src_reg| {
12937 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),13420 const mem_size = switch (base) {
12938 .reg => |reg| switch (reg) {13421 .frame => |base_fi| mem_size: {
12939 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),13422 assert(disp >= 0);
12940 else => false,13423 const frame_abi_size = self.frame_allocs.items(.abi_size)[@intFromEnum(base_fi)];
12941 },13424 const frame_spill_pad = self.frame_allocs.items(.spill_pad)[@intFromEnum(base_fi)];
12942 .frame => |frame_index| self.getFrameAddrAlignment(13425 assert(frame_abi_size - frame_spill_pad - disp >= abi_size);
12943 .{ .index = frame_index, .off = disp },13426 break :mem_size if (frame_abi_size - frame_spill_pad - disp == abi_size)
12944 ).compare(.gte, ty.abiAlignment(mod)),13427 frame_abi_size
12945 .reloc => false,13428 else
12946 })).write(13429 abi_size;
12947 self,13430 },
12948 .{ .base = base, .mod = .{ .rm = .{13431 else => abi_size,
12949 .size = self.memSize(ty),13432 };
12950 .disp = disp,13433 const src_alias = registerAlias(src_reg, abi_size);
12951 } } },13434 const src_size: u32 = @intCast(switch (src_alias.class()) {
12952 registerAlias(src_reg, abi_size),13435 .general_purpose, .segment, .x87 => @divExact(src_alias.bitSize(), 8),
12953 ),13436 .mmx, .sse => abi_size,
12954 .register_pair => |src_regs| for (src_regs, 0..) |src_reg, src_reg_i| {13437 });
12955 const part_size: u16 = @min(abi_size - src_reg_i * 8, 8);13438 if (src_size > mem_size) {
12956 try (try self.moveStrategy(13439 const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{
12957 try mod.intType(.unsigned, part_size * 8),13440 .size = src_size,
12958 src_reg.class(),13441 .alignment = Alignment.fromNonzeroByteUnits(src_size),
12959 switch (base) {13442 }));
12960 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),13443 const frame_mcv: MCValue = .{ .load_frame = .{ .index = frame_index } };
12961 .reg => |reg| switch (reg) {13444 try (try self.moveStrategy(ty, src_alias.class(), true)).write(
12962 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),13445 self,
12963 else => false,13446 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{
12964 },13447 .size = Memory.Size.fromSize(src_size),
12965 .frame => |frame_index| self.getFrameAddrAlignment(13448 } } },
12966 .{ .index = frame_index, .off = disp },13449 src_alias,
12967 ).compare(.gte, ty.abiAlignment(mod)),13450 );
12968 .reloc => false,13451 try self.genSetMem(base, disp, ty, frame_mcv);
13452 try self.freeValue(frame_mcv);
13453 } else try (try self.moveStrategy(ty, src_alias.class(), switch (base) {
13454 .none => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
13455 .reg => |reg| switch (reg) {
13456 .es, .cs, .ss, .ds => ty.abiAlignment(mod).check(@as(u32, @bitCast(disp))),
13457 else => false,
12969 },13458 },
12970 )).write(self, .{ .base = base, .mod = .{ .rm = .{13459 .frame => |frame_index| self.getFrameAddrAlignment(
12971 .size = Memory.Size.fromSize(part_size),13460 .{ .index = frame_index, .off = disp },
12972 .disp = disp + @as(i32, @intCast(src_reg_i * 8)),13461 ).compare(.gte, ty.abiAlignment(mod)),
12973 } } }, registerAlias(src_reg, part_size));13462 .reloc => false,
13463 })).write(
13464 self,
13465 .{ .base = base, .mod = .{ .rm = .{
13466 .size = self.memSize(ty),
13467 .disp = disp,
13468 } } },
13469 src_alias,
13470 );
13471 },
13472 .register_pair => |src_regs| {
13473 var part_disp: i32 = disp;
13474 for (try self.splitType(ty), src_regs) |src_ty, src_reg| {
13475 try self.genSetMem(base, part_disp, src_ty, .{ .register = src_reg });
13476 part_disp += @intCast(src_ty.abiSize(mod));
13477 }
12974 },13478 },
12975 .register_overflow => |ro| switch (ty.zigTypeTag(mod)) {13479 .register_overflow => |ro| switch (ty.zigTypeTag(mod)) {
12976 .Struct => {13480 .Struct => {
...@@ -13226,50 +13730,43 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -13226,50 +13730,43 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
13226 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;13730 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
13227 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);13731 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
1322813732
13229 const dst_mcv = if (dst_rc.supersetOf(src_rc) and13733 const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(mod) <= src_ty.abiSize(mod) and
13230 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))13734 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
13231 src_mcv
13232 else dst: {
13233 const dst_mcv = try self.allocRegOrMem(inst, true);13735 const dst_mcv = try self.allocRegOrMem(inst, true);
13234 try self.genCopy(13736 try self.genCopy(switch (math.order(dst_ty.abiSize(mod), src_ty.abiSize(mod))) {
13235 if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,13737 .lt => dst_ty,
13236 dst_mcv,13738 .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
13237 src_mcv,13739 .gt => src_ty,
13238 );13740 }, dst_mcv, src_mcv);
13239 break :dst dst_mcv;13741 break :dst dst_mcv;
13240 };13742 };
1324113743
13242 if (dst_ty.isRuntimeFloat()) break :result dst_mcv;13744 if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
1324313745
13244 const dst_signedness =13746 if (dst_ty.isAbiInt(mod) and src_ty.isAbiInt(mod) and
13245 if (dst_ty.isAbiInt(mod)) dst_ty.intInfo(mod).signedness else .unsigned;13747 dst_ty.intInfo(mod).signedness == src_ty.intInfo(mod).signedness) break :result dst_mcv;
13246 if (!src_ty.isRuntimeFloat() or src_ty.floatBits(self.target.*) != 80) {
13247 const src_signedness =
13248 if (src_ty.isAbiInt(mod)) src_ty.intInfo(mod).signedness else .unsigned;
13249 if (dst_signedness == src_signedness) break :result dst_mcv;
13250 }
1325113748
13252 const abi_size: u16 = @intCast(dst_ty.abiSize(mod));13749 const abi_size = dst_ty.abiSize(mod);
13253 const bit_size: u16 = @intCast(dst_ty.bitSize(mod));13750 const bit_size = dst_ty.bitSize(mod);
13254 if (abi_size * 8 <= bit_size) break :result dst_mcv;13751 if (abi_size * 8 <= bit_size or dst_ty.isVector(mod)) break :result dst_mcv;
1325513752
13256 const dst_limbs_len = math.divCeil(i32, bit_size, 64) catch unreachable;13753 const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable;
13257 const high_reg = if (dst_mcv.isRegister())13754 const high_mcv: MCValue = switch (dst_mcv) {
13258 dst_mcv.getReg().?13755 .register => |dst_reg| .{ .register = dst_reg },
13756 .register_pair => |dst_regs| .{ .register = dst_regs[1] },
13757 else => dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
13758 };
13759 const high_reg = if (high_mcv.isRegister())
13760 high_mcv.getReg().?
13259 else13761 else
13260 try self.copyToTmpRegister(13762 try self.copyToTmpRegister(Type.usize, high_mcv);
13261 Type.usize,
13262 dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
13263 );
13264 const high_lock = self.register_manager.lockReg(high_reg);13763 const high_lock = self.register_manager.lockReg(high_reg);
13265 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);13764 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
1326613765
13267 const high_ty = try mod.intType(dst_signedness, bit_size % 64);13766 try self.truncateRegister(dst_ty, high_reg);
1326813767 if (!high_mcv.isRegister()) try self.genCopy(
13269 try self.truncateRegister(high_ty, high_reg);13768 if (abi_size <= 8) dst_ty else Type.usize,
13270 if (!dst_mcv.isRegister()) try self.genCopy(13769 high_mcv,
13271 Type.usize,
13272 dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
13273 .{ .register = high_reg },13770 .{ .register = high_reg },
13274 );13771 );
13275 break :result dst_mcv;13772 break :result dst_mcv;
...@@ -13287,7 +13784,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {...@@ -13287,7 +13784,7 @@ fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void {
13287 const array_ty = ptr_ty.childType(mod);13784 const array_ty = ptr_ty.childType(mod);
13288 const array_len = array_ty.arrayLen(mod);13785 const array_len = array_ty.arrayLen(mod);
1328913786
13290 const frame_index = try self.allocFrameIndex(FrameAlloc.initType(slice_ty, mod));13787 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod));
13291 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);13788 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr);
13292 try self.genSetMem(13789 try self.genSetMem(
13293 .{ .frame = frame_index },13790 .{ .frame = frame_index },
...@@ -13497,7 +13994,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -13497,7 +13994,7 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
13497 const ptr_mcv = try self.resolveInst(extra.ptr);13994 const ptr_mcv = try self.resolveInst(extra.ptr);
13498 const mem_size = Memory.Size.fromSize(val_abi_size);13995 const mem_size = Memory.Size.fromSize(val_abi_size);
13499 const ptr_mem: Memory = switch (ptr_mcv) {13996 const ptr_mem: Memory = switch (ptr_mcv) {
13500 .immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),13997 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
13501 else => .{13998 else => .{
13502 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },13999 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13503 .mod = .{ .rm = .{ .size = mem_size } },14000 .mod = .{ .rm = .{ .size = mem_size } },
...@@ -13563,7 +14060,7 @@ fn atomicOp(...@@ -13563,7 +14060,7 @@ fn atomicOp(
13563 const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));14060 const val_abi_size: u32 = @intCast(val_ty.abiSize(mod));
13564 const mem_size = Memory.Size.fromSize(val_abi_size);14061 const mem_size = Memory.Size.fromSize(val_abi_size);
13565 const ptr_mem: Memory = switch (ptr_mcv) {14062 const ptr_mem: Memory = switch (ptr_mcv) {
13566 .immediate, .register, .register_offset, .lea_frame => ptr_mcv.deref().mem(mem_size),14063 .immediate, .register, .register_offset, .lea_frame => try ptr_mcv.deref().mem(self, mem_size),
13567 else => .{14064 else => .{
13568 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },14065 .base = .{ .reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
13569 .mod = .{ .rm = .{ .size = mem_size } },14066 .mod = .{ .rm = .{ .size = mem_size } },
...@@ -13671,27 +14168,41 @@ fn atomicOp(...@@ -13671,27 +14168,41 @@ fn atomicOp(
13671 },14168 },
13672 };14169 };
1367314170
13674 try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
13675 const cmov_abi_size = @max(val_abi_size, 2);14171 const cmov_abi_size = @max(val_abi_size, 2);
13676 switch (val_mcv) {14172 switch (val_mcv) {
13677 .register => |val_reg| try self.asmCmovccRegisterRegister(14173 .register => |val_reg| {
13678 cc,14174 try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
13679 registerAlias(tmp_reg, cmov_abi_size),
13680 registerAlias(val_reg, cmov_abi_size),
13681 ),
13682 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
13683 cc,
13684 registerAlias(tmp_reg, cmov_abi_size),
13685 val_mcv.mem(Memory.Size.fromSize(cmov_abi_size)),
13686 ),
13687 else => {
13688 const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
13689 try self.asmCmovccRegisterRegister(14175 try self.asmCmovccRegisterRegister(
13690 cc,14176 cc,
13691 registerAlias(tmp_reg, cmov_abi_size),14177 registerAlias(tmp_reg, cmov_abi_size),
13692 registerAlias(val_reg, cmov_abi_size),14178 registerAlias(val_reg, cmov_abi_size),
13693 );14179 );
13694 },14180 },
14181 .memory, .indirect, .load_frame => {
14182 try self.genBinOpMir(.{ ._, .cmp }, val_ty, tmp_mcv, val_mcv);
14183 try self.asmCmovccRegisterMemory(
14184 cc,
14185 registerAlias(tmp_reg, cmov_abi_size),
14186 try val_mcv.mem(self, Memory.Size.fromSize(cmov_abi_size)),
14187 );
14188 },
14189 else => {
14190 const mat_reg = try self.copyToTmpRegister(val_ty, val_mcv);
14191 const mat_lock = self.register_manager.lockRegAssumeUnused(mat_reg);
14192 defer self.register_manager.unlockReg(mat_lock);
14193
14194 try self.genBinOpMir(
14195 .{ ._, .cmp },
14196 val_ty,
14197 tmp_mcv,
14198 .{ .register = mat_reg },
14199 );
14200 try self.asmCmovccRegisterRegister(
14201 cc,
14202 registerAlias(tmp_reg, cmov_abi_size),
14203 registerAlias(mat_reg, cmov_abi_size),
14204 );
14205 },
13695 }14206 }
13696 },14207 },
13697 };14208 };
...@@ -13728,8 +14239,8 @@ fn atomicOp(...@@ -13728,8 +14239,8 @@ fn atomicOp(
13728 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),14239 .reg = try self.copyToTmpRegister(Type.usize, val_mcv.address()),
13729 } },14240 } },
13730 };14241 };
13731 const val_lo_mem = val_mem_mcv.mem(.qword);14242 const val_lo_mem = try val_mem_mcv.mem(self, .qword);
13732 const val_hi_mem = val_mem_mcv.address().offset(8).deref().mem(.qword);14243 const val_hi_mem = try val_mem_mcv.address().offset(8).deref().mem(self, .qword);
13733 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {14244 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
13734 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);14245 try self.asmRegisterRegister(.{ ._, .mov }, .rbx, .rax);
13735 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);14246 try self.asmRegisterRegister(.{ ._, .mov }, .rcx, .rdx);
...@@ -14000,7 +14511,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -14000,7 +14511,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
14000 try self.asmRegisterMemoryImmediate(14511 try self.asmRegisterMemoryImmediate(
14001 .{ .i_, .mul },14512 .{ .i_, .mul },
14002 len_reg,14513 len_reg,
14003 dst_ptr.address().offset(8).deref().mem(.qword),14514 try dst_ptr.address().offset(8).deref().mem(self, .qword),
14004 Immediate.s(@intCast(dst_ptr_ty.childType(mod).abiSize(mod))),14515 Immediate.s(@intCast(dst_ptr_ty.childType(mod).abiSize(mod))),
14005 );14516 );
14006 break :len .{ .register = len_reg };14517 break :len .{ .register = len_reg };
...@@ -14171,28 +14682,162 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14171,28 +14682,162 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14171 const mod = self.bin_file.options.module.?;14682 const mod = self.bin_file.options.module.?;
14172 const ty_op = self.air.instructions.items(.data)[inst].ty_op;14683 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
14173 const vector_ty = self.typeOfIndex(inst);14684 const vector_ty = self.typeOfIndex(inst);
14685 const vector_len = vector_ty.vectorLen(mod);
14174 const dst_rc = self.regClassForType(vector_ty);14686 const dst_rc = self.regClassForType(vector_ty);
14175 const scalar_ty = vector_ty.scalarType(mod);14687 const scalar_ty = self.typeOf(ty_op.operand);
1417614688
14177 const src_mcv = try self.resolveInst(ty_op.operand);
14178 const result: MCValue = result: {14689 const result: MCValue = result: {
14179 switch (scalar_ty.zigTypeTag(mod)) {14690 switch (scalar_ty.zigTypeTag(mod)) {
14180 else => {},14691 else => {},
14692 .Bool => {
14693 const regs =
14694 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp);
14695 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
14696 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
14697
14698 try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 });
14699 try self.genSetReg(
14700 regs[1],
14701 vector_ty,
14702 .{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) },
14703 );
14704 const src_mcv = try self.resolveInst(ty_op.operand);
14705 const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4);
14706 try self.asmCmovccRegisterRegister(
14707 switch (src_mcv) {
14708 .eflags => |cc| cc,
14709 .register => |src_reg| cc: {
14710 try self.asmRegisterImmediate(
14711 .{ ._, .@"test" },
14712 src_reg.to8(),
14713 Immediate.u(1),
14714 );
14715 break :cc .nz;
14716 },
14717 else => cc: {
14718 try self.asmMemoryImmediate(
14719 .{ ._, .@"test" },
14720 try src_mcv.mem(self, .byte),
14721 Immediate.u(1),
14722 );
14723 break :cc .nz;
14724 },
14725 },
14726 registerAlias(regs[0], abi_size),
14727 registerAlias(regs[1], abi_size),
14728 );
14729 break :result .{ .register = regs[0] };
14730 },
14731 .Int => if (self.hasFeature(.avx2)) avx2: {
14732 const mir_tag = @as(?Mir.Inst.FixedTag, switch (scalar_ty.intInfo(mod).bits) {
14733 else => null,
14734 1...8 => switch (vector_len) {
14735 else => null,
14736 1...32 => .{ .vp_b, .broadcast },
14737 },
14738 9...16 => switch (vector_len) {
14739 else => null,
14740 1...16 => .{ .vp_w, .broadcast },
14741 },
14742 17...32 => switch (vector_len) {
14743 else => null,
14744 1...8 => .{ .vp_d, .broadcast },
14745 },
14746 33...64 => switch (vector_len) {
14747 else => null,
14748 1...4 => .{ .vp_q, .broadcast },
14749 },
14750 65...128 => switch (vector_len) {
14751 else => null,
14752 1...2 => .{ .vp_i128, .broadcast },
14753 },
14754 }) orelse break :avx2;
14755
14756 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
14757 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
14758 defer self.register_manager.unlockReg(dst_lock);
14759
14760 const src_mcv = try self.resolveInst(ty_op.operand);
14761 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14762 mir_tag,
14763 registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
14764 try src_mcv.mem(self, self.memSize(scalar_ty)),
14765 ) else {
14766 if (mir_tag[0] == .vp_i128) break :avx2;
14767 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14768 try self.asmRegisterRegister(
14769 mir_tag,
14770 registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))),
14771 registerAlias(dst_reg, @intCast(scalar_ty.abiSize(mod))),
14772 );
14773 }
14774 break :result .{ .register = dst_reg };
14775 } else {
14776 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
14777 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
14778 defer self.register_manager.unlockReg(dst_lock);
14779
14780 try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand });
14781 if (vector_len == 1) break :result .{ .register = dst_reg };
14782
14783 const dst_alias = registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod)));
14784 const scalar_bits = scalar_ty.intInfo(mod).bits;
14785 if (switch (scalar_bits) {
14786 1...8 => true,
14787 9...128 => false,
14788 else => unreachable,
14789 }) if (self.hasFeature(.avx)) try self.asmRegisterRegisterRegister(
14790 .{ .vp_, .unpcklbw },
14791 dst_alias,
14792 dst_alias,
14793 dst_alias,
14794 ) else try self.asmRegisterRegister(
14795 .{ .p_, .unpcklbw },
14796 dst_alias,
14797 dst_alias,
14798 );
14799 if (switch (scalar_bits) {
14800 1...8 => vector_len > 2,
14801 9...16 => true,
14802 17...128 => false,
14803 else => unreachable,
14804 }) try self.asmRegisterRegisterImmediate(
14805 .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
14806 dst_alias,
14807 dst_alias,
14808 Immediate.u(0),
14809 );
14810 if (switch (scalar_bits) {
14811 1...8 => vector_len > 4,
14812 9...16 => vector_len > 2,
14813 17...64 => true,
14814 65...128 => false,
14815 else => unreachable,
14816 }) try self.asmRegisterRegisterImmediate(
14817 .{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
14818 dst_alias,
14819 dst_alias,
14820 Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
14821 );
14822 break :result .{ .register = dst_reg };
14823 },
14181 .Float => switch (scalar_ty.floatBits(self.target.*)) {14824 .Float => switch (scalar_ty.floatBits(self.target.*)) {
14182 32 => switch (vector_ty.vectorLen(mod)) {14825 32 => switch (vector_len) {
14183 1 => {14826 1 => {
14827 const src_mcv = try self.resolveInst(ty_op.operand);
14184 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;14828 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14185 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14829 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14186 try self.genSetReg(dst_reg, scalar_ty, src_mcv);14830 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14187 break :result .{ .register = dst_reg };14831 break :result .{ .register = dst_reg };
14188 },14832 },
14189 2...4 => {14833 2...4 => {
14834 const src_mcv = try self.resolveInst(ty_op.operand);
14190 if (self.hasFeature(.avx)) {14835 if (self.hasFeature(.avx)) {
14191 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14836 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14192 if (src_mcv.isMemory()) try self.asmRegisterMemory(14837 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14193 .{ .v_ss, .broadcast },14838 .{ .v_ss, .broadcast },
14194 dst_reg.to128(),14839 dst_reg.to128(),
14195 src_mcv.mem(.dword),14840 try src_mcv.mem(self, .dword),
14196 ) else {14841 ) else {
14197 const src_reg = if (src_mcv.isRegister())14842 const src_reg = if (src_mcv.isRegister())
14198 src_mcv.getReg().?14843 src_mcv.getReg().?
...@@ -14224,11 +14869,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14224,11 +14869,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14224 }14869 }
14225 },14870 },
14226 5...8 => if (self.hasFeature(.avx)) {14871 5...8 => if (self.hasFeature(.avx)) {
14872 const src_mcv = try self.resolveInst(ty_op.operand);
14227 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14873 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14228 if (src_mcv.isMemory()) try self.asmRegisterMemory(14874 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14229 .{ .v_ss, .broadcast },14875 .{ .v_ss, .broadcast },
14230 dst_reg.to256(),14876 dst_reg.to256(),
14231 src_mcv.mem(.dword),14877 try src_mcv.mem(self, .dword),
14232 ) else {14878 ) else {
14233 const src_reg = if (src_mcv.isRegister())14879 const src_reg = if (src_mcv.isRegister())
14234 src_mcv.getReg().?14880 src_mcv.getReg().?
...@@ -14259,20 +14905,22 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14259,20 +14905,22 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14259 },14905 },
14260 else => {},14906 else => {},
14261 },14907 },
14262 64 => switch (vector_ty.vectorLen(mod)) {14908 64 => switch (vector_len) {
14263 1 => {14909 1 => {
14910 const src_mcv = try self.resolveInst(ty_op.operand);
14264 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;14911 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14265 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14912 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14266 try self.genSetReg(dst_reg, scalar_ty, src_mcv);14913 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14267 break :result .{ .register = dst_reg };14914 break :result .{ .register = dst_reg };
14268 },14915 },
14269 2 => {14916 2 => {
14917 const src_mcv = try self.resolveInst(ty_op.operand);
14270 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14918 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14271 if (self.hasFeature(.sse3)) {14919 if (self.hasFeature(.sse3)) {
14272 if (src_mcv.isMemory()) try self.asmRegisterMemory(14920 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14273 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },14921 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
14274 dst_reg.to128(),14922 dst_reg.to128(),
14275 src_mcv.mem(.qword),14923 try src_mcv.mem(self, .qword),
14276 ) else try self.asmRegisterRegister(14924 ) else try self.asmRegisterRegister(
14277 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },14925 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
14278 dst_reg.to128(),14926 dst_reg.to128(),
...@@ -14292,11 +14940,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14292,11 +14940,12 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14292 );14940 );
14293 },14941 },
14294 3...4 => if (self.hasFeature(.avx)) {14942 3...4 => if (self.hasFeature(.avx)) {
14943 const src_mcv = try self.resolveInst(ty_op.operand);
14295 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14944 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14296 if (src_mcv.isMemory()) try self.asmRegisterMemory(14945 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14297 .{ .v_sd, .broadcast },14946 .{ .v_sd, .broadcast },
14298 dst_reg.to256(),14947 dst_reg.to256(),
14299 src_mcv.mem(.qword),14948 try src_mcv.mem(self, .qword),
14300 ) else {14949 ) else {
14301 const src_reg = if (src_mcv.isRegister())14950 const src_reg = if (src_mcv.isRegister())
14302 src_mcv.getReg().?14951 src_mcv.getReg().?
...@@ -14325,19 +14974,21 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -14325,19 +14974,21 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
14325 },14974 },
14326 else => {},14975 else => {},
14327 },14976 },
14328 128 => switch (vector_ty.vectorLen(mod)) {14977 128 => switch (vector_len) {
14329 1 => {14978 1 => {
14979 const src_mcv = try self.resolveInst(ty_op.operand);
14330 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;14980 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
14331 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14981 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14332 try self.genSetReg(dst_reg, scalar_ty, src_mcv);14982 try self.genSetReg(dst_reg, scalar_ty, src_mcv);
14333 break :result .{ .register = dst_reg };14983 break :result .{ .register = dst_reg };
14334 },14984 },
14335 2 => if (self.hasFeature(.avx)) {14985 2 => if (self.hasFeature(.avx)) {
14986 const src_mcv = try self.resolveInst(ty_op.operand);
14336 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);14987 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
14337 if (src_mcv.isMemory()) try self.asmRegisterMemory(14988 if (src_mcv.isMemory()) try self.asmRegisterMemory(
14338 .{ .v_f128, .broadcast },14989 .{ .v_f128, .broadcast },
14339 dst_reg.to256(),14990 dst_reg.to256(),
14340 src_mcv.mem(.xword),14991 try src_mcv.mem(self, .xword),
14341 ) else {14992 ) else {
14342 const src_reg = if (src_mcv.isRegister())14993 const src_reg = if (src_mcv.isRegister())
14343 src_mcv.getReg().?14994 src_mcv.getReg().?
...@@ -14389,7 +15040,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {...@@ -14389,7 +15040,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14389 try self.spillEflagsIfOccupied();15040 try self.spillEflagsIfOccupied();
1439015041
14391 const operand_mcv = try self.resolveInst(reduce.operand);15042 const operand_mcv = try self.resolveInst(reduce.operand);
14392 const mask_len = (std.math.cast(u6, operand_ty.vectorLen(mod)) orelse15043 const mask_len = (math.cast(u6, operand_ty.vectorLen(mod)) orelse
14393 return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(mod)}));15044 return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(mod)}));
14394 const mask = (@as(u64, 1) << mask_len) - 1;15045 const mask = (@as(u64, 1) << mask_len) - 1;
14395 const abi_size: u32 = @intCast(operand_ty.abiSize(mod));15046 const abi_size: u32 = @intCast(operand_ty.abiSize(mod));
...@@ -14397,7 +15048,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {...@@ -14397,7 +15048,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void {
14397 .Or => {15048 .Or => {
14398 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(15049 if (operand_mcv.isMemory()) try self.asmMemoryImmediate(
14399 .{ ._, .@"test" },15050 .{ ._, .@"test" },
14400 operand_mcv.mem(Memory.Size.fromSize(abi_size)),15051 try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)),
14401 Immediate.u(mask),15052 Immediate.u(mask),
14402 ) else {15053 ) else {
14403 const operand_reg = registerAlias(if (operand_mcv.isRegister())15054 const operand_reg = registerAlias(if (operand_mcv.isRegister())
...@@ -14445,8 +15096,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -14445,8 +15096,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
14445 const result: MCValue = result: {15096 const result: MCValue = result: {
14446 switch (result_ty.zigTypeTag(mod)) {15097 switch (result_ty.zigTypeTag(mod)) {
14447 .Struct => {15098 .Struct => {
14448 const frame_index =15099 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod));
14449 try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
14450 if (result_ty.containerLayout(mod) == .Packed) {15100 if (result_ty.containerLayout(mod) == .Packed) {
14451 const struct_type = mod.typeToStruct(result_ty).?;15101 const struct_type = mod.typeToStruct(result_ty).?;
14452 try self.genInlineMemset(15102 try self.genInlineMemset(
...@@ -14542,8 +15192,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -14542,8 +15192,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
14542 break :result .{ .load_frame = .{ .index = frame_index } };15192 break :result .{ .load_frame = .{ .index = frame_index } };
14543 },15193 },
14544 .Array => {15194 .Array => {
14545 const frame_index =15195 const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod));
14546 try self.allocFrameIndex(FrameAlloc.initType(result_ty, mod));
14547 const elem_ty = result_ty.childType(mod);15196 const elem_ty = result_ty.childType(mod);
14548 const elem_size: u32 = @intCast(elem_ty.abiSize(mod));15197 const elem_size: u32 = @intCast(elem_ty.abiSize(mod));
1454915198
...@@ -14789,7 +15438,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {...@@ -14789,7 +15438,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
14789 mir_tag,15438 mir_tag,
14790 mop1_reg,15439 mop1_reg,
14791 mop2_reg,15440 mop2_reg,
14792 mops[2].mem(Memory.Size.fromSize(abi_size)),15441 try mops[2].mem(self, Memory.Size.fromSize(abi_size)),
14793 );15442 );
14794 break :result mops[0];15443 break :result mops[0];
14795 };15444 };
...@@ -14807,7 +15456,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {...@@ -14807,7 +15456,7 @@ fn airVaStart(self: *Self, inst: Air.Inst.Index) !void {
14807 )) {15456 )) {
14808 .SysV => result: {15457 .SysV => result: {
14809 const info = self.va_info.sysv;15458 const info = self.va_info.sysv;
14810 const dst_fi = try self.allocFrameIndex(FrameAlloc.initType(va_list_ty, mod));15459 const dst_fi = try self.allocFrameIndex(FrameAlloc.initSpill(va_list_ty, mod));
14811 var field_off: u31 = 0;15460 var field_off: u31 = 0;
14812 // gp_offset: c_uint,15461 // gp_offset: c_uint,
14813 try self.genSetMem(15462 try self.genSetMem(
...@@ -15015,7 +15664,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -15015,7 +15664,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
15015 .{ .v_ss, .cvtsd2 },15664 .{ .v_ss, .cvtsd2 },
15016 dst_reg,15665 dst_reg,
15017 dst_reg,15666 dst_reg,
15018 promote_mcv.mem(.qword),15667 try promote_mcv.mem(self, .qword),
15019 ) else try self.asmRegisterRegisterRegister(15668 ) else try self.asmRegisterRegisterRegister(
15020 .{ .v_ss, .cvtsd2 },15669 .{ .v_ss, .cvtsd2 },
15021 dst_reg,15670 dst_reg,
...@@ -15027,7 +15676,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {...@@ -15027,7 +15676,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void {
15027 ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(15676 ) else if (promote_mcv.isMemory()) try self.asmRegisterMemory(
15028 .{ ._ss, .cvtsd2 },15677 .{ ._ss, .cvtsd2 },
15029 dst_reg,15678 dst_reg,
15030 promote_mcv.mem(.qword),15679 try promote_mcv.mem(self, .qword),
15031 ) else try self.asmRegisterRegister(15680 ) else try self.asmRegisterRegister(
15032 .{ ._ss, .cvtsd2 },15681 .{ ._ss, .cvtsd2 },
15033 dst_reg,15682 dst_reg,
...@@ -15473,6 +16122,33 @@ fn memSize(self: *Self, ty: Type) Memory.Size {...@@ -15473,6 +16122,33 @@ fn memSize(self: *Self, ty: Type) Memory.Size {
15473 };16122 };
15474}16123}
1547516124
16125fn splitType(self: *Self, ty: Type) ![2]Type {
16126 const mod = self.bin_file.options.module.?;
16127 const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none);
16128 var parts: [2]Type = undefined;
16129 if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| {
16130 part.* = switch (class) {
16131 .integer => switch (part_i) {
16132 0 => Type.u64,
16133 1 => part: {
16134 const elem_size = ty.abiAlignment(mod).minStrict(.@"8").toByteUnitsOptional().?;
16135 const elem_ty = try mod.intType(.unsigned, @intCast(elem_size * 8));
16136 break :part switch (@divExact(ty.abiSize(mod) - 8, elem_size)) {
16137 1 => elem_ty,
16138 else => |len| try mod.arrayType(.{ .len = len, .child = elem_ty.toIntern() }),
16139 };
16140 },
16141 else => unreachable,
16142 },
16143 .float => Type.f32,
16144 .float_combine => try mod.vectorType(.{ .len = 2, .child = .f32_type }),
16145 .sse => Type.f64,
16146 else => break,
16147 };
16148 } else if (parts[0].abiSize(mod) + parts[1].abiSize(mod) == ty.abiSize(mod)) return parts;
16149 return self.fail("TODO implement splitType for {}", .{ty.fmt(mod)});
16150}
16151
15476/// Truncates the value in the register in place.16152/// Truncates the value in the register in place.
15477/// Clobbers any remaining bits.16153/// Clobbers any remaining bits.
15478fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {16154fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
src/arch/x86_64/Encoding.zig+10-8
...@@ -410,6 +410,8 @@ pub const Mnemonic = enum {...@@ -410,6 +410,8 @@ pub const Mnemonic = enum {
410 vfmadd132ps, vfmadd213ps, vfmadd231ps,410 vfmadd132ps, vfmadd213ps, vfmadd231ps,
411 vfmadd132sd, vfmadd213sd, vfmadd231sd,411 vfmadd132sd, vfmadd213sd, vfmadd231sd,
412 vfmadd132ss, vfmadd213ss, vfmadd231ss,412 vfmadd132ss, vfmadd213ss, vfmadd231ss,
413 // AVX2
414 vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw,
413 // zig fmt: on415 // zig fmt: on
414};416};
415417
...@@ -444,7 +446,7 @@ pub const Op = enum {...@@ -444,7 +446,7 @@ pub const Op = enum {
444 moffs,446 moffs,
445 sreg,447 sreg,
446 st, mm, mm_m64,448 st, mm, mm_m64,
447 xmm0, xmm, xmm_m32, xmm_m64, xmm_m128,449 xmm0, xmm, xmm_m8, xmm_m16, xmm_m32, xmm_m64, xmm_m128,
448 ymm, ymm_m256,450 ymm, ymm_m256,
449 // zig fmt: on451 // zig fmt: on
450452
...@@ -534,7 +536,7 @@ pub const Op = enum {...@@ -534,7 +536,7 @@ pub const Op = enum {
534 .eax, .r32, .rm32, .r32_m16 => unreachable,536 .eax, .r32, .rm32, .r32_m16 => unreachable,
535 .rax, .r64, .rm64, .r64_m16 => unreachable,537 .rax, .r64, .rm64, .r64_m16 => unreachable,
536 .st, .mm, .mm_m64 => unreachable,538 .st, .mm, .mm_m64 => unreachable,
537 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,539 .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
538 .ymm, .ymm_m256 => unreachable,540 .ymm, .ymm_m256 => unreachable,
539 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,541 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
540 .unity => 1,542 .unity => 1,
...@@ -556,7 +558,7 @@ pub const Op = enum {...@@ -556,7 +558,7 @@ pub const Op = enum {
556 .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,558 .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
557 .rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,559 .rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,
558 .st => 80,560 .st => 80,
559 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,561 .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
560 .ymm, .ymm_m256 => 256,562 .ymm, .ymm_m256 => 256,
561 };563 };
562 }564 }
...@@ -568,8 +570,8 @@ pub const Op = enum {...@@ -568,8 +570,8 @@ pub const Op = enum {
568 .rel8, .rel16, .rel32 => unreachable,570 .rel8, .rel16, .rel32 => unreachable,
569 .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64 => unreachable,571 .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64 => unreachable,
570 .st, .mm, .xmm0, .xmm, .ymm => unreachable,572 .st, .mm, .xmm0, .xmm, .ymm => unreachable,
571 .m8, .rm8, .r32_m8 => 8,573 .m8, .rm8, .r32_m8, .xmm_m8 => 8,
572 .m16, .rm16, .r32_m16, .r64_m16 => 16,574 .m16, .rm16, .r32_m16, .r64_m16, .xmm_m16 => 16,
573 .m32, .rm32, .xmm_m32 => 32,575 .m32, .rm32, .xmm_m32 => 32,
574 .m64, .rm64, .mm_m64, .xmm_m64 => 64,576 .m64, .rm64, .mm_m64, .xmm_m64 => 64,
575 .m80 => 80,577 .m80 => 80,
...@@ -600,7 +602,7 @@ pub const Op = enum {...@@ -600,7 +602,7 @@ pub const Op = enum {
600 .rm8, .rm16, .rm32, .rm64,602 .rm8, .rm16, .rm32, .rm64,
601 .r32_m8, .r32_m16, .r64_m16,603 .r32_m8, .r32_m16, .r64_m16,
602 .st, .mm, .mm_m64,604 .st, .mm, .mm_m64,
603 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128,605 .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
604 .ymm, .ymm_m256,606 .ymm, .ymm_m256,
605 => true,607 => true,
606 else => false,608 else => false,
...@@ -629,7 +631,7 @@ pub const Op = enum {...@@ -629,7 +631,7 @@ pub const Op = enum {
629 .m8, .m16, .m32, .m64, .m80, .m128, .m256,631 .m8, .m16, .m32, .m64, .m80, .m128, .m256,
630 .m,632 .m,
631 .mm_m64,633 .mm_m64,
632 .xmm_m32, .xmm_m64, .xmm_m128,634 .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
633 .ymm_m256,635 .ymm_m256,
634 => true,636 => true,
635 else => false,637 else => false,
...@@ -654,7 +656,7 @@ pub const Op = enum {...@@ -654,7 +656,7 @@ pub const Op = enum {
654 .sreg => .segment,656 .sreg => .segment,
655 .st => .x87,657 .st => .x87,
656 .mm, .mm_m64 => .mmx,658 .mm, .mm_m64 => .mmx,
657 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,659 .xmm0, .xmm, .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,
658 .ymm, .ymm_m256 => .sse,660 .ymm, .ymm_m256 => .sse,
659 };661 };
660 }662 }
src/arch/x86_64/Mir.zig+2
...@@ -255,6 +255,8 @@ pub const Inst = struct {...@@ -255,6 +255,8 @@ pub const Inst = struct {
255 vp_q,255 vp_q,
256 /// VEX-Encoded Packed ___ Double Quadword256 /// VEX-Encoded Packed ___ Double Quadword
257 vp_dq,257 vp_dq,
258 /// VEX-Encoded Packed ___ Integer Data
259 vp_i128,
258 /// VEX-Encoded ___ Scalar Single-Precision Values260 /// VEX-Encoded ___ Scalar Single-Precision Values
259 v_ss,261 v_ss,
260 /// VEX-Encoded ___ Packed Single-Precision Values262 /// VEX-Encoded ___ Packed Single-Precision Values
src/arch/x86_64/bits.zig+1-1
...@@ -237,7 +237,7 @@ pub const Register = enum(u7) {...@@ -237,7 +237,7 @@ pub const Register = enum(u7) {
237 return @intCast(@intFromEnum(reg) - base);237 return @intCast(@intFromEnum(reg) - base);
238 }238 }
239239
240 pub fn bitSize(reg: Register) u64 {240 pub fn bitSize(reg: Register) u10 {
241 return switch (@intFromEnum(reg)) {241 return switch (@intFromEnum(reg)) {
242 // zig fmt: off242 // zig fmt: off
243 @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => 64,243 @intFromEnum(Register.rax) ... @intFromEnum(Register.r15) => 64,
src/arch/x86_64/encodings.zig+10
...@@ -1742,6 +1742,16 @@ pub const table = [_]Entry{...@@ -1742,6 +1742,16 @@ pub const table = [_]Entry{
17421742
1743 .{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },1743 .{ .vpandn, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xdf }, 0, .vex_256_wig, .avx2 },
17441744
1745 .{ .vpbroadcastb, .rm, &.{ .xmm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_128_w0, .avx2 },
1746 .{ .vpbroadcastb, .rm, &.{ .ymm, .xmm_m8 }, &.{ 0x66, 0x0f, 0x38, 0x78 }, 0, .vex_256_w0, .avx2 },
1747 .{ .vpbroadcastw, .rm, &.{ .xmm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_128_w0, .avx2 },
1748 .{ .vpbroadcastw, .rm, &.{ .ymm, .xmm_m16 }, &.{ 0x66, 0x0f, 0x38, 0x79 }, 0, .vex_256_w0, .avx2 },
1749 .{ .vpbroadcastd, .rm, &.{ .xmm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_128_w0, .avx2 },
1750 .{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 },
1751 .{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 },
1752 .{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 },
1753 .{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 },
1754
1745 .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },1755 .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 },
1746 .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },1756 .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 },
1747 .{ .vpcmpeqd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x76 }, 0, .vex_256_wig, .avx2 },1757 .{ .vpcmpeqd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x76 }, 0, .vex_256_wig, .avx2 },
src/codegen.zig+4-1
...@@ -376,7 +376,10 @@ pub fn generateSymbol(...@@ -376,7 +376,10 @@ pub fn generateSymbol(
376 .val = switch (aggregate.storage) {376 .val = switch (aggregate.storage) {
377 .bytes => unreachable,377 .bytes => unreachable,
378 .elems => |elems| elems[@as(usize, @intCast(index))],378 .elems => |elems| elems[@as(usize, @intCast(index))],
379 .repeated_elem => |elem| elem,379 .repeated_elem => |elem| if (index < array_type.len)
380 elem
381 else
382 array_type.sentinel,
380 }.toValue(),383 }.toValue(),
381 }, code, debug_output, reloc_info)) {384 }, code, debug_output, reloc_info)) {
382 .ok => {},385 .ok => {},
src/link/Coff.zig+1
...@@ -388,6 +388,7 @@ fn populateMissingMetadata(self: *Coff) !void {...@@ -388,6 +388,7 @@ fn populateMissingMetadata(self: *Coff) !void {
388 self.rdata_section_index = try self.allocateSection(".rdata", file_size, .{388 self.rdata_section_index = try self.allocateSection(".rdata", file_size, .{
389 .CNT_INITIALIZED_DATA = 1,389 .CNT_INITIALIZED_DATA = 1,
390 .MEM_READ = 1,390 .MEM_READ = 1,
391 .MEM_WRITE = 1,
391 });392 });
392 }393 }
393394
test/behavior/vector.zig-1
...@@ -1260,7 +1260,6 @@ test "zero multiplicand" {...@@ -1260,7 +1260,6 @@ test "zero multiplicand" {
12601260
1261test "@intCast to u0" {1261test "@intCast to u0" {
1262 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1262 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1263 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1264 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1263 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1265 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1264 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1266 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1265 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO