authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-10-30 09:11:36+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-11-12 16:00:15+00:00
log9a7d28fe58c0dbd34b0e41fe0e4fed6d57c7ad45
treee991c9a41bd17eecc626138f9bc8c720f08ff276
parentee0a0f119c25cb5e4436aa705f672316e80a74d1
signaturelock-open Commit is signed but in an unrecognized format.

Legalize: rewrite several legalizations

The main goal of this change was to avoid emitting the `vector_store_elem` AIR tag, because this represents an operation which Zig no longer supports (and hence Sema no longer emits) as of 010d9a6 (because runtime vector indices are now forbidden). Backends should not need to lower this operation, so I rewrote the legalizations which emitted it (scalarizations of vector operations) to instead unroll the loop and hence emit comptime-known vector indices. In doing this, I actually reworked those legalizations to use a different strategy; instead of using an `alloc` and storing to individual vector elements, the vector is constructed by-val, for instance by performing the scalar operation on all elements and passing them to an `aggregate_init`. This is vastly simpler to implement in Legalize, conceptually simpler, and doesn't severely pessimise memory usage, because a non-optimizing backend will store the full vector on the stack either way. Given the above rationale, I also ended up reworking several other legalizations to use simpler lowerings. The legalizations in question were bitcast scalarization, `struct_field_val` of `packed struct`s (where we just bitcast to an integer and perform the appropriate shift/trunc sequence), and `aggregate_init` of a `packed struct` (also implemented in terms of integer bitwise operations with bitcasts to and from the actual types). This hugely simplified some parts of `Legalize`. So, `Legalize` is now much simpler, and the `vector_store_elem` instruction is no longer emitted by any part of the compiler so can be removed in a future commit.

1 files changed, 480 insertions(+), 1383 deletions(-)

src/Air/Legalize.zig+480-1383
...@@ -14,7 +14,7 @@ features: if (switch (dev.env) {...@@ -14,7 +14,7 @@ features: if (switch (dev.env) {
14 return comptime bootstrap_features.contains(feature);14 return comptime bootstrap_features.contains(feature);
15 }15 }
16 /// `inline` to propagate comptime-known result.16 /// `inline` to propagate comptime-known result.
17 fn hasAny(_: @This(), comptime features: []const Feature) bool {17 inline fn hasAny(_: @This(), comptime features: []const Feature) bool {
18 return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.initEmpty());18 return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.initEmpty());
19 }19 }
20} else struct {20} else struct {
...@@ -154,9 +154,9 @@ pub const Feature = enum {...@@ -154,9 +154,9 @@ pub const Feature = enum {
154 /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the154 /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
155 /// first byte of memory until bit pointers know their backing type.155 /// first byte of memory until bit pointers know their backing type.
156 expand_packed_store,156 expand_packed_store,
157 /// Replace `struct_field_val` of a packed field with a `store` and packed `load`.157 /// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.
158 expand_packed_struct_field_val,158 expand_packed_struct_field_val,
159 /// Replace `aggregate_init` of a packed aggregate with a series a packed `store`s followed by a `load`.159 /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.
160 expand_packed_aggregate_init,160 expand_packed_aggregate_init,
161161
162 fn scalarize(tag: Air.Inst.Tag) Feature {162 fn scalarize(tag: Air.Inst.Tag) Feature {
...@@ -409,40 +409,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -409,40 +409,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
409 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);409 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
410 },410 },
411 .bitcast => if (l.features.has(.scalarize_bitcast)) {411 .bitcast => if (l.features.has(.scalarize_bitcast)) {
412 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;412 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
413413 continue :inst l.replaceInst(inst, .block, payload);
414 const to_ty = ty_op.ty.toType();414 }
415 const to_ty_tag = to_ty.zigTypeTag(zcu);
416 const to_ty_legal = legal: switch (to_ty_tag) {
417 else => true,
418 .array, .vector => {
419 if (to_ty.arrayLen(zcu) == 1) break :legal true;
420 const to_elem_ty = to_ty.childType(zcu);
421 break :legal to_elem_ty.bitSize(zcu) == 8 * to_elem_ty.abiSize(zcu);
422 },
423 };
424
425 const from_ty = l.typeOf(ty_op.operand);
426 const from_ty_legal = legal: switch (from_ty.zigTypeTag(zcu)) {
427 else => true,
428 .array, .vector => {
429 if (from_ty.arrayLen(zcu) == 1) break :legal true;
430 const from_elem_ty = from_ty.childType(zcu);
431 break :legal from_elem_ty.bitSize(zcu) == 8 * from_elem_ty.abiSize(zcu);
432 },
433 };
434
435 if (!to_ty_legal and !from_ty_legal and to_ty.arrayLen(zcu) == from_ty.arrayLen(zcu)) switch (to_ty_tag) {
436 else => unreachable,
437 .array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastToArrayBlockPayload(inst)),
438 .vector => continue :inst try l.scalarize(inst, .bitcast),
439 };
440 if (!to_ty_legal) switch (to_ty_tag) {
441 else => unreachable,
442 .array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultArrayBlockPayload(inst)),
443 .vector => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultVectorBlockPayload(inst)),
444 };
445 if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst));
446 },415 },
447 .intcast_safe => if (l.features.has(.expand_intcast_safe)) {416 .intcast_safe => if (l.features.has(.expand_intcast_safe)) {
448 assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both417 assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both
...@@ -570,13 +539,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -570,13 +539,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
570 .load => if (l.features.has(.expand_packed_load)) {539 .load => if (l.features.has(.expand_packed_load)) {
571 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;540 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
572 const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu);541 const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu);
573 if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));542 if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
543 continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));
544 }
574 },545 },
575 .ret, .ret_safe, .ret_load => {},546 .ret, .ret_safe, .ret_load => {},
576 .store, .store_safe => if (l.features.has(.expand_packed_store)) {547 .store, .store_safe => if (l.features.has(.expand_packed_store)) {
577 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;548 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
578 const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu);549 const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu);
579 if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));550 if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
551 continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));
552 }
580 },553 },
581 .unreach,554 .unreach,
582 .optional_payload,555 .optional_payload,
...@@ -624,7 +597,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -624,7 +597,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
624 switch (vector_ty.vectorLen(zcu)) {597 switch (vector_ty.vectorLen(zcu)) {
625 0 => unreachable,598 0 => unreachable,
626 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{599 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
627 .ty = Air.internedToRef(vector_ty.childType(zcu).toIntern()),600 .ty = .fromType(vector_ty.childType(zcu)),
628 .operand = reduce.operand,601 .operand = reduce.operand,
629 } }),602 } }),
630 else => {},603 else => {},
...@@ -666,9 +639,18 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -666,9 +639,18 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
666 const agg_ty = ty_pl.ty.toType();639 const agg_ty = ty_pl.ty.toType();
667 switch (agg_ty.zigTypeTag(zcu)) {640 switch (agg_ty.zigTypeTag(zcu)) {
668 else => {},641 else => {},
669 .@"struct", .@"union" => switch (agg_ty.containerLayout(zcu)) {642 .@"union" => unreachable,
643 .@"struct" => switch (agg_ty.containerLayout(zcu)) {
670 .auto, .@"extern" => {},644 .auto, .@"extern" => {},
671 .@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)),645 .@"packed" => switch (agg_ty.structFieldCount(zcu)) {
646 0 => unreachable,
647 // An `aggregate_init` of a packed struct with 1 field is just a fancy bitcast.
648 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
649 .ty = .fromType(agg_ty),
650 .operand = @enumFromInt(l.air_extra.items[ty_pl.payload]),
651 } }),
652 else => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)),
653 },
672 },654 },
673 }655 }
674 },656 },
...@@ -685,7 +667,6 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -685,7 +667,6 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
685 .set_err_return_trace,667 .set_err_return_trace,
686 .addrspace_cast,668 .addrspace_cast,
687 .save_err_return_trace_index,669 .save_err_return_trace_index,
688 .vector_store_elem,
689 .runtime_nav_ptr,670 .runtime_nav_ptr,
690 .c_va_arg,671 .c_va_arg,
691 .c_va_copy,672 .c_va_copy,
...@@ -699,7 +680,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -699,7 +680,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
699 }680 }
700}681}
701682
702const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select };683const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, shuffle_one, shuffle_two, select };
703/// inline to propagate comptime-known `replaceInst` result.684/// inline to propagate comptime-known `replaceInst` result.
704inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag {685inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag {
705 return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form));686 return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form));
...@@ -707,1160 +688,420 @@ inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: Scal...@@ -707,1160 +688,420 @@ inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: Scal
707fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Data {688fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Data {
708 const pt = l.pt;689 const pt = l.pt;
709 const zcu = pt.zcu;690 const zcu = pt.zcu;
691 const gpa = zcu.gpa;
710692
711 const orig = l.air_instructions.get(@intFromEnum(orig_inst));693 const orig = l.air_instructions.get(@intFromEnum(orig_inst));
712 const res_ty = l.typeOfIndex(orig_inst);694 const res_ty = l.typeOfIndex(orig_inst);
713 const res_len = res_ty.vectorLen(zcu);695 const res_len = res_ty.vectorLen(zcu);
714696
715 const extra_insts = switch (form) {697 const inst_per_elem = switch (form) {
716 .un_op, .ty_op, .bitcast => 1,698 .un_op, .ty_op => 2,
717 .bin_op, .cmp_vector => 2,699 .bin_op, .cmp_vector => 3,
718 .pl_op_bin => 3,700 .pl_op_bin => 4,
719 .shuffle_one, .shuffle_two => 13,701 .shuffle_one, .shuffle_two => 1,
720 .select => 6,702 .select => 7,
721 };703 };
722 var inst_buf: [5 + extra_insts + 9]Air.Inst.Index = undefined;
723 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
724704
725 var res_block: Block = .init(&inst_buf);705 var sfba_state = std.heap.stackFallback(@sizeOf([inst_per_elem * 32 + 2]Air.Inst.Index) + @sizeOf([32]Air.Inst.Ref), gpa);
726 {706 const sfba = sfba_state.get();
727 const res_alloc_inst = res_block.add(l, .{
728 .tag = .alloc,
729 .data = .{ .ty = try pt.singleMutPtrType(res_ty) },
730 });
731 const index_alloc_inst = res_block.add(l, .{
732 .tag = .alloc,
733 .data = .{ .ty = .ptr_usize },
734 });
735 _ = res_block.add(l, .{
736 .tag = .store,
737 .data = .{ .bin_op = .{
738 .lhs = index_alloc_inst.toRef(),
739 .rhs = .zero_usize,
740 } },
741 });
742707
743 var loop: Loop = .init(l, &res_block);708 // Plus 2 extra instructions for `aggregate_init` and `br`.
744 loop.block = .init(res_block.stealRemainingCapacity());709 const inst_buf = try sfba.alloc(Air.Inst.Index, inst_per_elem * res_len + 2);
745 {710 defer sfba.free(inst_buf);
746 const cur_index_inst = loop.block.add(l, .{
747 .tag = .load,
748 .data = .{ .ty_op = .{
749 .ty = .usize_type,
750 .operand = index_alloc_inst.toRef(),
751 } },
752 });
753 _ = loop.block.add(l, .{
754 .tag = .vector_store_elem,
755 .data = .{ .vector_store_elem = .{
756 .vector_ptr = res_alloc_inst.toRef(),
757 .payload = try l.addExtra(Air.Bin, .{
758 .lhs = cur_index_inst.toRef(),
759 .rhs = res_elem: switch (form) {
760 .un_op => loop.block.add(l, .{
761 .tag = orig.tag,
762 .data = .{ .un_op = loop.block.add(l, .{
763 .tag = .array_elem_val,
764 .data = .{ .bin_op = .{
765 .lhs = orig.data.un_op,
766 .rhs = cur_index_inst.toRef(),
767 } },
768 }).toRef() },
769 }).toRef(),
770 .ty_op => loop.block.add(l, .{
771 .tag = orig.tag,
772 .data = .{ .ty_op = .{
773 .ty = Air.internedToRef(res_ty.childType(zcu).toIntern()),
774 .operand = loop.block.add(l, .{
775 .tag = .array_elem_val,
776 .data = .{ .bin_op = .{
777 .lhs = orig.data.ty_op.operand,
778 .rhs = cur_index_inst.toRef(),
779 } },
780 }).toRef(),
781 } },
782 }).toRef(),
783 .bin_op => loop.block.add(l, .{
784 .tag = orig.tag,
785 .data = .{ .bin_op = .{
786 .lhs = loop.block.add(l, .{
787 .tag = .array_elem_val,
788 .data = .{ .bin_op = .{
789 .lhs = orig.data.bin_op.lhs,
790 .rhs = cur_index_inst.toRef(),
791 } },
792 }).toRef(),
793 .rhs = loop.block.add(l, .{
794 .tag = .array_elem_val,
795 .data = .{ .bin_op = .{
796 .lhs = orig.data.bin_op.rhs,
797 .rhs = cur_index_inst.toRef(),
798 } },
799 }).toRef(),
800 } },
801 }).toRef(),
802 .pl_op_bin => {
803 const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
804 break :res_elem loop.block.add(l, .{
805 .tag = orig.tag,
806 .data = .{ .pl_op = .{
807 .payload = try l.addExtra(Air.Bin, .{
808 .lhs = loop.block.add(l, .{
809 .tag = .array_elem_val,
810 .data = .{ .bin_op = .{
811 .lhs = extra.lhs,
812 .rhs = cur_index_inst.toRef(),
813 } },
814 }).toRef(),
815 .rhs = loop.block.add(l, .{
816 .tag = .array_elem_val,
817 .data = .{ .bin_op = .{
818 .lhs = extra.rhs,
819 .rhs = cur_index_inst.toRef(),
820 } },
821 }).toRef(),
822 }),
823 .operand = loop.block.add(l, .{
824 .tag = .array_elem_val,
825 .data = .{ .bin_op = .{
826 .lhs = orig.data.pl_op.operand,
827 .rhs = cur_index_inst.toRef(),
828 } },
829 }).toRef(),
830 } },
831 }).toRef();
832 },
833 .bitcast => loop.block.addBitCast(l, res_ty.childType(zcu), loop.block.add(l, .{
834 .tag = .array_elem_val,
835 .data = .{ .bin_op = .{
836 .lhs = orig.data.ty_op.operand,
837 .rhs = cur_index_inst.toRef(),
838 } },
839 }).toRef()),
840 .cmp_vector => {
841 const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
842 break :res_elem (try loop.block.addCmp(
843 l,
844 extra.compareOperator(),
845 loop.block.add(l, .{
846 .tag = .array_elem_val,
847 .data = .{ .bin_op = .{
848 .lhs = extra.lhs,
849 .rhs = cur_index_inst.toRef(),
850 } },
851 }).toRef(),
852 loop.block.add(l, .{
853 .tag = .array_elem_val,
854 .data = .{ .bin_op = .{
855 .lhs = extra.rhs,
856 .rhs = cur_index_inst.toRef(),
857 } },
858 }).toRef(),
859 .{ .optimized = switch (orig.tag) {
860 else => unreachable,
861 .cmp_vector => false,
862 .cmp_vector_optimized => true,
863 } },
864 )).toRef();
865 },
866 .shuffle_one, .shuffle_two => {
867 const ip = &zcu.intern_pool;
868 const unwrapped = switch (form) {
869 else => comptime unreachable,
870 .shuffle_one => l.getTmpAir().unwrapShuffleOne(zcu, orig_inst),
871 .shuffle_two => l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst),
872 };
873 const operand_a = switch (form) {
874 else => comptime unreachable,
875 .shuffle_one => unwrapped.operand,
876 .shuffle_two => unwrapped.operand_a,
877 };
878 const operand_a_len = l.typeOf(operand_a).vectorLen(zcu);
879 const elem_ty = res_ty.childType(zcu);
880 var res_elem: Result = .init(l, elem_ty, &loop.block);
881 res_elem.block = .init(loop.block.stealCapacity(extra_insts));
882 {
883 const ExpectedContents = extern struct {
884 mask_elems: [128]InternPool.Index,
885 ct_elems: switch (form) {
886 else => unreachable,
887 .shuffle_one => extern struct {
888 keys: [152]InternPool.Index,
889 header: u8 align(@alignOf(u32)),
890 index: [256][2]u8,
891 },
892 .shuffle_two => void,
893 },
894 };
895 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
896 std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa);
897 const gpa = stack.get();
898711
899 const mask_elems = try gpa.alloc(InternPool.Index, res_len);712 var main_block: Block = .init(inst_buf);
900 defer gpa.free(mask_elems);713 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
901
902 var ct_elems: switch (form) {
903 else => unreachable,
904 .shuffle_one => std.AutoArrayHashMapUnmanaged(InternPool.Index, void),
905 .shuffle_two => struct {
906 const empty: @This() = .{};
907 inline fn deinit(_: @This(), _: std.mem.Allocator) void {}
908 inline fn ensureTotalCapacity(_: @This(), _: std.mem.Allocator, _: usize) error{}!void {}
909 },
910 } = .empty;
911 defer ct_elems.deinit(gpa);
912 try ct_elems.ensureTotalCapacity(gpa, res_len);
913
914 const mask_elem_ty = try pt.intType(.signed, 1 + Type.smallestUnsignedBits(@max(operand_a_len, switch (form) {
915 else => comptime unreachable,
916 .shuffle_one => res_len,
917 .shuffle_two => l.typeOf(unwrapped.operand_b).vectorLen(zcu),
918 })));
919 for (mask_elems, unwrapped.mask) |*mask_elem_val, mask_elem| mask_elem_val.* = (try pt.intValue(mask_elem_ty, switch (form) {
920 else => comptime unreachable,
921 .shuffle_one => switch (mask_elem.unwrap()) {
922 .elem => |index| index,
923 .value => |elem_val| if (ip.isUndef(elem_val))
924 operand_a_len
925 else
926 ~@as(i33, @intCast((ct_elems.getOrPutAssumeCapacity(elem_val)).index)),
927 },
928 .shuffle_two => switch (mask_elem.unwrap()) {
929 .a_elem => |a_index| a_index,
930 .b_elem => |b_index| ~@as(i33, b_index),
931 .undef => operand_a_len,
932 },
933 })).toIntern();
934 const mask_ty = try pt.arrayType(.{
935 .len = res_len,
936 .child = mask_elem_ty.toIntern(),
937 });
938 const mask_elem_inst = res_elem.block.add(l, .{
939 .tag = .ptr_elem_val,
940 .data = .{ .bin_op = .{
941 .lhs = Air.internedToRef(try pt.intern(.{ .ptr = .{
942 .ty = (try pt.manyConstPtrType(mask_elem_ty)).toIntern(),
943 .base_addr = .{ .uav = .{
944 .val = (try pt.aggregateValue(mask_ty, mask_elems)).toIntern(),
945 .orig_ty = (try pt.singleConstPtrType(mask_ty)).toIntern(),
946 } },
947 .byte_offset = 0,
948 } })),
949 .rhs = cur_index_inst.toRef(),
950 } },
951 });
952 var def_cond_br: CondBr = .init(l, (try res_elem.block.addCmp(
953 l,
954 .lt,
955 mask_elem_inst.toRef(),
956 try pt.intRef(mask_elem_ty, operand_a_len),
957 .{},
958 )).toRef(), &res_elem.block, .{});
959 def_cond_br.then_block = .init(res_elem.block.stealRemainingCapacity());
960 {
961 const operand_b_used = switch (form) {
962 else => comptime unreachable,
963 .shuffle_one => ct_elems.count() > 0,
964 .shuffle_two => true,
965 };
966 var operand_cond_br: CondBr = undefined;
967 operand_cond_br.then_block = if (operand_b_used) then_block: {
968 operand_cond_br = .init(l, (try def_cond_br.then_block.addCmp(
969 l,
970 .gte,
971 mask_elem_inst.toRef(),
972 try pt.intRef(mask_elem_ty, 0),
973 .{},
974 )).toRef(), &def_cond_br.then_block, .{});
975 break :then_block .init(def_cond_br.then_block.stealRemainingCapacity());
976 } else def_cond_br.then_block;
977 _ = operand_cond_br.then_block.add(l, .{
978 .tag = .br,
979 .data = .{ .br = .{
980 .block_inst = res_elem.inst,
981 .operand = operand_cond_br.then_block.add(l, .{
982 .tag = .array_elem_val,
983 .data = .{ .bin_op = .{
984 .lhs = operand_a,
985 .rhs = operand_cond_br.then_block.add(l, .{
986 .tag = .intcast,
987 .data = .{ .ty_op = .{
988 .ty = .usize_type,
989 .operand = mask_elem_inst.toRef(),
990 } },
991 }).toRef(),
992 } },
993 }).toRef(),
994 } },
995 });
996 if (operand_b_used) {
997 operand_cond_br.else_block = .init(operand_cond_br.then_block.stealRemainingCapacity());
998 _ = operand_cond_br.else_block.add(l, .{
999 .tag = .br,
1000 .data = .{ .br = .{
1001 .block_inst = res_elem.inst,
1002 .operand = if (switch (form) {
1003 else => comptime unreachable,
1004 .shuffle_one => ct_elems.count() > 1,
1005 .shuffle_two => true,
1006 }) operand_cond_br.else_block.add(l, .{
1007 .tag = switch (form) {
1008 else => comptime unreachable,
1009 .shuffle_one => .ptr_elem_val,
1010 .shuffle_two => .array_elem_val,
1011 },
1012 .data = .{ .bin_op = .{
1013 .lhs = operand_b: switch (form) {
1014 else => comptime unreachable,
1015 .shuffle_one => {
1016 const ct_elems_ty = try pt.arrayType(.{
1017 .len = ct_elems.count(),
1018 .child = elem_ty.toIntern(),
1019 });
1020 break :operand_b Air.internedToRef(try pt.intern(.{ .ptr = .{
1021 .ty = (try pt.manyConstPtrType(elem_ty)).toIntern(),
1022 .base_addr = .{ .uav = .{
1023 .val = (try pt.aggregateValue(ct_elems_ty, ct_elems.keys())).toIntern(),
1024 .orig_ty = (try pt.singleConstPtrType(ct_elems_ty)).toIntern(),
1025 } },
1026 .byte_offset = 0,
1027 } }));
1028 },
1029 .shuffle_two => unwrapped.operand_b,
1030 },
1031 .rhs = operand_cond_br.else_block.add(l, .{
1032 .tag = .intcast,
1033 .data = .{ .ty_op = .{
1034 .ty = .usize_type,
1035 .operand = operand_cond_br.else_block.add(l, .{
1036 .tag = .not,
1037 .data = .{ .ty_op = .{
1038 .ty = Air.internedToRef(mask_elem_ty.toIntern()),
1039 .operand = mask_elem_inst.toRef(),
1040 } },
1041 }).toRef(),
1042 } },
1043 }).toRef(),
1044 } },
1045 }).toRef() else res_elem_br: {
1046 _ = operand_cond_br.else_block.stealCapacity(3);
1047 break :res_elem_br Air.internedToRef(ct_elems.keys()[0]);
1048 },
1049 } },
1050 });
1051 def_cond_br.else_block = .init(operand_cond_br.else_block.stealRemainingCapacity());
1052 try operand_cond_br.finish(l);
1053 } else {
1054 def_cond_br.then_block = operand_cond_br.then_block;
1055 _ = def_cond_br.then_block.stealCapacity(6);
1056 def_cond_br.else_block = .init(def_cond_br.then_block.stealRemainingCapacity());
1057 }
1058 }
1059 _ = def_cond_br.else_block.add(l, .{
1060 .tag = .br,
1061 .data = .{ .br = .{
1062 .block_inst = res_elem.inst,
1063 .operand = try pt.undefRef(elem_ty),
1064 } },
1065 });
1066 try def_cond_br.finish(l);
1067 }
1068 try res_elem.finish(l);
1069 break :res_elem res_elem.inst.toRef();
1070 },
1071 .select => {
1072 const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
1073 var res_elem: Result = .init(l, l.typeOf(extra.lhs).childType(zcu), &loop.block);
1074 res_elem.block = .init(loop.block.stealCapacity(extra_insts));
1075 {
1076 var select_cond_br: CondBr = .init(l, res_elem.block.add(l, .{
1077 .tag = .array_elem_val,
1078 .data = .{ .bin_op = .{
1079 .lhs = orig.data.pl_op.operand,
1080 .rhs = cur_index_inst.toRef(),
1081 } },
1082 }).toRef(), &res_elem.block, .{});
1083 select_cond_br.then_block = .init(res_elem.block.stealRemainingCapacity());
1084 _ = select_cond_br.then_block.add(l, .{
1085 .tag = .br,
1086 .data = .{ .br = .{
1087 .block_inst = res_elem.inst,
1088 .operand = select_cond_br.then_block.add(l, .{
1089 .tag = .array_elem_val,
1090 .data = .{ .bin_op = .{
1091 .lhs = extra.lhs,
1092 .rhs = cur_index_inst.toRef(),
1093 } },
1094 }).toRef(),
1095 } },
1096 });
1097 select_cond_br.else_block = .init(select_cond_br.then_block.stealRemainingCapacity());
1098 _ = select_cond_br.else_block.add(l, .{
1099 .tag = .br,
1100 .data = .{ .br = .{
1101 .block_inst = res_elem.inst,
1102 .operand = select_cond_br.else_block.add(l, .{
1103 .tag = .array_elem_val,
1104 .data = .{ .bin_op = .{
1105 .lhs = extra.rhs,
1106 .rhs = cur_index_inst.toRef(),
1107 } },
1108 }).toRef(),
1109 } },
1110 });
1111 try select_cond_br.finish(l);
1112 }
1113 try res_elem.finish(l);
1114 break :res_elem res_elem.inst.toRef();
1115 },
1116 },
1117 }),
1118 } },
1119 });
1120714
1121 var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(715 const elem_buf = try sfba.alloc(Air.Inst.Ref, res_len);
1122 l,716 defer sfba.free(elem_buf);
1123 .lt,717
1124 cur_index_inst.toRef(),718 switch (form) {
1125 try pt.intRef(.usize, res_len - 1),719 .un_op => {
1126 .{},720 const orig_operand = orig.data.un_op;
1127 )).toRef(), &loop.block, .{});721 const un_op_tag = orig.tag;
1128 loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());722 for (elem_buf, 0..) |*elem, elem_idx| {
1129 {723 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
1130 _ = loop_cond_br.then_block.add(l, .{724 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
1131 .tag = .store,725 elem.* = main_block.addUnOp(l, un_op_tag, operand).toRef();
1132 .data = .{ .bin_op = .{726 }
1133 .lhs = index_alloc_inst.toRef(),727 },
1134 .rhs = loop_cond_br.then_block.add(l, .{728 .ty_op => {
1135 .tag = .add,729 const orig_operand = orig.data.ty_op.operand;
1136 .data = .{ .bin_op = .{730 const orig_ty: Type = .fromInterned(orig.data.ty_op.ty.toInterned().?);
1137 .lhs = cur_index_inst.toRef(),731 const scalar_ty = orig_ty.childType(zcu);
1138 .rhs = .one_usize,732 const ty_op_tag = orig.tag;
1139 } },733 for (elem_buf, 0..) |*elem, elem_idx| {
1140 }).toRef(),734 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
735 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
736 elem.* = main_block.addTyOp(l, ty_op_tag, scalar_ty, operand).toRef();
737 }
738 },
739 .bin_op => {
740 const orig_operands = orig.data.bin_op;
741 const bin_op_tag = orig.tag;
742 for (elem_buf, 0..) |*elem, elem_idx| {
743 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
744 const lhs = main_block.addBinOp(l, .array_elem_val, orig_operands.lhs, elem_idx_ref).toRef();
745 const rhs = main_block.addBinOp(l, .array_elem_val, orig_operands.rhs, elem_idx_ref).toRef();
746 elem.* = main_block.addBinOp(l, bin_op_tag, lhs, rhs).toRef();
747 }
748 },
749 .pl_op_bin => {
750 const orig_operand = orig.data.pl_op.operand;
751 const orig_payload = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
752 const pl_op_tag = orig.tag;
753 for (elem_buf, 0..) |*elem, elem_idx| {
754 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
755 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
756 const lhs = main_block.addBinOp(l, .array_elem_val, orig_payload.lhs, elem_idx_ref).toRef();
757 const rhs = main_block.addBinOp(l, .array_elem_val, orig_payload.rhs, elem_idx_ref).toRef();
758 elem.* = main_block.add(l, .{
759 .tag = pl_op_tag,
760 .data = .{ .pl_op = .{
761 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
762 .operand = operand,
1141 } },763 } },
1142 });764 }).toRef();
1143 _ = loop_cond_br.then_block.add(l, .{
1144 .tag = .repeat,
1145 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1146 });
1147 }765 }
1148 loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());766 },
1149 _ = loop_cond_br.else_block.add(l, .{767 .cmp_vector => {
1150 .tag = .br,768 const orig_payload = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
1151 .data = .{ .br = .{769 const cmp_op = orig_payload.compareOperator();
1152 .block_inst = orig_inst,770 const optimized = switch (orig.tag) {
1153 .operand = loop_cond_br.else_block.add(l, .{771 .cmp_vector => false,
1154 .tag = .load,772 .cmp_vector_optimized => true,
1155 .data = .{ .ty_op = .{773 else => unreachable,
1156 .ty = Air.internedToRef(res_ty.toIntern()),774 };
1157 .operand = res_alloc_inst.toRef(),775 for (elem_buf, 0..) |*elem, elem_idx| {
1158 } },776 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
1159 }).toRef(),777 const lhs = main_block.addBinOp(l, .array_elem_val, orig_payload.lhs, elem_idx_ref).toRef();
1160 } },778 const rhs = main_block.addBinOp(l, .array_elem_val, orig_payload.rhs, elem_idx_ref).toRef();
1161 });779 elem.* = main_block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
1162 try loop_cond_br.finish(l);780 }
1163 }781 },
1164 try loop.finish(l);782 .shuffle_one => {
1165 }783 const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
1166 return .{ .ty_pl = .{784 for (elem_buf, shuffle.mask) |*elem, mask| elem.* = switch (mask.unwrap()) {
1167 .ty = Air.internedToRef(res_ty.toIntern()),785 .value => |val| .fromIntern(val),
1168 .payload = try l.addBlockBody(res_block.body()),786 .elem => |src_idx| elem: {
1169 } };787 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
1170}788 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand, src_idx_ref).toRef();
1171fn scalarizeBitcastToArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {789 },
1172 const pt = l.pt;790 };
1173 const zcu = pt.zcu;791 },
792 .shuffle_two => {
793 const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
794 const scalar_ty = res_ty.childType(zcu);
795 for (elem_buf, shuffle.mask) |*elem, mask| elem.* = switch (mask.unwrap()) {
796 .undef => .fromValue(try pt.undefValue(scalar_ty)),
797 .a_elem => |src_idx| elem: {
798 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
799 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand_a, src_idx_ref).toRef();
800 },
801 .b_elem => |src_idx| elem: {
802 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
803 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand_b, src_idx_ref).toRef();
804 },
805 };
806 },
807 .select => {
808 const orig_cond = orig.data.pl_op.operand;
809 const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
810 const res_scalar_ty = res_ty.childType(zcu);
811 for (elem_buf, 0..) |*elem, elem_idx| {
812 // Payload to be populated later; we need the index early for `br`s.
813 const elem_block_inst = main_block.add(l, .{
814 .tag = .block,
815 .data = .{ .ty_pl = .{
816 .ty = .fromType(res_scalar_ty),
817 .payload = undefined,
818 } },
819 });
820 var elem_block: Block = .init(main_block.stealCapacity(2));
1174821
1175 const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;822 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
1176 const res_ty = orig_ty_op.ty.toType();823 const cond = elem_block.addBinOp(l, .array_elem_val, orig_cond, elem_idx_ref).toRef();
1177 const res_elem_ty = res_ty.childType(zcu);824 var condbr: CondBr = .init(l, cond, &elem_block, .{});
1178 const res_len = res_ty.arrayLen(zcu);
1179825
1180 var inst_buf: [16]Air.Inst.Index = undefined;826 condbr.then_block = .init(main_block.stealCapacity(2));
1181 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);827 const lhs = condbr.then_block.addBinOp(l, .array_elem_val, orig_bin.lhs, elem_idx_ref).toRef();
828 condbr.then_block.addBr(l, elem_block_inst, lhs);
1182829
1183 var res_block: Block = .init(&inst_buf);830 condbr.else_block = .init(main_block.stealCapacity(2));
1184 {831 const rhs = condbr.else_block.addBinOp(l, .array_elem_val, orig_bin.rhs, elem_idx_ref).toRef();
1185 const res_alloc_inst = res_block.add(l, .{832 condbr.else_block.addBr(l, elem_block_inst, rhs);
1186 .tag = .alloc,
1187 .data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1188 });
1189 const index_alloc_inst = res_block.add(l, .{
1190 .tag = .alloc,
1191 .data = .{ .ty = .ptr_usize },
1192 });
1193 _ = res_block.add(l, .{
1194 .tag = .store,
1195 .data = .{ .bin_op = .{
1196 .lhs = index_alloc_inst.toRef(),
1197 .rhs = .zero_usize,
1198 } },
1199 });
1200833
1201 var loop: Loop = .init(l, &res_block);834 try condbr.finish(l);
1202 loop.block = .init(res_block.stealRemainingCapacity());
1203 {
1204 const cur_index_inst = loop.block.add(l, .{
1205 .tag = .load,
1206 .data = .{ .ty_op = .{
1207 .ty = .usize_type,
1208 .operand = index_alloc_inst.toRef(),
1209 } },
1210 });
1211 _ = loop.block.add(l, .{
1212 .tag = .store,
1213 .data = .{ .bin_op = .{
1214 .lhs = loop.block.add(l, .{
1215 .tag = .ptr_elem_ptr,
1216 .data = .{ .ty_pl = .{
1217 .ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()),
1218 .payload = try l.addExtra(Air.Bin, .{
1219 .lhs = res_alloc_inst.toRef(),
1220 .rhs = cur_index_inst.toRef(),
1221 }),
1222 } },
1223 }).toRef(),
1224 .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1225 .tag = .array_elem_val,
1226 .data = .{ .bin_op = .{
1227 .lhs = orig_ty_op.operand,
1228 .rhs = cur_index_inst.toRef(),
1229 } },
1230 }).toRef()),
1231 } },
1232 });
1233835
1234 var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(836 const inst_data = l.air_instructions.items(.data);
1235 l,837 inst_data[@intFromEnum(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());
1236 .lt,838
1237 cur_index_inst.toRef(),839 elem.* = elem_block_inst.toRef();
1238 try pt.intRef(.usize, res_len - 1),
1239 .{},
1240 )).toRef(), &loop.block, .{});
1241 loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1242 {
1243 _ = loop_cond_br.then_block.add(l, .{
1244 .tag = .store,
1245 .data = .{ .bin_op = .{
1246 .lhs = index_alloc_inst.toRef(),
1247 .rhs = loop_cond_br.then_block.add(l, .{
1248 .tag = .add,
1249 .data = .{ .bin_op = .{
1250 .lhs = cur_index_inst.toRef(),
1251 .rhs = .one_usize,
1252 } },
1253 }).toRef(),
1254 } },
1255 });
1256 _ = loop_cond_br.then_block.add(l, .{
1257 .tag = .repeat,
1258 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1259 });
1260 }840 }
1261 loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());841 },
1262 _ = loop_cond_br.else_block.add(l, .{
1263 .tag = .br,
1264 .data = .{ .br = .{
1265 .block_inst = orig_inst,
1266 .operand = loop_cond_br.else_block.add(l, .{
1267 .tag = .load,
1268 .data = .{ .ty_op = .{
1269 .ty = Air.internedToRef(res_ty.toIntern()),
1270 .operand = res_alloc_inst.toRef(),
1271 } },
1272 }).toRef(),
1273 } },
1274 });
1275 try loop_cond_br.finish(l);
1276 }
1277 try loop.finish(l);
1278 }842 }
843
844 const result = main_block.add(l, .{
845 .tag = .aggregate_init,
846 .data = .{ .ty_pl = .{
847 .ty = .fromType(res_ty),
848 .payload = payload: {
849 const idx = l.air_extra.items.len;
850 try l.air_extra.appendSlice(gpa, @ptrCast(elem_buf));
851 break :payload @intCast(idx);
852 },
853 } },
854 }).toRef();
855
856 main_block.addBr(l, orig_inst, result);
857
858 // Some `form` values may intentionally not use the full instruction buffer.
859 switch (form) {
860 .un_op,
861 .ty_op,
862 .bin_op,
863 .pl_op_bin,
864 .cmp_vector,
865 .select,
866 => {},
867 .shuffle_one,
868 .shuffle_two,
869 => _ = main_block.stealRemainingCapacity(),
870 }
871
1279 return .{ .ty_pl = .{872 return .{ .ty_pl = .{
1280 .ty = Air.internedToRef(res_ty.toIntern()),873 .ty = .fromType(res_ty),
1281 .payload = try l.addBlockBody(res_block.body()),874 .payload = try l.addBlockBody(main_block.body()),
1282 } };875 } };
1283}876}
1284fn scalarizeBitcastOperandBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {877fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
1285 const pt = l.pt;878 const pt = l.pt;
1286 const zcu = pt.zcu;879 const zcu = pt.zcu;
880 const gpa = zcu.gpa;
1287881
1288 const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;882 var sfba_state = std.heap.stackFallback(512, gpa);
1289 const res_ty = orig_ty_op.ty.toType();883 const sfba = sfba_state.get();
1290 const operand_ty = l.typeOf(orig_ty_op.operand);
1291 const int_bits: u16 = @intCast(operand_ty.bitSize(zcu));
1292 const int_ty = try pt.intType(.unsigned, int_bits);
1293 const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1294 const elem_bits: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu));
1295 const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1296
1297 var inst_buf: [22]Air.Inst.Index = undefined;
1298 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1299884
1300 var res_block: Block = .init(&inst_buf);885 const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1301 {
1302 const int_alloc_inst = res_block.add(l, .{
1303 .tag = .alloc,
1304 .data = .{ .ty = try pt.singleMutPtrType(int_ty) },
1305 });
1306 _ = res_block.add(l, .{
1307 .tag = .store,
1308 .data = .{ .bin_op = .{
1309 .lhs = int_alloc_inst.toRef(),
1310 .rhs = try pt.intRef(int_ty, 0),
1311 } },
1312 });
1313 const index_alloc_inst = res_block.add(l, .{
1314 .tag = .alloc,
1315 .data = .{ .ty = .ptr_usize },
1316 });
1317 _ = res_block.add(l, .{
1318 .tag = .store,
1319 .data = .{ .bin_op = .{
1320 .lhs = index_alloc_inst.toRef(),
1321 .rhs = .zero_usize,
1322 } },
1323 });
1324886
1325 var loop: Loop = .init(l, &res_block);887 const dest_ty = ty_op.ty.toType();
1326 loop.block = .init(res_block.stealRemainingCapacity());888 const dest_legal = switch (dest_ty.zigTypeTag(zcu)) {
1327 {889 else => true,
1328 const cur_index_inst = loop.block.add(l, .{890 .array, .vector => legal: {
1329 .tag = .load,891 if (dest_ty.arrayLen(zcu) == 1) break :legal true;
1330 .data = .{ .ty_op = .{892 const dest_elem_ty = dest_ty.childType(zcu);
1331 .ty = .usize_type,893 break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu);
1332 .operand = index_alloc_inst.toRef(),894 },
1333 } },895 };
1334 });
1335 const cur_int_inst = loop.block.add(l, .{
1336 .tag = .bit_or,
1337 .data = .{ .bin_op = .{
1338 .lhs = loop.block.add(l, .{
1339 .tag = .shl_exact,
1340 .data = .{ .bin_op = .{
1341 .lhs = loop.block.add(l, .{
1342 .tag = .intcast,
1343 .data = .{ .ty_op = .{
1344 .ty = Air.internedToRef(int_ty.toIntern()),
1345 .operand = loop.block.addBitCast(l, elem_int_ty, loop.block.add(l, .{
1346 .tag = .array_elem_val,
1347 .data = .{ .bin_op = .{
1348 .lhs = orig_ty_op.operand,
1349 .rhs = cur_index_inst.toRef(),
1350 } },
1351 }).toRef()),
1352 } },
1353 }).toRef(),
1354 .rhs = loop.block.add(l, .{
1355 .tag = .mul,
1356 .data = .{ .bin_op = .{
1357 .lhs = loop.block.add(l, .{
1358 .tag = .intcast,
1359 .data = .{ .ty_op = .{
1360 .ty = Air.internedToRef(shift_ty.toIntern()),
1361 .operand = cur_index_inst.toRef(),
1362 } },
1363 }).toRef(),
1364 .rhs = try pt.intRef(shift_ty, elem_bits),
1365 } },
1366 }).toRef(),
1367 } },
1368 }).toRef(),
1369 .rhs = loop.block.add(l, .{
1370 .tag = .load,
1371 .data = .{ .ty_op = .{
1372 .ty = Air.internedToRef(int_ty.toIntern()),
1373 .operand = int_alloc_inst.toRef(),
1374 } },
1375 }).toRef(),
1376 } },
1377 });
1378896
1379 var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(897 const operand_ty = l.typeOf(ty_op.operand);
1380 l,898 const operand_legal = switch (operand_ty.zigTypeTag(zcu)) {
1381 .lt,899 else => true,
1382 cur_index_inst.toRef(),900 .array, .vector => legal: {
1383 try pt.intRef(.usize, operand_ty.arrayLen(zcu) - 1),901 if (operand_ty.arrayLen(zcu) == 1) break :legal true;
1384 .{},902 const operand_elem_ty = operand_ty.childType(zcu);
1385 )).toRef(), &loop.block, .{});903 break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu);
1386 loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());904 },
1387 {905 };
1388 _ = loop_cond_br.then_block.add(l, .{
1389 .tag = .store,
1390 .data = .{ .bin_op = .{
1391 .lhs = int_alloc_inst.toRef(),
1392 .rhs = cur_int_inst.toRef(),
1393 } },
1394 });
1395 _ = loop_cond_br.then_block.add(l, .{
1396 .tag = .store,
1397 .data = .{ .bin_op = .{
1398 .lhs = index_alloc_inst.toRef(),
1399 .rhs = loop_cond_br.then_block.add(l, .{
1400 .tag = .add,
1401 .data = .{ .bin_op = .{
1402 .lhs = cur_index_inst.toRef(),
1403 .rhs = .one_usize,
1404 } },
1405 }).toRef(),
1406 } },
1407 });
1408 _ = loop_cond_br.then_block.add(l, .{
1409 .tag = .repeat,
1410 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1411 });
1412 }
1413 loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1414 _ = loop_cond_br.else_block.add(l, .{
1415 .tag = .br,
1416 .data = .{ .br = .{
1417 .block_inst = orig_inst,
1418 .operand = loop_cond_br.else_block.addBitCast(l, res_ty, cur_int_inst.toRef()),
1419 } },
1420 });
1421 try loop_cond_br.finish(l);
1422 }
1423 try loop.finish(l);
1424 }
1425 return .{ .ty_pl = .{
1426 .ty = Air.internedToRef(res_ty.toIntern()),
1427 .payload = try l.addBlockBody(res_block.body()),
1428 } };
1429}
1430fn scalarizeBitcastResultArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1431 const pt = l.pt;
1432 const zcu = pt.zcu;
1433906
1434 const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;907 if (dest_legal and operand_legal) return null;
1435 const res_ty = orig_ty_op.ty.toType();
1436 const int_bits: u16 = @intCast(res_ty.bitSize(zcu));
1437 const int_ty = try pt.intType(.unsigned, int_bits);
1438 const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1439 const res_elem_ty = res_ty.childType(zcu);
1440 const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu));
1441 const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1442
1443 var inst_buf: [20]Air.Inst.Index = undefined;
1444 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1445908
1446 var res_block: Block = .init(&inst_buf);909 if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) {
1447 {910 // from_ty and to_ty are both arrays or vectors of types with the same bit size,
1448 const res_alloc_inst = res_block.add(l, .{911 // so we can do an elementwise bitcast.
1449 .tag = .alloc,912 return try l.scalarizeBlockPayload(orig_inst, .ty_op);
1450 .data = .{ .ty = try pt.singleMutPtrType(res_ty) },913 }
1451 });
1452 const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand);
1453 const index_alloc_inst = res_block.add(l, .{
1454 .tag = .alloc,
1455 .data = .{ .ty = .ptr_usize },
1456 });
1457 _ = res_block.add(l, .{
1458 .tag = .store,
1459 .data = .{ .bin_op = .{
1460 .lhs = index_alloc_inst.toRef(),
1461 .rhs = .zero_usize,
1462 } },
1463 });
1464914
1465 var loop: Loop = .init(l, &res_block);915 // Fallback path. Our strategy is to use an unsigned integer type as an intermediate
1466 loop.block = .init(res_block.stealRemainingCapacity());916 // "bag of bits" representation which can be manipulated by bitwise operations.
1467 {
1468 const cur_index_inst = loop.block.add(l, .{
1469 .tag = .load,
1470 .data = .{ .ty_op = .{
1471 .ty = .usize_type,
1472 .operand = index_alloc_inst.toRef(),
1473 } },
1474 });
1475 _ = loop.block.add(l, .{
1476 .tag = .store,
1477 .data = .{ .bin_op = .{
1478 .lhs = loop.block.add(l, .{
1479 .tag = .ptr_elem_ptr,
1480 .data = .{ .ty_pl = .{
1481 .ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()),
1482 .payload = try l.addExtra(Air.Bin, .{
1483 .lhs = res_alloc_inst.toRef(),
1484 .rhs = cur_index_inst.toRef(),
1485 }),
1486 } },
1487 }).toRef(),
1488 .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1489 .tag = .trunc,
1490 .data = .{ .ty_op = .{
1491 .ty = Air.internedToRef(elem_int_ty.toIntern()),
1492 .operand = loop.block.add(l, .{
1493 .tag = .shr,
1494 .data = .{ .bin_op = .{
1495 .lhs = int_ref,
1496 .rhs = loop.block.add(l, .{
1497 .tag = .mul,
1498 .data = .{ .bin_op = .{
1499 .lhs = loop.block.add(l, .{
1500 .tag = .intcast,
1501 .data = .{ .ty_op = .{
1502 .ty = Air.internedToRef(shift_ty.toIntern()),
1503 .operand = cur_index_inst.toRef(),
1504 } },
1505 }).toRef(),
1506 .rhs = try pt.intRef(shift_ty, elem_bits),
1507 } },
1508 }).toRef(),
1509 } },
1510 }).toRef(),
1511 } },
1512 }).toRef()),
1513 } },
1514 });
1515917
1516 var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(918 const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
1517 l,919 assert(operand_ty.bitSize(zcu) == num_bits);
1518 .lt,920 const uint_ty = try pt.intType(.unsigned, num_bits);
1519 cur_index_inst.toRef(),921 const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
1520 try pt.intRef(.usize, res_ty.arrayLen(zcu) - 1),922
1521 .{},923 const inst_buf = try sfba.alloc(Air.Inst.Index, len: {
1522 )).toRef(), &loop.block, .{});924 const operand_to_uint_len: u64 = if (operand_legal) 1 else (operand_ty.arrayLen(zcu) * 5);
1523 loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());925 const uint_to_dest_len: u64 = if (dest_legal) 1 else (dest_ty.arrayLen(zcu) * 3 + 1);
1524 {926 break :len @intCast(operand_to_uint_len + uint_to_dest_len + 1);
1525 _ = loop_cond_br.then_block.add(l, .{927 });
1526 .tag = .store,928 defer sfba.free(inst_buf);
1527 .data = .{ .bin_op = .{929 var main_block: Block = .init(inst_buf);
1528 .lhs = index_alloc_inst.toRef(),930 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
1529 .rhs = loop_cond_br.then_block.add(l, .{931
1530 .tag = .add,932 // First, convert `operand_ty` to `uint_ty` (`uN`).
1531 .data = .{ .bin_op = .{933
1532 .lhs = cur_index_inst.toRef(),934 const uint_val: Air.Inst.Ref = uint_val: {
1533 .rhs = .one_usize,935 if (operand_legal) break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
1534 } },936
1535 }).toRef(),937 const bits_per_elem: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu));
1536 } },938 const bits_per_elem_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, bits_per_elem));
1537 });939 const elem_uint_ty = try pt.intType(.unsigned, bits_per_elem);
1538 _ = loop_cond_br.then_block.add(l, .{940
1539 .tag = .repeat,941 var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
1540 .data = .{ .repeat = .{ .loop_inst = loop.inst } },942 var elem_idx = operand_ty.arrayLen(zcu);
1541 });943 while (elem_idx > 0) {
1542 }944 elem_idx -= 1;
1543 loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());945 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
1544 _ = loop_cond_br.else_block.add(l, .{946 const orig_elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, elem_idx_ref).toRef();
1545 .tag = .br,947 const elem_as_uint = main_block.addBitCast(l, elem_uint_ty, orig_elem);
1546 .data = .{ .br = .{948 const elem_extended = main_block.addTyOp(l, .intcast, uint_ty, elem_as_uint).toRef();
1547 .block_inst = orig_inst,949 cur_uint = main_block.addBinOp(l, .shl_exact, cur_uint, bits_per_elem_ref).toRef();
1548 .operand = loop_cond_br.else_block.add(l, .{950 cur_uint = main_block.addBinOp(l, .bit_or, cur_uint, elem_extended).toRef();
1549 .tag = .load,
1550 .data = .{ .ty_op = .{
1551 .ty = Air.internedToRef(res_ty.toIntern()),
1552 .operand = res_alloc_inst.toRef(),
1553 } },
1554 }).toRef(),
1555 } },
1556 });
1557 try loop_cond_br.finish(l);
1558 }951 }
1559 try loop.finish(l);952 break :uint_val cur_uint;
1560 }953 };
1561 return .{ .ty_pl = .{
1562 .ty = Air.internedToRef(res_ty.toIntern()),
1563 .payload = try l.addBlockBody(res_block.body()),
1564 } };
1565}
1566fn scalarizeBitcastResultVectorBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1567 const pt = l.pt;
1568 const zcu = pt.zcu;
1569954
1570 const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;955 // Now convert `uint_ty` (`uN`) to `dest_ty`.
1571 const res_ty = orig_ty_op.ty.toType();
1572 const int_bits: u16 = @intCast(res_ty.bitSize(zcu));
1573 const int_ty = try pt.intType(.unsigned, int_bits);
1574 const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1575 const res_elem_ty = res_ty.childType(zcu);
1576 const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu));
1577 const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1578
1579 var inst_buf: [19]Air.Inst.Index = undefined;
1580 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1581956
1582 var res_block: Block = .init(&inst_buf);957 const result: Air.Inst.Ref = result: {
1583 {958 if (dest_legal) break :result main_block.addBitCast(l, dest_ty, uint_val);
1584 const res_alloc_inst = res_block.add(l, .{
1585 .tag = .alloc,
1586 .data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1587 });
1588 const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand);
1589 const index_alloc_inst = res_block.add(l, .{
1590 .tag = .alloc,
1591 .data = .{ .ty = .ptr_usize },
1592 });
1593 _ = res_block.add(l, .{
1594 .tag = .store,
1595 .data = .{ .bin_op = .{
1596 .lhs = index_alloc_inst.toRef(),
1597 .rhs = .zero_usize,
1598 } },
1599 });
1600959
1601 var loop: Loop = .init(l, &res_block);960 const elem_ty = dest_ty.childType(zcu);
1602 loop.block = .init(res_block.stealRemainingCapacity());961 const bits_per_elem: u16 = @intCast(elem_ty.bitSize(zcu));
1603 {962 const bits_per_elem_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, bits_per_elem));
1604 const cur_index_inst = loop.block.add(l, .{963 const elem_uint_ty = try pt.intType(.unsigned, bits_per_elem);
1605 .tag = .load,
1606 .data = .{ .ty_op = .{
1607 .ty = .usize_type,
1608 .operand = index_alloc_inst.toRef(),
1609 } },
1610 });
1611 _ = loop.block.add(l, .{
1612 .tag = .vector_store_elem,
1613 .data = .{ .vector_store_elem = .{
1614 .vector_ptr = res_alloc_inst.toRef(),
1615 .payload = try l.addExtra(Air.Bin, .{
1616 .lhs = cur_index_inst.toRef(),
1617 .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1618 .tag = .trunc,
1619 .data = .{ .ty_op = .{
1620 .ty = Air.internedToRef(elem_int_ty.toIntern()),
1621 .operand = loop.block.add(l, .{
1622 .tag = .shr,
1623 .data = .{ .bin_op = .{
1624 .lhs = int_ref,
1625 .rhs = loop.block.add(l, .{
1626 .tag = .mul,
1627 .data = .{ .bin_op = .{
1628 .lhs = loop.block.add(l, .{
1629 .tag = .intcast,
1630 .data = .{ .ty_op = .{
1631 .ty = Air.internedToRef(shift_ty.toIntern()),
1632 .operand = cur_index_inst.toRef(),
1633 } },
1634 }).toRef(),
1635 .rhs = try pt.intRef(shift_ty, elem_bits),
1636 } },
1637 }).toRef(),
1638 } },
1639 }).toRef(),
1640 } },
1641 }).toRef()),
1642 }),
1643 } },
1644 });
1645964
1646 var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(965 const elem_buf = try sfba.alloc(Air.Inst.Ref, dest_ty.arrayLen(zcu));
1647 l,966 defer sfba.free(elem_buf);
1648 .lt,967
1649 cur_index_inst.toRef(),968 var cur_uint = uint_val;
1650 try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1),969 for (elem_buf) |*elem| {
1651 .{},970 const elem_as_uint = main_block.addTyOp(l, .trunc, elem_uint_ty, cur_uint).toRef();
1652 )).toRef(), &loop.block, .{});971 elem.* = main_block.addBitCast(l, elem_ty, elem_as_uint);
1653 loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());972 cur_uint = main_block.addBinOp(l, .shr, cur_uint, bits_per_elem_ref).toRef();
1654 {
1655 _ = loop_cond_br.then_block.add(l, .{
1656 .tag = .store,
1657 .data = .{ .bin_op = .{
1658 .lhs = index_alloc_inst.toRef(),
1659 .rhs = loop_cond_br.then_block.add(l, .{
1660 .tag = .add,
1661 .data = .{ .bin_op = .{
1662 .lhs = cur_index_inst.toRef(),
1663 .rhs = .one_usize,
1664 } },
1665 }).toRef(),
1666 } },
1667 });
1668 _ = loop_cond_br.then_block.add(l, .{
1669 .tag = .repeat,
1670 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1671 });
1672 }
1673 loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1674 _ = loop_cond_br.else_block.add(l, .{
1675 .tag = .br,
1676 .data = .{ .br = .{
1677 .block_inst = orig_inst,
1678 .operand = loop_cond_br.else_block.add(l, .{
1679 .tag = .load,
1680 .data = .{ .ty_op = .{
1681 .ty = Air.internedToRef(res_ty.toIntern()),
1682 .operand = res_alloc_inst.toRef(),
1683 } },
1684 }).toRef(),
1685 } },
1686 });
1687 try loop_cond_br.finish(l);
1688 }973 }
1689 try loop.finish(l);974
1690 }975 break :result main_block.add(l, .{
976 .tag = .aggregate_init,
977 .data = .{ .ty_pl = .{
978 .ty = .fromType(dest_ty),
979 .payload = payload: {
980 const idx = l.air_extra.items.len;
981 try l.air_extra.appendSlice(gpa, @ptrCast(elem_buf));
982 break :payload @intCast(idx);
983 },
984 } },
985 }).toRef();
986 };
987
988 main_block.addBr(l, orig_inst, result);
989
1691 return .{ .ty_pl = .{990 return .{ .ty_pl = .{
1692 .ty = Air.internedToRef(res_ty.toIntern()),991 .ty = .fromType(dest_ty),
1693 .payload = try l.addBlockBody(res_block.body()),992 .payload = try l.addBlockBody(main_block.body()),
1694 } };993 } };
1695}994}
1696fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {995fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1697 const pt = l.pt;996 const pt = l.pt;
1698 const zcu = pt.zcu;997 const zcu = pt.zcu;
998 const gpa = zcu.gpa;
999
1000 var sfba_state = std.heap.stackFallback(512, gpa);
1001 const sfba = sfba_state.get();
16991002
1700 const orig = l.air_instructions.get(@intFromEnum(orig_inst));1003 const orig = l.air_instructions.get(@intFromEnum(orig_inst));
1701 const res_ty = l.typeOfIndex(orig_inst);1004 const orig_operands = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
1702 const wrapped_res_ty = res_ty.fieldType(0, zcu);
1703 const wrapped_res_scalar_ty = wrapped_res_ty.childType(zcu);
1704 const res_len = wrapped_res_ty.vectorLen(zcu);
17051005
1706 var inst_buf: [21]Air.Inst.Index = undefined;1006 const vec_tuple_ty = l.typeOfIndex(orig_inst);
1707 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);1007 const vec_int_ty = vec_tuple_ty.fieldType(0, zcu);
1008 const vec_overflow_ty = vec_tuple_ty.fieldType(1, zcu);
17081009
1709 var res_block: Block = .init(&inst_buf);1010 assert(l.typeOf(orig_operands.lhs).toIntern() == vec_int_ty.toIntern());
1710 {1011 if (orig.tag != .shl_with_overflow) {
1711 const res_alloc_inst = res_block.add(l, .{1012 assert(l.typeOf(orig_operands.rhs).toIntern() == vec_int_ty.toIntern());
1712 .tag = .alloc,1013 }
1713 .data = .{ .ty = try pt.singleMutPtrType(res_ty) },1014
1714 });1015 const scalar_int_ty = vec_int_ty.childType(zcu);
1715 const ptr_wrapped_res_inst = res_block.add(l, .{1016 const scalar_tuple_ty = try pt.overflowArithmeticTupleType(scalar_int_ty);
1716 .tag = .struct_field_ptr_index_0,1017
1717 .data = .{ .ty_op = .{1018 const elems_len = vec_int_ty.vectorLen(zcu);
1718 .ty = Air.internedToRef((try pt.singleMutPtrType(wrapped_res_ty)).toIntern()),1019
1719 .operand = res_alloc_inst.toRef(),1020 const inst_buf = try sfba.alloc(Air.Inst.Index, 5 * elems_len + 4);
1021 defer sfba.free(inst_buf);
1022
1023 var main_block: Block = .init(inst_buf);
1024 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
1025
1026 const int_elem_buf = try sfba.alloc(Air.Inst.Ref, elems_len);
1027 defer sfba.free(int_elem_buf);
1028 const overflow_elem_buf = try sfba.alloc(Air.Inst.Ref, elems_len);
1029 defer sfba.free(overflow_elem_buf);
1030
1031 for (int_elem_buf, overflow_elem_buf, 0..) |*int_elem, *overflow_elem, elem_idx| {
1032 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
1033 const lhs = main_block.addBinOp(l, .array_elem_val, orig_operands.lhs, elem_idx_ref).toRef();
1034 const rhs = main_block.addBinOp(l, .array_elem_val, orig_operands.rhs, elem_idx_ref).toRef();
1035 const elem_result = main_block.add(l, .{
1036 .tag = orig.tag,
1037 .data = .{ .ty_pl = .{
1038 .ty = .fromType(scalar_tuple_ty),
1039 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1720 } },1040 } },
1721 });1041 }).toRef();
1722 const ptr_overflow_res_inst = res_block.add(l, .{1042 int_elem.* = main_block.add(l, .{
1723 .tag = .struct_field_ptr_index_1,1043 .tag = .struct_field_val,
1724 .data = .{ .ty_op = .{1044 .data = .{ .ty_pl = .{
1725 .ty = Air.internedToRef((try pt.singleMutPtrType(res_ty.fieldType(1, zcu))).toIntern()),1045 .ty = .fromType(scalar_int_ty),
1726 .operand = res_alloc_inst.toRef(),1046 .payload = try l.addExtra(Air.StructField, .{
1047 .struct_operand = elem_result,
1048 .field_index = 0,
1049 }),
1727 } },1050 } },
1728 });1051 }).toRef();
1729 const index_alloc_inst = res_block.add(l, .{1052 overflow_elem.* = main_block.add(l, .{
1730 .tag = .alloc,1053 .tag = .struct_field_val,
1731 .data = .{ .ty = .ptr_usize },1054 .data = .{ .ty_pl = .{
1732 });1055 .ty = .bool_type,
1733 _ = res_block.add(l, .{1056 .payload = try l.addExtra(Air.StructField, .{
1734 .tag = .store,1057 .struct_operand = elem_result,
1735 .data = .{ .bin_op = .{1058 .field_index = 1,
1736 .lhs = index_alloc_inst.toRef(),1059 }),
1737 .rhs = .zero_usize,
1738 } },1060 } },
1739 });1061 }).toRef();
1062 }
17401063
1741 var loop: Loop = .init(l, &res_block);1064 const int_vec = main_block.add(l, .{
1742 loop.block = .init(res_block.stealRemainingCapacity());1065 .tag = .aggregate_init,
1743 {1066 .data = .{ .ty_pl = .{
1744 const cur_index_inst = loop.block.add(l, .{1067 .ty = .fromType(vec_int_ty),
1745 .tag = .load,1068 .payload = payload: {
1746 .data = .{ .ty_op = .{1069 const idx = l.air_extra.items.len;
1747 .ty = .usize_type,1070 try l.air_extra.appendSlice(gpa, @ptrCast(int_elem_buf));
1748 .operand = index_alloc_inst.toRef(),1071 break :payload @intCast(idx);
1749 } },1072 },
1750 });1073 } },
1751 const extra = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;1074 }).toRef();
1752 const res_elem = loop.block.add(l, .{1075 const overflow_vec = main_block.add(l, .{
1753 .tag = orig.tag,1076 .tag = .aggregate_init,
1754 .data = .{ .ty_pl = .{1077 .data = .{ .ty_pl = .{
1755 .ty = Air.internedToRef(try zcu.intern_pool.getTupleType(zcu.gpa, pt.tid, .{1078 .ty = .fromType(vec_overflow_ty),
1756 .types = &.{ wrapped_res_scalar_ty.toIntern(), .u1_type },1079 .payload = payload: {
1757 .values = &(.{.none} ** 2),1080 const idx = l.air_extra.items.len;
1758 })),1081 try l.air_extra.appendSlice(gpa, @ptrCast(overflow_elem_buf));
1759 .payload = try l.addExtra(Air.Bin, .{1082 break :payload @intCast(idx);
1760 .lhs = loop.block.add(l, .{1083 },
1761 .tag = .array_elem_val,1084 } },
1762 .data = .{ .bin_op = .{1085 }).toRef();
1763 .lhs = extra.lhs,1086
1764 .rhs = cur_index_inst.toRef(),1087 const tuple_elems: [2]Air.Inst.Ref = .{ int_vec, overflow_vec };
1765 } },1088 const result = main_block.add(l, .{
1766 }).toRef(),1089 .tag = .aggregate_init,
1767 .rhs = loop.block.add(l, .{1090 .data = .{ .ty_pl = .{
1768 .tag = .array_elem_val,1091 .ty = .fromType(vec_tuple_ty),
1769 .data = .{ .bin_op = .{1092 .payload = payload: {
1770 .lhs = extra.rhs,1093 const idx = l.air_extra.items.len;
1771 .rhs = cur_index_inst.toRef(),1094 try l.air_extra.appendSlice(gpa, @ptrCast(&tuple_elems));
1772 } },1095 break :payload @intCast(idx);
1773 }).toRef(),1096 },
1774 }),1097 } },
1775 } },1098 }).toRef();
1776 });1099
1777 _ = loop.block.add(l, .{1100 main_block.addBr(l, orig_inst, result);
1778 .tag = .vector_store_elem,
1779 .data = .{ .vector_store_elem = .{
1780 .vector_ptr = ptr_overflow_res_inst.toRef(),
1781 .payload = try l.addExtra(Air.Bin, .{
1782 .lhs = cur_index_inst.toRef(),
1783 .rhs = loop.block.add(l, .{
1784 .tag = .struct_field_val,
1785 .data = .{ .ty_pl = .{
1786 .ty = .u1_type,
1787 .payload = try l.addExtra(Air.StructField, .{
1788 .struct_operand = res_elem.toRef(),
1789 .field_index = 1,
1790 }),
1791 } },
1792 }).toRef(),
1793 }),
1794 } },
1795 });
1796 _ = loop.block.add(l, .{
1797 .tag = .vector_store_elem,
1798 .data = .{ .vector_store_elem = .{
1799 .vector_ptr = ptr_wrapped_res_inst.toRef(),
1800 .payload = try l.addExtra(Air.Bin, .{
1801 .lhs = cur_index_inst.toRef(),
1802 .rhs = loop.block.add(l, .{
1803 .tag = .struct_field_val,
1804 .data = .{ .ty_pl = .{
1805 .ty = Air.internedToRef(wrapped_res_scalar_ty.toIntern()),
1806 .payload = try l.addExtra(Air.StructField, .{
1807 .struct_operand = res_elem.toRef(),
1808 .field_index = 0,
1809 }),
1810 } },
1811 }).toRef(),
1812 }),
1813 } },
1814 });
18151101
1816 var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1817 l,
1818 .lt,
1819 cur_index_inst.toRef(),
1820 try pt.intRef(.usize, res_len - 1),
1821 .{},
1822 )).toRef(), &loop.block, .{});
1823 loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1824 {
1825 _ = loop_cond_br.then_block.add(l, .{
1826 .tag = .store,
1827 .data = .{ .bin_op = .{
1828 .lhs = index_alloc_inst.toRef(),
1829 .rhs = loop_cond_br.then_block.add(l, .{
1830 .tag = .add,
1831 .data = .{ .bin_op = .{
1832 .lhs = cur_index_inst.toRef(),
1833 .rhs = .one_usize,
1834 } },
1835 }).toRef(),
1836 } },
1837 });
1838 _ = loop_cond_br.then_block.add(l, .{
1839 .tag = .repeat,
1840 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1841 });
1842 }
1843 loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1844 _ = loop_cond_br.else_block.add(l, .{
1845 .tag = .br,
1846 .data = .{ .br = .{
1847 .block_inst = orig_inst,
1848 .operand = loop_cond_br.else_block.add(l, .{
1849 .tag = .load,
1850 .data = .{ .ty_op = .{
1851 .ty = Air.internedToRef(res_ty.toIntern()),
1852 .operand = res_alloc_inst.toRef(),
1853 } },
1854 }).toRef(),
1855 } },
1856 });
1857 try loop_cond_br.finish(l);
1858 }
1859 try loop.finish(l);
1860 }
1861 return .{ .ty_pl = .{1102 return .{ .ty_pl = .{
1862 .ty = Air.internedToRef(res_ty.toIntern()),1103 .ty = .fromType(vec_tuple_ty),
1863 .payload = try l.addBlockBody(res_block.body()),1104 .payload = try l.addBlockBody(main_block.body()),
1864 } };1105 } };
1865}1106}
18661107
...@@ -2231,37 +1472,6 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_...@@ -2231,37 +1472,6 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
2231 } };1472 } };
2232}1473}
22331474
2234fn expandBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2235 const pt = l.pt;
2236 const zcu = pt.zcu;
2237 const ip = &zcu.intern_pool;
2238
2239 const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
2240 const res_ty = orig_ty_op.ty.toType();
2241 const res_ty_key = ip.indexToKey(res_ty.toIntern());
2242 const operand_ty = l.typeOf(orig_ty_op.operand);
2243 const operand_ty_key = ip.indexToKey(operand_ty.toIntern());
2244 _ = res_ty_key;
2245 _ = operand_ty_key;
2246
2247 var inst_buf: [1]Air.Inst.Index = undefined;
2248 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2249
2250 var res_block: Block = .init(&inst_buf);
2251 {
2252 _ = res_block.add(l, .{
2253 .tag = .br,
2254 .data = .{ .br = .{
2255 .block_inst = orig_inst,
2256 .operand = try pt.undefRef(res_ty),
2257 } },
2258 });
2259 }
2260 return .{ .ty_pl = .{
2261 .ty = Air.internedToRef(res_ty.toIntern()),
2262 .payload = try l.addBlockBody(res_block.body()),
2263 } };
2264}
2265fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {1475fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2266 const pt = l.pt;1476 const pt = l.pt;
2267 const zcu = pt.zcu;1477 const zcu = pt.zcu;
...@@ -2431,89 +1641,73 @@ fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Err...@@ -2431,89 +1641,73 @@ fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Err
2431 const field_ty = orig_ty_pl.ty.toType();1641 const field_ty = orig_ty_pl.ty.toType();
2432 const agg_ty = l.typeOf(orig_extra.struct_operand);1642 const agg_ty = l.typeOf(orig_extra.struct_operand);
24331643
1644 const agg_bits: u16 = @intCast(agg_ty.bitSize(zcu));
1645 const bit_offset = zcu.structPackedFieldBitOffset(zcu.typeToStruct(agg_ty).?, orig_extra.field_index);
1646
1647 const agg_int_ty = try pt.intType(.unsigned, agg_bits);
1648 const field_int_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
1649
1650 const agg_shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, agg_bits));
1651 const bit_offset_ref: Air.Inst.Ref = .fromValue(try pt.intValue(agg_shift_ty, bit_offset));
1652
2434 var inst_buf: [5]Air.Inst.Index = undefined;1653 var inst_buf: [5]Air.Inst.Index = undefined;
1654 var main_block: Block = .init(&inst_buf);
2435 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);1655 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
24361656
2437 var res_block: Block = .init(&inst_buf);1657 const agg_int = main_block.addBitCast(l, agg_int_ty, orig_extra.struct_operand);
2438 {1658 const shifted_agg_int = main_block.addBinOp(l, .shr, agg_int, bit_offset_ref).toRef();
2439 const agg_alloc_inst = res_block.add(l, .{1659 const field_int = main_block.addTyOp(l, .trunc, field_int_ty, shifted_agg_int).toRef();
2440 .tag = .alloc,1660 const field_val = main_block.addBitCast(l, field_ty, field_int);
2441 .data = .{ .ty = try pt.singleMutPtrType(agg_ty) },1661 main_block.addBr(l, orig_inst, field_val);
2442 });1662
2443 _ = res_block.add(l, .{
2444 .tag = .store,
2445 .data = .{ .bin_op = .{
2446 .lhs = agg_alloc_inst.toRef(),
2447 .rhs = orig_extra.struct_operand,
2448 } },
2449 });
2450 _ = res_block.add(l, .{
2451 .tag = .br,
2452 .data = .{ .br = .{
2453 .block_inst = orig_inst,
2454 .operand = res_block.add(l, .{
2455 .tag = .load,
2456 .data = .{ .ty_op = .{
2457 .ty = Air.internedToRef(field_ty.toIntern()),
2458 .operand = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), orig_extra.field_index)).toRef(),
2459 } },
2460 }).toRef(),
2461 } },
2462 });
2463 }
2464 return .{ .ty_pl = .{1663 return .{ .ty_pl = .{
2465 .ty = Air.internedToRef(field_ty.toIntern()),1664 .ty = .fromType(field_ty),
2466 .payload = try l.addBlockBody(res_block.body()),1665 .payload = try l.addBlockBody(main_block.body()),
2467 } };1666 } };
2468}1667}
2469fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {1668fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2470 const pt = l.pt;1669 const pt = l.pt;
2471 const zcu = pt.zcu;1670 const zcu = pt.zcu;
1671 const gpa = zcu.gpa;
24721672
2473 const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl;1673 const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl;
2474 const field_ty = orig_ty_pl.ty.toType();
2475 const agg_ty = orig_ty_pl.ty.toType();1674 const agg_ty = orig_ty_pl.ty.toType();
2476 const agg_field_count = agg_ty.structFieldCount(zcu);1675 const agg_field_count = agg_ty.structFieldCount(zcu);
24771676
2478 const ExpectedContents = [1 + 2 * 32 + 2]Air.Inst.Index;1677 var sfba_state = std.heap.stackFallback(@sizeOf([4 * 32 + 2]Air.Inst.Index), gpa);
2479 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =1678 const sfba = sfba_state.get();
2480 std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa);
2481 const gpa = stack.get();
24821679
2483 const inst_buf = try gpa.alloc(Air.Inst.Index, 1 + 2 * agg_field_count + 2);1680 const inst_buf = try sfba.alloc(Air.Inst.Index, 4 * agg_field_count + 2);
2484 defer gpa.free(inst_buf);1681 defer sfba.free(inst_buf);
2485 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
24861682
2487 var res_block: Block = .init(inst_buf);1683 var main_block: Block = .init(inst_buf);
2488 {1684 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2489 const agg_alloc_inst = res_block.add(l, .{1685
2490 .tag = .alloc,1686 const num_bits: u16 = @intCast(agg_ty.bitSize(zcu));
2491 .data = .{ .ty = try pt.singleMutPtrType(agg_ty) },1687 const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
2492 });1688 const uint_ty = try pt.intType(.unsigned, num_bits);
2493 for (0..agg_field_count, orig_ty_pl.payload..) |field_index, extra_index| _ = res_block.add(l, .{1689 var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
2494 .tag = .store,1690
2495 .data = .{ .bin_op = .{1691 var field_idx = agg_field_count;
2496 .lhs = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), field_index)).toRef(),1692 while (field_idx > 0) {
2497 .rhs = @enumFromInt(l.air_extra.items[extra_index]),1693 field_idx -= 1;
2498 } },1694 const field_ty = agg_ty.fieldType(field_idx, zcu);
2499 });1695 const field_uint_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
2500 _ = res_block.add(l, .{1696 const field_bit_size_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, field_ty.bitSize(zcu)));
2501 .tag = .br,1697 const field_val: Air.Inst.Ref = @enumFromInt(l.air_extra.items[orig_ty_pl.payload + field_idx]);
2502 .data = .{ .br = .{1698
2503 .block_inst = orig_inst,1699 const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
2504 .operand = res_block.add(l, .{1700 const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
2505 .tag = .load,1701 const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();
2506 .data = .{ .ty_op = .{1702 cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2507 .ty = Air.internedToRef(field_ty.toIntern()),
2508 .operand = agg_alloc_inst.toRef(),
2509 } },
2510 }).toRef(),
2511 } },
2512 });
2513 }1703 }
1704
1705 const result = main_block.addBitCast(l, agg_ty, cur_uint);
1706 main_block.addBr(l, orig_inst, result);
1707
2514 return .{ .ty_pl = .{1708 return .{ .ty_pl = .{
2515 .ty = Air.internedToRef(field_ty.toIntern()),1709 .ty = .fromType(agg_ty),
2516 .payload = try l.addBlockBody(res_block.body()),1710 .payload = try l.addBlockBody(main_block.body()),
2517 } };1711 } };
2518}1712}
25191713
...@@ -2571,6 +1765,33 @@ const Block = struct {...@@ -2571,6 +1765,33 @@ const Block = struct {
2571 b.len += 1;1765 b.len += 1;
2572 return inst;1766 return inst;
2573 }1767 }
1768 fn addBr(b: *Block, l: *Legalize, target: Air.Inst.Index, operand: Air.Inst.Ref) void {
1769 _ = b.add(l, .{
1770 .tag = .br,
1771 .data = .{ .br = .{ .block_inst = target, .operand = operand } },
1772 });
1773 }
1774 fn addBinOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) Air.Inst.Index {
1775 return b.add(l, .{
1776 .tag = tag,
1777 .data = .{ .bin_op = .{ .lhs = lhs, .rhs = rhs } },
1778 });
1779 }
1780 fn addUnOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, operand: Air.Inst.Ref) Air.Inst.Index {
1781 return b.add(l, .{
1782 .tag = tag,
1783 .data = .{ .un_op = operand },
1784 });
1785 }
1786 fn addTyOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, ty: Type, operand: Air.Inst.Ref) Air.Inst.Index {
1787 return b.add(l, .{
1788 .tag = tag,
1789 .data = .{ .ty_op = .{
1790 .ty = .fromType(ty),
1791 .operand = operand,
1792 } },
1793 });
1794 }
25741795
2575 /// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,1796 /// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,
2576 /// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.1797 /// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.
...@@ -2625,14 +1846,27 @@ const Block = struct {...@@ -2625,14 +1846,27 @@ const Block = struct {
2625 } },1846 } },
2626 });1847 });
2627 }1848 }
1849 return addCmpScalar(b, l, op, lhs, rhs, opts.optimized);
1850 }
1851
1852 /// Similar to `addCmp`, but for scalars only. Unlike `addCmp`, this function is
1853 /// infallible, because it doesn't need to add entries to `extra`.
1854 fn addCmpScalar(
1855 b: *Block,
1856 l: *Legalize,
1857 op: std.math.CompareOperator,
1858 lhs: Air.Inst.Ref,
1859 rhs: Air.Inst.Ref,
1860 optimized: bool,
1861 ) Air.Inst.Index {
2628 return b.add(l, .{1862 return b.add(l, .{
2629 .tag = switch (op) {1863 .tag = switch (op) {
2630 .lt => if (opts.optimized) .cmp_lt_optimized else .cmp_lt,1864 .lt => if (optimized) .cmp_lt_optimized else .cmp_lt,
2631 .lte => if (opts.optimized) .cmp_lte_optimized else .cmp_lte,1865 .lte => if (optimized) .cmp_lte_optimized else .cmp_lte,
2632 .eq => if (opts.optimized) .cmp_eq_optimized else .cmp_eq,1866 .eq => if (optimized) .cmp_eq_optimized else .cmp_eq,
2633 .gte => if (opts.optimized) .cmp_gte_optimized else .cmp_gte,1867 .gte => if (optimized) .cmp_gte_optimized else .cmp_gte,
2634 .gt => if (opts.optimized) .cmp_gt_optimized else .cmp_gt,1868 .gt => if (optimized) .cmp_gt_optimized else .cmp_gt,
2635 .neq => if (opts.optimized) .cmp_neq_optimized else .cmp_neq,1869 .neq => if (optimized) .cmp_neq_optimized else .cmp_neq,
2636 },1870 },
2637 .data = .{ .bin_op = .{1871 .data = .{ .bin_op = .{
2638 .lhs = lhs,1872 .lhs = lhs,
...@@ -2641,93 +1875,6 @@ const Block = struct {...@@ -2641,93 +1875,6 @@ const Block = struct {
2641 });1875 });
2642 }1876 }
26431877
2644 /// Adds a `struct_field_ptr*` instruction to `b`. This is a fairly thin wrapper around `add`
2645 /// that selects the optimized instruction encoding to use, although it does compute the
2646 /// proper field pointer type.
2647 fn addStructFieldPtr(
2648 b: *Block,
2649 l: *Legalize,
2650 struct_operand: Air.Inst.Ref,
2651 field_index: usize,
2652 ) Error!Air.Inst.Index {
2653 const pt = l.pt;
2654 const zcu = pt.zcu;
2655
2656 const agg_ptr_ty = l.typeOf(struct_operand);
2657 const agg_ptr_info = agg_ptr_ty.ptrInfo(zcu);
2658 const agg_ty: Type = .fromInterned(agg_ptr_info.child);
2659 const agg_ptr_align = switch (agg_ptr_info.flags.alignment) {
2660 .none => agg_ty.abiAlignment(zcu),
2661 else => |agg_ptr_align| agg_ptr_align,
2662 };
2663 const agg_layout = agg_ty.containerLayout(zcu);
2664 const field_ty = agg_ty.fieldType(field_index, zcu);
2665 var field_ptr_info: InternPool.Key.PtrType = .{
2666 .child = field_ty.toIntern(),
2667 .flags = .{
2668 .is_const = agg_ptr_info.flags.is_const,
2669 .is_volatile = agg_ptr_info.flags.is_volatile,
2670 .address_space = agg_ptr_info.flags.address_space,
2671 },
2672 };
2673 field_ptr_info.flags.alignment = field_ptr_align: switch (agg_layout) {
2674 .auto => agg_ty.fieldAlignment(field_index, zcu).min(agg_ptr_align),
2675 .@"extern" => switch (agg_ty.zigTypeTag(zcu)) {
2676 else => unreachable,
2677 .@"struct" => .fromLog2Units(@min(
2678 agg_ptr_align.toLog2Units(),
2679 @ctz(agg_ty.structFieldOffset(field_index, zcu)),
2680 )),
2681 .@"union" => agg_ptr_align,
2682 },
2683 .@"packed" => switch (agg_ty.zigTypeTag(zcu)) {
2684 else => unreachable,
2685 .@"struct" => {
2686 const packed_offset = agg_ty.packedStructFieldPtrInfo(agg_ptr_ty, @intCast(field_index), pt);
2687 field_ptr_info.packed_offset = packed_offset;
2688 break :field_ptr_align agg_ptr_align;
2689 },
2690 .@"union" => {
2691 field_ptr_info.packed_offset = .{
2692 .host_size = switch (agg_ptr_info.packed_offset.host_size) {
2693 0 => @intCast(agg_ty.abiSize(zcu)),
2694 else => |host_size| host_size,
2695 },
2696 .bit_offset = agg_ptr_info.packed_offset.bit_offset,
2697 };
2698 break :field_ptr_align agg_ptr_align;
2699 },
2700 },
2701 };
2702 const field_ptr_ty = try pt.ptrType(field_ptr_info);
2703 const field_ptr_ty_ref = Air.internedToRef(field_ptr_ty.toIntern());
2704 return switch (field_index) {
2705 inline 0...3 => |ct_field_index| b.add(l, .{
2706 .tag = switch (ct_field_index) {
2707 0 => .struct_field_ptr_index_0,
2708 1 => .struct_field_ptr_index_1,
2709 2 => .struct_field_ptr_index_2,
2710 3 => .struct_field_ptr_index_3,
2711 else => comptime unreachable,
2712 },
2713 .data = .{ .ty_op = .{
2714 .ty = field_ptr_ty_ref,
2715 .operand = struct_operand,
2716 } },
2717 }),
2718 else => b.add(l, .{
2719 .tag = .struct_field_ptr,
2720 .data = .{ .ty_pl = .{
2721 .ty = field_ptr_ty_ref,
2722 .payload = try l.addExtra(Air.StructField, .{
2723 .struct_operand = struct_operand,
2724 .field_index = @intCast(field_index),
2725 }),
2726 } },
2727 }),
2728 };
2729 }
2730
2731 /// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for1878 /// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for
2732 /// no-op casts.1879 /// no-op casts.
2733 fn addBitCast(1880 fn addBitCast(
...@@ -2774,56 +1921,6 @@ const Block = struct {...@@ -2774,56 +1921,6 @@ const Block = struct {
2774 }1921 }
2775};1922};
27761923
2777const Result = struct {
2778 inst: Air.Inst.Index,
2779 block: Block,
2780
2781 /// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
2782 /// to initialize it.
2783 fn init(l: *Legalize, ty: Type, parent_block: *Block) Result {
2784 return .{
2785 .inst = parent_block.add(l, .{
2786 .tag = .block,
2787 .data = .{ .ty_pl = .{
2788 .ty = Air.internedToRef(ty.toIntern()),
2789 .payload = undefined,
2790 } },
2791 }),
2792 .block = undefined,
2793 };
2794 }
2795
2796 fn finish(res: Result, l: *Legalize) Error!void {
2797 const data = &l.air_instructions.items(.data)[@intFromEnum(res.inst)];
2798 data.ty_pl.payload = try l.addBlockBody(res.block.body());
2799 }
2800};
2801
2802const Loop = struct {
2803 inst: Air.Inst.Index,
2804 block: Block,
2805
2806 /// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
2807 /// to initialize it.
2808 fn init(l: *Legalize, parent_block: *Block) Loop {
2809 return .{
2810 .inst = parent_block.add(l, .{
2811 .tag = .loop,
2812 .data = .{ .ty_pl = .{
2813 .ty = .noreturn_type,
2814 .payload = undefined,
2815 } },
2816 }),
2817 .block = undefined,
2818 };
2819 }
2820
2821 fn finish(loop: Loop, l: *Legalize) Error!void {
2822 const data = &l.air_instructions.items(.data)[@intFromEnum(loop.inst)];
2823 data.ty_pl.payload = try l.addBlockBody(loop.block.body());
2824 }
2825};
2826
2827const CondBr = struct {1924const CondBr = struct {
2828 inst: Air.Inst.Index,1925 inst: Air.Inst.Index,
2829 hints: Air.CondBr.BranchHints,1926 hints: Air.CondBr.BranchHints,