| author |  Matthew Lugg <mlugg@mlugg.co.uk> 2025-10-30 09:20:04+00:00 |
| committer | Matthew Lugg <mlugg@mlugg.co.uk> 2025-11-12 16:00:15+00:00 |
| log | c091e27aac9d51cb3af06904c3039a8c316e5b89 |
| tree | edac857449576ad9d759e13aa97d261125f3068a |
| parent | 9a7d28fe58c0dbd34b0e41fe0e4fed6d57c7ad45 |
| signature |  Commit is signed but in an unrecognized format. |
compiler: spring cleaning
I started this diff trying to remove a little dead code from the C
backend, but ended up finding a bunch of dead code sprinkled all over
the place:
* `packed` handling in the C backend which was made dead by `Legalize`
* Representation of pointers to runtime-known vector indices
* Handling for the `vector_store_elem` AIR instruction (now removed)
* Old tuple handling from when they used the InternPool repr of structs
* Straightforward unused functions
* TODOs in the LLVM backend for features which Zig just does not support21 files changed, 210 insertions(+), 2442 deletions(-)
src/Air.zig-11
| ... | ... | @@ -874,10 +874,6 @@ pub const Inst = struct { |
| 874 | 874 | /// Uses the `ty_pl` field. |
| 875 | 875 | save_err_return_trace_index, |
| 876 | 876 | |
| 877 | /// Store an element to a vector pointer at an index. | |
| 878 | /// Uses the `vector_store_elem` field. | |
| 879 | vector_store_elem, | |
| 880 | ||
| 881 | 877 | /// Compute a pointer to a `Nav` at runtime, always one of: |
| 882 | 878 | /// |
| 883 | 879 | /// * `threadlocal var` |
| ... | ... | @@ -1220,11 +1216,6 @@ pub const Inst = struct { |
| 1220 | 1216 | operand: Ref, |
| 1221 | 1217 | operation: std.builtin.ReduceOp, |
| 1222 | 1218 | }, |
| 1223 | vector_store_elem: struct { | |
| 1224 | vector_ptr: Ref, | |
| 1225 | // Index into a different array. | |
| 1226 | payload: u32, | |
| 1227 | }, | |
| 1228 | 1219 | ty_nav: struct { |
| 1229 | 1220 | ty: InternPool.Index, |
| 1230 | 1221 | nav: InternPool.Nav.Index, |
| ... | ... | @@ -1689,7 +1680,6 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool) |
| 1689 | 1680 | .set_union_tag, |
| 1690 | 1681 | .prefetch, |
| 1691 | 1682 | .set_err_return_trace, |
| 1692 | .vector_store_elem, | |
| 1693 | 1683 | .c_va_end, |
| 1694 | 1684 | => return .void, |
| 1695 | 1685 | |
| ... | ... | @@ -1857,7 +1847,6 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool { |
| 1857 | 1847 | .prefetch, |
| 1858 | 1848 | .wasm_memory_grow, |
| 1859 | 1849 | .set_err_return_trace, |
| 1860 | .vector_store_elem, | |
| 1861 | 1850 | .c_va_arg, |
| 1862 | 1851 | .c_va_copy, |
| 1863 | 1852 | .c_va_end, |
src/Air/Liveness.zig-6
| ... | ... | @@ -463,12 +463,6 @@ fn analyzeInst( |
| 463 | 463 | return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none }); |
| 464 | 464 | }, |
| 465 | 465 | |
| 466 | .vector_store_elem => { | |
| 467 | const o = inst_datas[@intFromEnum(inst)].vector_store_elem; | |
| 468 | const extra = a.air.extraData(Air.Bin, o.payload).data; | |
| 469 | return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs }); | |
| 470 | }, | |
| 471 | ||
| 472 | 466 | .arg, |
| 473 | 467 | .alloc, |
| 474 | 468 | .ret_ptr, |
src/Air/Liveness/Verify.zig-5
| ... | ... | @@ -322,11 +322,6 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { |
| 322 | 322 | const extra = self.air.extraData(Air.Bin, pl_op.payload).data; |
| 323 | 323 | try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand }); |
| 324 | 324 | }, |
| 325 | .vector_store_elem => { | |
| 326 | const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem; | |
| 327 | const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data; | |
| 328 | try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs }); | |
| 329 | }, | |
| 330 | 325 | .cmpxchg_strong, |
| 331 | 326 | .cmpxchg_weak, |
| 332 | 327 | => { |
src/Air/print.zig-12
| ... | ... | @@ -330,7 +330,6 @@ const Writer = struct { |
| 330 | 330 | .shuffle_two => try w.writeShuffleTwo(s, inst), |
| 331 | 331 | .reduce, .reduce_optimized => try w.writeReduce(s, inst), |
| 332 | 332 | .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst), |
| 333 | .vector_store_elem => try w.writeVectorStoreElem(s, inst), | |
| 334 | 333 | .runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst), |
| 335 | 334 | |
| 336 | 335 | .work_item_id, |
| ... | ... | @@ -576,17 +575,6 @@ const Writer = struct { |
| 576 | 575 | try w.writeOperand(s, inst, 1, extra.rhs); |
| 577 | 576 | } |
| 578 | 577 | |
| 579 | fn writeVectorStoreElem(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void { | |
| 580 | const data = w.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem; | |
| 581 | const extra = w.air.extraData(Air.VectorCmp, data.payload).data; | |
| 582 | ||
| 583 | try w.writeOperand(s, inst, 0, data.vector_ptr); | |
| 584 | try s.writeAll(", "); | |
| 585 | try w.writeOperand(s, inst, 1, extra.lhs); | |
| 586 | try s.writeAll(", "); | |
| 587 | try w.writeOperand(s, inst, 2, extra.rhs); | |
| 588 | } | |
| 589 | ||
| 590 | 578 | fn writeRuntimeNavPtr(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void { |
| 591 | 579 | const ip = &w.pt.zcu.intern_pool; |
| 592 | 580 | const ty_nav = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; |
src/Air/types_resolved.zig-7
| ... | ... | @@ -316,13 +316,6 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { |
| 316 | 316 | if (!checkRef(data.prefetch.ptr, zcu)) return false; |
| 317 | 317 | }, |
| 318 | 318 | |
| 319 | .vector_store_elem => { | |
| 320 | const bin = air.extraData(Air.Bin, data.vector_store_elem.payload).data; | |
| 321 | if (!checkRef(data.vector_store_elem.vector_ptr, zcu)) return false; | |
| 322 | if (!checkRef(bin.lhs, zcu)) return false; | |
| 323 | if (!checkRef(bin.rhs, zcu)) return false; | |
| 324 | }, | |
| 325 | ||
| 326 | 319 | .runtime_nav_ptr => { |
| 327 | 320 | if (!checkType(.fromInterned(data.ty_nav.ty), zcu)) return false; |
| 328 | 321 | }, |
src/InternPool.zig+2-5
| ... | ... | @@ -2104,7 +2104,6 @@ pub const Key = union(enum) { |
| 2104 | 2104 | |
| 2105 | 2105 | pub const VectorIndex = enum(u16) { |
| 2106 | 2106 | none = std.math.maxInt(u16), |
| 2107 | runtime = std.math.maxInt(u16) - 1, | |
| 2108 | 2107 | _, |
| 2109 | 2108 | }; |
| 2110 | 2109 | |
| ... | ... | @@ -3739,10 +3738,8 @@ pub const LoadedStructType = struct { |
| 3739 | 3738 | return s.field_inits.get(ip)[i]; |
| 3740 | 3739 | } |
| 3741 | 3740 | |
| 3742 | /// Returns `none` in the case the struct is a tuple. | |
| 3743 | pub fn fieldName(s: LoadedStructType, ip: *const InternPool, i: usize) OptionalNullTerminatedString { | |
| 3744 | if (s.field_names.len == 0) return .none; | |
| 3745 | return s.field_names.get(ip)[i].toOptional(); | |
| 3741 | pub fn fieldName(s: LoadedStructType, ip: *const InternPool, i: usize) NullTerminatedString { | |
| 3742 | return s.field_names.get(ip)[i]; | |
| 3746 | 3743 | } |
| 3747 | 3744 | |
| 3748 | 3745 | pub fn fieldIsComptime(s: LoadedStructType, ip: *const InternPool, i: usize) bool { |
src/Sema.zig+27-95
| ... | ... | @@ -15919,26 +15919,27 @@ fn zirOverflowArithmetic( |
| 15919 | 15919 | }, |
| 15920 | 15920 | .mul_with_overflow => { |
| 15921 | 15921 | // If either of the arguments is zero, the result is zero and no overflow occured. |
| 15922 | if (maybe_lhs_val) |lhs_val| { | |
| 15923 | if (!lhs_val.isUndef(zcu) and try lhs_val.compareAllWithZeroSema(.eq, pt)) { | |
| 15924 | break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs }; | |
| 15925 | } | |
| 15926 | } | |
| 15927 | if (maybe_rhs_val) |rhs_val| { | |
| 15928 | if (!rhs_val.isUndef(zcu) and try rhs_val.compareAllWithZeroSema(.eq, pt)) { | |
| 15929 | break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs }; | |
| 15930 | } | |
| 15931 | } | |
| 15922 | 15932 | // If either of the arguments is one, the result is the other and no overflow occured. |
| 15923 | // Otherwise, if either of the arguments is undefined, both results are undefined. | |
| 15924 | 15933 | const scalar_one = try pt.intValue(dest_ty.scalarType(zcu), 1); |
| 15934 | const vec_one = try sema.splat(dest_ty, scalar_one); | |
| 15925 | 15935 | if (maybe_lhs_val) |lhs_val| { |
| 15926 | if (!lhs_val.isUndef(zcu)) { | |
| 15927 | if (try lhs_val.compareAllWithZeroSema(.eq, pt)) { | |
| 15928 | break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs }; | |
| 15929 | } else if (try sema.compareAll(lhs_val, .eq, try sema.splat(dest_ty, scalar_one), dest_ty)) { | |
| 15930 | break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs }; | |
| 15931 | } | |
| 15936 | if (!lhs_val.isUndef(zcu) and try sema.compareAll(lhs_val, .eq, vec_one, dest_ty)) { | |
| 15937 | break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs }; | |
| 15932 | 15938 | } |
| 15933 | 15939 | } |
| 15934 | ||
| 15935 | 15940 | if (maybe_rhs_val) |rhs_val| { |
| 15936 | if (!rhs_val.isUndef(zcu)) { | |
| 15937 | if (try rhs_val.compareAllWithZeroSema(.eq, pt)) { | |
| 15938 | break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs }; | |
| 15939 | } else if (try sema.compareAll(rhs_val, .eq, try sema.splat(dest_ty, scalar_one), dest_ty)) { | |
| 15940 | break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs }; | |
| 15941 | } | |
| 15941 | if (!rhs_val.isUndef(zcu) and try sema.compareAll(rhs_val, .eq, vec_one, dest_ty)) { | |
| 15942 | break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs }; | |
| 15942 | 15943 | } |
| 15943 | 15944 | } |
| 15944 | 15945 | |
| ... | ... | @@ -15947,7 +15948,6 @@ fn zirOverflowArithmetic( |
| 15947 | 15948 | if (lhs_val.isUndef(zcu) or rhs_val.isUndef(zcu)) { |
| 15948 | 15949 | break :result .{ .overflow_bit = .undef, .wrapped = .undef }; |
| 15949 | 15950 | } |
| 15950 | ||
| 15951 | 15951 | const result = try arith.mulWithOverflow(sema, dest_ty, lhs_val, rhs_val); |
| 15952 | 15952 | break :result .{ .overflow_bit = result.overflow_bit, .wrapped = result.wrapped_result }; |
| 15953 | 15953 | } |
| ... | ... | @@ -17751,10 +17751,7 @@ fn zirTypeInfo(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai |
| 17751 | 17751 | try ty.resolveStructFieldInits(pt); |
| 17752 | 17752 | |
| 17753 | 17753 | for (struct_field_vals, 0..) |*field_val, field_index| { |
| 17754 | const field_name = if (struct_type.fieldName(ip, field_index).unwrap()) |field_name| | |
| 17755 | field_name | |
| 17756 | else | |
| 17757 | try ip.getOrPutStringFmt(gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls); | |
| 17754 | const field_name = struct_type.fieldName(ip, field_index); | |
| 17758 | 17755 | const field_name_len = field_name.length(ip); |
| 17759 | 17756 | const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]); |
| 17760 | 17757 | const field_init = struct_type.fieldInit(ip, field_index); |
| ... | ... | @@ -28347,6 +28344,10 @@ fn elemPtrArray( |
| 28347 | 28344 | break :o index; |
| 28348 | 28345 | } else null; |
| 28349 | 28346 | |
| 28347 | if (offset == null and array_ty.zigTypeTag(zcu) == .vector) { | |
| 28348 | return sema.fail(block, elem_index_src, "vector index not comptime known", .{}); | |
| 28349 | } | |
| 28350 | ||
| 28350 | 28351 | const elem_ptr_ty = try array_ptr_ty.elemPtrType(offset, pt); |
| 28351 | 28352 | |
| 28352 | 28353 | if (maybe_undef_array_ptr_val) |array_ptr_val| { |
| ... | ... | @@ -28364,10 +28365,6 @@ fn elemPtrArray( |
| 28364 | 28365 | try sema.validateRuntimeValue(block, array_ptr_src, array_ptr); |
| 28365 | 28366 | } |
| 28366 | 28367 | |
| 28367 | if (offset == null and array_ty.zigTypeTag(zcu) == .vector) { | |
| 28368 | return sema.fail(block, elem_index_src, "vector index not comptime known", .{}); | |
| 28369 | } | |
| 28370 | ||
| 28371 | 28368 | // Runtime check is only needed if unable to comptime check. |
| 28372 | 28369 | if (oob_safety and block.wantSafety() and offset == null) { |
| 28373 | 28370 | const len_inst = try pt.intRef(.usize, array_len); |
| ... | ... | @@ -30399,22 +30396,6 @@ fn storePtr2( |
| 30399 | 30396 | |
| 30400 | 30397 | const is_ret = air_tag == .ret_ptr; |
| 30401 | 30398 | |
| 30402 | // Detect if we are storing an array operand to a bitcasted vector pointer. | |
| 30403 | // If so, we instead reach through the bitcasted pointer to the vector pointer, | |
| 30404 | // bitcast the array operand to a vector, and then lower this as a store of | |
| 30405 | // a vector value to a vector pointer. This generally results in better code, | |
| 30406 | // as well as working around an LLVM bug: | |
| 30407 | // https://github.com/ziglang/zig/issues/11154 | |
| 30408 | if (sema.obtainBitCastedVectorPtr(ptr)) |vector_ptr| { | |
| 30409 | const vector_ty = sema.typeOf(vector_ptr).childType(zcu); | |
| 30410 | const vector = sema.coerceExtra(block, vector_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch |err| switch (err) { | |
| 30411 | error.NotCoercible => unreachable, | |
| 30412 | else => |e| return e, | |
| 30413 | }; | |
| 30414 | try sema.storePtr2(block, src, vector_ptr, ptr_src, vector, operand_src, .store); | |
| 30415 | return; | |
| 30416 | } | |
| 30417 | ||
| 30418 | 30399 | const operand = sema.coerceExtra(block, elem_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch |err| switch (err) { |
| 30419 | 30400 | error.NotCoercible => unreachable, |
| 30420 | 30401 | else => |e| return e, |
| ... | ... | @@ -30447,29 +30428,6 @@ fn storePtr2( |
| 30447 | 30428 | |
| 30448 | 30429 | try sema.requireRuntimeBlock(block, src, runtime_src); |
| 30449 | 30430 | |
| 30450 | if (ptr_ty.ptrInfo(zcu).flags.vector_index == .runtime) { | |
| 30451 | const ptr_inst = ptr.toIndex().?; | |
| 30452 | const air_tags = sema.air_instructions.items(.tag); | |
| 30453 | if (air_tags[@intFromEnum(ptr_inst)] == .ptr_elem_ptr) { | |
| 30454 | const ty_pl = sema.air_instructions.items(.data)[@intFromEnum(ptr_inst)].ty_pl; | |
| 30455 | const bin_op = sema.getTmpAir().extraData(Air.Bin, ty_pl.payload).data; | |
| 30456 | _ = try block.addInst(.{ | |
| 30457 | .tag = .vector_store_elem, | |
| 30458 | .data = .{ .vector_store_elem = .{ | |
| 30459 | .vector_ptr = bin_op.lhs, | |
| 30460 | .payload = try block.sema.addExtra(Air.Bin{ | |
| 30461 | .lhs = bin_op.rhs, | |
| 30462 | .rhs = operand, | |
| 30463 | }), | |
| 30464 | } }, | |
| 30465 | }); | |
| 30466 | return; | |
| 30467 | } | |
| 30468 | return sema.fail(block, ptr_src, "unable to determine vector element index of type '{f}'", .{ | |
| 30469 | ptr_ty.fmt(pt), | |
| 30470 | }); | |
| 30471 | } | |
| 30472 | ||
| 30473 | 30431 | const store_inst = if (is_ret) |
| 30474 | 30432 | try block.addBinOp(.store, ptr, operand) |
| 30475 | 30433 | else |
| ... | ... | @@ -30569,37 +30527,6 @@ fn markMaybeComptimeAllocRuntime(sema: *Sema, block: *Block, alloc_inst: Air.Ins |
| 30569 | 30527 | } |
| 30570 | 30528 | } |
| 30571 | 30529 | |
| 30572 | /// Traverse an arbitrary number of bitcasted pointers and return the underyling vector | |
| 30573 | /// pointer. Only if the final element type matches the vector element type, and the | |
| 30574 | /// lengths match. | |
| 30575 | fn obtainBitCastedVectorPtr(sema: *Sema, ptr: Air.Inst.Ref) ?Air.Inst.Ref { | |
| 30576 | const pt = sema.pt; | |
| 30577 | const zcu = pt.zcu; | |
| 30578 | const array_ty = sema.typeOf(ptr).childType(zcu); | |
| 30579 | if (array_ty.zigTypeTag(zcu) != .array) return null; | |
| 30580 | var ptr_ref = ptr; | |
| 30581 | var ptr_inst = ptr_ref.toIndex() orelse return null; | |
| 30582 | const air_datas = sema.air_instructions.items(.data); | |
| 30583 | const air_tags = sema.air_instructions.items(.tag); | |
| 30584 | const vector_ty = while (air_tags[@intFromEnum(ptr_inst)] == .bitcast) { | |
| 30585 | ptr_ref = air_datas[@intFromEnum(ptr_inst)].ty_op.operand; | |
| 30586 | if (!sema.isKnownZigType(ptr_ref, .pointer)) return null; | |
| 30587 | const child_ty = sema.typeOf(ptr_ref).childType(zcu); | |
| 30588 | if (child_ty.zigTypeTag(zcu) == .vector) break child_ty; | |
| 30589 | ptr_inst = ptr_ref.toIndex() orelse return null; | |
| 30590 | } else return null; | |
| 30591 | ||
| 30592 | // We have a pointer-to-array and a pointer-to-vector. If the elements and | |
| 30593 | // lengths match, return the result. | |
| 30594 | if (array_ty.childType(zcu).eql(vector_ty.childType(zcu), zcu) and | |
| 30595 | array_ty.arrayLen(zcu) == vector_ty.vectorLen(zcu)) | |
| 30596 | { | |
| 30597 | return ptr_ref; | |
| 30598 | } else { | |
| 30599 | return null; | |
| 30600 | } | |
| 30601 | } | |
| 30602 | ||
| 30603 | 30530 | /// Call when you have Value objects rather than Air instructions, and you want to |
| 30604 | 30531 | /// assert the store must be done at comptime. |
| 30605 | 30532 | fn storePtrVal( |
| ... | ... | @@ -35579,8 +35506,13 @@ fn structFieldInits( |
| 35579 | 35506 | const default_val = try sema.resolveConstValue(&block_scope, init_src, coerced, null); |
| 35580 | 35507 | |
| 35581 | 35508 | if (default_val.canMutateComptimeVarState(zcu)) { |
| 35582 | const field_name = struct_type.fieldName(ip, field_i).unwrap().?; | |
| 35583 | return sema.failWithContainsReferenceToComptimeVar(&block_scope, init_src, field_name, "field default value", default_val); | |
| 35509 | return sema.failWithContainsReferenceToComptimeVar( | |
| 35510 | &block_scope, | |
| 35511 | init_src, | |
| 35512 | struct_type.fieldName(ip, field_i), | |
| 35513 | "field default value", | |
| 35514 | default_val, | |
| 35515 | ); | |
| 35584 | 35516 | } |
| 35585 | 35517 | struct_type.field_inits.get(ip)[field_i] = default_val.toIntern(); |
| 35586 | 35518 | } |
src/Sema/comptime_ptr_access.zig-2
| ... | ... | @@ -24,7 +24,6 @@ pub fn loadComptimePtr(sema: *Sema, block: *Block, src: LazySrcLoc, ptr: Value) |
| 24 | 24 | const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu); |
| 25 | 25 | const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { |
| 26 | 26 | .none => 0, |
| 27 | .runtime => return .runtime_load, | |
| 28 | 27 | else => |idx| switch (pt.zcu.getTarget().cpu.arch.endian()) { |
| 29 | 28 | .little => child_bits * @intFromEnum(idx), |
| 30 | 29 | .big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian |
| ... | ... | @@ -81,7 +80,6 @@ pub fn storeComptimePtr( |
| 81 | 80 | }; |
| 82 | 81 | const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { |
| 83 | 82 | .none => 0, |
| 84 | .runtime => return .runtime_store, | |
| 85 | 83 | else => |idx| switch (zcu.getTarget().cpu.arch.endian()) { |
| 86 | 84 | .little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx), |
| 87 | 85 | .big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian |
src/Type.zig+3-5
| ... | ... | @@ -198,9 +198,7 @@ pub fn print(ty: Type, writer: *std.Io.Writer, pt: Zcu.PerThread) std.Io.Writer. |
| 198 | 198 | info.packed_offset.bit_offset, info.packed_offset.host_size, |
| 199 | 199 | }); |
| 200 | 200 | } |
| 201 | if (info.flags.vector_index == .runtime) { | |
| 202 | try writer.writeAll(":?"); | |
| 203 | } else if (info.flags.vector_index != .none) { | |
| 201 | if (info.flags.vector_index != .none) { | |
| 204 | 202 | try writer.print(":{d}", .{@intFromEnum(info.flags.vector_index)}); |
| 205 | 203 | } |
| 206 | 204 | try writer.writeAll(") "); |
| ... | ... | @@ -3113,7 +3111,7 @@ pub fn enumTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 { |
| 3113 | 3111 | pub fn structFieldName(ty: Type, index: usize, zcu: *const Zcu) InternPool.OptionalNullTerminatedString { |
| 3114 | 3112 | const ip = &zcu.intern_pool; |
| 3115 | 3113 | return switch (ip.indexToKey(ty.toIntern())) { |
| 3116 | .struct_type => ip.loadStructType(ty.toIntern()).fieldName(ip, index), | |
| 3114 | .struct_type => ip.loadStructType(ty.toIntern()).fieldName(ip, index).toOptional(), | |
| 3117 | 3115 | .tuple_type => .none, |
| 3118 | 3116 | else => unreachable, |
| 3119 | 3117 | }; |
| ... | ... | @@ -3985,7 +3983,7 @@ pub fn elemPtrType(ptr_ty: Type, offset: ?usize, pt: Zcu.PerThread) !Type { |
| 3985 | 3983 | break :blk .{ |
| 3986 | 3984 | .host_size = @intCast(parent_ty.arrayLen(zcu)), |
| 3987 | 3985 | .alignment = parent_ty.abiAlignment(zcu), |
| 3988 | .vector_index = if (offset) |some| @enumFromInt(some) else .runtime, | |
| 3986 | .vector_index = @enumFromInt(offset.?), | |
| 3989 | 3987 | }; |
| 3990 | 3988 | } else .{}; |
| 3991 | 3989 |
src/Value.zig+21-150
| ... | ... | @@ -574,166 +574,37 @@ pub fn writeToPackedMemory( |
| 574 | 574 | } |
| 575 | 575 | } |
| 576 | 576 | |
| 577 | /// Load a Value from the contents of `buffer`. | |
| 577 | /// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type. | |
| 578 | 578 | /// |
| 579 | 579 | /// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past |
| 580 | 580 | /// the end of the value in memory. |
| 581 | pub fn readFromMemory( | |
| 581 | pub fn readUintFromMemory( | |
| 582 | 582 | ty: Type, |
| 583 | 583 | pt: Zcu.PerThread, |
| 584 | 584 | buffer: []const u8, |
| 585 | 585 | arena: Allocator, |
| 586 | ) error{ | |
| 587 | IllDefinedMemoryLayout, | |
| 588 | Unimplemented, | |
| 589 | OutOfMemory, | |
| 590 | }!Value { | |
| 586 | ) Allocator.Error!Value { | |
| 591 | 587 | const zcu = pt.zcu; |
| 592 | const ip = &zcu.intern_pool; | |
| 593 | const target = zcu.getTarget(); | |
| 594 | const endian = target.cpu.arch.endian(); | |
| 595 | switch (ty.zigTypeTag(zcu)) { | |
| 596 | .void => return Value.void, | |
| 597 | .bool => { | |
| 598 | if (buffer[0] == 0) { | |
| 599 | return Value.false; | |
| 600 | } else { | |
| 601 | return Value.true; | |
| 602 | } | |
| 603 | }, | |
| 604 | .int, .@"enum" => |ty_tag| { | |
| 605 | const int_ty = switch (ty_tag) { | |
| 606 | .int => ty, | |
| 607 | .@"enum" => ty.intTagType(zcu), | |
| 608 | else => unreachable, | |
| 609 | }; | |
| 610 | const int_info = int_ty.intInfo(zcu); | |
| 611 | const bits = int_info.bits; | |
| 612 | const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8); | |
| 613 | if (bits == 0 or buffer.len == 0) return zcu.getCoerced(try zcu.intValue(int_ty, 0), ty); | |
| 588 | const endian = zcu.getTarget().cpu.arch.endian(); | |
| 614 | 589 | |
| 615 | if (bits <= 64) switch (int_info.signedness) { // Fast path for integers <= u64 | |
| 616 | .signed => { | |
| 617 | const val = std.mem.readVarInt(i64, buffer[0..byte_count], endian); | |
| 618 | const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); | |
| 619 | return zcu.getCoerced(try zcu.intValue(int_ty, result), ty); | |
| 620 | }, | |
| 621 | .unsigned => { | |
| 622 | const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian); | |
| 623 | const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); | |
| 624 | return zcu.getCoerced(try zcu.intValue(int_ty, result), ty); | |
| 625 | }, | |
| 626 | } else { // Slow path, we have to construct a big-int | |
| 627 | const Limb = std.math.big.Limb; | |
| 628 | const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb); | |
| 629 | const limbs_buffer = try arena.alloc(Limb, limb_count); | |
| 630 | ||
| 631 | var bigint = BigIntMutable.init(limbs_buffer, 0); | |
| 632 | bigint.readTwosComplement(buffer[0..byte_count], bits, endian, int_info.signedness); | |
| 633 | return zcu.getCoerced(try zcu.intValue_big(int_ty, bigint.toConst()), ty); | |
| 634 | } | |
| 635 | }, | |
| 636 | .float => return Value.fromInterned(try pt.intern(.{ .float = .{ | |
| 637 | .ty = ty.toIntern(), | |
| 638 | .storage = switch (ty.floatBits(target)) { | |
| 639 | 16 => .{ .f16 = @bitCast(std.mem.readInt(u16, buffer[0..2], endian)) }, | |
| 640 | 32 => .{ .f32 = @bitCast(std.mem.readInt(u32, buffer[0..4], endian)) }, | |
| 641 | 64 => .{ .f64 = @bitCast(std.mem.readInt(u64, buffer[0..8], endian)) }, | |
| 642 | 80 => .{ .f80 = @bitCast(std.mem.readInt(u80, buffer[0..10], endian)) }, | |
| 643 | 128 => .{ .f128 = @bitCast(std.mem.readInt(u128, buffer[0..16], endian)) }, | |
| 644 | else => unreachable, | |
| 645 | }, | |
| 646 | } })), | |
| 647 | .array => { | |
| 648 | const elem_ty = ty.childType(zcu); | |
| 649 | const elem_size = elem_ty.abiSize(zcu); | |
| 650 | const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu))); | |
| 651 | var offset: usize = 0; | |
| 652 | for (elems) |*elem| { | |
| 653 | elem.* = (try readFromMemory(elem_ty, zcu, buffer[offset..], arena)).toIntern(); | |
| 654 | offset += @intCast(elem_size); | |
| 655 | } | |
| 656 | return pt.aggregateValue(ty, elems); | |
| 657 | }, | |
| 658 | .vector => { | |
| 659 | // We use byte_count instead of abi_size here, so that any padding bytes | |
| 660 | // follow the data bytes, on both big- and little-endian systems. | |
| 661 | const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8; | |
| 662 | return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena); | |
| 663 | }, | |
| 664 | .@"struct" => { | |
| 665 | const struct_type = zcu.typeToStruct(ty).?; | |
| 666 | switch (struct_type.layout) { | |
| 667 | .auto => unreachable, // Sema is supposed to have emitted a compile error already | |
| 668 | .@"extern" => { | |
| 669 | const field_types = struct_type.field_types; | |
| 670 | const field_vals = try arena.alloc(InternPool.Index, field_types.len); | |
| 671 | for (field_vals, 0..) |*field_val, i| { | |
| 672 | const field_ty = Type.fromInterned(field_types.get(ip)[i]); | |
| 673 | const off: usize = @intCast(ty.structFieldOffset(i, zcu)); | |
| 674 | const sz: usize = @intCast(field_ty.abiSize(zcu)); | |
| 675 | field_val.* = (try readFromMemory(field_ty, zcu, buffer[off..(off + sz)], arena)).toIntern(); | |
| 676 | } | |
| 677 | return pt.aggregateValue(ty, field_vals); | |
| 678 | }, | |
| 679 | .@"packed" => { | |
| 680 | const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8; | |
| 681 | return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena); | |
| 682 | }, | |
| 683 | } | |
| 684 | }, | |
| 685 | .error_set => { | |
| 686 | const bits = zcu.errorSetBits(); | |
| 687 | const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8); | |
| 688 | const int = std.mem.readVarInt(u64, buffer[0..byte_count], endian); | |
| 689 | const index = (int << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); | |
| 690 | const name = zcu.global_error_set.keys()[@intCast(index)]; | |
| 590 | assert(ty.isUnsignedInt(zcu)); | |
| 591 | const bits = ty.intInfo(zcu).bits; | |
| 592 | const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8); | |
| 691 | 593 | |
| 692 | return Value.fromInterned(try pt.intern(.{ .err = .{ | |
| 693 | .ty = ty.toIntern(), | |
| 694 | .name = name, | |
| 695 | } })); | |
| 696 | }, | |
| 697 | .@"union" => switch (ty.containerLayout(zcu)) { | |
| 698 | .auto => return error.IllDefinedMemoryLayout, | |
| 699 | .@"extern" => { | |
| 700 | const union_size = ty.abiSize(zcu); | |
| 701 | const array_ty = try zcu.arrayType(.{ .len = union_size, .child = .u8_type }); | |
| 702 | const val = (try readFromMemory(array_ty, zcu, buffer, arena)).toIntern(); | |
| 703 | return Value.fromInterned(try pt.internUnion(.{ | |
| 704 | .ty = ty.toIntern(), | |
| 705 | .tag = .none, | |
| 706 | .val = val, | |
| 707 | })); | |
| 708 | }, | |
| 709 | .@"packed" => { | |
| 710 | const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8; | |
| 711 | return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena); | |
| 712 | }, | |
| 713 | }, | |
| 714 | .pointer => { | |
| 715 | assert(!ty.isSlice(zcu)); // No well defined layout. | |
| 716 | const int_val = try readFromMemory(Type.usize, zcu, buffer, arena); | |
| 717 | return Value.fromInterned(try pt.intern(.{ .ptr = .{ | |
| 718 | .ty = ty.toIntern(), | |
| 719 | .base_addr = .int, | |
| 720 | .byte_offset = int_val.toUnsignedInt(zcu), | |
| 721 | } })); | |
| 722 | }, | |
| 723 | .optional => { | |
| 724 | assert(ty.isPtrLikeOptional(zcu)); | |
| 725 | const child_ty = ty.optionalChild(zcu); | |
| 726 | const child_val = try readFromMemory(child_ty, zcu, buffer, arena); | |
| 727 | return Value.fromInterned(try pt.intern(.{ .opt = .{ | |
| 728 | .ty = ty.toIntern(), | |
| 729 | .val = switch (child_val.orderAgainstZero(pt)) { | |
| 730 | .lt => unreachable, | |
| 731 | .eq => .none, | |
| 732 | .gt => child_val.toIntern(), | |
| 733 | }, | |
| 734 | } })); | |
| 735 | }, | |
| 736 | else => return error.Unimplemented, | |
| 594 | assert(buffer.len >= byte_count); | |
| 595 | ||
| 596 | if (bits <= 64) { | |
| 597 | const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian); | |
| 598 | const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); | |
| 599 | return pt.intValue(ty, result); | |
| 600 | } else { | |
| 601 | const Limb = std.math.big.Limb; | |
| 602 | const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb); | |
| 603 | const limbs_buffer = try arena.alloc(Limb, limb_count); | |
| 604 | ||
| 605 | var bigint: BigIntMutable = .init(limbs_buffer, 0); | |
| 606 | bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned); | |
| 607 | return pt.intValue_big(ty, bigint.toConst()); | |
| 737 | 608 | } |
| 738 | 609 | } |
| 739 | 610 |
src/Zcu/PerThread.zig+22-4
| ... | ... | @@ -3512,7 +3512,6 @@ pub fn ptrType(pt: Zcu.PerThread, info: InternPool.Key.PtrType) Allocator.Error! |
| 3512 | 3512 | canon_info.packed_offset.host_size = 0; |
| 3513 | 3513 | } |
| 3514 | 3514 | }, |
| 3515 | .runtime => {}, | |
| 3516 | 3515 | _ => assert(@intFromEnum(info.flags.vector_index) < info.packed_offset.host_size), |
| 3517 | 3516 | } |
| 3518 | 3517 | |
| ... | ... | @@ -3663,21 +3662,40 @@ pub fn intRef(pt: Zcu.PerThread, ty: Type, x: anytype) Allocator.Error!Air.Inst. |
| 3663 | 3662 | } |
| 3664 | 3663 | |
| 3665 | 3664 | pub fn intValue_big(pt: Zcu.PerThread, ty: Type, x: BigIntConst) Allocator.Error!Value { |
| 3666 | return Value.fromInterned(try pt.intern(.{ .int = .{ | |
| 3665 | if (ty.toIntern() != .comptime_int_type) { | |
| 3666 | const int_info = ty.intInfo(pt.zcu); | |
| 3667 | assert(x.fitsInTwosComp(int_info.signedness, int_info.bits)); | |
| 3668 | } | |
| 3669 | return .fromInterned(try pt.intern(.{ .int = .{ | |
| 3667 | 3670 | .ty = ty.toIntern(), |
| 3668 | 3671 | .storage = .{ .big_int = x }, |
| 3669 | 3672 | } })); |
| 3670 | 3673 | } |
| 3671 | 3674 | |
| 3672 | 3675 | pub fn intValue_u64(pt: Zcu.PerThread, ty: Type, x: u64) Allocator.Error!Value { |
| 3673 | return Value.fromInterned(try pt.intern(.{ .int = .{ | |
| 3676 | if (ty.toIntern() != .comptime_int_type and x != 0) { | |
| 3677 | const int_info = ty.intInfo(pt.zcu); | |
| 3678 | const unsigned_bits = int_info.bits - @intFromBool(int_info.signedness == .signed); | |
| 3679 | assert(unsigned_bits >= std.math.log2(x) + 1); | |
| 3680 | } | |
| 3681 | return .fromInterned(try pt.intern(.{ .int = .{ | |
| 3674 | 3682 | .ty = ty.toIntern(), |
| 3675 | 3683 | .storage = .{ .u64 = x }, |
| 3676 | 3684 | } })); |
| 3677 | 3685 | } |
| 3678 | 3686 | |
| 3679 | 3687 | pub fn intValue_i64(pt: Zcu.PerThread, ty: Type, x: i64) Allocator.Error!Value { |
| 3680 | return Value.fromInterned(try pt.intern(.{ .int = .{ | |
| 3688 | if (ty.toIntern() != .comptime_int_type and x != 0) { | |
| 3689 | const int_info = ty.intInfo(pt.zcu); | |
| 3690 | const unsigned_bits = int_info.bits - @intFromBool(int_info.signedness == .signed); | |
| 3691 | if (x > 0) { | |
| 3692 | assert(unsigned_bits >= std.math.log2(x) + 1); | |
| 3693 | } else { | |
| 3694 | assert(int_info.signedness == .signed); | |
| 3695 | assert(unsigned_bits >= std.math.log2_int_ceil(u64, @abs(x))); | |
| 3696 | } | |
| 3697 | } | |
| 3698 | return .fromInterned(try pt.intern(.{ .int = .{ | |
| 3681 | 3699 | .ty = ty.toIntern(), |
| 3682 | 3700 | .storage = .{ .i64 = x }, |
| 3683 | 3701 | } })); |
src/codegen/aarch64/Select.zig-12
| ... | ... | @@ -826,18 +826,6 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { |
| 826 | 826 | |
| 827 | 827 | try isel.analyzeUse(un_op); |
| 828 | 828 | |
| 829 | air_body_index += 1; | |
| 830 | air_inst_index = air_body[air_body_index]; | |
| 831 | continue :air_tag air_tags[@intFromEnum(air_inst_index)]; | |
| 832 | }, | |
| 833 | .vector_store_elem => { | |
| 834 | const vector_store_elem = air_data[@intFromEnum(air_inst_index)].vector_store_elem; | |
| 835 | const bin_op = isel.air.extraData(Air.Bin, vector_store_elem.payload).data; | |
| 836 | ||
| 837 | try isel.analyzeUse(vector_store_elem.vector_ptr); | |
| 838 | try isel.analyzeUse(bin_op.lhs); | |
| 839 | try isel.analyzeUse(bin_op.rhs); | |
| 840 | ||
| 841 | 829 | air_body_index += 1; |
| 842 | 830 | air_inst_index = air_body[air_body_index]; |
| 843 | 831 | continue :air_tag air_tags[@intFromEnum(air_inst_index)]; |
src/codegen/c.zig+79-449
| ... | ... | @@ -37,6 +37,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { |
| 37 | 37 | .expand_packed_load = true, |
| 38 | 38 | .expand_packed_store = true, |
| 39 | 39 | .expand_packed_struct_field_val = true, |
| 40 | .expand_packed_aggregate_init = true, | |
| 40 | 41 | }), |
| 41 | 42 | }; |
| 42 | 43 | } |
| ... | ... | @@ -1392,114 +1393,21 @@ pub const DeclGen = struct { |
| 1392 | 1393 | try w.writeByte('}'); |
| 1393 | 1394 | }, |
| 1394 | 1395 | .@"packed" => { |
| 1395 | const int_info = ty.intInfo(zcu); | |
| 1396 | ||
| 1397 | const bits = Type.smallestUnsignedBits(int_info.bits - 1); | |
| 1398 | const bit_offset_ty = try pt.intType(.unsigned, bits); | |
| 1399 | ||
| 1400 | var bit_offset: u64 = 0; | |
| 1401 | var eff_num_fields: usize = 0; | |
| 1402 | ||
| 1403 | for (0..loaded_struct.field_types.len) |field_index| { | |
| 1404 | const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); | |
| 1405 | if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; | |
| 1406 | eff_num_fields += 1; | |
| 1407 | } | |
| 1408 | ||
| 1409 | if (eff_num_fields == 0) { | |
| 1410 | try w.writeByte('('); | |
| 1411 | try dg.renderUndefValue(w, ty, location); | |
| 1412 | try w.writeByte(')'); | |
| 1413 | } else if (ty.bitSize(zcu) > 64) { | |
| 1414 | // zig_or_u128(zig_or_u128(zig_shl_u128(a, a_off), zig_shl_u128(b, b_off)), zig_shl_u128(c, c_off)) | |
| 1415 | var num_or = eff_num_fields - 1; | |
| 1416 | while (num_or > 0) : (num_or -= 1) { | |
| 1417 | try w.writeAll("zig_or_"); | |
| 1418 | try dg.renderTypeForBuiltinFnName(w, ty); | |
| 1419 | try w.writeByte('('); | |
| 1420 | } | |
| 1421 | ||
| 1422 | var eff_index: usize = 0; | |
| 1423 | var needs_closing_paren = false; | |
| 1424 | for (0..loaded_struct.field_types.len) |field_index| { | |
| 1425 | const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); | |
| 1426 | if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; | |
| 1427 | ||
| 1428 | const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) { | |
| 1429 | .bytes => |bytes| try pt.intern(.{ .int = .{ | |
| 1430 | .ty = field_ty.toIntern(), | |
| 1431 | .storage = .{ .u64 = bytes.at(field_index, ip) }, | |
| 1432 | } }), | |
| 1433 | .elems => |elems| elems[field_index], | |
| 1434 | .repeated_elem => |elem| elem, | |
| 1435 | }; | |
| 1436 | const cast_context = IntCastContext{ .value = .{ .value = Value.fromInterned(field_val) } }; | |
| 1437 | if (bit_offset != 0) { | |
| 1438 | try w.writeAll("zig_shl_"); | |
| 1439 | try dg.renderTypeForBuiltinFnName(w, ty); | |
| 1440 | try w.writeByte('('); | |
| 1441 | try dg.renderIntCast(w, ty, cast_context, field_ty, .FunctionArgument); | |
| 1442 | try w.writeAll(", "); | |
| 1443 | try dg.renderValue(w, try pt.intValue(bit_offset_ty, bit_offset), .FunctionArgument); | |
| 1444 | try w.writeByte(')'); | |
| 1445 | } else { | |
| 1446 | try dg.renderIntCast(w, ty, cast_context, field_ty, .FunctionArgument); | |
| 1447 | } | |
| 1448 | ||
| 1449 | if (needs_closing_paren) try w.writeByte(')'); | |
| 1450 | if (eff_index != eff_num_fields - 1) try w.writeAll(", "); | |
| 1451 | ||
| 1452 | bit_offset += field_ty.bitSize(zcu); | |
| 1453 | needs_closing_paren = true; | |
| 1454 | eff_index += 1; | |
| 1455 | } | |
| 1456 | } else { | |
| 1457 | try w.writeByte('('); | |
| 1458 | // a << a_off | b << b_off | c << c_off | |
| 1459 | var empty = true; | |
| 1460 | for (0..loaded_struct.field_types.len) |field_index| { | |
| 1461 | const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); | |
| 1462 | if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; | |
| 1463 | ||
| 1464 | if (!empty) try w.writeAll(" | "); | |
| 1465 | try w.writeByte('('); | |
| 1466 | try dg.renderCType(w, ctype); | |
| 1467 | try w.writeByte(')'); | |
| 1468 | ||
| 1469 | const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) { | |
| 1470 | .bytes => |bytes| try pt.intern(.{ .int = .{ | |
| 1471 | .ty = field_ty.toIntern(), | |
| 1472 | .storage = .{ .u64 = bytes.at(field_index, ip) }, | |
| 1473 | } }), | |
| 1474 | .elems => |elems| elems[field_index], | |
| 1475 | .repeated_elem => |elem| elem, | |
| 1476 | }; | |
| 1477 | ||
| 1478 | const field_int_info: std.builtin.Type.Int = if (field_ty.isAbiInt(zcu)) | |
| 1479 | field_ty.intInfo(zcu) | |
| 1480 | else | |
| 1481 | .{ .signedness = .unsigned, .bits = undefined }; | |
| 1482 | switch (field_int_info.signedness) { | |
| 1483 | .signed => { | |
| 1484 | try w.writeByte('('); | |
| 1485 | try dg.renderValue(w, Value.fromInterned(field_val), .Other); | |
| 1486 | try w.writeAll(" & "); | |
| 1487 | const field_uint_ty = try pt.intType(.unsigned, field_int_info.bits); | |
| 1488 | try dg.renderValue(w, try field_uint_ty.maxIntScalar(pt, field_uint_ty), .Other); | |
| 1489 | try w.writeByte(')'); | |
| 1490 | }, | |
| 1491 | .unsigned => try dg.renderValue(w, Value.fromInterned(field_val), .Other), | |
| 1492 | } | |
| 1493 | if (bit_offset != 0) { | |
| 1494 | try w.writeAll(" << "); | |
| 1495 | try dg.renderValue(w, try pt.intValue(bit_offset_ty, bit_offset), .FunctionArgument); | |
| 1496 | } | |
| 1497 | ||
| 1498 | bit_offset += field_ty.bitSize(zcu); | |
| 1499 | empty = false; | |
| 1500 | } | |
| 1501 | try w.writeByte(')'); | |
| 1502 | } | |
| 1396 | // https://github.com/ziglang/zig/issues/24657 will eliminate most of the | |
| 1397 | // following logic, leaving only the recursive `renderValue` call. Once | |
| 1398 | // that proposal is implemented, a `packed struct` will literally be | |
| 1399 | // represented in the InternPool by its comptime-known backing integer. | |
| 1400 | var arena: std.heap.ArenaAllocator = .init(zcu.gpa); | |
| 1401 | defer arena.deinit(); | |
| 1402 | const backing_ty: Type = .fromInterned(loaded_struct.backingIntTypeUnordered(ip)); | |
| 1403 | const buf = try arena.allocator().alloc(u8, @intCast(ty.abiSize(zcu))); | |
| 1404 | val.writeToMemory(pt, buf) catch |err| switch (err) { | |
| 1405 | error.IllDefinedMemoryLayout => unreachable, | |
| 1406 | error.OutOfMemory => |e| return e, | |
| 1407 | error.ReinterpretDeclRef, error.Unimplemented => return dg.fail("TODO: C backend: lower packed struct value", .{}), | |
| 1408 | }; | |
| 1409 | const backing_val: Value = try .readUintFromMemory(backing_ty, pt, buf, arena.allocator()); | |
| 1410 | return dg.renderValue(w, backing_val, location); | |
| 1503 | 1411 | }, |
| 1504 | 1412 | } |
| 1505 | 1413 | }, |
| ... | ... | @@ -1507,33 +1415,38 @@ pub const DeclGen = struct { |
| 1507 | 1415 | }, |
| 1508 | 1416 | .un => |un| { |
| 1509 | 1417 | const loaded_union = ip.loadUnionType(ty.toIntern()); |
| 1418 | if (loaded_union.flagsUnordered(ip).layout == .@"packed") { | |
| 1419 | // https://github.com/ziglang/zig/issues/24657 will eliminate most of the | |
| 1420 | // following logic, leaving only the recursive `renderValue` call. Once | |
| 1421 | // that proposal is implemented, a `packed union` will literally be | |
| 1422 | // represented in the InternPool by its comptime-known backing integer. | |
| 1423 | var arena: std.heap.ArenaAllocator = .init(zcu.gpa); | |
| 1424 | defer arena.deinit(); | |
| 1425 | const backing_ty = try ty.unionBackingType(pt); | |
| 1426 | const buf = try arena.allocator().alloc(u8, @intCast(ty.abiSize(zcu))); | |
| 1427 | val.writeToMemory(pt, buf) catch |err| switch (err) { | |
| 1428 | error.IllDefinedMemoryLayout => unreachable, | |
| 1429 | error.OutOfMemory => |e| return e, | |
| 1430 | error.ReinterpretDeclRef, error.Unimplemented => return dg.fail("TODO: C backend: lower packed union value", .{}), | |
| 1431 | }; | |
| 1432 | const backing_val: Value = try .readUintFromMemory(backing_ty, pt, buf, arena.allocator()); | |
| 1433 | return dg.renderValue(w, backing_val, location); | |
| 1434 | } | |
| 1510 | 1435 | if (un.tag == .none) { |
| 1511 | 1436 | const backing_ty = try ty.unionBackingType(pt); |
| 1512 | switch (loaded_union.flagsUnordered(ip).layout) { | |
| 1513 | .@"packed" => { | |
| 1514 | if (!location.isInitializer()) { | |
| 1515 | try w.writeByte('('); | |
| 1516 | try dg.renderType(w, backing_ty); | |
| 1517 | try w.writeByte(')'); | |
| 1518 | } | |
| 1519 | try dg.renderValue(w, Value.fromInterned(un.val), location); | |
| 1520 | }, | |
| 1521 | .@"extern" => { | |
| 1522 | if (location == .StaticInitializer) { | |
| 1523 | return dg.fail("TODO: C backend: implement extern union backing type rendering in static initializers", .{}); | |
| 1524 | } | |
| 1525 | ||
| 1526 | const ptr_ty = try pt.singleConstPtrType(ty); | |
| 1527 | try w.writeAll("*(("); | |
| 1528 | try dg.renderType(w, ptr_ty); | |
| 1529 | try w.writeAll(")("); | |
| 1530 | try dg.renderType(w, backing_ty); | |
| 1531 | try w.writeAll("){"); | |
| 1532 | try dg.renderValue(w, Value.fromInterned(un.val), location); | |
| 1533 | try w.writeAll("})"); | |
| 1534 | }, | |
| 1535 | else => unreachable, | |
| 1437 | assert(loaded_union.flagsUnordered(ip).layout == .@"extern"); | |
| 1438 | if (location == .StaticInitializer) { | |
| 1439 | return dg.fail("TODO: C backend: implement extern union backing type rendering in static initializers", .{}); | |
| 1536 | 1440 | } |
| 1441 | ||
| 1442 | const ptr_ty = try pt.singleConstPtrType(ty); | |
| 1443 | try w.writeAll("*(("); | |
| 1444 | try dg.renderType(w, ptr_ty); | |
| 1445 | try w.writeAll(")("); | |
| 1446 | try dg.renderType(w, backing_ty); | |
| 1447 | try w.writeAll("){"); | |
| 1448 | try dg.renderValue(w, Value.fromInterned(un.val), location); | |
| 1449 | try w.writeAll("})"); | |
| 1537 | 1450 | } else { |
| 1538 | 1451 | if (!location.isInitializer()) { |
| 1539 | 1452 | try w.writeByte('('); |
| ... | ... | @@ -1544,21 +1457,6 @@ pub const DeclGen = struct { |
| 1544 | 1457 | const field_index = zcu.unionTagFieldIndex(loaded_union, Value.fromInterned(un.tag)).?; |
| 1545 | 1458 | const field_ty: Type = .fromInterned(loaded_union.field_types.get(ip)[field_index]); |
| 1546 | 1459 | const field_name = loaded_union.loadTagType(ip).names.get(ip)[field_index]; |
| 1547 | if (loaded_union.flagsUnordered(ip).layout == .@"packed") { | |
| 1548 | if (field_ty.hasRuntimeBits(zcu)) { | |
| 1549 | if (field_ty.isPtrAtRuntime(zcu)) { | |
| 1550 | try w.writeByte('('); | |
| 1551 | try dg.renderCType(w, ctype); | |
| 1552 | try w.writeByte(')'); | |
| 1553 | } else if (field_ty.zigTypeTag(zcu) == .float) { | |
| 1554 | try w.writeByte('('); | |
| 1555 | try dg.renderCType(w, ctype); | |
| 1556 | try w.writeByte(')'); | |
| 1557 | } | |
| 1558 | try dg.renderValue(w, Value.fromInterned(un.val), location); | |
| 1559 | } else try w.writeByte('0'); | |
| 1560 | return; | |
| 1561 | } | |
| 1562 | 1460 | |
| 1563 | 1461 | const has_tag = loaded_union.hasTag(ip); |
| 1564 | 1462 | if (has_tag) try w.writeByte('{'); |
| ... | ... | @@ -1745,9 +1643,11 @@ pub const DeclGen = struct { |
| 1745 | 1643 | } |
| 1746 | 1644 | return w.writeByte('}'); |
| 1747 | 1645 | }, |
| 1748 | .@"packed" => return w.print("{f}", .{ | |
| 1749 | try dg.fmtIntLiteralHex(try pt.undefValue(ty), .Other), | |
| 1750 | }), | |
| 1646 | .@"packed" => return dg.renderUndefValue( | |
| 1647 | w, | |
| 1648 | .fromInterned(loaded_struct.backingIntTypeUnordered(ip)), | |
| 1649 | location, | |
| 1650 | ), | |
| 1751 | 1651 | } |
| 1752 | 1652 | }, |
| 1753 | 1653 | .tuple_type => |tuple_info| { |
| ... | ... | @@ -1815,9 +1715,11 @@ pub const DeclGen = struct { |
| 1815 | 1715 | } |
| 1816 | 1716 | if (has_tag) try w.writeByte('}'); |
| 1817 | 1717 | }, |
| 1818 | .@"packed" => return w.print("{f}", .{ | |
| 1819 | try dg.fmtIntLiteralHex(try pt.undefValue(ty), .Other), | |
| 1820 | }), | |
| 1718 | .@"packed" => return dg.renderUndefValue( | |
| 1719 | w, | |
| 1720 | try ty.unionBackingType(pt), | |
| 1721 | location, | |
| 1722 | ), | |
| 1821 | 1723 | } |
| 1822 | 1724 | }, |
| 1823 | 1725 | .error_union_type => |error_union_type| switch (ctype.info(ctype_pool)) { |
| ... | ... | @@ -2445,10 +2347,7 @@ pub const DeclGen = struct { |
| 2445 | 2347 | const ty = val.typeOf(zcu); |
| 2446 | 2348 | return .{ .data = .{ |
| 2447 | 2349 | .dg = dg, |
| 2448 | .int_info = if (ty.zigTypeTag(zcu) == .@"union" and ty.containerLayout(zcu) == .@"packed") | |
| 2449 | .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) } | |
| 2450 | else | |
| 2451 | ty.intInfo(zcu), | |
| 2350 | .int_info = ty.intInfo(zcu), | |
| 2452 | 2351 | .kind = kind, |
| 2453 | 2352 | .ctype = try dg.ctypeFromType(ty, kind), |
| 2454 | 2353 | .val = val, |
| ... | ... | @@ -3656,7 +3555,6 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void { |
| 3656 | 3555 | |
| 3657 | 3556 | .is_named_enum_value => return f.fail("TODO: C backend: implement is_named_enum_value", .{}), |
| 3658 | 3557 | .error_set_has_value => return f.fail("TODO: C backend: implement error_set_has_value", .{}), |
| 3659 | .vector_store_elem => return f.fail("TODO: C backend: implement vector_store_elem", .{}), | |
| 3660 | 3558 | |
| 3661 | 3559 | .runtime_nav_ptr => try airRuntimeNavPtr(f, inst), |
| 3662 | 3560 | |
| ... | ... | @@ -3956,6 +3854,10 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue { |
| 3956 | 3854 | const ptr_info = ptr_scalar_ty.ptrInfo(zcu); |
| 3957 | 3855 | const src_ty: Type = .fromInterned(ptr_info.child); |
| 3958 | 3856 | |
| 3857 | // `Air.Legalize.Feature.expand_packed_load` should ensure that the only | |
| 3858 | // bit-pointers we see here are vector element pointers. | |
| 3859 | assert(ptr_info.packed_offset.host_size == 0 or ptr_info.flags.vector_index != .none); | |
| 3860 | ||
| 3959 | 3861 | if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) { |
| 3960 | 3862 | try reap(f, inst, &.{ty_op.operand}); |
| 3961 | 3863 | return .none; |
| ... | ... | @@ -3987,40 +3889,6 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue { |
| 3987 | 3889 | try w.writeAll(", sizeof("); |
| 3988 | 3890 | try f.renderType(w, src_ty); |
| 3989 | 3891 | try w.writeAll("))"); |
| 3990 | } else if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) { | |
| 3991 | const host_bits: u16 = ptr_info.packed_offset.host_size * 8; | |
| 3992 | const host_ty = try pt.intType(.unsigned, host_bits); | |
| 3993 | ||
| 3994 | const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(host_bits - 1)); | |
| 3995 | const bit_offset_val = try pt.intValue(bit_offset_ty, ptr_info.packed_offset.bit_offset); | |
| 3996 | ||
| 3997 | const field_ty = try pt.intType(.unsigned, @as(u16, @intCast(src_ty.bitSize(zcu)))); | |
| 3998 | ||
| 3999 | try f.writeCValue(w, local, .Other); | |
| 4000 | try v.elem(f, w); | |
| 4001 | try w.writeAll(" = ("); | |
| 4002 | try f.renderType(w, src_ty); | |
| 4003 | try w.writeAll(")zig_wrap_"); | |
| 4004 | try f.object.dg.renderTypeForBuiltinFnName(w, field_ty); | |
| 4005 | try w.writeAll("(("); | |
| 4006 | try f.renderType(w, field_ty); | |
| 4007 | try w.writeByte(')'); | |
| 4008 | const cant_cast = host_ty.isInt(zcu) and host_ty.bitSize(zcu) > 64; | |
| 4009 | if (cant_cast) { | |
| 4010 | if (field_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{}); | |
| 4011 | try w.writeAll("zig_lo_"); | |
| 4012 | try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); | |
| 4013 | try w.writeByte('('); | |
| 4014 | } | |
| 4015 | try w.writeAll("zig_shr_"); | |
| 4016 | try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); | |
| 4017 | try w.writeByte('('); | |
| 4018 | try f.writeCValueDeref(w, operand); | |
| 4019 | try v.elem(f, w); | |
| 4020 | try w.print(", {f})", .{try f.fmtIntLiteralDec(bit_offset_val)}); | |
| 4021 | if (cant_cast) try w.writeByte(')'); | |
| 4022 | try f.object.dg.renderBuiltinInfo(w, field_ty, .bits); | |
| 4023 | try w.writeByte(')'); | |
| 4024 | 3892 | } else { |
| 4025 | 3893 | try f.writeCValue(w, local, .Other); |
| 4026 | 3894 | try v.elem(f, w); |
| ... | ... | @@ -4213,6 +4081,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { |
| 4213 | 4081 | const ptr_scalar_ty = ptr_ty.scalarType(zcu); |
| 4214 | 4082 | const ptr_info = ptr_scalar_ty.ptrInfo(zcu); |
| 4215 | 4083 | |
| 4084 | // `Air.Legalize.Feature.expand_packed_store` should ensure that the only | |
| 4085 | // bit-pointers we see here are vector element pointers. | |
| 4086 | assert(ptr_info.packed_offset.host_size == 0 or ptr_info.flags.vector_index != .none); | |
| 4087 | ||
| 4216 | 4088 | const ptr_val = try f.resolveInst(bin_op.lhs); |
| 4217 | 4089 | const src_ty = f.typeOf(bin_op.rhs); |
| 4218 | 4090 | |
| ... | ... | @@ -4277,66 +4149,6 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { |
| 4277 | 4149 | try w.writeByte(';'); |
| 4278 | 4150 | try f.object.newline(); |
| 4279 | 4151 | try v.end(f, inst, w); |
| 4280 | } else if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) { | |
| 4281 | const host_bits = ptr_info.packed_offset.host_size * 8; | |
| 4282 | const host_ty = try pt.intType(.unsigned, host_bits); | |
| 4283 | ||
| 4284 | const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(host_bits - 1)); | |
| 4285 | const bit_offset_val = try pt.intValue(bit_offset_ty, ptr_info.packed_offset.bit_offset); | |
| 4286 | ||
| 4287 | const src_bits = src_ty.bitSize(zcu); | |
| 4288 | ||
| 4289 | const ExpectedContents = [BigInt.Managed.default_capacity]BigIntLimb; | |
| 4290 | var stack align(@alignOf(ExpectedContents)) = | |
| 4291 | std.heap.stackFallback(@sizeOf(ExpectedContents), f.object.dg.gpa); | |
| 4292 | ||
| 4293 | var mask = try BigInt.Managed.initCapacity(stack.get(), BigInt.calcTwosCompLimbCount(host_bits)); | |
| 4294 | defer mask.deinit(); | |
| 4295 | ||
| 4296 | try mask.setTwosCompIntLimit(.max, .unsigned, @intCast(src_bits)); | |
| 4297 | try mask.shiftLeft(&mask, ptr_info.packed_offset.bit_offset); | |
| 4298 | try mask.bitNotWrap(&mask, .unsigned, host_bits); | |
| 4299 | ||
| 4300 | const mask_val = try pt.intValue_big(host_ty, mask.toConst()); | |
| 4301 | ||
| 4302 | const v = try Vectorize.start(f, inst, w, ptr_ty); | |
| 4303 | const a = try Assignment.start(f, w, src_scalar_ctype); | |
| 4304 | try f.writeCValueDeref(w, ptr_val); | |
| 4305 | try v.elem(f, w); | |
| 4306 | try a.assign(f, w); | |
| 4307 | try w.writeAll("zig_or_"); | |
| 4308 | try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); | |
| 4309 | try w.writeAll("(zig_and_"); | |
| 4310 | try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); | |
| 4311 | try w.writeByte('('); | |
| 4312 | try f.writeCValueDeref(w, ptr_val); | |
| 4313 | try v.elem(f, w); | |
| 4314 | try w.print(", {f}), zig_shl_", .{try f.fmtIntLiteralHex(mask_val)}); | |
| 4315 | try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); | |
| 4316 | try w.writeByte('('); | |
| 4317 | const cant_cast = host_ty.isInt(zcu) and host_ty.bitSize(zcu) > 64; | |
| 4318 | if (cant_cast) { | |
| 4319 | if (src_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{}); | |
| 4320 | try w.writeAll("zig_make_"); | |
| 4321 | try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); | |
| 4322 | try w.writeAll("(0, "); | |
| 4323 | } else { | |
| 4324 | try w.writeByte('('); | |
| 4325 | try f.renderType(w, host_ty); | |
| 4326 | try w.writeByte(')'); | |
| 4327 | } | |
| 4328 | ||
| 4329 | if (src_ty.isPtrAtRuntime(zcu)) { | |
| 4330 | try w.writeByte('('); | |
| 4331 | try f.renderType(w, .usize); | |
| 4332 | try w.writeByte(')'); | |
| 4333 | } | |
| 4334 | try f.writeCValue(w, src_val, .Other); | |
| 4335 | try v.elem(f, w); | |
| 4336 | if (cant_cast) try w.writeByte(')'); | |
| 4337 | try w.print(", {f}))", .{try f.fmtIntLiteralDec(bit_offset_val)}); | |
| 4338 | try a.end(f, w); | |
| 4339 | try v.end(f, inst, w); | |
| 4340 | 4152 | } else { |
| 4341 | 4153 | switch (ptr_val) { |
| 4342 | 4154 | .local_ref => |ptr_local_index| switch (src_val) { |
| ... | ... | @@ -6015,10 +5827,7 @@ fn fieldLocation( |
| 6015 | 5827 | else if (!field_ptr_ty.childType(zcu).hasRuntimeBitsIgnoreComptime(zcu)) |
| 6016 | 5828 | .{ .byte_offset = loaded_struct.offsets.get(ip)[field_index] } |
| 6017 | 5829 | else |
| 6018 | .{ .field = if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| | |
| 6019 | .{ .identifier = field_name.toSlice(ip) } | |
| 6020 | else | |
| 6021 | .{ .field = field_index } }, | |
| 5830 | .{ .field = .{ .identifier = loaded_struct.fieldName(ip, field_index).toSlice(ip) } }, | |
| 6022 | 5831 | .@"packed" => if (field_ptr_ty.ptrInfo(zcu).packed_offset.host_size == 0) |
| 6023 | 5832 | .{ .byte_offset = @divExact(zcu.structPackedFieldBitOffset(loaded_struct, field_index) + |
| 6024 | 5833 | container_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset, 8) } |
| ... | ... | @@ -6202,115 +6011,20 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { |
| 6202 | 6011 | // Ensure complete type definition is visible before accessing fields. |
| 6203 | 6012 | _ = try f.ctypeFromType(struct_ty, .complete); |
| 6204 | 6013 | |
| 6014 | assert(struct_ty.containerLayout(zcu) != .@"packed"); // `Air.Legalize.Feature.expand_packed_struct_field_val` handles this case | |
| 6205 | 6015 | const field_name: CValue = switch (ip.indexToKey(struct_ty.toIntern())) { |
| 6206 | .struct_type => field_name: { | |
| 6207 | const loaded_struct = ip.loadStructType(struct_ty.toIntern()); | |
| 6208 | switch (loaded_struct.layout) { | |
| 6209 | .auto, .@"extern" => break :field_name if (loaded_struct.fieldName(ip, extra.field_index).unwrap()) |field_name| | |
| 6210 | .{ .identifier = field_name.toSlice(ip) } | |
| 6211 | else | |
| 6212 | .{ .field = extra.field_index }, | |
| 6213 | .@"packed" => { | |
| 6214 | const int_info = struct_ty.intInfo(zcu); | |
| 6215 | ||
| 6216 | const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1)); | |
| 6217 | ||
| 6218 | const bit_offset = zcu.structPackedFieldBitOffset(loaded_struct, extra.field_index); | |
| 6219 | ||
| 6220 | const field_int_signedness = if (inst_ty.isAbiInt(zcu)) | |
| 6221 | inst_ty.intInfo(zcu).signedness | |
| 6222 | else | |
| 6223 | .unsigned; | |
| 6224 | const field_int_ty = try pt.intType(field_int_signedness, @as(u16, @intCast(inst_ty.bitSize(zcu)))); | |
| 6225 | ||
| 6226 | const temp_local = try f.allocLocal(inst, field_int_ty); | |
| 6227 | try f.writeCValue(w, temp_local, .Other); | |
| 6228 | try w.writeAll(" = zig_wrap_"); | |
| 6229 | try f.object.dg.renderTypeForBuiltinFnName(w, field_int_ty); | |
| 6230 | try w.writeAll("(("); | |
| 6231 | try f.renderType(w, field_int_ty); | |
| 6232 | try w.writeByte(')'); | |
| 6233 | const cant_cast = int_info.bits > 64; | |
| 6234 | if (cant_cast) { | |
| 6235 | if (field_int_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{}); | |
| 6236 | try w.writeAll("zig_lo_"); | |
| 6237 | try f.object.dg.renderTypeForBuiltinFnName(w, struct_ty); | |
| 6238 | try w.writeByte('('); | |
| 6239 | } | |
| 6240 | if (bit_offset > 0) { | |
| 6241 | try w.writeAll("zig_shr_"); | |
| 6242 | try f.object.dg.renderTypeForBuiltinFnName(w, struct_ty); | |
| 6243 | try w.writeByte('('); | |
| 6244 | } | |
| 6245 | try f.writeCValue(w, struct_byval, .Other); | |
| 6246 | if (bit_offset > 0) try w.print(", {f})", .{ | |
| 6247 | try f.fmtIntLiteralDec(try pt.intValue(bit_offset_ty, bit_offset)), | |
| 6248 | }); | |
| 6249 | if (cant_cast) try w.writeByte(')'); | |
| 6250 | try f.object.dg.renderBuiltinInfo(w, field_int_ty, .bits); | |
| 6251 | try w.writeAll(");"); | |
| 6252 | try f.object.newline(); | |
| 6253 | if (inst_ty.eql(field_int_ty, zcu)) return temp_local; | |
| 6254 | ||
| 6255 | const local = try f.allocLocal(inst, inst_ty); | |
| 6256 | if (local.new_local != temp_local.new_local) { | |
| 6257 | try w.writeAll("memcpy("); | |
| 6258 | try f.writeCValue(w, .{ .local_ref = local.new_local }, .FunctionArgument); | |
| 6259 | try w.writeAll(", "); | |
| 6260 | try f.writeCValue(w, .{ .local_ref = temp_local.new_local }, .FunctionArgument); | |
| 6261 | try w.writeAll(", sizeof("); | |
| 6262 | try f.renderType(w, inst_ty); | |
| 6263 | try w.writeAll("));"); | |
| 6264 | try f.object.newline(); | |
| 6265 | } | |
| 6266 | try freeLocal(f, inst, temp_local.new_local, null); | |
| 6267 | return local; | |
| 6268 | }, | |
| 6016 | .struct_type => .{ .identifier = struct_ty.structFieldName(extra.field_index, zcu).unwrap().?.toSlice(ip) }, | |
| 6017 | .union_type => name: { | |
| 6018 | const union_type = ip.loadUnionType(struct_ty.toIntern()); | |
| 6019 | const enum_tag_ty: Type = .fromInterned(union_type.enum_tag_ty); | |
| 6020 | const field_name_str = enum_tag_ty.enumFieldName(extra.field_index, zcu).toSlice(ip); | |
| 6021 | if (union_type.hasTag(ip)) { | |
| 6022 | break :name .{ .payload_identifier = field_name_str }; | |
| 6023 | } else { | |
| 6024 | break :name .{ .identifier = field_name_str }; | |
| 6269 | 6025 | } |
| 6270 | 6026 | }, |
| 6271 | 6027 | .tuple_type => .{ .field = extra.field_index }, |
| 6272 | .union_type => field_name: { | |
| 6273 | const loaded_union = ip.loadUnionType(struct_ty.toIntern()); | |
| 6274 | switch (loaded_union.flagsUnordered(ip).layout) { | |
| 6275 | .auto, .@"extern" => { | |
| 6276 | const name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index]; | |
| 6277 | break :field_name if (loaded_union.hasTag(ip)) | |
| 6278 | .{ .payload_identifier = name.toSlice(ip) } | |
| 6279 | else | |
| 6280 | .{ .identifier = name.toSlice(ip) }; | |
| 6281 | }, | |
| 6282 | .@"packed" => { | |
| 6283 | const operand_lval = if (struct_byval == .constant) blk: { | |
| 6284 | const operand_local = try f.allocLocal(inst, struct_ty); | |
| 6285 | try f.writeCValue(w, operand_local, .Other); | |
| 6286 | try w.writeAll(" = "); | |
| 6287 | try f.writeCValue(w, struct_byval, .Other); | |
| 6288 | try w.writeByte(';'); | |
| 6289 | try f.object.newline(); | |
| 6290 | break :blk operand_local; | |
| 6291 | } else struct_byval; | |
| 6292 | const local = try f.allocLocal(inst, inst_ty); | |
| 6293 | if (switch (local) { | |
| 6294 | .new_local, .local => |local_index| switch (operand_lval) { | |
| 6295 | .new_local, .local => |operand_local_index| local_index != operand_local_index, | |
| 6296 | else => true, | |
| 6297 | }, | |
| 6298 | else => true, | |
| 6299 | }) { | |
| 6300 | try w.writeAll("memcpy(&"); | |
| 6301 | try f.writeCValue(w, local, .Other); | |
| 6302 | try w.writeAll(", &"); | |
| 6303 | try f.writeCValue(w, operand_lval, .Other); | |
| 6304 | try w.writeAll(", sizeof("); | |
| 6305 | try f.renderType(w, inst_ty); | |
| 6306 | try w.writeAll("));"); | |
| 6307 | try f.object.newline(); | |
| 6308 | } | |
| 6309 | try f.freeCValue(inst, operand_lval); | |
| 6310 | return local; | |
| 6311 | }, | |
| 6312 | } | |
| 6313 | }, | |
| 6314 | 6028 | else => unreachable, |
| 6315 | 6029 | }; |
| 6316 | 6030 | |
| ... | ... | @@ -7702,98 +7416,13 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { |
| 7702 | 7416 | if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; |
| 7703 | 7417 | |
| 7704 | 7418 | const a = try Assignment.start(f, w, try f.ctypeFromType(field_ty, .complete)); |
| 7705 | try f.writeCValueMember(w, local, if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| | |
| 7706 | .{ .identifier = field_name.toSlice(ip) } | |
| 7707 | else | |
| 7708 | .{ .field = field_index }); | |
| 7419 | try f.writeCValueMember(w, local, .{ .identifier = loaded_struct.fieldName(ip, field_index).toSlice(ip) }); | |
| 7709 | 7420 | try a.assign(f, w); |
| 7710 | 7421 | try f.writeCValue(w, resolved_elements[field_index], .Other); |
| 7711 | 7422 | try a.end(f, w); |
| 7712 | 7423 | } |
| 7713 | 7424 | }, |
| 7714 | .@"packed" => { | |
| 7715 | try f.writeCValue(w, local, .Other); | |
| 7716 | try w.writeAll(" = "); | |
| 7717 | ||
| 7718 | const backing_int_ty: Type = .fromInterned(loaded_struct.backingIntTypeUnordered(ip)); | |
| 7719 | const int_info = backing_int_ty.intInfo(zcu); | |
| 7720 | ||
| 7721 | const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1)); | |
| 7722 | ||
| 7723 | var bit_offset: u64 = 0; | |
| 7724 | ||
| 7725 | var empty = true; | |
| 7726 | for (0..elements.len) |field_index| { | |
| 7727 | if (inst_ty.structFieldIsComptime(field_index, zcu)) continue; | |
| 7728 | const field_ty = inst_ty.fieldType(field_index, zcu); | |
| 7729 | if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; | |
| 7730 | ||
| 7731 | if (!empty) { | |
| 7732 | try w.writeAll("zig_or_"); | |
| 7733 | try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty); | |
| 7734 | try w.writeByte('('); | |
| 7735 | } | |
| 7736 | empty = false; | |
| 7737 | } | |
| 7738 | empty = true; | |
| 7739 | for (resolved_elements, 0..) |element, field_index| { | |
| 7740 | if (inst_ty.structFieldIsComptime(field_index, zcu)) continue; | |
| 7741 | const field_ty = inst_ty.fieldType(field_index, zcu); | |
| 7742 | if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; | |
| 7743 | ||
| 7744 | if (!empty) try w.writeAll(", "); | |
| 7745 | // TODO: Skip this entire shift if val is 0? | |
| 7746 | try w.writeAll("zig_shlw_"); | |
| 7747 | try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty); | |
| 7748 | try w.writeByte('('); | |
| 7749 | ||
| 7750 | if (field_ty.isAbiInt(zcu)) { | |
| 7751 | try w.writeAll("zig_and_"); | |
| 7752 | try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty); | |
| 7753 | try w.writeByte('('); | |
| 7754 | } | |
| 7755 | ||
| 7756 | if (inst_ty.isAbiInt(zcu) and (field_ty.isAbiInt(zcu) or field_ty.isPtrAtRuntime(zcu))) { | |
| 7757 | try f.renderIntCast(w, inst_ty, element, .{}, field_ty, .FunctionArgument); | |
| 7758 | } else { | |
| 7759 | try w.writeByte('('); | |
| 7760 | try f.renderType(w, inst_ty); | |
| 7761 | try w.writeByte(')'); | |
| 7762 | if (field_ty.isPtrAtRuntime(zcu)) { | |
| 7763 | try w.writeByte('('); | |
| 7764 | try f.renderType(w, switch (int_info.signedness) { | |
| 7765 | .unsigned => .usize, | |
| 7766 | .signed => .isize, | |
| 7767 | }); | |
| 7768 | try w.writeByte(')'); | |
| 7769 | } | |
| 7770 | try f.writeCValue(w, element, .Other); | |
| 7771 | } | |
| 7772 | ||
| 7773 | if (field_ty.isAbiInt(zcu)) { | |
| 7774 | try w.writeAll(", "); | |
| 7775 | const field_int_info = field_ty.intInfo(zcu); | |
| 7776 | const field_mask = if (int_info.signedness == .signed and int_info.bits == field_int_info.bits) | |
| 7777 | try pt.intValue(backing_int_ty, -1) | |
| 7778 | else | |
| 7779 | try (try pt.intType(.unsigned, field_int_info.bits)).maxIntScalar(pt, backing_int_ty); | |
| 7780 | try f.object.dg.renderValue(w, field_mask, .FunctionArgument); | |
| 7781 | try w.writeByte(')'); | |
| 7782 | } | |
| 7783 | ||
| 7784 | try w.print(", {f}", .{ | |
| 7785 | try f.fmtIntLiteralDec(try pt.intValue(bit_offset_ty, bit_offset)), | |
| 7786 | }); | |
| 7787 | try f.object.dg.renderBuiltinInfo(w, inst_ty, .bits); | |
| 7788 | try w.writeByte(')'); | |
| 7789 | if (!empty) try w.writeByte(')'); | |
| 7790 | ||
| 7791 | bit_offset += field_ty.bitSize(zcu); | |
| 7792 | empty = false; | |
| 7793 | } | |
| 7794 | try w.writeByte(';'); | |
| 7795 | try f.object.newline(); | |
| 7796 | }, | |
| 7425 | .@"packed" => unreachable, // `Air.Legalize.Feature.expand_packed_struct_init` handles this case | |
| 7797 | 7426 | } |
| 7798 | 7427 | }, |
| 7799 | 7428 | .tuple_type => |tuple_info| for (0..tuple_info.types.len) |field_index| { |
| ... | ... | @@ -7828,9 +7457,10 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue { |
| 7828 | 7457 | try reap(f, inst, &.{extra.init}); |
| 7829 | 7458 | |
| 7830 | 7459 | const w = &f.object.code.writer; |
| 7831 | const local = try f.allocLocal(inst, union_ty); | |
| 7832 | 7460 | if (loaded_union.flagsUnordered(ip).layout == .@"packed") return f.moveCValue(inst, union_ty, payload); |
| 7833 | 7461 | |
| 7462 | const local = try f.allocLocal(inst, union_ty); | |
| 7463 | ||
| 7834 | 7464 | const field: CValue = if (union_ty.unionTagTypeSafety(zcu)) |tag_ty| field: { |
| 7835 | 7465 | const layout = union_ty.unionGetLayout(zcu); |
| 7836 | 7466 | if (layout.tag_size != 0) { |
src/codegen/c/Type.zig+1-5
| ... | ... | @@ -2514,11 +2514,7 @@ pub const Pool = struct { |
| 2514 | 2514 | kind.noParameter(), |
| 2515 | 2515 | ); |
| 2516 | 2516 | if (field_ctype.index == .void) continue; |
| 2517 | const field_name = if (loaded_struct.fieldName(ip, field_index) | |
| 2518 | .unwrap()) |field_name| | |
| 2519 | try pool.string(allocator, field_name.toSlice(ip)) | |
| 2520 | else | |
| 2521 | String.fromUnnamed(@intCast(field_index)); | |
| 2517 | const field_name = try pool.string(allocator, loaded_struct.fieldName(ip, field_index).toSlice(ip)); | |
| 2522 | 2518 | const field_alignas = AlignAs.fromAlignment(.{ |
| 2523 | 2519 | .@"align" = loaded_struct.fieldAlign(ip, field_index), |
| 2524 | 2520 | .abi = field_type.abiAlignment(zcu), |
src/codegen/llvm.zig+8-45
| ... | ... | @@ -2411,8 +2411,7 @@ pub const Object = struct { |
| 2411 | 2411 | const field_size = field_ty.abiSize(zcu); |
| 2412 | 2412 | const field_align = ty.fieldAlignment(field_index, zcu); |
| 2413 | 2413 | const field_offset = ty.structFieldOffset(field_index, zcu); |
| 2414 | const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse | |
| 2415 | try ip.getOrPutStringFmt(gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls); | |
| 2414 | const field_name = struct_type.fieldName(ip, field_index); | |
| 2416 | 2415 | fields.appendAssumeCapacity(try o.builder.debugMemberType( |
| 2417 | 2416 | try o.builder.metadataString(field_name.toSlice(ip)), |
| 2418 | 2417 | null, // File |
| ... | ... | @@ -5093,8 +5092,6 @@ pub const FuncGen = struct { |
| 5093 | 5092 | .wasm_memory_size => try self.airWasmMemorySize(inst), |
| 5094 | 5093 | .wasm_memory_grow => try self.airWasmMemoryGrow(inst), |
| 5095 | 5094 | |
| 5096 | .vector_store_elem => try self.airVectorStoreElem(inst), | |
| 5097 | ||
| 5098 | 5095 | .runtime_nav_ptr => try self.airRuntimeNavPtr(inst), |
| 5099 | 5096 | |
| 5100 | 5097 | .inferred_alloc, .inferred_alloc_comptime => unreachable, |
| ... | ... | @@ -6873,16 +6870,14 @@ pub const FuncGen = struct { |
| 6873 | 6870 | const array_llvm_ty = try o.lowerType(pt, array_ty); |
| 6874 | 6871 | const elem_ty = array_ty.childType(zcu); |
| 6875 | 6872 | if (isByRef(array_ty, zcu)) { |
| 6876 | const indices: [2]Builder.Value = .{ | |
| 6877 | try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs, | |
| 6878 | }; | |
| 6873 | const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{ | |
| 6874 | try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), | |
| 6875 | rhs, | |
| 6876 | }, ""); | |
| 6879 | 6877 | if (isByRef(elem_ty, zcu)) { |
| 6880 | const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, ""); | |
| 6881 | 6878 | const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm(); |
| 6882 | 6879 | return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal); |
| 6883 | 6880 | } else { |
| 6884 | const elem_ptr = | |
| 6885 | try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, ""); | |
| 6886 | 6881 | return self.loadTruncate(.normal, elem_ty, elem_ptr, .default); |
| 6887 | 6882 | } |
| 6888 | 6883 | } |
| ... | ... | @@ -8140,33 +8135,6 @@ pub const FuncGen = struct { |
| 8140 | 8135 | }, ""); |
| 8141 | 8136 | } |
| 8142 | 8137 | |
| 8143 | fn airVectorStoreElem(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { | |
| 8144 | const o = self.ng.object; | |
| 8145 | const pt = self.ng.pt; | |
| 8146 | const zcu = pt.zcu; | |
| 8147 | const data = self.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem; | |
| 8148 | const extra = self.air.extraData(Air.Bin, data.payload).data; | |
| 8149 | ||
| 8150 | const vector_ptr = try self.resolveInst(data.vector_ptr); | |
| 8151 | const vector_ptr_ty = self.typeOf(data.vector_ptr); | |
| 8152 | const index = try self.resolveInst(extra.lhs); | |
| 8153 | const operand = try self.resolveInst(extra.rhs); | |
| 8154 | ||
| 8155 | self.maybeMarkAllowZeroAccess(vector_ptr_ty.ptrInfo(zcu)); | |
| 8156 | ||
| 8157 | // TODO: Emitting a load here is a violation of volatile semantics. Not fixable in general. | |
| 8158 | // https://github.com/ziglang/zig/issues/18652#issuecomment-2452844908 | |
| 8159 | const access_kind: Builder.MemoryAccessKind = | |
| 8160 | if (vector_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; | |
| 8161 | const elem_llvm_ty = try o.lowerType(pt, vector_ptr_ty.childType(zcu)); | |
| 8162 | const alignment = vector_ptr_ty.ptrAlignment(zcu).toLlvm(); | |
| 8163 | const loaded = try self.wip.load(access_kind, elem_llvm_ty, vector_ptr, alignment, ""); | |
| 8164 | ||
| 8165 | const new_vector = try self.wip.insertElement(loaded, operand, index, ""); | |
| 8166 | _ = try self.store(vector_ptr, vector_ptr_ty, new_vector, .none); | |
| 8167 | return .none; | |
| 8168 | } | |
| 8169 | ||
| 8170 | 8138 | fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 8171 | 8139 | const o = fg.ng.object; |
| 8172 | 8140 | const pt = fg.ng.pt; |
| ... | ... | @@ -8303,8 +8271,7 @@ pub const FuncGen = struct { |
| 8303 | 8271 | const rhs = try self.resolveInst(bin_op.rhs); |
| 8304 | 8272 | const inst_ty = self.typeOfIndex(inst); |
| 8305 | 8273 | const scalar_ty = inst_ty.scalarType(zcu); |
| 8306 | ||
| 8307 | if (scalar_ty.isAnyFloat()) return self.todo("saturating float add", .{}); | |
| 8274 | assert(scalar_ty.zigTypeTag(zcu) == .int); | |
| 8308 | 8275 | return self.wip.callIntrinsic( |
| 8309 | 8276 | .normal, |
| 8310 | 8277 | .none, |
| ... | ... | @@ -8344,8 +8311,7 @@ pub const FuncGen = struct { |
| 8344 | 8311 | const rhs = try self.resolveInst(bin_op.rhs); |
| 8345 | 8312 | const inst_ty = self.typeOfIndex(inst); |
| 8346 | 8313 | const scalar_ty = inst_ty.scalarType(zcu); |
| 8347 | ||
| 8348 | if (scalar_ty.isAnyFloat()) return self.todo("saturating float sub", .{}); | |
| 8314 | assert(scalar_ty.zigTypeTag(zcu) == .int); | |
| 8349 | 8315 | return self.wip.callIntrinsic( |
| 8350 | 8316 | .normal, |
| 8351 | 8317 | .none, |
| ... | ... | @@ -8385,8 +8351,7 @@ pub const FuncGen = struct { |
| 8385 | 8351 | const rhs = try self.resolveInst(bin_op.rhs); |
| 8386 | 8352 | const inst_ty = self.typeOfIndex(inst); |
| 8387 | 8353 | const scalar_ty = inst_ty.scalarType(zcu); |
| 8388 | ||
| 8389 | if (scalar_ty.isAnyFloat()) return self.todo("saturating float mul", .{}); | |
| 8354 | assert(scalar_ty.zigTypeTag(zcu) == .int); | |
| 8390 | 8355 | return self.wip.callIntrinsic( |
| 8391 | 8356 | .normal, |
| 8392 | 8357 | .none, |
| ... | ... | @@ -11454,7 +11419,6 @@ pub const FuncGen = struct { |
| 11454 | 11419 | const access_kind: Builder.MemoryAccessKind = |
| 11455 | 11420 | if (info.flags.is_volatile) .@"volatile" else .normal; |
| 11456 | 11421 | |
| 11457 | assert(info.flags.vector_index != .runtime); | |
| 11458 | 11422 | if (info.flags.vector_index != .none) { |
| 11459 | 11423 | const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); |
| 11460 | 11424 | const vec_elem_ty = try o.lowerType(pt, elem_ty); |
| ... | ... | @@ -11524,7 +11488,6 @@ pub const FuncGen = struct { |
| 11524 | 11488 | const access_kind: Builder.MemoryAccessKind = |
| 11525 | 11489 | if (info.flags.is_volatile) .@"volatile" else .normal; |
| 11526 | 11490 | |
| 11527 | assert(info.flags.vector_index != .runtime); | |
| 11528 | 11491 | if (info.flags.vector_index != .none) { |
| 11529 | 11492 | const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); |
| 11530 | 11493 | const vec_elem_ty = try o.lowerType(pt, elem_ty); |
src/codegen/riscv64/CodeGen.zig-1
| ... | ... | @@ -1633,7 +1633,6 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { |
| 1633 | 1633 | |
| 1634 | 1634 | .is_named_enum_value => return func.fail("TODO implement is_named_enum_value", .{}), |
| 1635 | 1635 | .error_set_has_value => return func.fail("TODO implement error_set_has_value", .{}), |
| 1636 | .vector_store_elem => return func.fail("TODO implement vector_store_elem", .{}), | |
| 1637 | 1636 | |
| 1638 | 1637 | .c_va_arg => return func.fail("TODO implement c_va_arg", .{}), |
| 1639 | 1638 | .c_va_copy => return func.fail("TODO implement c_va_copy", .{}), |
src/codegen/sparc64/CodeGen.zig-1
| ... | ... | @@ -702,7 +702,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 702 | 702 | |
| 703 | 703 | .is_named_enum_value => @panic("TODO implement is_named_enum_value"), |
| 704 | 704 | .error_set_has_value => @panic("TODO implement error_set_has_value"), |
| 705 | .vector_store_elem => @panic("TODO implement vector_store_elem"), | |
| 706 | 705 | .runtime_nav_ptr => @panic("TODO implement runtime_nav_ptr"), |
| 707 | 706 | |
| 708 | 707 | .c_va_arg => return self.fail("TODO implement c_va_arg", .{}), |
src/codegen/spirv/CodeGen.zig+1-27
| ... | ... | @@ -1520,8 +1520,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { |
| 1520 | 1520 | const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]); |
| 1521 | 1521 | if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; |
| 1522 | 1522 | |
| 1523 | const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse | |
| 1524 | try ip.getOrPutStringFmt(zcu.gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls); | |
| 1523 | const field_name = struct_type.fieldName(ip, field_index); | |
| 1525 | 1524 | try member_types.append(try cg.resolveType(field_ty, .indirect)); |
| 1526 | 1525 | try member_names.append(field_name.toSlice(ip)); |
| 1527 | 1526 | try member_offsets.append(@intCast(ty.structFieldOffset(field_index, zcu))); |
| ... | ... | @@ -2726,8 +2725,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void { |
| 2726 | 2725 | .ptr_elem_val => try cg.airPtrElemVal(inst), |
| 2727 | 2726 | .array_elem_val => try cg.airArrayElemVal(inst), |
| 2728 | 2727 | |
| 2729 | .vector_store_elem => return cg.airVectorStoreElem(inst), | |
| 2730 | ||
| 2731 | 2728 | .set_union_tag => return cg.airSetUnionTag(inst), |
| 2732 | 2729 | .get_union_tag => try cg.airGetUnionTag(inst), |
| 2733 | 2730 | .union_init => try cg.airUnionInit(inst), |
| ... | ... | @@ -4446,29 +4443,6 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id { |
| 4446 | 4443 | return try cg.load(elem_ty, elem_ptr_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) }); |
| 4447 | 4444 | } |
| 4448 | 4445 | |
| 4449 | fn airVectorStoreElem(cg: *CodeGen, inst: Air.Inst.Index) !void { | |
| 4450 | const zcu = cg.module.zcu; | |
| 4451 | const data = cg.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem; | |
| 4452 | const extra = cg.air.extraData(Air.Bin, data.payload).data; | |
| 4453 | ||
| 4454 | const vector_ptr_ty = cg.typeOf(data.vector_ptr); | |
| 4455 | const vector_ty = vector_ptr_ty.childType(zcu); | |
| 4456 | const scalar_ty = vector_ty.scalarType(zcu); | |
| 4457 | ||
| 4458 | const scalar_ty_id = try cg.resolveType(scalar_ty, .indirect); | |
| 4459 | const storage_class = cg.module.storageClass(vector_ptr_ty.ptrAddressSpace(zcu)); | |
| 4460 | const scalar_ptr_ty_id = try cg.module.ptrType(scalar_ty_id, storage_class); | |
| 4461 | ||
| 4462 | const vector_ptr = try cg.resolve(data.vector_ptr); | |
| 4463 | const index = try cg.resolve(extra.lhs); | |
| 4464 | const operand = try cg.resolve(extra.rhs); | |
| 4465 | ||
| 4466 | const elem_ptr_id = try cg.accessChainId(scalar_ptr_ty_id, vector_ptr, &.{index}); | |
| 4467 | try cg.store(scalar_ty, elem_ptr_id, operand, .{ | |
| 4468 | .is_volatile = vector_ptr_ty.isVolatilePtr(zcu), | |
| 4469 | }); | |
| 4470 | } | |
| 4471 | ||
| 4472 | 4446 | fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !void { |
| 4473 | 4447 | const zcu = cg.module.zcu; |
| 4474 | 4448 | const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
src/codegen/wasm/CodeGen.zig-1
| ... | ... | @@ -1978,7 +1978,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { |
| 1978 | 1978 | .save_err_return_trace_index, |
| 1979 | 1979 | .is_named_enum_value, |
| 1980 | 1980 | .addrspace_cast, |
| 1981 | .vector_store_elem, | |
| 1982 | 1981 | .c_va_arg, |
| 1983 | 1982 | .c_va_copy, |
| 1984 | 1983 | .c_va_end, |
src/codegen/x86_64/CodeGen.zig+40-1577
| ... | ... | @@ -854,12 +854,6 @@ const FrameAlloc = struct { |
| 854 | 854 | } |
| 855 | 855 | }; |
| 856 | 856 | |
| 857 | const StackAllocation = struct { | |
| 858 | inst: ?Air.Inst.Index, | |
| 859 | /// TODO do we need size? should be determined by inst.ty.abiSize(zcu) | |
| 860 | size: u32, | |
| 861 | }; | |
| 862 | ||
| 863 | 857 | const BlockData = struct { |
| 864 | 858 | relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, |
| 865 | 859 | state: State, |
| ... | ... | @@ -89326,7 +89320,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89326 | 89320 | error.SelectFailed => res[0] = try ops[0].load(val_ty, .{ |
| 89327 | 89321 | .disp = switch (cg.typeOf(ty_op.operand).ptrInfo(zcu).flags.vector_index) { |
| 89328 | 89322 | .none => 0, |
| 89329 | .runtime => unreachable, | |
| 89330 | 89323 | else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), |
| 89331 | 89324 | }, |
| 89332 | 89325 | }, cg), |
| ... | ... | @@ -89569,7 +89562,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89569 | 89562 | error.SelectFailed => try ops[0].store(&ops[1], .{ |
| 89570 | 89563 | .disp = switch (cg.typeOf(bin_op.lhs).ptrInfo(zcu).flags.vector_index) { |
| 89571 | 89564 | .none => 0, |
| 89572 | .runtime => unreachable, | |
| 89573 | 89565 | else => |vector_index| @intCast(cg.typeOf(bin_op.rhs).abiSize(zcu) * @intFromEnum(vector_index)), |
| 89574 | 89566 | }, |
| 89575 | 89567 | .safe = switch (air_tag) { |
| ... | ... | @@ -171402,8 +171394,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 171402 | 171394 | .aggregate_init => |air_tag| fallback: { |
| 171403 | 171395 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 171404 | 171396 | const agg_ty = ty_pl.ty.toType(); |
| 171405 | if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or | |
| 171406 | (agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst); | |
| 171397 | if (agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) { | |
| 171398 | break :fallback try cg.airAggregateInitBoolVec(inst); | |
| 171399 | } | |
| 171407 | 171400 | var res = try cg.tempAllocMem(agg_ty); |
| 171408 | 171401 | const reset_index = cg.next_temp_index; |
| 171409 | 171402 | var bt = cg.liveness.iterateBigTomb(inst); |
| ... | ... | @@ -171441,10 +171434,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 171441 | 171434 | } |
| 171442 | 171435 | } |
| 171443 | 171436 | }, |
| 171444 | .@"packed" => return cg.fail("failed to select {s} {f}", .{ | |
| 171445 | @tagName(air_tag), | |
| 171446 | agg_ty.fmt(pt), | |
| 171447 | }), | |
| 171437 | .@"packed" => unreachable, | |
| 171448 | 171438 | } |
| 171449 | 171439 | }, |
| 171450 | 171440 | .tuple_type => |tuple_type| { |
| ... | ... | @@ -173054,633 +173044,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 173054 | 173044 | try ert.die(cg); |
| 173055 | 173045 | try res.finish(inst, &.{}, &.{}, cg); |
| 173056 | 173046 | }, |
| 173057 | .vector_store_elem => { | |
| 173058 | const vector_store_elem = air_datas[@intFromEnum(inst)].vector_store_elem; | |
| 173059 | const bin_op = cg.air.extraData(Air.Bin, vector_store_elem.payload).data; | |
| 173060 | var ops = try cg.tempsFromOperands(inst, .{ vector_store_elem.vector_ptr, bin_op.lhs, bin_op.rhs }); | |
| 173061 | cg.select(&.{}, &.{}, &ops, comptime &.{ .{ | |
| 173062 | .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, | |
| 173063 | .patterns = &.{ | |
| 173064 | .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, | |
| 173065 | }, | |
| 173066 | .extra_temps = .{ | |
| 173067 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 173068 | .unused, | |
| 173069 | .unused, | |
| 173070 | .unused, | |
| 173071 | .unused, | |
| 173072 | .unused, | |
| 173073 | .unused, | |
| 173074 | .unused, | |
| 173075 | .unused, | |
| 173076 | .unused, | |
| 173077 | .unused, | |
| 173078 | }, | |
| 173079 | .clobbers = .{ .eflags = true }, | |
| 173080 | .each = .{ .once = &.{ | |
| 173081 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, | |
| 173082 | .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ }, | |
| 173083 | .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, | |
| 173084 | } }, | |
| 173085 | }, .{ | |
| 173086 | .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, | |
| 173087 | .patterns = &.{ | |
| 173088 | .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, | |
| 173089 | }, | |
| 173090 | .extra_temps = .{ | |
| 173091 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 173092 | .unused, | |
| 173093 | .unused, | |
| 173094 | .unused, | |
| 173095 | .unused, | |
| 173096 | .unused, | |
| 173097 | .unused, | |
| 173098 | .unused, | |
| 173099 | .unused, | |
| 173100 | .unused, | |
| 173101 | .unused, | |
| 173102 | }, | |
| 173103 | .clobbers = .{ .eflags = true }, | |
| 173104 | .each = .{ .once = &.{ | |
| 173105 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, | |
| 173106 | .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, | |
| 173107 | .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, | |
| 173108 | } }, | |
| 173109 | }, .{ | |
| 173110 | .required_features = .{ .cmov, null, null, null }, | |
| 173111 | .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, | |
| 173112 | .patterns = &.{ | |
| 173113 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173114 | }, | |
| 173115 | .extra_temps = .{ | |
| 173116 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 173117 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 173118 | .unused, | |
| 173119 | .unused, | |
| 173120 | .unused, | |
| 173121 | .unused, | |
| 173122 | .unused, | |
| 173123 | .unused, | |
| 173124 | .unused, | |
| 173125 | .unused, | |
| 173126 | .unused, | |
| 173127 | }, | |
| 173128 | .clobbers = .{ .eflags = true }, | |
| 173129 | .each = .{ .once = &.{ | |
| 173130 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, | |
| 173131 | .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, | |
| 173132 | .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, | |
| 173133 | .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, | |
| 173134 | .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, | |
| 173135 | .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, | |
| 173136 | .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, | |
| 173137 | } }, | |
| 173138 | }, .{ | |
| 173139 | .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, | |
| 173140 | .patterns = &.{ | |
| 173141 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173142 | }, | |
| 173143 | .extra_temps = .{ | |
| 173144 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, | |
| 173145 | .unused, | |
| 173146 | .unused, | |
| 173147 | .unused, | |
| 173148 | .unused, | |
| 173149 | .unused, | |
| 173150 | .unused, | |
| 173151 | .unused, | |
| 173152 | .unused, | |
| 173153 | .unused, | |
| 173154 | .unused, | |
| 173155 | }, | |
| 173156 | .clobbers = .{ .eflags = true }, | |
| 173157 | .each = .{ .once = &.{ | |
| 173158 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, | |
| 173159 | .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, | |
| 173160 | .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, | |
| 173161 | .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ }, | |
| 173162 | .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, | |
| 173163 | .{ .@"0:", ._s, .bt, .tmp0d, .src1d, ._, ._ }, | |
| 173164 | .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, | |
| 173165 | } }, | |
| 173166 | }, .{ | |
| 173167 | .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, | |
| 173168 | .patterns = &.{ | |
| 173169 | .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, | |
| 173170 | }, | |
| 173171 | .clobbers = .{ .eflags = true }, | |
| 173172 | .each = .{ .once = &.{ | |
| 173173 | .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ }, | |
| 173174 | } }, | |
| 173175 | }, .{ | |
| 173176 | .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, | |
| 173177 | .patterns = &.{ | |
| 173178 | .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, | |
| 173179 | }, | |
| 173180 | .clobbers = .{ .eflags = true }, | |
| 173181 | .each = .{ .once = &.{ | |
| 173182 | .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, | |
| 173183 | } }, | |
| 173184 | }, .{ | |
| 173185 | .required_features = .{ .cmov, null, null, null }, | |
| 173186 | .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, | |
| 173187 | .patterns = &.{ | |
| 173188 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173189 | }, | |
| 173190 | .extra_temps = .{ | |
| 173191 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, | |
| 173192 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, | |
| 173193 | .unused, | |
| 173194 | .unused, | |
| 173195 | .unused, | |
| 173196 | .unused, | |
| 173197 | .unused, | |
| 173198 | .unused, | |
| 173199 | .unused, | |
| 173200 | .unused, | |
| 173201 | .unused, | |
| 173202 | }, | |
| 173203 | .clobbers = .{ .eflags = true }, | |
| 173204 | .each = .{ .once = &.{ | |
| 173205 | .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ }, | |
| 173206 | .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, | |
| 173207 | .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, | |
| 173208 | .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, | |
| 173209 | .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, | |
| 173210 | .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, | |
| 173211 | .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ }, | |
| 173212 | } }, | |
| 173213 | }, .{ | |
| 173214 | .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, | |
| 173215 | .patterns = &.{ | |
| 173216 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173217 | }, | |
| 173218 | .clobbers = .{ .eflags = true }, | |
| 173219 | .each = .{ .once = &.{ | |
| 173220 | .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, | |
| 173221 | .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, | |
| 173222 | .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ }, | |
| 173223 | .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, | |
| 173224 | .{ .@"1:", ._s, .bt, .lea(.src0w), .src1w, ._, ._ }, | |
| 173225 | } }, | |
| 173226 | }, .{ | |
| 173227 | .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, | |
| 173228 | .patterns = &.{ | |
| 173229 | .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, | |
| 173230 | }, | |
| 173231 | .clobbers = .{ .eflags = true }, | |
| 173232 | .each = .{ .once = &.{ | |
| 173233 | .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ }, | |
| 173234 | } }, | |
| 173235 | }, .{ | |
| 173236 | .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, | |
| 173237 | .patterns = &.{ | |
| 173238 | .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, | |
| 173239 | }, | |
| 173240 | .clobbers = .{ .eflags = true }, | |
| 173241 | .each = .{ .once = &.{ | |
| 173242 | .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, | |
| 173243 | } }, | |
| 173244 | }, .{ | |
| 173245 | .required_features = .{ .cmov, null, null, null }, | |
| 173246 | .src_constraints = .{ .{ .ptr_bool_vec = .dword }, .any, .bool }, | |
| 173247 | .patterns = &.{ | |
| 173248 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173249 | }, | |
| 173250 | .extra_temps = .{ | |
| 173251 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | |
| 173252 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | |
| 173253 | .unused, | |
| 173254 | .unused, | |
| 173255 | .unused, | |
| 173256 | .unused, | |
| 173257 | .unused, | |
| 173258 | .unused, | |
| 173259 | .unused, | |
| 173260 | .unused, | |
| 173261 | .unused, | |
| 173262 | }, | |
| 173263 | .clobbers = .{ .eflags = true }, | |
| 173264 | .each = .{ .once = &.{ | |
| 173265 | .{ ._, ._, .mov, .tmp0d, .lea(.src0d), ._, ._ }, | |
| 173266 | .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, | |
| 173267 | .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, | |
| 173268 | .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, | |
| 173269 | .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, | |
| 173270 | .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, | |
| 173271 | .{ ._, ._, .mov, .lea(.src0d), .tmp0d, ._, ._ }, | |
| 173272 | } }, | |
| 173273 | }, .{ | |
| 173274 | .required_features = .{ .@"64bit", .cmov, null, null }, | |
| 173275 | .src_constraints = .{ .{ .ptr_bool_vec = .qword }, .any, .bool }, | |
| 173276 | .patterns = &.{ | |
| 173277 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173278 | }, | |
| 173279 | .extra_temps = .{ | |
| 173280 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 173281 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | |
| 173282 | .unused, | |
| 173283 | .unused, | |
| 173284 | .unused, | |
| 173285 | .unused, | |
| 173286 | .unused, | |
| 173287 | .unused, | |
| 173288 | .unused, | |
| 173289 | .unused, | |
| 173290 | .unused, | |
| 173291 | }, | |
| 173292 | .clobbers = .{ .eflags = true }, | |
| 173293 | .each = .{ .once = &.{ | |
| 173294 | .{ ._, ._, .mov, .tmp0q, .lea(.src0q), ._, ._ }, | |
| 173295 | .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, | |
| 173296 | .{ ._, ._r, .bt, .tmp1q, .src1q, ._, ._ }, | |
| 173297 | .{ ._, ._s, .bt, .tmp0q, .src1q, ._, ._ }, | |
| 173298 | .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, | |
| 173299 | .{ ._, ._z, .cmov, .tmp0q, .tmp1q, ._, ._ }, | |
| 173300 | .{ ._, ._, .mov, .lea(.src0q), .tmp0q, ._, ._ }, | |
| 173301 | } }, | |
| 173302 | }, .{ | |
| 173303 | .required_features = .{ .cmov, null, null, null }, | |
| 173304 | .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, | |
| 173305 | .patterns = &.{ | |
| 173306 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173307 | }, | |
| 173308 | .extra_temps = .{ | |
| 173309 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | |
| 173310 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | |
| 173311 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | |
| 173312 | .unused, | |
| 173313 | .unused, | |
| 173314 | .unused, | |
| 173315 | .unused, | |
| 173316 | .unused, | |
| 173317 | .unused, | |
| 173318 | .unused, | |
| 173319 | .unused, | |
| 173320 | }, | |
| 173321 | .clobbers = .{ .eflags = true }, | |
| 173322 | .each = .{ .once = &.{ | |
| 173323 | .{ ._, ._, .mov, .tmp0d, .src1d, ._, ._ }, | |
| 173324 | .{ ._, ._r, .sh, .tmp0d, .ui(5), ._, ._ }, | |
| 173325 | .{ ._, ._, .mov, .tmp1d, .leasi(.src0d, .@"4", .tmp0), ._, ._ }, | |
| 173326 | .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, | |
| 173327 | .{ ._, ._r, .bt, .tmp2d, .src1d, ._, ._ }, | |
| 173328 | .{ ._, ._s, .bt, .tmp1d, .src1d, ._, ._ }, | |
| 173329 | .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, | |
| 173330 | .{ ._, ._z, .cmov, .tmp1d, .tmp2d, ._, ._ }, | |
| 173331 | .{ ._, ._, .mov, .leasi(.src0d, .@"4", .tmp0), .tmp1d, ._, ._ }, | |
| 173332 | } }, | |
| 173333 | }, .{ | |
| 173334 | .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, | |
| 173335 | .patterns = &.{ | |
| 173336 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173337 | }, | |
| 173338 | .clobbers = .{ .eflags = true }, | |
| 173339 | .each = .{ .once = &.{ | |
| 173340 | .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, | |
| 173341 | .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, | |
| 173342 | .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ }, | |
| 173343 | .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, | |
| 173344 | .{ .@"1:", ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, | |
| 173345 | } }, | |
| 173346 | }, .{ | |
| 173347 | .src_constraints = .{ .any, .any, .{ .int = .byte } }, | |
| 173348 | .patterns = &.{ | |
| 173349 | .{ .src = .{ .to_gpr, .simm32, .imm8 } }, | |
| 173350 | .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, | |
| 173351 | }, | |
| 173352 | .each = .{ .once = &.{ | |
| 173353 | .{ ._, ._, .mov, .leaa(.src0b, .add_src0_elem_size_mul_src1), .src2b, ._, ._ }, | |
| 173354 | } }, | |
| 173355 | }, .{ | |
| 173356 | .src_constraints = .{ .any, .any, .{ .int = .byte } }, | |
| 173357 | .patterns = &.{ | |
| 173358 | .{ .src = .{ .to_gpr, .to_gpr, .imm8 } }, | |
| 173359 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173360 | }, | |
| 173361 | .each = .{ .once = &.{ | |
| 173362 | .{ ._, ._, .mov, .leai(.src0b, .src1), .src2b, ._, ._ }, | |
| 173363 | } }, | |
| 173364 | }, .{ | |
| 173365 | .src_constraints = .{ .any, .any, .{ .int = .word } }, | |
| 173366 | .patterns = &.{ | |
| 173367 | .{ .src = .{ .to_gpr, .simm32, .imm16 } }, | |
| 173368 | .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, | |
| 173369 | }, | |
| 173370 | .each = .{ .once = &.{ | |
| 173371 | .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2w, ._, ._ }, | |
| 173372 | } }, | |
| 173373 | }, .{ | |
| 173374 | .src_constraints = .{ .any, .any, .{ .int = .word } }, | |
| 173375 | .patterns = &.{ | |
| 173376 | .{ .src = .{ .to_gpr, .to_gpr, .imm16 } }, | |
| 173377 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173378 | }, | |
| 173379 | .each = .{ .once = &.{ | |
| 173380 | .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .src2w, ._, ._ }, | |
| 173381 | } }, | |
| 173382 | }, .{ | |
| 173383 | .required_features = .{ .avx, null, null, null }, | |
| 173384 | .src_constraints = .{ .any, .any, .{ .float = .word } }, | |
| 173385 | .patterns = &.{ | |
| 173386 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173387 | }, | |
| 173388 | .each = .{ .once = &.{ | |
| 173389 | .{ ._, .vp_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ }, | |
| 173390 | } }, | |
| 173391 | }, .{ | |
| 173392 | .required_features = .{ .sse4_1, null, null, null }, | |
| 173393 | .src_constraints = .{ .any, .any, .{ .float = .word } }, | |
| 173394 | .patterns = &.{ | |
| 173395 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173396 | }, | |
| 173397 | .each = .{ .once = &.{ | |
| 173398 | .{ ._, .p_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ }, | |
| 173399 | } }, | |
| 173400 | }, .{ | |
| 173401 | .required_features = .{ .sse2, null, null, null }, | |
| 173402 | .src_constraints = .{ .any, .any, .{ .float = .word } }, | |
| 173403 | .patterns = &.{ | |
| 173404 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173405 | }, | |
| 173406 | .extra_temps = .{ | |
| 173407 | .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, | |
| 173408 | .unused, | |
| 173409 | .unused, | |
| 173410 | .unused, | |
| 173411 | .unused, | |
| 173412 | .unused, | |
| 173413 | .unused, | |
| 173414 | .unused, | |
| 173415 | .unused, | |
| 173416 | .unused, | |
| 173417 | .unused, | |
| 173418 | }, | |
| 173419 | .each = .{ .once = &.{ | |
| 173420 | .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ }, | |
| 173421 | .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp0w, ._, ._ }, | |
| 173422 | } }, | |
| 173423 | }, .{ | |
| 173424 | .required_features = .{ .sse, null, null, null }, | |
| 173425 | .src_constraints = .{ .any, .any, .{ .float = .word } }, | |
| 173426 | .patterns = &.{ | |
| 173427 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173428 | }, | |
| 173429 | .extra_temps = .{ | |
| 173430 | .{ .type = .f32, .kind = .mem }, | |
| 173431 | .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, | |
| 173432 | .unused, | |
| 173433 | .unused, | |
| 173434 | .unused, | |
| 173435 | .unused, | |
| 173436 | .unused, | |
| 173437 | .unused, | |
| 173438 | .unused, | |
| 173439 | .unused, | |
| 173440 | .unused, | |
| 173441 | }, | |
| 173442 | .each = .{ .once = &.{ | |
| 173443 | .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ }, | |
| 173444 | .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ }, | |
| 173445 | .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp1w, ._, ._ }, | |
| 173446 | } }, | |
| 173447 | }, .{ | |
| 173448 | .required_features = .{ .avx, null, null, null }, | |
| 173449 | .src_constraints = .{ .any, .any, .{ .float = .word } }, | |
| 173450 | .patterns = &.{ | |
| 173451 | .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, | |
| 173452 | }, | |
| 173453 | .each = .{ .once = &.{ | |
| 173454 | .{ ._, .vp_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ }, | |
| 173455 | } }, | |
| 173456 | }, .{ | |
| 173457 | .required_features = .{ .sse4_1, null, null, null }, | |
| 173458 | .src_constraints = .{ .any, .any, .{ .float = .word } }, | |
| 173459 | .patterns = &.{ | |
| 173460 | .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, | |
| 173461 | }, | |
| 173462 | .each = .{ .once = &.{ | |
| 173463 | .{ ._, .p_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ }, | |
| 173464 | } }, | |
| 173465 | }, .{ | |
| 173466 | .required_features = .{ .sse2, null, null, null }, | |
| 173467 | .src_constraints = .{ .any, .any, .{ .float = .word } }, | |
| 173468 | .patterns = &.{ | |
| 173469 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173470 | }, | |
| 173471 | .extra_temps = .{ | |
| 173472 | .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, | |
| 173473 | .unused, | |
| 173474 | .unused, | |
| 173475 | .unused, | |
| 173476 | .unused, | |
| 173477 | .unused, | |
| 173478 | .unused, | |
| 173479 | .unused, | |
| 173480 | .unused, | |
| 173481 | .unused, | |
| 173482 | .unused, | |
| 173483 | }, | |
| 173484 | .each = .{ .once = &.{ | |
| 173485 | .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ }, | |
| 173486 | .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp0w, ._, ._ }, | |
| 173487 | } }, | |
| 173488 | }, .{ | |
| 173489 | .required_features = .{ .sse, null, null, null }, | |
| 173490 | .src_constraints = .{ .any, .any, .{ .float = .word } }, | |
| 173491 | .patterns = &.{ | |
| 173492 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173493 | }, | |
| 173494 | .extra_temps = .{ | |
| 173495 | .{ .type = .f32, .kind = .mem }, | |
| 173496 | .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, | |
| 173497 | .unused, | |
| 173498 | .unused, | |
| 173499 | .unused, | |
| 173500 | .unused, | |
| 173501 | .unused, | |
| 173502 | .unused, | |
| 173503 | .unused, | |
| 173504 | .unused, | |
| 173505 | .unused, | |
| 173506 | }, | |
| 173507 | .each = .{ .once = &.{ | |
| 173508 | .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ }, | |
| 173509 | .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ }, | |
| 173510 | .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp1w, ._, ._ }, | |
| 173511 | } }, | |
| 173512 | }, .{ | |
| 173513 | .src_constraints = .{ .any, .any, .{ .int = .dword } }, | |
| 173514 | .patterns = &.{ | |
| 173515 | .{ .src = .{ .to_gpr, .simm32, .imm32 } }, | |
| 173516 | .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, | |
| 173517 | }, | |
| 173518 | .each = .{ .once = &.{ | |
| 173519 | .{ ._, ._, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2d, ._, ._ }, | |
| 173520 | } }, | |
| 173521 | }, .{ | |
| 173522 | .src_constraints = .{ .any, .any, .{ .int = .dword } }, | |
| 173523 | .patterns = &.{ | |
| 173524 | .{ .src = .{ .to_gpr, .to_gpr, .imm32 } }, | |
| 173525 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173526 | }, | |
| 173527 | .each = .{ .once = &.{ | |
| 173528 | .{ ._, ._, .mov, .leasi(.src0d, .@"4", .src1), .src2d, ._, ._ }, | |
| 173529 | } }, | |
| 173530 | }, .{ | |
| 173531 | .required_features = .{ .avx, null, null, null }, | |
| 173532 | .src_constraints = .{ .any, .any, .{ .float = .dword } }, | |
| 173533 | .patterns = &.{ | |
| 173534 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173535 | }, | |
| 173536 | .each = .{ .once = &.{ | |
| 173537 | .{ ._, .v_ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, | |
| 173538 | } }, | |
| 173539 | }, .{ | |
| 173540 | .required_features = .{ .sse, null, null, null }, | |
| 173541 | .src_constraints = .{ .any, .any, .{ .float = .dword } }, | |
| 173542 | .patterns = &.{ | |
| 173543 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173544 | }, | |
| 173545 | .each = .{ .once = &.{ | |
| 173546 | .{ ._, ._ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, | |
| 173547 | } }, | |
| 173548 | }, .{ | |
| 173549 | .required_features = .{ .avx, null, null, null }, | |
| 173550 | .src_constraints = .{ .any, .any, .{ .float = .dword } }, | |
| 173551 | .patterns = &.{ | |
| 173552 | .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, | |
| 173553 | }, | |
| 173554 | .each = .{ .once = &.{ | |
| 173555 | .{ ._, .v_ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ }, | |
| 173556 | } }, | |
| 173557 | }, .{ | |
| 173558 | .required_features = .{ .sse, null, null, null }, | |
| 173559 | .src_constraints = .{ .any, .any, .{ .float = .dword } }, | |
| 173560 | .patterns = &.{ | |
| 173561 | .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, | |
| 173562 | }, | |
| 173563 | .each = .{ .once = &.{ | |
| 173564 | .{ ._, ._ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ }, | |
| 173565 | } }, | |
| 173566 | }, .{ | |
| 173567 | .required_features = .{ .@"64bit", null, null, null }, | |
| 173568 | .src_constraints = .{ .any, .any, .{ .int = .qword } }, | |
| 173569 | .patterns = &.{ | |
| 173570 | .{ .src = .{ .to_gpr, .simm32, .simm32 } }, | |
| 173571 | .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, | |
| 173572 | }, | |
| 173573 | .each = .{ .once = &.{ | |
| 173574 | .{ ._, ._, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2q, ._, ._ }, | |
| 173575 | } }, | |
| 173576 | }, .{ | |
| 173577 | .required_features = .{ .@"64bit", null, null, null }, | |
| 173578 | .src_constraints = .{ .any, .any, .{ .int = .qword } }, | |
| 173579 | .patterns = &.{ | |
| 173580 | .{ .src = .{ .to_gpr, .to_gpr, .simm32 } }, | |
| 173581 | .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, | |
| 173582 | }, | |
| 173583 | .each = .{ .once = &.{ | |
| 173584 | .{ ._, ._, .mov, .leasi(.src0q, .@"8", .src1), .src2q, ._, ._ }, | |
| 173585 | } }, | |
| 173586 | }, .{ | |
| 173587 | .required_features = .{ .avx, null, null, null }, | |
| 173588 | .src_constraints = .{ .any, .any, .{ .float = .qword } }, | |
| 173589 | .patterns = &.{ | |
| 173590 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173591 | }, | |
| 173592 | .each = .{ .once = &.{ | |
| 173593 | .{ ._, .v_sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, | |
| 173594 | } }, | |
| 173595 | }, .{ | |
| 173596 | .required_features = .{ .sse2, null, null, null }, | |
| 173597 | .src_constraints = .{ .any, .any, .{ .float = .qword } }, | |
| 173598 | .patterns = &.{ | |
| 173599 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173600 | }, | |
| 173601 | .each = .{ .once = &.{ | |
| 173602 | .{ ._, ._sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, | |
| 173603 | } }, | |
| 173604 | }, .{ | |
| 173605 | .required_features = .{ .sse, null, null, null }, | |
| 173606 | .src_constraints = .{ .any, .any, .{ .float = .qword } }, | |
| 173607 | .patterns = &.{ | |
| 173608 | .{ .src = .{ .to_gpr, .simm32, .to_sse } }, | |
| 173609 | }, | |
| 173610 | .each = .{ .once = &.{ | |
| 173611 | .{ ._, ._ps, .movl, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, | |
| 173612 | } }, | |
| 173613 | }, .{ | |
| 173614 | .required_features = .{ .avx, null, null, null }, | |
| 173615 | .src_constraints = .{ .any, .any, .{ .float = .qword } }, | |
| 173616 | .patterns = &.{ | |
| 173617 | .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, | |
| 173618 | }, | |
| 173619 | .each = .{ .once = &.{ | |
| 173620 | .{ ._, .v_sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, | |
| 173621 | } }, | |
| 173622 | }, .{ | |
| 173623 | .required_features = .{ .sse2, null, null, null }, | |
| 173624 | .src_constraints = .{ .any, .any, .{ .float = .qword } }, | |
| 173625 | .patterns = &.{ | |
| 173626 | .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, | |
| 173627 | }, | |
| 173628 | .each = .{ .once = &.{ | |
| 173629 | .{ ._, ._sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, | |
| 173630 | } }, | |
| 173631 | }, .{ | |
| 173632 | .required_features = .{ .sse, null, null, null }, | |
| 173633 | .src_constraints = .{ .any, .any, .{ .float = .qword } }, | |
| 173634 | .patterns = &.{ | |
| 173635 | .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, | |
| 173636 | }, | |
| 173637 | .each = .{ .once = &.{ | |
| 173638 | .{ ._, ._ps, .movl, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, | |
| 173639 | } }, | |
| 173640 | } }) catch |err| switch (err) { | |
| 173641 | error.SelectFailed => { | |
| 173642 | const elem_size = cg.typeOf(bin_op.rhs).abiSize(zcu); | |
| 173643 | while (try ops[0].toRegClass(true, .general_purpose, cg) or | |
| 173644 | try ops[1].toRegClass(true, .general_purpose, cg)) | |
| 173645 | {} | |
| 173646 | const base_reg = ops[0].tracking(cg).short.register.to64(); | |
| 173647 | const rhs_reg = ops[1].tracking(cg).short.register.to64(); | |
| 173648 | if (!std.math.isPowerOfTwo(elem_size)) { | |
| 173649 | try cg.spillEflagsIfOccupied(); | |
| 173650 | try cg.asmRegisterRegisterImmediate( | |
| 173651 | .{ .i_, .mul }, | |
| 173652 | rhs_reg, | |
| 173653 | rhs_reg, | |
| 173654 | .u(elem_size), | |
| 173655 | ); | |
| 173656 | try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ | |
| 173657 | .base = .{ .reg = base_reg }, | |
| 173658 | .mod = .{ .rm = .{ .index = rhs_reg } }, | |
| 173659 | }); | |
| 173660 | } else if (elem_size > 8) { | |
| 173661 | try cg.spillEflagsIfOccupied(); | |
| 173662 | try cg.asmRegisterImmediate( | |
| 173663 | .{ ._l, .sh }, | |
| 173664 | rhs_reg, | |
| 173665 | .u(std.math.log2_int(u64, elem_size)), | |
| 173666 | ); | |
| 173667 | try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ | |
| 173668 | .base = .{ .reg = base_reg }, | |
| 173669 | .mod = .{ .rm = .{ .index = rhs_reg } }, | |
| 173670 | }); | |
| 173671 | } else try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ | |
| 173672 | .base = .{ .reg = base_reg }, | |
| 173673 | .mod = .{ .rm = .{ | |
| 173674 | .index = rhs_reg, | |
| 173675 | .scale = .fromFactor(@intCast(elem_size)), | |
| 173676 | } }, | |
| 173677 | }); | |
| 173678 | try ops[0].store(&ops[2], .{}, cg); | |
| 173679 | }, | |
| 173680 | else => |e| return e, | |
| 173681 | }; | |
| 173682 | for (ops) |op| try op.die(cg); | |
| 173683 | }, | |
| 173684 | 173047 | .runtime_nav_ptr => { |
| 173685 | 173048 | const ty_nav = air_datas[@intFromEnum(inst)].ty_nav; |
| 173686 | 173049 | const nav = ip.getNav(ty_nav.nav); |
| ... | ... | @@ -180646,944 +180009,55 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 180646 | 180009 | return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); |
| 180647 | 180010 | } |
| 180648 | 180011 | |
| 180649 | fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { | |
| 180012 | fn airAggregateInitBoolVec(self: *CodeGen, inst: Air.Inst.Index) !void { | |
| 180650 | 180013 | const pt = self.pt; |
| 180651 | 180014 | const zcu = pt.zcu; |
| 180015 | const result_ty = self.typeOfIndex(inst); | |
| 180016 | const len: usize = @intCast(result_ty.arrayLen(zcu)); | |
| 180652 | 180017 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 180653 | const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data; | |
| 180654 | ||
| 180655 | const dst_ty = self.typeOfIndex(inst); | |
| 180656 | const elem_ty = dst_ty.childType(zcu); | |
| 180657 | const elem_abi_size: u16 = @intCast(elem_ty.abiSize(zcu)); | |
| 180658 | const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); | |
| 180659 | const lhs_ty = self.typeOf(extra.a); | |
| 180660 | const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); | |
| 180661 | const rhs_ty = self.typeOf(extra.b); | |
| 180662 | const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(zcu)); | |
| 180663 | const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size); | |
| 180664 | ||
| 180665 | const ExpectedContents = [32]?i32; | |
| 180666 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = | |
| 180667 | std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); | |
| 180668 | const allocator = stack.get(); | |
| 180669 | ||
| 180670 | const mask_elems = try allocator.alloc(?i32, extra.mask_len); | |
| 180671 | defer allocator.free(mask_elems); | |
| 180672 | for (mask_elems, 0..) |*mask_elem, elem_index| { | |
| 180673 | const mask_elem_val = | |
| 180674 | Value.fromInterned(extra.mask).elemValue(pt, elem_index) catch unreachable; | |
| 180675 | mask_elem.* = if (mask_elem_val.isUndef(zcu)) | |
| 180676 | null | |
| 180677 | else | |
| 180678 | @intCast(mask_elem_val.toSignedInt(zcu)); | |
| 180679 | } | |
| 180680 | ||
| 180681 | const has_avx = self.hasFeature(.avx); | |
| 180682 | const result = @as(?MCValue, result: { | |
| 180683 | for (mask_elems) |mask_elem| { | |
| 180684 | if (mask_elem) |_| break; | |
| 180685 | } else break :result try self.allocRegOrMem(inst, true); | |
| 180686 | ||
| 180687 | for (mask_elems, 0..) |mask_elem, elem_index| { | |
| 180688 | if (mask_elem orelse continue != elem_index) break; | |
| 180689 | } else { | |
| 180690 | const lhs_mcv = try self.resolveInst(extra.a); | |
| 180691 | if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv; | |
| 180692 | const dst_mcv = try self.allocRegOrMem(inst, true); | |
| 180693 | try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{}); | |
| 180694 | break :result dst_mcv; | |
| 180695 | } | |
| 180696 | ||
| 180697 | for (mask_elems, 0..) |mask_elem, elem_index| { | |
| 180698 | if (~(mask_elem orelse continue) != elem_index) break; | |
| 180699 | } else { | |
| 180700 | const rhs_mcv = try self.resolveInst(extra.b); | |
| 180701 | if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv; | |
| 180702 | const dst_mcv = try self.allocRegOrMem(inst, true); | |
| 180703 | try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{}); | |
| 180704 | break :result dst_mcv; | |
| 180705 | } | |
| 180706 | ||
| 180707 | for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: { | |
| 180708 | if (elem_abi_size > 8) break :unpck; | |
| 180709 | if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck; | |
| 180710 | ||
| 180711 | var sources: [2]?u1 = @splat(null); | |
| 180712 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { | |
| 180713 | const mask_elem = maybe_mask_elem orelse continue; | |
| 180714 | const mask_elem_index = | |
| 180715 | std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck; | |
| 180716 | const elem_byte = (elem_index >> 1) * elem_abi_size; | |
| 180717 | if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) { | |
| 180718 | .unpckl => 0b0000, | |
| 180719 | .unpckh => 0b1000, | |
| 180720 | else => unreachable, | |
| 180721 | }) | (elem_byte << 1 & 0b10000)) break :unpck; | |
| 180722 | ||
| 180723 | const source = @intFromBool(mask_elem < 0); | |
| 180724 | if (sources[elem_index & 0b00001]) |prev_source| { | |
| 180725 | if (source != prev_source) break :unpck; | |
| 180726 | } else sources[elem_index & 0b00001] = source; | |
| 180727 | } | |
| 180728 | if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck; | |
| 180729 | ||
| 180730 | const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; | |
| 180731 | const operand_tys = [2]Type{ lhs_ty, rhs_ty }; | |
| 180732 | const lhs_mcv = try self.resolveInst(operands[sources[0].?]); | |
| 180733 | const rhs_mcv = try self.resolveInst(operands[sources[1].?]); | |
| 180734 | ||
| 180735 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and | |
| 180736 | self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) | |
| 180737 | lhs_mcv | |
| 180738 | else if (has_avx and lhs_mcv.isRegister()) | |
| 180739 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } | |
| 180740 | else | |
| 180741 | try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); | |
| 180742 | const dst_reg = dst_mcv.getReg().?; | |
| 180743 | const dst_alias = registerAlias(dst_reg, max_abi_size); | |
| 180744 | ||
| 180745 | const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or | |
| 180746 | (dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) { | |
| 180747 | 4 => if (has_avx) .v_ps else ._ps, | |
| 180748 | 8 => if (has_avx) .v_pd else ._pd, | |
| 180749 | else => unreachable, | |
| 180750 | }, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) { | |
| 180751 | .unpckl => switch (elem_abi_size) { | |
| 180752 | 1 => .unpcklbw, | |
| 180753 | 2 => .unpcklwd, | |
| 180754 | 4 => .unpckldq, | |
| 180755 | 8 => .unpcklqdq, | |
| 180756 | else => unreachable, | |
| 180757 | }, | |
| 180758 | .unpckh => switch (elem_abi_size) { | |
| 180759 | 1 => .unpckhbw, | |
| 180760 | 2 => .unpckhwd, | |
| 180761 | 4 => .unpckhdq, | |
| 180762 | 8 => .unpckhqdq, | |
| 180763 | else => unreachable, | |
| 180764 | }, | |
| 180765 | else => unreachable, | |
| 180766 | } }; | |
| 180767 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory( | |
| 180768 | mir_tag, | |
| 180769 | dst_alias, | |
| 180770 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | |
| 180771 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | |
| 180772 | ) else try self.asmRegisterRegisterRegister( | |
| 180773 | mir_tag, | |
| 180774 | dst_alias, | |
| 180775 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | |
| 180776 | registerAlias(if (rhs_mcv.isRegister()) | |
| 180777 | rhs_mcv.getReg().? | |
| 180778 | else | |
| 180779 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | |
| 180780 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemory( | |
| 180781 | mir_tag, | |
| 180782 | dst_alias, | |
| 180783 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | |
| 180784 | ) else try self.asmRegisterRegister( | |
| 180785 | mir_tag, | |
| 180786 | dst_alias, | |
| 180787 | registerAlias(if (rhs_mcv.isRegister()) | |
| 180788 | rhs_mcv.getReg().? | |
| 180789 | else | |
| 180790 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | |
| 180791 | ); | |
| 180792 | break :result dst_mcv; | |
| 180793 | } | |
| 180794 | ||
| 180795 | pshufd: { | |
| 180796 | if (elem_abi_size != 4) break :pshufd; | |
| 180797 | if (max_abi_size > self.vectorSize(.float)) break :pshufd; | |
| 180798 | ||
| 180799 | var control: u8 = 0b00_00_00_00; | |
| 180800 | var sources: [1]?u1 = @splat(null); | |
| 180801 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { | |
| 180802 | const mask_elem = maybe_mask_elem orelse continue; | |
| 180803 | const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); | |
| 180804 | if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd; | |
| 180805 | ||
| 180806 | const source = @intFromBool(mask_elem < 0); | |
| 180807 | if (sources[0]) |prev_source| { | |
| 180808 | if (source != prev_source) break :pshufd; | |
| 180809 | } else sources[(elem_index & 0b010) >> 1] = source; | |
| 180810 | ||
| 180811 | const select_bit: u3 = @intCast((elem_index & 0b011) << 1); | |
| 180812 | const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; | |
| 180813 | if (elem_index & 0b100 == 0) | |
| 180814 | control |= select_mask | |
| 180815 | else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd; | |
| 180816 | } | |
| 180817 | ||
| 180818 | const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; | |
| 180819 | const operand_tys = [2]Type{ lhs_ty, rhs_ty }; | |
| 180820 | const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]); | |
| 180821 | ||
| 180822 | const dst_reg = if (src_mcv.isRegister() and | |
| 180823 | self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv)) | |
| 180824 | src_mcv.getReg().? | |
| 180825 | else | |
| 180826 | try self.register_manager.allocReg(inst, abi.RegisterClass.sse); | |
| 180827 | const dst_alias = registerAlias(dst_reg, max_abi_size); | |
| 180828 | ||
| 180829 | if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( | |
| 180830 | .{ if (has_avx) .vp_d else .p_d, .shuf }, | |
| 180831 | dst_alias, | |
| 180832 | try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | |
| 180833 | .u(control), | |
| 180834 | ) else try self.asmRegisterRegisterImmediate( | |
| 180835 | .{ if (has_avx) .vp_d else .p_d, .shuf }, | |
| 180836 | dst_alias, | |
| 180837 | registerAlias(if (src_mcv.isRegister()) | |
| 180838 | src_mcv.getReg().? | |
| 180839 | else | |
| 180840 | try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size), | |
| 180841 | .u(control), | |
| 180842 | ); | |
| 180843 | break :result .{ .register = dst_reg }; | |
| 180844 | } | |
| 180845 | ||
| 180846 | shufps: { | |
| 180847 | if (elem_abi_size != 4) break :shufps; | |
| 180848 | if (max_abi_size > self.vectorSize(.float)) break :shufps; | |
| 180849 | ||
| 180850 | var control: u8 = 0b00_00_00_00; | |
| 180851 | var sources: [2]?u1 = @splat(null); | |
| 180852 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { | |
| 180853 | const mask_elem = maybe_mask_elem orelse continue; | |
| 180854 | const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); | |
| 180855 | if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps; | |
| 180856 | ||
| 180857 | const source = @intFromBool(mask_elem < 0); | |
| 180858 | if (sources[(elem_index & 0b010) >> 1]) |prev_source| { | |
| 180859 | if (source != prev_source) break :shufps; | |
| 180860 | } else sources[(elem_index & 0b010) >> 1] = source; | |
| 180861 | ||
| 180862 | const select_bit: u3 = @intCast((elem_index & 0b011) << 1); | |
| 180863 | const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; | |
| 180864 | if (elem_index & 0b100 == 0) | |
| 180865 | control |= select_mask | |
| 180866 | else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps; | |
| 180867 | } | |
| 180868 | if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps; | |
| 180869 | ||
| 180870 | const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; | |
| 180871 | const operand_tys = [2]Type{ lhs_ty, rhs_ty }; | |
| 180872 | const lhs_mcv = try self.resolveInst(operands[sources[0].?]); | |
| 180873 | const rhs_mcv = try self.resolveInst(operands[sources[1].?]); | |
| 180874 | ||
| 180875 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and | |
| 180876 | self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) | |
| 180877 | lhs_mcv | |
| 180878 | else if (has_avx and lhs_mcv.isRegister()) | |
| 180879 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } | |
| 180880 | else | |
| 180881 | try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); | |
| 180882 | const dst_reg = dst_mcv.getReg().?; | |
| 180883 | const dst_alias = registerAlias(dst_reg, max_abi_size); | |
| 180884 | ||
| 180885 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( | |
| 180886 | .{ .v_ps, .shuf }, | |
| 180887 | dst_alias, | |
| 180888 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | |
| 180889 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | |
| 180890 | .u(control), | |
| 180891 | ) else try self.asmRegisterRegisterRegisterImmediate( | |
| 180892 | .{ .v_ps, .shuf }, | |
| 180893 | dst_alias, | |
| 180894 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | |
| 180895 | registerAlias(if (rhs_mcv.isRegister()) | |
| 180896 | rhs_mcv.getReg().? | |
| 180897 | else | |
| 180898 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | |
| 180899 | .u(control), | |
| 180900 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( | |
| 180901 | .{ ._ps, .shuf }, | |
| 180902 | dst_alias, | |
| 180903 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | |
| 180904 | .u(control), | |
| 180905 | ) else try self.asmRegisterRegisterImmediate( | |
| 180906 | .{ ._ps, .shuf }, | |
| 180907 | dst_alias, | |
| 180908 | registerAlias(if (rhs_mcv.isRegister()) | |
| 180909 | rhs_mcv.getReg().? | |
| 180910 | else | |
| 180911 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | |
| 180912 | .u(control), | |
| 180913 | ); | |
| 180914 | break :result dst_mcv; | |
| 180915 | } | |
| 180916 | ||
| 180917 | shufpd: { | |
| 180918 | if (elem_abi_size != 8) break :shufpd; | |
| 180919 | if (max_abi_size > self.vectorSize(.float)) break :shufpd; | |
| 180920 | ||
| 180921 | var control: u4 = 0b0_0_0_0; | |
| 180922 | var sources: [2]?u1 = @splat(null); | |
| 180923 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { | |
| 180924 | const mask_elem = maybe_mask_elem orelse continue; | |
| 180925 | const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); | |
| 180926 | if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd; | |
| 180927 | ||
| 180928 | const source = @intFromBool(mask_elem < 0); | |
| 180929 | if (sources[elem_index & 0b01]) |prev_source| { | |
| 180930 | if (source != prev_source) break :shufpd; | |
| 180931 | } else sources[elem_index & 0b01] = source; | |
| 180932 | ||
| 180933 | control |= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index); | |
| 180934 | } | |
| 180935 | if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd; | |
| 180936 | ||
| 180937 | const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b }; | |
| 180938 | const operand_tys: [2]Type = .{ lhs_ty, rhs_ty }; | |
| 180939 | const lhs_mcv = try self.resolveInst(operands[sources[0].?]); | |
| 180940 | const rhs_mcv = try self.resolveInst(operands[sources[1].?]); | |
| 180941 | ||
| 180942 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and | |
| 180943 | self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) | |
| 180944 | lhs_mcv | |
| 180945 | else if (has_avx and lhs_mcv.isRegister()) | |
| 180946 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } | |
| 180947 | else | |
| 180948 | try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); | |
| 180949 | const dst_reg = dst_mcv.getReg().?; | |
| 180950 | const dst_alias = registerAlias(dst_reg, max_abi_size); | |
| 180951 | ||
| 180952 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( | |
| 180953 | .{ .v_pd, .shuf }, | |
| 180954 | dst_alias, | |
| 180955 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | |
| 180956 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | |
| 180957 | .u(control), | |
| 180958 | ) else try self.asmRegisterRegisterRegisterImmediate( | |
| 180959 | .{ .v_pd, .shuf }, | |
| 180960 | dst_alias, | |
| 180961 | registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), | |
| 180962 | registerAlias(if (rhs_mcv.isRegister()) | |
| 180963 | rhs_mcv.getReg().? | |
| 180964 | else | |
| 180965 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | |
| 180966 | .u(control), | |
| 180967 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( | |
| 180968 | .{ ._pd, .shuf }, | |
| 180969 | dst_alias, | |
| 180970 | try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), | |
| 180971 | .u(control), | |
| 180972 | ) else try self.asmRegisterRegisterImmediate( | |
| 180973 | .{ ._pd, .shuf }, | |
| 180974 | dst_alias, | |
| 180975 | registerAlias(if (rhs_mcv.isRegister()) | |
| 180976 | rhs_mcv.getReg().? | |
| 180977 | else | |
| 180978 | try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), | |
| 180979 | .u(control), | |
| 180980 | ); | |
| 180981 | break :result dst_mcv; | |
| 180982 | } | |
| 180983 | ||
| 180984 | blend: { | |
| 180985 | if (elem_abi_size < 2) break :blend; | |
| 180986 | if (dst_abi_size > self.vectorSize(.float)) break :blend; | |
| 180987 | if (!self.hasFeature(.sse4_1)) break :blend; | |
| 180988 | ||
| 180989 | var control: u8 = 0b0_0_0_0_0_0_0_0; | |
| 180990 | for (mask_elems, 0..) |maybe_mask_elem, elem_index| { | |
| 180991 | const mask_elem = maybe_mask_elem orelse continue; | |
| 180992 | const mask_elem_index = | |
| 180993 | std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend; | |
| 180994 | if (mask_elem_index != elem_index) break :blend; | |
| 180995 | ||
| 180996 | const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index); | |
| 180997 | if (elem_index & 0b1000 == 0) | |
| 180998 | control |= select_mask | |
| 180999 | else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend; | |
| 181000 | } | |
| 181001 | ||
| 181002 | if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: { | |
| 181003 | const expanded_control = switch (elem_abi_size) { | |
| 181004 | 4 => control, | |
| 181005 | 8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | | |
| 181006 | @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | | |
| 181007 | @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | | |
| 181008 | @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00), | |
| 181009 | else => break :vpblendd, | |
| 181010 | }; | |
| 181011 | ||
| 181012 | const lhs_mcv = try self.resolveInst(extra.a); | |
| 181013 | const lhs_reg = if (lhs_mcv.isRegister()) | |
| 181014 | lhs_mcv.getReg().? | |
| 181015 | else | |
| 181016 | try self.copyToTmpRegister(dst_ty, lhs_mcv); | |
| 181017 | const lhs_lock = self.register_manager.lockReg(lhs_reg); | |
| 181018 | defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); | |
| 181019 | ||
| 181020 | const rhs_mcv = try self.resolveInst(extra.b); | |
| 181021 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); | |
| 181022 | if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( | |
| 181023 | .{ .vp_d, .blend }, | |
| 181024 | registerAlias(dst_reg, dst_abi_size), | |
| 181025 | registerAlias(lhs_reg, dst_abi_size), | |
| 181026 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | |
| 181027 | .u(expanded_control), | |
| 181028 | ) else try self.asmRegisterRegisterRegisterImmediate( | |
| 181029 | .{ .vp_d, .blend }, | |
| 181030 | registerAlias(dst_reg, dst_abi_size), | |
| 181031 | registerAlias(lhs_reg, dst_abi_size), | |
| 181032 | registerAlias(if (rhs_mcv.isRegister()) | |
| 181033 | rhs_mcv.getReg().? | |
| 181034 | else | |
| 181035 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | |
| 181036 | .u(expanded_control), | |
| 181037 | ); | |
| 181038 | break :result .{ .register = dst_reg }; | |
| 181039 | } | |
| 181040 | ||
| 181041 | if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: { | |
| 181042 | const expanded_control = switch (elem_abi_size) { | |
| 181043 | 2 => control, | |
| 181044 | 4 => if (dst_abi_size <= 16 or | |
| 181045 | @as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0))) | |
| 181046 | @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | | |
| 181047 | @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | | |
| 181048 | @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | | |
| 181049 | @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00) | |
| 181050 | else | |
| 181051 | break :pblendw, | |
| 181052 | 8 => if (dst_abi_size <= 16 or | |
| 181053 | @as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0))) | |
| 181054 | @as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) | | |
| 181055 | @as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000) | |
| 181056 | else | |
| 181057 | break :pblendw, | |
| 181058 | 16 => break :pblendw, | |
| 181059 | else => unreachable, | |
| 181060 | }; | |
| 181061 | ||
| 181062 | const lhs_mcv = try self.resolveInst(extra.a); | |
| 181063 | const rhs_mcv = try self.resolveInst(extra.b); | |
| 181064 | ||
| 181065 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and | |
| 181066 | self.reuseOperand(inst, extra.a, 0, lhs_mcv)) | |
| 181067 | lhs_mcv | |
| 181068 | else if (has_avx and lhs_mcv.isRegister()) | |
| 181069 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } | |
| 181070 | else | |
| 181071 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); | |
| 181072 | const dst_reg = dst_mcv.getReg().?; | |
| 181073 | ||
| 181074 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( | |
| 181075 | .{ .vp_w, .blend }, | |
| 181076 | registerAlias(dst_reg, dst_abi_size), | |
| 181077 | registerAlias(if (lhs_mcv.isRegister()) | |
| 181078 | lhs_mcv.getReg().? | |
| 181079 | else | |
| 181080 | dst_reg, dst_abi_size), | |
| 181081 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | |
| 181082 | .u(expanded_control), | |
| 181083 | ) else try self.asmRegisterRegisterRegisterImmediate( | |
| 181084 | .{ .vp_w, .blend }, | |
| 181085 | registerAlias(dst_reg, dst_abi_size), | |
| 181086 | registerAlias(if (lhs_mcv.isRegister()) | |
| 181087 | lhs_mcv.getReg().? | |
| 181088 | else | |
| 181089 | dst_reg, dst_abi_size), | |
| 181090 | registerAlias(if (rhs_mcv.isRegister()) | |
| 181091 | rhs_mcv.getReg().? | |
| 181092 | else | |
| 181093 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | |
| 181094 | .u(expanded_control), | |
| 181095 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( | |
| 181096 | .{ .p_w, .blend }, | |
| 181097 | registerAlias(dst_reg, dst_abi_size), | |
| 181098 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | |
| 181099 | .u(expanded_control), | |
| 181100 | ) else try self.asmRegisterRegisterImmediate( | |
| 181101 | .{ .p_w, .blend }, | |
| 181102 | registerAlias(dst_reg, dst_abi_size), | |
| 181103 | registerAlias(if (rhs_mcv.isRegister()) | |
| 181104 | rhs_mcv.getReg().? | |
| 181105 | else | |
| 181106 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | |
| 181107 | .u(expanded_control), | |
| 181108 | ); | |
| 181109 | break :result .{ .register = dst_reg }; | |
| 181110 | } | |
| 181111 | ||
| 181112 | const expanded_control = switch (elem_abi_size) { | |
| 181113 | 4, 8 => control, | |
| 181114 | 16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) | | |
| 181115 | @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00), | |
| 181116 | else => unreachable, | |
| 181117 | }; | |
| 181118 | ||
| 181119 | const lhs_mcv = try self.resolveInst(extra.a); | |
| 181120 | const rhs_mcv = try self.resolveInst(extra.b); | |
| 181121 | ||
| 181122 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and | |
| 181123 | self.reuseOperand(inst, extra.a, 0, lhs_mcv)) | |
| 181124 | lhs_mcv | |
| 181125 | else if (has_avx and lhs_mcv.isRegister()) | |
| 181126 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } | |
| 181127 | else | |
| 181128 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); | |
| 181129 | const dst_reg = dst_mcv.getReg().?; | |
| 181130 | ||
| 181131 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( | |
| 181132 | switch (elem_abi_size) { | |
| 181133 | 4 => .{ .v_ps, .blend }, | |
| 181134 | 8, 16 => .{ .v_pd, .blend }, | |
| 181135 | else => unreachable, | |
| 181136 | }, | |
| 181137 | registerAlias(dst_reg, dst_abi_size), | |
| 181138 | registerAlias(if (lhs_mcv.isRegister()) | |
| 181139 | lhs_mcv.getReg().? | |
| 181140 | else | |
| 181141 | dst_reg, dst_abi_size), | |
| 181142 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | |
| 181143 | .u(expanded_control), | |
| 181144 | ) else try self.asmRegisterRegisterRegisterImmediate( | |
| 181145 | switch (elem_abi_size) { | |
| 181146 | 4 => .{ .v_ps, .blend }, | |
| 181147 | 8, 16 => .{ .v_pd, .blend }, | |
| 181148 | else => unreachable, | |
| 181149 | }, | |
| 181150 | registerAlias(dst_reg, dst_abi_size), | |
| 181151 | registerAlias(if (lhs_mcv.isRegister()) | |
| 181152 | lhs_mcv.getReg().? | |
| 181153 | else | |
| 181154 | dst_reg, dst_abi_size), | |
| 181155 | registerAlias(if (rhs_mcv.isRegister()) | |
| 181156 | rhs_mcv.getReg().? | |
| 181157 | else | |
| 181158 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | |
| 181159 | .u(expanded_control), | |
| 181160 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( | |
| 181161 | switch (elem_abi_size) { | |
| 181162 | 4 => .{ ._ps, .blend }, | |
| 181163 | 8, 16 => .{ ._pd, .blend }, | |
| 181164 | else => unreachable, | |
| 181165 | }, | |
| 181166 | registerAlias(dst_reg, dst_abi_size), | |
| 181167 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | |
| 181168 | .u(expanded_control), | |
| 181169 | ) else try self.asmRegisterRegisterImmediate( | |
| 181170 | switch (elem_abi_size) { | |
| 181171 | 4 => .{ ._ps, .blend }, | |
| 181172 | 8, 16 => .{ ._pd, .blend }, | |
| 181173 | else => unreachable, | |
| 181174 | }, | |
| 181175 | registerAlias(dst_reg, dst_abi_size), | |
| 181176 | registerAlias(if (rhs_mcv.isRegister()) | |
| 181177 | rhs_mcv.getReg().? | |
| 181178 | else | |
| 181179 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | |
| 181180 | .u(expanded_control), | |
| 181181 | ); | |
| 181182 | break :result .{ .register = dst_reg }; | |
| 181183 | } | |
| 181184 | ||
| 181185 | blendv: { | |
| 181186 | if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv; | |
| 181187 | ||
| 181188 | const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8); | |
| 181189 | const select_mask_ty = try pt.vectorType(.{ | |
| 181190 | .len = @intCast(mask_elems.len), | |
| 181191 | .child = select_mask_elem_ty.toIntern(), | |
| 181192 | }); | |
| 181193 | var select_mask_elems: [32]InternPool.Index = undefined; | |
| 181194 | for ( | |
| 181195 | select_mask_elems[0..mask_elems.len], | |
| 181196 | mask_elems, | |
| 181197 | 0.., | |
| 181198 | ) |*select_mask_elem, maybe_mask_elem, elem_index| { | |
| 181199 | const mask_elem = maybe_mask_elem orelse continue; | |
| 181200 | const mask_elem_index = | |
| 181201 | std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv; | |
| 181202 | if (mask_elem_index != elem_index) break :blendv; | |
| 181203 | ||
| 181204 | select_mask_elem.* = (if (mask_elem < 0) | |
| 181205 | try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty) | |
| 181206 | else | |
| 181207 | try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern(); | |
| 181208 | } | |
| 181209 | const select_mask_mcv = try self.lowerValue( | |
| 181210 | try pt.aggregateValue(select_mask_ty, select_mask_elems[0..mask_elems.len]), | |
| 181211 | ); | |
| 181212 | ||
| 181213 | if (self.hasFeature(.sse4_1)) { | |
| 181214 | const mir_tag: Mir.Inst.FixedTag = .{ | |
| 181215 | if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or | |
| 181216 | (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) { | |
| 181217 | 4 => if (has_avx) .v_ps else ._ps, | |
| 181218 | 8 => if (has_avx) .v_pd else ._pd, | |
| 181219 | else => unreachable, | |
| 181220 | } else if (has_avx) .vp_b else .p_b, | |
| 181221 | .blendv, | |
| 181222 | }; | |
| 181223 | ||
| 181224 | const select_mask_reg = if (!has_avx) reg: { | |
| 181225 | try self.register_manager.getKnownReg(.xmm0, null); | |
| 181226 | try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{}); | |
| 181227 | break :reg .xmm0; | |
| 181228 | } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); | |
| 181229 | const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size); | |
| 181230 | const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg); | |
| 181231 | defer self.register_manager.unlockReg(select_mask_lock); | |
| 181232 | ||
| 181233 | const lhs_mcv = try self.resolveInst(extra.a); | |
| 181234 | const rhs_mcv = try self.resolveInst(extra.b); | |
| 181235 | ||
| 181236 | const dst_mcv: MCValue = if (lhs_mcv.isRegister() and | |
| 181237 | self.reuseOperand(inst, extra.a, 0, lhs_mcv)) | |
| 181238 | lhs_mcv | |
| 181239 | else if (has_avx and lhs_mcv.isRegister()) | |
| 181240 | .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } | |
| 181241 | else | |
| 181242 | try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); | |
| 181243 | const dst_reg = dst_mcv.getReg().?; | |
| 181244 | const dst_alias = registerAlias(dst_reg, dst_abi_size); | |
| 181245 | ||
| 181246 | if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( | |
| 181247 | mir_tag, | |
| 181248 | dst_alias, | |
| 181249 | if (lhs_mcv.isRegister()) | |
| 181250 | registerAlias(lhs_mcv.getReg().?, dst_abi_size) | |
| 181251 | else | |
| 181252 | dst_alias, | |
| 181253 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | |
| 181254 | select_mask_alias, | |
| 181255 | ) else try self.asmRegisterRegisterRegisterRegister( | |
| 181256 | mir_tag, | |
| 181257 | dst_alias, | |
| 181258 | if (lhs_mcv.isRegister()) | |
| 181259 | registerAlias(lhs_mcv.getReg().?, dst_abi_size) | |
| 181260 | else | |
| 181261 | dst_alias, | |
| 181262 | registerAlias(if (rhs_mcv.isRegister()) | |
| 181263 | rhs_mcv.getReg().? | |
| 181264 | else | |
| 181265 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | |
| 181266 | select_mask_alias, | |
| 181267 | ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister( | |
| 181268 | mir_tag, | |
| 181269 | dst_alias, | |
| 181270 | try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | |
| 181271 | select_mask_alias, | |
| 181272 | ) else try self.asmRegisterRegisterRegister( | |
| 181273 | mir_tag, | |
| 181274 | dst_alias, | |
| 181275 | registerAlias(if (rhs_mcv.isRegister()) | |
| 181276 | rhs_mcv.getReg().? | |
| 181277 | else | |
| 181278 | try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), | |
| 181279 | select_mask_alias, | |
| 181280 | ); | |
| 181281 | break :result dst_mcv; | |
| 181282 | } | |
| 180018 | const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); | |
| 181283 | 180019 | |
| 181284 | const lhs_mcv = try self.resolveInst(extra.a); | |
| 181285 | const rhs_mcv = try self.resolveInst(extra.b); | |
| 180020 | assert(result_ty.zigTypeTag(zcu) == .vector); | |
| 180021 | assert(result_ty.childType(zcu).toIntern() == .bool_type); | |
| 181286 | 180022 | |
| 181287 | const dst_mcv: MCValue = if (rhs_mcv.isRegister() and | |
| 181288 | self.reuseOperand(inst, extra.b, 1, rhs_mcv)) | |
| 181289 | rhs_mcv | |
| 181290 | else | |
| 181291 | try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv); | |
| 181292 | const dst_reg = dst_mcv.getReg().?; | |
| 181293 | const dst_alias = registerAlias(dst_reg, dst_abi_size); | |
| 180023 | const result_size = result_ty.abiSize(zcu); | |
| 180024 | if (result_size > 8) return self.fail("TODO airAggregateInitBoolVec over 8 bytes", .{}); | |
| 181294 | 180025 | |
| 181295 | const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); | |
| 181296 | const mask_alias = registerAlias(mask_reg, dst_abi_size); | |
| 181297 | const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); | |
| 181298 | defer self.register_manager.unlockReg(mask_lock); | |
| 180026 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); | |
| 181299 | 180027 | |
| 181300 | const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat()) | |
| 181301 | switch (elem_ty.floatBits(self.target)) { | |
| 181302 | 16, 80, 128 => .p_, | |
| 181303 | 32 => ._ps, | |
| 181304 | 64 => ._pd, | |
| 181305 | else => unreachable, | |
| 181306 | } | |
| 181307 | else | |
| 181308 | .p_; | |
| 181309 | try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); | |
| 181310 | if (lhs_mcv.isBase()) try self.asmRegisterMemory( | |
| 181311 | .{ mir_fixes, .andn }, | |
| 181312 | mask_alias, | |
| 181313 | try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), | |
| 181314 | ) else try self.asmRegisterRegister( | |
| 181315 | .{ mir_fixes, .andn }, | |
| 181316 | mask_alias, | |
| 181317 | if (lhs_mcv.isRegister()) | |
| 181318 | lhs_mcv.getReg().? | |
| 181319 | else | |
| 181320 | try self.copyToTmpRegister(dst_ty, lhs_mcv), | |
| 181321 | ); | |
| 181322 | try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias); | |
| 181323 | break :result dst_mcv; | |
| 181324 | } | |
| 180028 | { | |
| 180029 | const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); | |
| 180030 | defer self.register_manager.unlockReg(dst_lock); | |
| 180031 | try self.asmRegisterRegister( | |
| 180032 | .{ ._, .xor }, | |
| 180033 | registerAlias(dst_reg, @min(result_size, 4)), | |
| 180034 | registerAlias(dst_reg, @min(result_size, 4)), | |
| 180035 | ); | |
| 181325 | 180036 | |
| 181326 | pshufb: { | |
| 181327 | if (max_abi_size > 16) break :pshufb; | |
| 181328 | if (!self.hasFeature(.ssse3)) break :pshufb; | |
| 181329 | ||
| 181330 | const temp_regs = | |
| 181331 | try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse); | |
| 181332 | const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs); | |
| 181333 | defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); | |
| 181334 | ||
| 181335 | const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size); | |
| 181336 | try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{}); | |
| 181337 | ||
| 181338 | const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size); | |
| 181339 | try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{}); | |
| 181340 | ||
| 181341 | var lhs_mask_elems: [16]InternPool.Index = undefined; | |
| 181342 | for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| { | |
| 181343 | const elem_index = byte_index / elem_abi_size; | |
| 181344 | lhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: { | |
| 181345 | const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000; | |
| 181346 | if (mask_elem < 0) break :elem 0b1_00_00000; | |
| 181347 | const mask_elem_index: u31 = @intCast(mask_elem); | |
| 181348 | const byte_off: u32 = @intCast(byte_index % elem_abi_size); | |
| 181349 | break :elem mask_elem_index * elem_abi_size + byte_off; | |
| 181350 | })).toIntern(); | |
| 181351 | } | |
| 181352 | const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type }); | |
| 181353 | const lhs_mask_mcv = try self.lowerValue( | |
| 181354 | try pt.aggregateValue(lhs_mask_ty, lhs_mask_elems[0..max_abi_size]), | |
| 181355 | ); | |
| 181356 | const lhs_mask_mem: Memory = .{ | |
| 181357 | .base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) }, | |
| 181358 | .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } }, | |
| 181359 | }; | |
| 181360 | if (has_avx) try self.asmRegisterRegisterMemory( | |
| 181361 | .{ .vp_b, .shuf }, | |
| 181362 | lhs_temp_alias, | |
| 181363 | lhs_temp_alias, | |
| 181364 | lhs_mask_mem, | |
| 181365 | ) else try self.asmRegisterMemory( | |
| 181366 | .{ .p_b, .shuf }, | |
| 181367 | lhs_temp_alias, | |
| 181368 | lhs_mask_mem, | |
| 181369 | ); | |
| 180037 | for (elements, 0..) |elem, elem_i| { | |
| 180038 | const elem_reg = try self.copyToTmpRegister(.bool, .{ .air_ref = elem }); | |
| 180039 | const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg); | |
| 180040 | defer self.register_manager.unlockReg(elem_lock); | |
| 181370 | 180041 | |
| 181371 | var rhs_mask_elems: [16]InternPool.Index = undefined; | |
| 181372 | for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| { | |
| 181373 | const elem_index = byte_index / elem_abi_size; | |
| 181374 | rhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: { | |
| 181375 | const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000; | |
| 181376 | if (mask_elem >= 0) break :elem 0b1_00_00000; | |
| 181377 | const mask_elem_index: u31 = @intCast(~mask_elem); | |
| 181378 | const byte_off: u32 = @intCast(byte_index % elem_abi_size); | |
| 181379 | break :elem mask_elem_index * elem_abi_size + byte_off; | |
| 181380 | })).toIntern(); | |
| 181381 | } | |
| 181382 | const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type }); | |
| 181383 | const rhs_mask_mcv = try self.lowerValue( | |
| 181384 | try pt.aggregateValue(rhs_mask_ty, rhs_mask_elems[0..max_abi_size]), | |
| 180042 | try self.asmRegisterImmediate( | |
| 180043 | .{ ._, .@"and" }, | |
| 180044 | registerAlias(elem_reg, @min(result_size, 4)), | |
| 180045 | .u(1), | |
| 181385 | 180046 | ); |
| 181386 | const rhs_mask_mem: Memory = .{ | |
| 181387 | .base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) }, | |
| 181388 | .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } }, | |
| 181389 | }; | |
| 181390 | if (has_avx) try self.asmRegisterRegisterMemory( | |
| 181391 | .{ .vp_b, .shuf }, | |
| 181392 | rhs_temp_alias, | |
| 181393 | rhs_temp_alias, | |
| 181394 | rhs_mask_mem, | |
| 181395 | ) else try self.asmRegisterMemory( | |
| 181396 | .{ .p_b, .shuf }, | |
| 181397 | rhs_temp_alias, | |
| 181398 | rhs_mask_mem, | |
| 180047 | if (elem_i > 0) try self.asmRegisterImmediate( | |
| 180048 | .{ ._l, .sh }, | |
| 180049 | registerAlias(elem_reg, @intCast(result_size)), | |
| 180050 | .u(@intCast(elem_i)), | |
| 181399 | 180051 | ); |
| 181400 | ||
| 181401 | if (has_avx) try self.asmRegisterRegisterRegister( | |
| 181402 | .{ switch (elem_ty.zigTypeTag(zcu)) { | |
| 181403 | else => break :result null, | |
| 181404 | .int => .vp_, | |
| 181405 | .float => switch (elem_ty.floatBits(self.target)) { | |
| 181406 | 32 => .v_ps, | |
| 181407 | 64 => .v_pd, | |
| 181408 | 16, 80, 128 => break :result null, | |
| 181409 | else => unreachable, | |
| 181410 | }, | |
| 181411 | }, .@"or" }, | |
| 181412 | lhs_temp_alias, | |
| 181413 | lhs_temp_alias, | |
| 181414 | rhs_temp_alias, | |
| 181415 | ) else try self.asmRegisterRegister( | |
| 181416 | .{ switch (elem_ty.zigTypeTag(zcu)) { | |
| 181417 | else => break :result null, | |
| 181418 | .int => .p_, | |
| 181419 | .float => switch (elem_ty.floatBits(self.target)) { | |
| 181420 | 32 => ._ps, | |
| 181421 | 64 => ._pd, | |
| 181422 | 16, 80, 128 => break :result null, | |
| 181423 | else => unreachable, | |
| 181424 | }, | |
| 181425 | }, .@"or" }, | |
| 181426 | lhs_temp_alias, | |
| 181427 | rhs_temp_alias, | |
| 180052 | try self.asmRegisterRegister( | |
| 180053 | .{ ._, .@"or" }, | |
| 180054 | registerAlias(dst_reg, @intCast(result_size)), | |
| 180055 | registerAlias(elem_reg, @intCast(result_size)), | |
| 181428 | 180056 | ); |
| 181429 | break :result .{ .register = temp_regs[0] }; | |
| 181430 | 180057 | } |
| 180058 | } | |
| 181431 | 180059 | |
| 181432 | break :result null; | |
| 181433 | }) orelse return self.fail("TODO implement airShuffle from {f} and {f} to {f} with {f}", .{ | |
| 181434 | lhs_ty.fmt(pt), | |
| 181435 | rhs_ty.fmt(pt), | |
| 181436 | dst_ty.fmt(pt), | |
| 181437 | Value.fromInterned(extra.mask).fmtValue(pt), | |
| 181438 | }); | |
| 181439 | return self.finishAir(inst, result, .{ extra.a, extra.b, .none }); | |
| 181440 | } | |
| 181441 | ||
| 181442 | fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void { | |
| 181443 | const pt = self.pt; | |
| 181444 | const zcu = pt.zcu; | |
| 181445 | const result_ty = self.typeOfIndex(inst); | |
| 181446 | const len: usize = @intCast(result_ty.arrayLen(zcu)); | |
| 181447 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | |
| 181448 | const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); | |
| 181449 | const result: MCValue = result: { | |
| 181450 | switch (result_ty.zigTypeTag(zcu)) { | |
| 181451 | .@"struct" => { | |
| 181452 | if (result_ty.containerLayout(zcu) == .@"packed") return self.fail( | |
| 181453 | "TODO implement airAggregateInit for {f}", | |
| 181454 | .{result_ty.fmt(pt)}, | |
| 181455 | ); | |
| 181456 | const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu)); | |
| 181457 | const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern()); | |
| 181458 | try self.genInlineMemset( | |
| 181459 | .{ .lea_frame = .{ .index = frame_index } }, | |
| 181460 | .{ .immediate = 0 }, | |
| 181461 | .{ .immediate = result_ty.abiSize(zcu) }, | |
| 181462 | .{}, | |
| 181463 | ); | |
| 181464 | for (elements, 0..) |elem, elem_i_usize| { | |
| 181465 | const elem_i: u32 = @intCast(elem_i_usize); | |
| 181466 | if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; | |
| 181467 | ||
| 181468 | const elem_ty = result_ty.fieldType(elem_i, zcu); | |
| 181469 | const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu)); | |
| 181470 | if (elem_bit_size > 64) { | |
| 181471 | return self.fail( | |
| 181472 | "TODO airAggregateInit implement packed structs with large fields", | |
| 181473 | .{}, | |
| 181474 | ); | |
| 181475 | } | |
| 181476 | const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); | |
| 181477 | const elem_abi_bits = elem_abi_size * 8; | |
| 181478 | const elem_off = zcu.structPackedFieldBitOffset(loaded_struct, elem_i); | |
| 181479 | const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size); | |
| 181480 | const elem_bit_off = elem_off % elem_abi_bits; | |
| 181481 | const elem_mcv = try self.resolveInst(elem); | |
| 181482 | const elem_lock = switch (elem_mcv) { | |
| 181483 | .register => |reg| self.register_manager.lockReg(reg), | |
| 181484 | .immediate => |imm| lock: { | |
| 181485 | if (imm == 0) continue; | |
| 181486 | break :lock null; | |
| 181487 | }, | |
| 181488 | else => null, | |
| 181489 | }; | |
| 181490 | defer if (elem_lock) |lock| self.register_manager.unlockReg(lock); | |
| 181491 | ||
| 181492 | const elem_extra_bits = self.regExtraBits(elem_ty); | |
| 181493 | { | |
| 181494 | const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv); | |
| 181495 | const temp_alias = registerAlias(temp_reg, elem_abi_size); | |
| 181496 | const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); | |
| 181497 | defer self.register_manager.unlockReg(temp_lock); | |
| 181498 | ||
| 181499 | if (elem_bit_off < elem_extra_bits) { | |
| 181500 | try self.truncateRegister(elem_ty, temp_alias); | |
| 181501 | } | |
| 181502 | if (elem_bit_off > 0) try self.genShiftBinOpMir( | |
| 181503 | .{ ._l, .sh }, | |
| 181504 | elem_ty, | |
| 181505 | .{ .register = temp_alias }, | |
| 181506 | .u8, | |
| 181507 | .{ .immediate = elem_bit_off }, | |
| 181508 | ); | |
| 181509 | try self.genBinOpMir( | |
| 181510 | .{ ._, .@"or" }, | |
| 181511 | elem_ty, | |
| 181512 | .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } }, | |
| 181513 | .{ .register = temp_alias }, | |
| 181514 | ); | |
| 181515 | } | |
| 181516 | if (elem_bit_off > elem_extra_bits) { | |
| 181517 | const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv); | |
| 181518 | const temp_alias = registerAlias(temp_reg, elem_abi_size); | |
| 181519 | const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); | |
| 181520 | defer self.register_manager.unlockReg(temp_lock); | |
| 181521 | ||
| 181522 | if (elem_extra_bits > 0) { | |
| 181523 | try self.truncateRegister(elem_ty, temp_alias); | |
| 181524 | } | |
| 181525 | try self.genShiftBinOpMir( | |
| 181526 | .{ ._r, .sh }, | |
| 181527 | elem_ty, | |
| 181528 | .{ .register = temp_reg }, | |
| 181529 | .u8, | |
| 181530 | .{ .immediate = elem_abi_bits - elem_bit_off }, | |
| 181531 | ); | |
| 181532 | try self.genBinOpMir( | |
| 181533 | .{ ._, .@"or" }, | |
| 181534 | elem_ty, | |
| 181535 | .{ .load_frame = .{ | |
| 181536 | .index = frame_index, | |
| 181537 | .off = elem_byte_off + @as(i32, @intCast(elem_abi_size)), | |
| 181538 | } }, | |
| 181539 | .{ .register = temp_alias }, | |
| 181540 | ); | |
| 181541 | } | |
| 181542 | } | |
| 181543 | break :result .{ .load_frame = .{ .index = frame_index } }; | |
| 181544 | }, | |
| 181545 | .vector => { | |
| 181546 | const elem_ty = result_ty.childType(zcu); | |
| 181547 | if (elem_ty.toIntern() != .bool_type) return self.fail( | |
| 181548 | "TODO implement airAggregateInit for {f}", | |
| 181549 | .{result_ty.fmt(pt)}, | |
| 181550 | ); | |
| 181551 | const result_size: u32 = @intCast(result_ty.abiSize(zcu)); | |
| 181552 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); | |
| 181553 | const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); | |
| 181554 | defer self.register_manager.unlockReg(dst_lock); | |
| 181555 | try self.asmRegisterRegister( | |
| 181556 | .{ ._, .xor }, | |
| 181557 | registerAlias(dst_reg, @min(result_size, 4)), | |
| 181558 | registerAlias(dst_reg, @min(result_size, 4)), | |
| 181559 | ); | |
| 181560 | ||
| 181561 | for (elements, 0..) |elem, elem_i| { | |
| 181562 | const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem }); | |
| 181563 | const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg); | |
| 181564 | defer self.register_manager.unlockReg(elem_lock); | |
| 181565 | ||
| 181566 | try self.asmRegisterImmediate( | |
| 181567 | .{ ._, .@"and" }, | |
| 181568 | registerAlias(elem_reg, @min(result_size, 4)), | |
| 181569 | .u(1), | |
| 181570 | ); | |
| 181571 | if (elem_i > 0) try self.asmRegisterImmediate( | |
| 181572 | .{ ._l, .sh }, | |
| 181573 | registerAlias(elem_reg, result_size), | |
| 181574 | .u(@intCast(elem_i)), | |
| 181575 | ); | |
| 181576 | try self.asmRegisterRegister( | |
| 181577 | .{ ._, .@"or" }, | |
| 181578 | registerAlias(dst_reg, result_size), | |
| 181579 | registerAlias(elem_reg, result_size), | |
| 181580 | ); | |
| 181581 | } | |
| 181582 | break :result .{ .register = dst_reg }; | |
| 181583 | }, | |
| 181584 | else => unreachable, | |
| 181585 | } | |
| 181586 | }; | |
| 180060 | const result: MCValue = .{ .register = dst_reg }; | |
| 181587 | 180061 | |
| 181588 | 180062 | if (elements.len <= Air.Liveness.bpi - 1) { |
| 181589 | 180063 | var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); |
| ... | ... | @@ -182269,15 +180743,6 @@ fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMe |
| 182269 | 180743 | }; |
| 182270 | 180744 | } |
| 182271 | 180745 | |
| 182272 | fn failMsg(cg: *CodeGen, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } { | |
| 182273 | @branchHint(.cold); | |
| 182274 | const zcu = cg.pt.zcu; | |
| 182275 | return switch (cg.owner) { | |
| 182276 | .nav_index => |i| zcu.codegenFailMsg(i, msg), | |
| 182277 | .lazy_sym => |s| zcu.codegenFailTypeMsg(s.ty, msg), | |
| 182278 | }; | |
| 182279 | } | |
| 182280 | ||
| 182281 | 180746 | fn parseRegName(name: []const u8) ?Register { |
| 182282 | 180747 | if (std.mem.startsWith(u8, name, "db")) return @enumFromInt( |
| 182283 | 180748 | @intFromEnum(Register.dr0) + (std.fmt.parseInt(u4, name["db".len..], 0) catch return null), |
| ... | ... | @@ -188819,7 +187284,6 @@ const Select = struct { |
| 188819 | 187284 | const ptr_info = ty.ptrInfo(zcu); |
| 188820 | 187285 | return switch (ptr_info.flags.vector_index) { |
| 188821 | 187286 | .none => false, |
| 188822 | .runtime => unreachable, | |
| 188823 | 187287 | else => ptr_info.child == .bool_type, |
| 188824 | 187288 | }; |
| 188825 | 187289 | }, |
| ... | ... | @@ -188827,7 +187291,6 @@ const Select = struct { |
| 188827 | 187291 | const ptr_info = ty.ptrInfo(zcu); |
| 188828 | 187292 | return switch (ptr_info.flags.vector_index) { |
| 188829 | 187293 | .none => false, |
| 188830 | .runtime => unreachable, | |
| 188831 | 187294 | else => ptr_info.child == .bool_type and size.bitSize(cg.target) >= ptr_info.packed_offset.host_size, |
| 188832 | 187295 | }; |
| 188833 | 187296 | }, |
| ... | ... | @@ -190814,7 +189277,7 @@ const Select = struct { |
| 190814 | 189277 | .src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * |
| 190815 | 189278 | Select.Operand.Ref.src1.valueOf(s).immediate), |
| 190816 | 189279 | .vector_index => switch (op.flags.base.ref.typeOf(s).ptrInfo(s.cg.pt.zcu).flags.vector_index) { |
| 190817 | .none, .runtime => unreachable, | |
| 189280 | .none => unreachable, | |
| 190818 | 189281 | else => |vector_index| @intFromEnum(vector_index), |
| 190819 | 189282 | }, |
| 190820 | 189283 | .src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate), |
src/link/Dwarf.zig+6-22
| ... | ... | @@ -3158,11 +3158,7 @@ fn updateComptimeNavInner(dwarf: *Dwarf, pt: Zcu.PerThread, nav_index: InternPoo |
| 3158 | 3158 | .struct_field |
| 3159 | 3159 | else |
| 3160 | 3160 | .struct_field); |
| 3161 | if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else { | |
| 3162 | var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined; | |
| 3163 | const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable; | |
| 3164 | try wip_nav.strp(field_name); | |
| 3165 | } | |
| 3161 | try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); | |
| 3166 | 3162 | try wip_nav.refType(field_type); |
| 3167 | 3163 | if (!is_comptime) { |
| 3168 | 3164 | try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]); |
| ... | ... | @@ -3187,7 +3183,7 @@ fn updateComptimeNavInner(dwarf: *Dwarf, pt: Zcu.PerThread, nav_index: InternPoo |
| 3187 | 3183 | var field_bit_offset: u16 = 0; |
| 3188 | 3184 | for (0..loaded_struct.field_types.len) |field_index| { |
| 3189 | 3185 | try wip_nav.abbrevCode(.packed_struct_field); |
| 3190 | try wip_nav.strp(loaded_struct.fieldName(ip, field_index).unwrap().?.toSlice(ip)); | |
| 3186 | try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); | |
| 3191 | 3187 | const field_type: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); |
| 3192 | 3188 | try wip_nav.refType(field_type); |
| 3193 | 3189 | try diw.writeUleb128(field_bit_offset); |
| ... | ... | @@ -4269,11 +4265,7 @@ fn updateLazyValue( |
| 4269 | 4265 | .comptime_value_field_runtime_bits |
| 4270 | 4266 | else |
| 4271 | 4267 | continue); |
| 4272 | if (loaded_struct_type.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else { | |
| 4273 | var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined; | |
| 4274 | const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable; | |
| 4275 | try wip_nav.strp(field_name); | |
| 4276 | } | |
| 4268 | try wip_nav.strp(loaded_struct_type.fieldName(ip, field_index).toSlice(ip)); | |
| 4277 | 4269 | const field_value: Value = .fromInterned(switch (aggregate.storage) { |
| 4278 | 4270 | .bytes => unreachable, |
| 4279 | 4271 | .elems => |elems| elems[field_index], |
| ... | ... | @@ -4467,11 +4459,7 @@ fn updateContainerTypeWriterError( |
| 4467 | 4459 | .struct_field |
| 4468 | 4460 | else |
| 4469 | 4461 | .struct_field); |
| 4470 | if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else { | |
| 4471 | var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined; | |
| 4472 | const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable; | |
| 4473 | try wip_nav.strp(field_name); | |
| 4474 | } | |
| 4462 | try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); | |
| 4475 | 4463 | try wip_nav.refType(field_type); |
| 4476 | 4464 | if (!is_comptime) { |
| 4477 | 4465 | try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]); |
| ... | ... | @@ -4573,11 +4561,7 @@ fn updateContainerTypeWriterError( |
| 4573 | 4561 | .struct_field |
| 4574 | 4562 | else |
| 4575 | 4563 | .struct_field); |
| 4576 | if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else { | |
| 4577 | var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined; | |
| 4578 | const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable; | |
| 4579 | try wip_nav.strp(field_name); | |
| 4580 | } | |
| 4564 | try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); | |
| 4581 | 4565 | try wip_nav.refType(field_type); |
| 4582 | 4566 | if (!is_comptime) { |
| 4583 | 4567 | try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]); |
| ... | ... | @@ -4600,7 +4584,7 @@ fn updateContainerTypeWriterError( |
| 4600 | 4584 | var field_bit_offset: u16 = 0; |
| 4601 | 4585 | for (0..loaded_struct.field_types.len) |field_index| { |
| 4602 | 4586 | try wip_nav.abbrevCode(.packed_struct_field); |
| 4603 | try wip_nav.strp(loaded_struct.fieldName(ip, field_index).unwrap().?.toSlice(ip)); | |
| 4587 | try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); | |
| 4604 | 4588 | const field_type: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); |
| 4605 | 4589 | try wip_nav.refType(field_type); |
| 4606 | 4590 | try diw.writeUleb128(field_bit_offset); |