| ... | ... | @@ -355,9 +355,10 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov |
| 355 | 355 | const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) { |
| 356 | 356 | // zig fmt: off |
| 357 | 357 | |
| 358 | | // No "scalarize" legalizations are enabled, so these instructions never appear. |
| 359 | | .legalize_vec_elem_val => unreachable, |
| 360 | | .legalize_vec_store_elem => unreachable, |
| 358 | // Required due to `.scalarize_bitcast_vector_non_elementwise` being enabled. |
| 359 | .legalize_vec_elem_val => try self.airLegalizeVecElemVal(inst), |
| 360 | .legalize_vec_store_elem => try self.airLegalizeVecStoreElem(inst), |
| 361 | |
| 361 | 362 | // No soft float legalizations are enabled. |
| 362 | 363 | .legalize_compiler_rt_call => unreachable, |
| 363 | 364 | |
| ... | ... | @@ -2312,6 +2313,34 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder |
| 2312 | 2313 | return self.wip.extractElement(array_llvm_val, rhs, ""); |
| 2313 | 2314 | } |
| 2314 | 2315 | |
| 2316 | fn airLegalizeVecElemVal(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { |
| 2317 | const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 2318 | const vec = try fg.resolveInst(bin_op.lhs); |
| 2319 | const index = try fg.resolveInst(bin_op.rhs); |
| 2320 | return fg.wip.extractElement(vec, index, ""); |
| 2321 | } |
| 2322 | fn airLegalizeVecStoreElem(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { |
| 2323 | const zcu = fg.object.zcu; |
| 2324 | |
| 2325 | const pl_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 2326 | const extra = fg.air.extraData(Air.Bin, pl_op.payload).data; |
| 2327 | |
| 2328 | const ptr_ty = fg.typeOf(pl_op.operand); |
| 2329 | const vec_ty = ptr_ty.childType(zcu); |
| 2330 | |
| 2331 | const ptr_align = ptr_ty.ptrAlignment(zcu); |
| 2332 | |
| 2333 | const vec_ptr = try fg.resolveInst(pl_op.operand); |
| 2334 | const index = try fg.resolveInst(extra.lhs); |
| 2335 | const elem = try fg.resolveInst(extra.rhs); |
| 2336 | |
| 2337 | const old_vec = try fg.load(vec_ptr, ptr_align, vec_ty, .normal); |
| 2338 | const new_vec = try fg.wip.insertElement(old_vec, elem, index, ""); |
| 2339 | try fg.store(vec_ptr, ptr_align, new_vec, vec_ty, .normal); |
| 2340 | |
| 2341 | return .none; |
| 2342 | } |
| 2343 | |
| 2315 | 2344 | fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { |
| 2316 | 2345 | const zcu = self.object.zcu; |
| 2317 | 2346 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| ... | ... | @@ -4559,116 +4588,50 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value |
| 4559 | 4588 | } |
| 4560 | 4589 | } |
| 4561 | 4590 | |
| 4562 | | fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { |
| 4563 | | const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 4564 | | const operand_ty = self.typeOf(ty_op.operand); |
| 4565 | | const inst_ty = self.typeOfIndex(inst); |
| 4566 | | const operand = try self.resolveInst(ty_op.operand); |
| 4567 | | return self.bitCast(operand, operand_ty, inst_ty); |
| 4568 | | } |
| 4569 | | |
| 4570 | | fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) Allocator.Error!Builder.Value { |
| 4571 | | const o = self.object; |
| 4591 | fn airBitCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { |
| 4592 | const o = fg.object; |
| 4572 | 4593 | const zcu = o.zcu; |
| 4573 | | const operand_is_ref = isByRef(operand_ty, zcu); |
| 4574 | | const result_is_ref = isByRef(inst_ty, zcu); |
| 4575 | | const llvm_dest_ty = try o.lowerType(inst_ty); |
| 4576 | 4594 | |
| 4577 | | if (operand_is_ref and result_is_ref) { |
| 4578 | | // They are both pointers, so just return the same opaque pointer :) |
| 4579 | | return operand; |
| 4580 | | } |
| 4595 | const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 4596 | const operand_ty = fg.typeOf(ty_op.operand); |
| 4597 | const dest_ty = fg.typeOfIndex(inst); |
| 4598 | const operand = try fg.resolveInst(ty_op.operand); |
| 4581 | 4599 | |
| 4582 | | if (inst_ty.isAbiInt(zcu) and operand_ty.isAbiInt(zcu)) { |
| 4583 | | assert(inst_ty.bitSize(zcu) == operand_ty.bitSize(zcu)); |
| 4600 | // We have the following `Air.Legalize` features enabled: |
| 4601 | // |
| 4602 | // * `.scalarize_bitcast_array` |
| 4603 | // * `.scalarize_bitcast_vector_non_elementwise` |
| 4604 | // |
| 4605 | // That means the set of bitcasts we might see is limited to the following: |
| 4606 | // |
| 4607 | // * bool/int/float <-> bool/int/float |
| 4608 | // * `@Vector(n, A)` <-> `@Vector(n, B)` |
| 4609 | // * pointer <-> pointer |
| 4610 | // * pointer <-> int |
| 4611 | // * slice <-> slice |
| 4612 | // |
| 4613 | // Most of these can be handled by LLVM's `bitcast` instruction. We will check for the few cases |
| 4614 | // that aren't, and otherwise use `bitcast`. |
| 4615 | |
| 4616 | if (operand_ty.isSlice(zcu) and dest_ty.isSlice(zcu)) { |
| 4617 | // The slice types are the same type in LLVM IR, so this conversion is a nop. |
| 4584 | 4618 | return operand; |
| 4585 | 4619 | } |
| 4586 | 4620 | |
| 4587 | | const operand_scalar_ty = operand_ty.scalarType(zcu); |
| 4588 | | const inst_scalar_ty = inst_ty.scalarType(zcu); |
| 4589 | | if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) { |
| 4590 | | return self.wip.cast(.inttoptr, operand, llvm_dest_ty, ""); |
| 4591 | | } |
| 4592 | | if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) { |
| 4593 | | return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, ""); |
| 4594 | | } |
| 4595 | | |
| 4596 | | if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) { |
| 4597 | | const elem_ty = operand_scalar_ty; |
| 4598 | | assert(result_is_ref); // arrays are always by-ref provided they have runtime bits |
| 4599 | | const alignment = inst_ty.abiAlignment(zcu); |
| 4600 | | const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment.toLlvm()); |
| 4601 | | const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; |
| 4602 | | if (bitcast_ok) { |
| 4603 | | try self.store(array_ptr, alignment, operand, operand_ty, .normal); |
| 4604 | | } else { |
| 4605 | | // If the ABI size of the element type is not evenly divisible by size in bits; |
| 4606 | | // a simple bitcast will not work, and we fall back to extractelement. |
| 4607 | | const elem_size = elem_ty.abiSize(zcu); |
| 4608 | | const vector_len = operand_ty.arrayLen(zcu); |
| 4609 | | var i: u64 = 0; |
| 4610 | | while (i < vector_len) : (i += 1) { |
| 4611 | | const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size); |
| 4612 | | const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), ""); |
| 4613 | | try self.store(arr_elem_ptr, .none, vec_elem, elem_ty, .normal); |
| 4614 | | } |
| 4615 | | } |
| 4616 | | return array_ptr; |
| 4617 | | } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) { |
| 4618 | | const elem_ty = operand_ty.childType(zcu); |
| 4619 | | assert(operand_is_ref); // arrays are always by-ref provided they have runtime bits |
| 4620 | | const llvm_vector_ty = try o.lowerType(inst_ty); |
| 4621 | | |
| 4622 | | const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; |
| 4623 | | if (bitcast_ok) { |
| 4624 | | // The array is aligned to the element's alignment, while the vector might have a completely |
| 4625 | | // different alignment. This means we need to enforce the alignment of this load. |
| 4626 | | return self.load(operand, elem_ty.abiAlignment(zcu), inst_ty, .normal); |
| 4627 | | } else { |
| 4628 | | // If the ABI size of the element type is not evenly divisible by size in bits; |
| 4629 | | // a simple bitcast will not work, and we fall back to extractelement. |
| 4630 | | const elem_size = elem_ty.abiSize(zcu); |
| 4631 | | const vector_len = operand_ty.arrayLen(zcu); |
| 4632 | | var vector = try o.builder.poisonValue(llvm_vector_ty); |
| 4633 | | var i: u64 = 0; |
| 4634 | | while (i < vector_len) : (i += 1) { |
| 4635 | | const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size); |
| 4636 | | const arr_elem = try self.load(arr_elem_ptr, .none, elem_ty, .normal); |
| 4637 | | vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), ""); |
| 4638 | | } |
| 4639 | | return vector; |
| 4640 | | } |
| 4641 | | } |
| 4621 | assert(!isByRef(operand_ty, zcu)); |
| 4622 | assert(!isByRef(dest_ty, zcu)); |
| 4642 | 4623 | |
| 4643 | | if (operand_is_ref) { |
| 4644 | | return self.load(operand, operand_ty.abiAlignment(zcu), inst_ty, .normal); |
| 4645 | | } |
| 4624 | const llvm_dest_ty = try o.lowerType(dest_ty); |
| 4646 | 4625 | |
| 4647 | | if (result_is_ref) { |
| 4648 | | const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)); |
| 4649 | | const llvm_alloc_ty = if (operand_ty.abiSize(zcu) > inst_ty.abiSize(zcu)) |
| 4650 | | try o.lowerType(operand_ty) |
| 4651 | | else |
| 4652 | | llvm_dest_ty; |
| 4653 | | const result_ptr = try self.buildAlloca(llvm_alloc_ty, alignment.toLlvm()); |
| 4654 | | try self.store(result_ptr, alignment, operand, operand_ty, .normal); |
| 4655 | | return result_ptr; |
| 4626 | if (operand_ty.scalarType(zcu).zigTypeTag(zcu) == .int and dest_ty.scalarType(zcu).isPtrAtRuntime(zcu)) { |
| 4627 | return fg.wip.cast(.inttoptr, operand, llvm_dest_ty, ""); |
| 4656 | 4628 | } |
| 4657 | 4629 | |
| 4658 | | if (inst_ty.isSliceAtRuntime(zcu) or |
| 4659 | | ((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and |
| 4660 | | operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu))) |
| 4661 | | { |
| 4662 | | // Both our operand and our result are values, not pointers, |
| 4663 | | // but LLVM won't let us bitcast struct values or vectors with padding bits. |
| 4664 | | // Therefore, we store operand to alloca, then load for result. |
| 4665 | | const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)); |
| 4666 | | const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment.toLlvm()); |
| 4667 | | try self.store(result_ptr, alignment, operand, operand_ty, .normal); |
| 4668 | | return self.load(result_ptr, alignment, inst_ty, .normal); |
| 4630 | if (operand_ty.scalarType(zcu).isPtrAtRuntime(zcu) and dest_ty.scalarType(zcu).zigTypeTag(zcu) == .int) { |
| 4631 | return fg.wip.cast(.ptrtoint, operand, llvm_dest_ty, ""); |
| 4669 | 4632 | } |
| 4670 | 4633 | |
| 4671 | | return self.wip.cast(.bitcast, operand, llvm_dest_ty, ""); |
| 4634 | return fg.wip.cast(.bitcast, operand, llvm_dest_ty, ""); |
| 4672 | 4635 | } |
| 4673 | 4636 | |
| 4674 | 4637 | fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { |
| ... | ... | @@ -5333,11 +5296,25 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error |
| 5333 | 5296 | const value = try self.resolveInst(bin_op.rhs); |
| 5334 | 5297 | const elem_abi_size = elem_ty.abiSize(zcu); |
| 5335 | 5298 | |
| 5336 | | if (allow_byte_memset and elem_abi_size == 1 and elem_ty.bitSize(zcu) == 8) { |
| 5337 | | // In this case we can take advantage of LLVM's intrinsic. |
| 5338 | | const fill_byte = try self.bitCast(value, elem_ty, Type.u8); |
| 5299 | intrinsic: { |
| 5300 | if (!allow_byte_memset) break :intrinsic; |
| 5301 | if (elem_abi_size != 1) break :intrinsic; |
| 5302 | // To use LLVM's intrinsic, we need to convert the operand to a raw 8-bit integer value. |
| 5303 | const fill_byte: Builder.Value = byte: { |
| 5304 | if (isByRef(elem_ty, zcu)) { |
| 5305 | break :byte try self.load(value, elem_ty.abiAlignment(zcu), .u8, .normal); |
| 5306 | } |
| 5307 | if (elem_ty.isAbiInt(zcu)) { |
| 5308 | const info = elem_ty.intInfo(zcu); |
| 5309 | break :byte self.wip.conv(info.signedness, value, .i8, ""); |
| 5310 | } |
| 5311 | if (elem_ty == .bool) { |
| 5312 | break :byte self.wip.cast(.zext, value, .i8, ""); |
| 5313 | } |
| 5314 | break :intrinsic; |
| 5315 | }; |
| 5316 | // Great, we can use the intrinsic! |
| 5339 | 5317 | const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); |
| 5340 | | |
| 5341 | 5318 | _ = try self.wip.callMemSet( |
| 5342 | 5319 | dest_ptr, |
| 5343 | 5320 | dest_ptr_align.toLlvm(), |
Matthew Lugg <mlugg@mlugg.co.uk> 2026-06-14 11:21:28+01:00
Commit is signed but in an unrecognized format.
src