| ... | ... | @@ -75,13 +75,6 @@ pub const Feature = enum { |
| 75 | 75 | scalarize_shl_sat, |
| 76 | 76 | scalarize_xor, |
| 77 | 77 | scalarize_not, |
| 78 | | /// Scalarize `bitcast` from or to an array or vector type to `bitcast`s of the elements. |
| 79 | | /// This does not apply if `@bitSizeOf(Elem) == 8 * @sizeOf(Elem)`. |
| 80 | | /// When this feature is enabled, all remaining `bitcast`s can be lowered using the old bitcast |
| 81 | | /// semantics (reinterpret memory) instead of the new bitcast semantics (copy logical bits) and |
| 82 | | /// the behavior will be equivalent. However, the behavior of `@bitSize` on arrays must be |
| 83 | | /// changed in `Type.zig` before enabling this feature to conform to the new bitcast semantics. |
| 84 | | scalarize_bitcast, |
| 85 | 78 | scalarize_clz, |
| 86 | 79 | scalarize_ctz, |
| 87 | 80 | scalarize_popcount, |
| ... | ... | @@ -122,6 +115,35 @@ pub const Feature = enum { |
| 122 | 115 | scalarize_select, |
| 123 | 116 | scalarize_mul_add, |
| 124 | 117 | |
| 118 | // Below are several different features for scalarizing `bitcast` in different scenarios. It is |
| 119 | // valid to enable any combination of these features. |
| 120 | |
| 121 | /// Scalarize `bitcast` where the operand or result type is an array. |
| 122 | scalarize_bitcast_array, |
| 123 | /// Scalarize `bitcast` where either: |
| 124 | /// |
| 125 | /// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or |
| 126 | /// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)` |
| 127 | /// |
| 128 | /// This effectively scalarizes any `bitcast` to/from a vector, *unless* the operation can be |
| 129 | /// performed by bitcasting each vector element and returning a vector of the results. |
| 130 | /// |
| 131 | /// If this feature is enabled, the following AIR instruction tags may be emitted: |
| 132 | /// * `.legalize_vec_elem_val` |
| 133 | /// * `.legalize_vec_store_elem` |
| 134 | scalarize_bitcast_vector_non_elementwise, |
| 135 | /// Scalarize `bitcast` where the operand or result type is an array or vector whose element |
| 136 | /// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may |
| 137 | /// need to sign- or zero-extend multiple elements to populate "padding" bits. |
| 138 | /// |
| 139 | /// Enabling this feature requires changing the behavior of `@bitSize` on arrays in `Type.zig` |
| 140 | /// to conform to the new bitcast semantics. |
| 141 | /// |
| 142 | /// If this feature is enabled, the following AIR instruction tags may be emitted: |
| 143 | /// * `.legalize_vec_elem_val` |
| 144 | /// * `.legalize_vec_store_elem` |
| 145 | scalarize_bitcast_padded_elems, |
| 146 | |
| 125 | 147 | /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs) |
| 126 | 148 | unsplat_shift_rhs, |
| 127 | 149 | /// Legalize reduce of a one element vector to a bitcast. |
| ... | ... | @@ -227,7 +249,6 @@ pub const Feature = enum { |
| 227 | 249 | .shl_sat => .scalarize_shl_sat, |
| 228 | 250 | .xor => .scalarize_xor, |
| 229 | 251 | .not => .scalarize_not, |
| 230 | | .bitcast => .scalarize_bitcast, |
| 231 | 252 | .clz => .scalarize_clz, |
| 232 | 253 | .ctz => .scalarize_ctz, |
| 233 | 254 | .popcount => .scalarize_popcount, |
| ... | ... | @@ -548,7 +569,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 548 | 569 | }, |
| 549 | 570 | } |
| 550 | 571 | }, |
| 551 | | .bitcast => if (l.features.has(.scalarize_bitcast)) { |
| 572 | .bitcast => if (l.features.hasAny(&.{ |
| 573 | .scalarize_bitcast_array, |
| 574 | .scalarize_bitcast_vector_non_elementwise, |
| 575 | .scalarize_bitcast_padded_elems, |
| 576 | })) { |
| 552 | 577 | if (try l.scalarizeBitcastBlockPayload(inst)) |payload| { |
| 553 | 578 | continue :inst l.replaceInst(inst, .block, payload); |
| 554 | 579 | } |
| ... | ... | @@ -1423,35 +1448,94 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? |
| 1423 | 1448 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1424 | 1449 | |
| 1425 | 1450 | const dest_ty = ty_op.ty.toType(); |
| 1426 | | const dest_legal = switch (dest_ty.zigTypeTag(zcu)) { |
| 1427 | | else => true, |
| 1428 | | .array, .vector => legal: { |
| 1429 | | if (dest_ty.arrayLen(zcu) == 1) break :legal true; |
| 1430 | | const dest_elem_ty = dest_ty.childType(zcu); |
| 1431 | | break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu); |
| 1432 | | }, |
| 1433 | | }; |
| 1434 | | |
| 1435 | 1451 | const operand_ty = l.typeOf(ty_op.operand); |
| 1436 | | const operand_legal = switch (operand_ty.zigTypeTag(zcu)) { |
| 1437 | | else => true, |
| 1438 | | .array, .vector => legal: { |
| 1439 | | if (operand_ty.arrayLen(zcu) == 1) break :legal true; |
| 1440 | | const operand_elem_ty = operand_ty.childType(zcu); |
| 1441 | | break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu); |
| 1442 | | }, |
| 1443 | | }; |
| 1444 | 1452 | |
| 1445 | | if (dest_legal and operand_legal) return null; |
| 1453 | // We exit this block only if the scalarization is actually necessary. Otherwise we will return |
| 1454 | // `null` from within the block. |
| 1455 | const operand_to_int_ok: bool, const int_to_dest_ok: bool = int_ok: { |
| 1456 | const operand_tag = operand_ty.zigTypeTag(zcu); |
| 1457 | const dest_tag = dest_ty.zigTypeTag(zcu); |
| 1458 | |
| 1459 | if (operand_tag != .array and |
| 1460 | operand_tag != .vector and |
| 1461 | dest_tag != .array and |
| 1462 | dest_tag != .vector) |
| 1463 | { |
| 1464 | return null; |
| 1465 | } |
| 1446 | 1466 | |
| 1447 | | if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) { |
| 1448 | | // from_ty and to_ty are both arrays or vectors of types with the same bit size, |
| 1449 | | // so we can do an elementwise bitcast. |
| 1450 | | return try l.scalarizeBlockPayload(orig_inst, .ty_op); |
| 1451 | | } |
| 1467 | // We track the validity of 3 different bitcast operations: |
| 1468 | // * operand -> dest |
| 1469 | // * operand -> uint |
| 1470 | // * uint -> dest |
| 1471 | // If operand->dest turns out to be valid, we don't need to scalarize. Otherwise, knowing |
| 1472 | // the validity of the other operations helps us lower the scalarization efficiently. |
| 1473 | var operand_to_dest: bool = true; |
| 1474 | var operand_to_int: bool = true; |
| 1475 | var int_to_dest: bool = true; |
| 1476 | |
| 1477 | if (l.features.has(.scalarize_bitcast_array)) { |
| 1478 | if (operand_tag == .array) { |
| 1479 | operand_to_dest = false; |
| 1480 | operand_to_int = false; |
| 1481 | } |
| 1482 | if (dest_tag == .array) { |
| 1483 | operand_to_dest = false; |
| 1484 | int_to_dest = false; |
| 1485 | } |
| 1486 | } |
| 1487 | |
| 1488 | if (l.features.has(.scalarize_bitcast_vector_non_elementwise)) { |
| 1489 | if (operand_tag == .vector) operand_to_int = false; |
| 1490 | if (dest_tag == .vector) int_to_dest = false; |
| 1491 | |
| 1492 | if (operand_tag == .vector or dest_tag == .vector) { |
| 1493 | if (operand_tag != .vector or |
| 1494 | dest_tag != .vector or |
| 1495 | operand_ty.vectorLen(zcu) != dest_ty.vectorLen(zcu)) |
| 1496 | { |
| 1497 | operand_to_dest = false; |
| 1498 | } |
| 1499 | } |
| 1500 | } |
| 1501 | |
| 1502 | if (l.features.has(.scalarize_bitcast_padded_elems)) { |
| 1503 | if (operand_tag == .array or operand_tag == .vector) { |
| 1504 | const elem_ty = operand_ty.childType(zcu); |
| 1505 | if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) { |
| 1506 | operand_to_int = false; |
| 1507 | operand_to_dest = false; |
| 1508 | } |
| 1509 | } |
| 1510 | if (dest_tag == .array or dest_tag == .vector) { |
| 1511 | const elem_ty = dest_ty.childType(zcu); |
| 1512 | if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) { |
| 1513 | int_to_dest = false; |
| 1514 | operand_to_dest = false; |
| 1515 | } |
| 1516 | } |
| 1517 | } |
| 1518 | |
| 1519 | if (operand_to_dest) { |
| 1520 | return null; // no scalarization needed! |
| 1521 | } |
| 1522 | |
| 1523 | // We need a scalarization, but before breaking from the block, check if we can do it |
| 1524 | // elementwise---if we can, that's preferable to the generic lowering. |
| 1525 | if ((operand_tag == .array or operand_tag == .vector) and |
| 1526 | (dest_tag == .array or dest_tag == .vector) and |
| 1527 | operand_ty.arrayLenIncludingSentinel(zcu) == dest_ty.arrayLenIncludingSentinel(zcu)) |
| 1528 | { |
| 1529 | // Operand and result types are both arrays/vectors whose element types have the same |
| 1530 | // bit size, so we can do an elementwise bitcast. |
| 1531 | return try l.scalarizeBlockPayload(orig_inst, .ty_op); |
| 1532 | } |
| 1533 | |
| 1534 | break :int_ok .{ operand_to_int, int_to_dest }; |
| 1535 | }; |
| 1452 | 1536 | |
| 1453 | | // Fallback path. Our strategy is to use an unsigned integer type as an intermediate |
| 1454 | | // "bag of bits" representation which can be manipulated by bitwise operations. |
| 1537 | // Generic scalarization implementation. Our strategy is to use an unsigned integer type as an |
| 1538 | // intermediate "bag of bits" representation which can be manipulated by bitwise operations. |
| 1455 | 1539 | |
| 1456 | 1540 | const num_bits: u16 = @intCast(dest_ty.bitSize(zcu)); |
| 1457 | 1541 | assert(operand_ty.bitSize(zcu) == num_bits); |
| ... | ... | @@ -1465,7 +1549,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? |
| 1465 | 1549 | // First, convert `operand_ty` to `uint_ty` (`uN`). |
| 1466 | 1550 | |
| 1467 | 1551 | const uint_val: Air.Inst.Ref = uint_val: { |
| 1468 | | if (operand_legal) { |
| 1552 | if (operand_to_int_ok) { |
| 1469 | 1553 | _ = main_block.stealCapacity(19); |
| 1470 | 1554 | break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand); |
| 1471 | 1555 | } |
| ... | ... | @@ -1560,7 +1644,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? |
| 1560 | 1644 | |
| 1561 | 1645 | // Now convert `uint_ty` (`uN`) to `dest_ty`. |
| 1562 | 1646 | |
| 1563 | | if (dest_legal) { |
| 1647 | if (int_to_dest_ok) { |
| 1564 | 1648 | _ = main_block.stealCapacity(17); |
| 1565 | 1649 | const result = main_block.addBitCast(l, dest_ty, uint_val); |
| 1566 | 1650 | main_block.addBr(l, orig_inst, result); |
Matthew Lugg <mlugg@mlugg.co.uk> 2026-06-14 11:21:11+01:00
Commit is signed but in an unrecognized format.
src