| ... | @@ -75,13 +75,6 @@ pub const Feature = enum { | ... | @@ -75,13 +75,6 @@ pub const Feature = enum { |
| 75 | scalarize_shl_sat, | 75 | scalarize_shl_sat, |
| 76 | scalarize_xor, | 76 | scalarize_xor, |
| 77 | scalarize_not, | 77 | scalarize_not, |
| 78 | /// Scalarize `bitcast` from or to an array or vector type to `bitcast`s of the elements. | | |
| 79 | /// This does not apply if `@bitSizeOf(Elem) == 8 * @sizeOf(Elem)`. | | |
| 80 | /// When this feature is enabled, all remaining `bitcast`s can be lowered using the old bitcast | | |
| 81 | /// semantics (reinterpret memory) instead of the new bitcast semantics (copy logical bits) and | | |
| 82 | /// the behavior will be equivalent. However, the behavior of `@bitSize` on arrays must be | | |
| 83 | /// changed in `Type.zig` before enabling this feature to conform to the new bitcast semantics. | | |
| 84 | scalarize_bitcast, | | |
| 85 | scalarize_clz, | 78 | scalarize_clz, |
| 86 | scalarize_ctz, | 79 | scalarize_ctz, |
| 87 | scalarize_popcount, | 80 | scalarize_popcount, |
| ... | @@ -122,6 +115,35 @@ pub const Feature = enum { | ... | @@ -122,6 +115,35 @@ pub const Feature = enum { |
| 122 | scalarize_select, | 115 | scalarize_select, |
| 123 | scalarize_mul_add, | 116 | scalarize_mul_add, |
| 124 | | 117 | |
| | 118 | // Below are several different features for scalarizing `bitcast` in different scenarios. It is |
| | 119 | // valid to enable any combination of these features. |
| | 120 | |
| | 121 | /// Scalarize `bitcast` where the operand or result type is an array. |
| | 122 | scalarize_bitcast_array, |
| | 123 | /// Scalarize `bitcast` where either: |
| | 124 | /// |
| | 125 | /// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or |
| | 126 | /// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)` |
| | 127 | /// |
| | 128 | /// This effectively scalarizes any `bitcast` to/from a vector, *unless* the operation can be |
| | 129 | /// performed by bitcasting each vector element and returning a vector of the results. |
| | 130 | /// |
| | 131 | /// If this feature is enabled, the following AIR instruction tags may be emitted: |
| | 132 | /// * `.legalize_vec_elem_val` |
| | 133 | /// * `.legalize_vec_store_elem` |
| | 134 | scalarize_bitcast_vector_non_elementwise, |
| | 135 | /// Scalarize `bitcast` where the operand or result type is an array or vector whose element |
| | 136 | /// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may |
| | 137 | /// need to sign- or zero-extend multiple elements to populate "padding" bits. |
| | 138 | /// |
| | 139 | /// Enabling this feature requires changing the behavior of `@bitSize` on arrays in `Type.zig` |
| | 140 | /// to conform to the new bitcast semantics. |
| | 141 | /// |
| | 142 | /// If this feature is enabled, the following AIR instruction tags may be emitted: |
| | 143 | /// * `.legalize_vec_elem_val` |
| | 144 | /// * `.legalize_vec_store_elem` |
| | 145 | scalarize_bitcast_padded_elems, |
| | 146 | |
| 125 | /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs) | 147 | /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs) |
| 126 | unsplat_shift_rhs, | 148 | unsplat_shift_rhs, |
| 127 | /// Legalize reduce of a one element vector to a bitcast. | 149 | /// Legalize reduce of a one element vector to a bitcast. |
| ... | @@ -227,7 +249,6 @@ pub const Feature = enum { | ... | @@ -227,7 +249,6 @@ pub const Feature = enum { |
| 227 | .shl_sat => .scalarize_shl_sat, | 249 | .shl_sat => .scalarize_shl_sat, |
| 228 | .xor => .scalarize_xor, | 250 | .xor => .scalarize_xor, |
| 229 | .not => .scalarize_not, | 251 | .not => .scalarize_not, |
| 230 | .bitcast => .scalarize_bitcast, | | |
| 231 | .clz => .scalarize_clz, | 252 | .clz => .scalarize_clz, |
| 232 | .ctz => .scalarize_ctz, | 253 | .ctz => .scalarize_ctz, |
| 233 | .popcount => .scalarize_popcount, | 254 | .popcount => .scalarize_popcount, |
| ... | @@ -548,7 +569,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { | ... | @@ -548,7 +569,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 548 | }, | 569 | }, |
| 549 | } | 570 | } |
| 550 | }, | 571 | }, |
| 551 | .bitcast => if (l.features.has(.scalarize_bitcast)) { | 572 | .bitcast => if (l.features.hasAny(&.{ |
| | 573 | .scalarize_bitcast_array, |
| | 574 | .scalarize_bitcast_vector_non_elementwise, |
| | 575 | .scalarize_bitcast_padded_elems, |
| | 576 | })) { |
| 552 | if (try l.scalarizeBitcastBlockPayload(inst)) |payload| { | 577 | if (try l.scalarizeBitcastBlockPayload(inst)) |payload| { |
| 553 | continue :inst l.replaceInst(inst, .block, payload); | 578 | continue :inst l.replaceInst(inst, .block, payload); |
| 554 | } | 579 | } |
| ... | @@ -1423,35 +1448,94 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? | ... | @@ -1423,35 +1448,94 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? |
| 1423 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; | 1448 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1424 | | 1449 | |
| 1425 | const dest_ty = ty_op.ty.toType(); | 1450 | const dest_ty = ty_op.ty.toType(); |
| 1426 | const dest_legal = switch (dest_ty.zigTypeTag(zcu)) { | | |
| 1427 | else => true, | | |
| 1428 | .array, .vector => legal: { | | |
| 1429 | if (dest_ty.arrayLen(zcu) == 1) break :legal true; | | |
| 1430 | const dest_elem_ty = dest_ty.childType(zcu); | | |
| 1431 | break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu); | | |
| 1432 | }, | | |
| 1433 | }; | | |
| 1434 | | | |
| 1435 | const operand_ty = l.typeOf(ty_op.operand); | 1451 | const operand_ty = l.typeOf(ty_op.operand); |
| 1436 | const operand_legal = switch (operand_ty.zigTypeTag(zcu)) { | | |
| 1437 | else => true, | | |
| 1438 | .array, .vector => legal: { | | |
| 1439 | if (operand_ty.arrayLen(zcu) == 1) break :legal true; | | |
| 1440 | const operand_elem_ty = operand_ty.childType(zcu); | | |
| 1441 | break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu); | | |
| 1442 | }, | | |
| 1443 | }; | | |
| 1444 | | 1452 | |
| 1445 | if (dest_legal and operand_legal) return null; | 1453 | // We exit this block only if the scalarization is actually necessary. Otherwise we will return |
| | 1454 | // `null` from within the block. |
| | 1455 | const operand_to_int_ok: bool, const int_to_dest_ok: bool = int_ok: { |
| | 1456 | const operand_tag = operand_ty.zigTypeTag(zcu); |
| | 1457 | const dest_tag = dest_ty.zigTypeTag(zcu); |
| | 1458 | |
| | 1459 | if (operand_tag != .array and |
| | 1460 | operand_tag != .vector and |
| | 1461 | dest_tag != .array and |
| | 1462 | dest_tag != .vector) |
| | 1463 | { |
| | 1464 | return null; |
| | 1465 | } |
| 1446 | | 1466 | |
| 1447 | if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) { | 1467 | // We track the validity of 3 different bitcast operations: |
| 1448 | // from_ty and to_ty are both arrays or vectors of types with the same bit size, | 1468 | // * operand -> dest |
| 1449 | // so we can do an elementwise bitcast. | 1469 | // * operand -> uint |
| 1450 | return try l.scalarizeBlockPayload(orig_inst, .ty_op); | 1470 | // * uint -> dest |
| 1451 | } | 1471 | // If operand->dest turns out to be valid, we don't need to scalarize. Otherwise, knowing |
| | 1472 | // the validity of the other operations helps us lower the scalarization efficiently. |
| | 1473 | var operand_to_dest: bool = true; |
| | 1474 | var operand_to_int: bool = true; |
| | 1475 | var int_to_dest: bool = true; |
| | 1476 | |
| | 1477 | if (l.features.has(.scalarize_bitcast_array)) { |
| | 1478 | if (operand_tag == .array) { |
| | 1479 | operand_to_dest = false; |
| | 1480 | operand_to_int = false; |
| | 1481 | } |
| | 1482 | if (dest_tag == .array) { |
| | 1483 | operand_to_dest = false; |
| | 1484 | int_to_dest = false; |
| | 1485 | } |
| | 1486 | } |
| | 1487 | |
| | 1488 | if (l.features.has(.scalarize_bitcast_vector_non_elementwise)) { |
| | 1489 | if (operand_tag == .vector) operand_to_int = false; |
| | 1490 | if (dest_tag == .vector) int_to_dest = false; |
| | 1491 | |
| | 1492 | if (operand_tag == .vector or dest_tag == .vector) { |
| | 1493 | if (operand_tag != .vector or |
| | 1494 | dest_tag != .vector or |
| | 1495 | operand_ty.vectorLen(zcu) != dest_ty.vectorLen(zcu)) |
| | 1496 | { |
| | 1497 | operand_to_dest = false; |
| | 1498 | } |
| | 1499 | } |
| | 1500 | } |
| | 1501 | |
| | 1502 | if (l.features.has(.scalarize_bitcast_padded_elems)) { |
| | 1503 | if (operand_tag == .array or operand_tag == .vector) { |
| | 1504 | const elem_ty = operand_ty.childType(zcu); |
| | 1505 | if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) { |
| | 1506 | operand_to_int = false; |
| | 1507 | operand_to_dest = false; |
| | 1508 | } |
| | 1509 | } |
| | 1510 | if (dest_tag == .array or dest_tag == .vector) { |
| | 1511 | const elem_ty = dest_ty.childType(zcu); |
| | 1512 | if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) { |
| | 1513 | int_to_dest = false; |
| | 1514 | operand_to_dest = false; |
| | 1515 | } |
| | 1516 | } |
| | 1517 | } |
| | 1518 | |
| | 1519 | if (operand_to_dest) { |
| | 1520 | return null; // no scalarization needed! |
| | 1521 | } |
| | 1522 | |
| | 1523 | // We need a scalarization, but before breaking from the block, check if we can do it |
| | 1524 | // elementwise---if we can, that's preferable to the generic lowering. |
| | 1525 | if ((operand_tag == .array or operand_tag == .vector) and |
| | 1526 | (dest_tag == .array or dest_tag == .vector) and |
| | 1527 | operand_ty.arrayLenIncludingSentinel(zcu) == dest_ty.arrayLenIncludingSentinel(zcu)) |
| | 1528 | { |
| | 1529 | // Operand and result types are both arrays/vectors whose element types have the same |
| | 1530 | // bit size, so we can do an elementwise bitcast. |
| | 1531 | return try l.scalarizeBlockPayload(orig_inst, .ty_op); |
| | 1532 | } |
| | 1533 | |
| | 1534 | break :int_ok .{ operand_to_int, int_to_dest }; |
| | 1535 | }; |
| 1452 | | 1536 | |
| 1453 | // Fallback path. Our strategy is to use an unsigned integer type as an intermediate | 1537 | // Generic scalarization implementation. Our strategy is to use an unsigned integer type as an |
| 1454 | // "bag of bits" representation which can be manipulated by bitwise operations. | 1538 | // intermediate "bag of bits" representation which can be manipulated by bitwise operations. |
| 1455 | | 1539 | |
| 1456 | const num_bits: u16 = @intCast(dest_ty.bitSize(zcu)); | 1540 | const num_bits: u16 = @intCast(dest_ty.bitSize(zcu)); |
| 1457 | assert(operand_ty.bitSize(zcu) == num_bits); | 1541 | assert(operand_ty.bitSize(zcu) == num_bits); |
| ... | @@ -1465,7 +1549,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? | ... | @@ -1465,7 +1549,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? |
| 1465 | // First, convert `operand_ty` to `uint_ty` (`uN`). | 1549 | // First, convert `operand_ty` to `uint_ty` (`uN`). |
| 1466 | | 1550 | |
| 1467 | const uint_val: Air.Inst.Ref = uint_val: { | 1551 | const uint_val: Air.Inst.Ref = uint_val: { |
| 1468 | if (operand_legal) { | 1552 | if (operand_to_int_ok) { |
| 1469 | _ = main_block.stealCapacity(19); | 1553 | _ = main_block.stealCapacity(19); |
| 1470 | break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand); | 1554 | break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand); |
| 1471 | } | 1555 | } |
| ... | @@ -1560,7 +1644,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? | ... | @@ -1560,7 +1644,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!? |
| 1560 | | 1644 | |
| 1561 | // Now convert `uint_ty` (`uN`) to `dest_ty`. | 1645 | // Now convert `uint_ty` (`uN`) to `dest_ty`. |
| 1562 | | 1646 | |
| 1563 | if (dest_legal) { | 1647 | if (int_to_dest_ok) { |
| 1564 | _ = main_block.stealCapacity(17); | 1648 | _ = main_block.stealCapacity(17); |
| 1565 | const result = main_block.addBitCast(l, dest_ty, uint_val); | 1649 | const result = main_block.addBitCast(l, dest_ty, uint_val); |
| 1566 | main_block.addBr(l, orig_inst, result); | 1650 | main_block.addBr(l, orig_inst, result); |
Matthew Lugg <mlugg@mlugg.co.uk> 2026-06-14 11:21:11+01:00
Commit is signed but in an unrecognized format.
src