| ... | ... | @@ -46,6 +46,12 @@ pub const Feature = enum { |
| 46 | 46 | scalarize_shl_sat, |
| 47 | 47 | scalarize_xor, |
| 48 | 48 | scalarize_not, |
| 49 | /// Scalarize `bitcast` from or to an array or vector type to `bitcast`s of the elements. |
| 50 | /// This does not apply if `@bitSizeOf(Elem) == 8 * @sizeOf(Elem)`. |
| 51 | /// When this feature is enabled, all remaining `bitcast`s can be lowered using the old bitcast |
| 52 | /// semantics (reinterpret memory) instead of the new bitcast semantics (copy logical bits) and |
| 53 | /// the behavior will be equivalent. However, the behavior of `@bitSize` on arrays must be |
| 54 | /// changed in `Type.zig` before enabling this feature to conform to the new bitcast semantics. |
| 49 | 55 | scalarize_bitcast, |
| 50 | 56 | scalarize_clz, |
| 51 | 57 | scalarize_ctz, |
| ... | ... | @@ -101,6 +107,19 @@ pub const Feature = enum { |
| 101 | 107 | /// Not compatible with `scalarize_mul_safe`. |
| 102 | 108 | expand_mul_safe, |
| 103 | 109 | |
| 110 | /// Replace `load` from a packed pointer with a non-packed `load`, `shr`, `truncate`. |
| 111 | /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the |
| 112 | /// first byte of memory until bit pointers know their backing type. |
| 113 | expand_packed_load, |
| 114 | /// Replace `store` and `store_safe` to a packed pointer with a non-packed `load`/`store`, `bit_and`, `bit_or`, and `shl`. |
| 115 | /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the |
| 116 | /// first byte of memory until bit pointers know their backing type. |
| 117 | expand_packed_store, |
| 118 | /// Replace `struct_field_val` of a packed field with a `store` and packed `load`. |
| 119 | expand_packed_struct_field_val, |
| 120 | /// Replace `aggregate_init` of a packed aggregate with a series a packed `store`s followed by a `load`. |
| 121 | expand_packed_aggregate_init, |
| 122 | |
| 104 | 123 | fn scalarize(tag: Air.Inst.Tag) Feature { |
| 105 | 124 | return switch (tag) { |
| 106 | 125 | else => unreachable, |
| ... | ... | @@ -192,7 +211,7 @@ pub const Error = std.mem.Allocator.Error; |
| 192 | 211 | |
| 193 | 212 | pub fn legalize(air: *Air, pt: Zcu.PerThread, features: *const Features) Error!void { |
| 194 | 213 | dev.check(.legalize); |
| 195 | | assert(!features.bits.eql(.initEmpty())); // backend asked to run legalize, but no features were enabled |
| 214 | assert(!features.eql(comptime .initEmpty())); // backend asked to run legalize, but no features were enabled |
| 196 | 215 | var l: Legalize = .{ |
| 197 | 216 | .pt = pt, |
| 198 | 217 | .air_instructions = air.instructions.toMultiArrayList(), |
| ... | ... | @@ -229,8 +248,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 229 | 248 | for (0..body_len) |body_index| { |
| 230 | 249 | const inst: Air.Inst.Index = @enumFromInt(l.air_extra.items[body_start + body_index]); |
| 231 | 250 | inst: switch (l.air_instructions.items(.tag)[@intFromEnum(inst)]) { |
| 232 | | .arg, |
| 233 | | => {}, |
| 251 | .arg => {}, |
| 234 | 252 | inline .add, |
| 235 | 253 | .add_optimized, |
| 236 | 254 | .add_wrap, |
| ... | ... | @@ -285,9 +303,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 285 | 303 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 286 | 304 | if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op); |
| 287 | 305 | }, |
| 288 | | .ptr_add, |
| 289 | | .ptr_sub, |
| 290 | | => {}, |
| 306 | .ptr_add, .ptr_sub => {}, |
| 291 | 307 | inline .add_with_overflow, |
| 292 | 308 | .sub_with_overflow, |
| 293 | 309 | .mul_with_overflow, |
| ... | ... | @@ -296,44 +312,43 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 296 | 312 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 297 | 313 | if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst)); |
| 298 | 314 | }, |
| 299 | | .alloc, |
| 300 | | => {}, |
| 301 | | .inferred_alloc, |
| 302 | | .inferred_alloc_comptime, |
| 303 | | => unreachable, |
| 304 | | .ret_ptr, |
| 305 | | .assembly, |
| 306 | | => {}, |
| 315 | .alloc => {}, |
| 316 | .inferred_alloc, .inferred_alloc_comptime => unreachable, |
| 317 | .ret_ptr, .assembly => {}, |
| 307 | 318 | inline .shr, |
| 308 | 319 | .shr_exact, |
| 309 | 320 | .shl, |
| 310 | 321 | .shl_exact, |
| 311 | 322 | .shl_sat, |
| 312 | | => |air_tag| done: { |
| 323 | => |air_tag| if (!l.features.intersectWith(comptime .initMany(&.{ |
| 324 | .unsplat_shift_rhs, |
| 325 | .scalarize(air_tag), |
| 326 | })).eql(comptime .initEmpty())) { |
| 313 | 327 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 314 | | if (!l.typeOf(bin_op.rhs).isVector(zcu)) break :done; |
| 315 | | if (l.features.contains(.unsplat_shift_rhs)) { |
| 316 | | if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) { |
| 317 | | else => {}, |
| 318 | | .aggregate => |aggregate| switch (aggregate.storage) { |
| 319 | | else => {}, |
| 320 | | .repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ |
| 321 | | .lhs = bin_op.lhs, |
| 322 | | .rhs = Air.internedToRef(splat), |
| 323 | | } }), |
| 324 | | }, |
| 325 | | } else { |
| 326 | | const rhs_inst = bin_op.rhs.toIndex().?; |
| 327 | | switch (l.air_instructions.items(.tag)[@intFromEnum(rhs_inst)]) { |
| 328 | if (l.typeOf(bin_op.rhs).isVector(zcu)) { |
| 329 | if (l.features.contains(.unsplat_shift_rhs)) { |
| 330 | if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) { |
| 328 | 331 | else => {}, |
| 329 | | .splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ |
| 330 | | .lhs = bin_op.lhs, |
| 331 | | .rhs = l.air_instructions.items(.data)[@intFromEnum(rhs_inst)].ty_op.operand, |
| 332 | | } }), |
| 332 | .aggregate => |aggregate| switch (aggregate.storage) { |
| 333 | else => {}, |
| 334 | .repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ |
| 335 | .lhs = bin_op.lhs, |
| 336 | .rhs = Air.internedToRef(splat), |
| 337 | } }), |
| 338 | }, |
| 339 | } else { |
| 340 | const rhs_inst = bin_op.rhs.toIndex().?; |
| 341 | switch (l.air_instructions.items(.tag)[@intFromEnum(rhs_inst)]) { |
| 342 | else => {}, |
| 343 | .splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{ |
| 344 | .lhs = bin_op.lhs, |
| 345 | .rhs = l.air_instructions.items(.data)[@intFromEnum(rhs_inst)].ty_op.operand, |
| 346 | } }), |
| 347 | } |
| 333 | 348 | } |
| 334 | 349 | } |
| 350 | if (l.features.contains(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op); |
| 335 | 351 | } |
| 336 | | if (l.features.contains(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op); |
| 337 | 352 | }, |
| 338 | 353 | inline .not, |
| 339 | 354 | .clz, |
| ... | ... | @@ -353,13 +368,41 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 353 | 368 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 354 | 369 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); |
| 355 | 370 | }, |
| 356 | | inline .bitcast, |
| 357 | | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { |
| 371 | .bitcast => if (l.features.contains(.scalarize_bitcast)) { |
| 358 | 372 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 373 | |
| 359 | 374 | const to_ty = ty_op.ty.toType(); |
| 375 | const to_ty_tag = to_ty.zigTypeTag(zcu); |
| 376 | const to_ty_legal = legal: switch (to_ty_tag) { |
| 377 | else => true, |
| 378 | .array, .vector => { |
| 379 | if (to_ty.arrayLen(zcu) == 1) break :legal true; |
| 380 | const to_elem_ty = to_ty.childType(zcu); |
| 381 | break :legal to_elem_ty.bitSize(zcu) == 8 * to_elem_ty.abiSize(zcu); |
| 382 | }, |
| 383 | }; |
| 384 | |
| 360 | 385 | const from_ty = l.typeOf(ty_op.operand); |
| 361 | | if (to_ty.isVector(zcu) and from_ty.isVector(zcu) and to_ty.vectorLen(zcu) == from_ty.vectorLen(zcu)) |
| 362 | | continue :inst try l.scalarize(inst, .ty_op); |
| 386 | const from_ty_legal = legal: switch (from_ty.zigTypeTag(zcu)) { |
| 387 | else => true, |
| 388 | .array, .vector => { |
| 389 | if (from_ty.arrayLen(zcu) == 1) break :legal true; |
| 390 | const from_elem_ty = from_ty.childType(zcu); |
| 391 | break :legal from_elem_ty.bitSize(zcu) == 8 * from_elem_ty.abiSize(zcu); |
| 392 | }, |
| 393 | }; |
| 394 | |
| 395 | if (!to_ty_legal and !from_ty_legal and to_ty.arrayLen(zcu) == from_ty.arrayLen(zcu)) switch (to_ty_tag) { |
| 396 | else => unreachable, |
| 397 | .array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastToArrayBlockPayload(inst)), |
| 398 | .vector => continue :inst try l.scalarize(inst, .bitcast), |
| 399 | }; |
| 400 | if (!to_ty_legal) switch (to_ty_tag) { |
| 401 | else => unreachable, |
| 402 | .array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultArrayBlockPayload(inst)), |
| 403 | .vector => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultVectorBlockPayload(inst)), |
| 404 | }; |
| 405 | if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst)); |
| 363 | 406 | }, |
| 364 | 407 | .intcast_safe => if (l.features.contains(.expand_intcast_safe)) { |
| 365 | 408 | assert(!l.features.contains(.scalarize_intcast_safe)); // it doesn't make sense to do both |
| ... | ... | @@ -368,9 +411,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 368 | 411 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 369 | 412 | if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op); |
| 370 | 413 | }, |
| 371 | | .block, |
| 372 | | .loop, |
| 373 | | => { |
| 414 | .block, .loop => { |
| 374 | 415 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 375 | 416 | const extra = l.extraData(Air.Block, ty_pl.payload); |
| 376 | 417 | try l.legalizeBody(extra.end, extra.data.body_len); |
| ... | ... | @@ -418,22 +459,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 418 | 459 | .cmp_neq, |
| 419 | 460 | .cmp_neq_optimized, |
| 420 | 461 | => {}, |
| 421 | | inline .cmp_vector, |
| 422 | | .cmp_vector_optimized, |
| 423 | | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { |
| 462 | inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { |
| 424 | 463 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 425 | | if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_pl_vector_cmp); |
| 464 | if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector); |
| 426 | 465 | }, |
| 427 | | .cond_br, |
| 428 | | => { |
| 466 | .cond_br => { |
| 429 | 467 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 430 | 468 | const extra = l.extraData(Air.CondBr, pl_op.payload); |
| 431 | 469 | try l.legalizeBody(extra.end, extra.data.then_body_len); |
| 432 | 470 | try l.legalizeBody(extra.end + extra.data.then_body_len, extra.data.else_body_len); |
| 433 | 471 | }, |
| 434 | | .switch_br, |
| 435 | | .loop_switch_br, |
| 436 | | => { |
| 472 | .switch_br, .loop_switch_br => { |
| 437 | 473 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 438 | 474 | const extra = l.extraData(Air.SwitchBr, pl_op.payload); |
| 439 | 475 | const hint_bag_count = std.math.divCeil(usize, extra.data.cases_len + 1, 10) catch unreachable; |
| ... | ... | @@ -446,27 +482,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 446 | 482 | } |
| 447 | 483 | try l.legalizeBody(extra_index, extra.data.else_body_len); |
| 448 | 484 | }, |
| 449 | | .switch_dispatch, |
| 450 | | => {}, |
| 451 | | .@"try", |
| 452 | | .try_cold, |
| 453 | | => { |
| 485 | .switch_dispatch => {}, |
| 486 | .@"try", .try_cold => { |
| 454 | 487 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 455 | 488 | const extra = l.extraData(Air.Try, pl_op.payload); |
| 456 | 489 | try l.legalizeBody(extra.end, extra.data.body_len); |
| 457 | 490 | }, |
| 458 | | .try_ptr, |
| 459 | | .try_ptr_cold, |
| 460 | | => { |
| 491 | .try_ptr, .try_ptr_cold => { |
| 461 | 492 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 462 | 493 | const extra = l.extraData(Air.TryPtr, ty_pl.payload); |
| 463 | 494 | try l.legalizeBody(extra.end, extra.data.body_len); |
| 464 | 495 | }, |
| 465 | | .dbg_stmt, |
| 466 | | .dbg_empty_stmt, |
| 467 | | => {}, |
| 468 | | .dbg_inline_block, |
| 469 | | => { |
| 496 | .dbg_stmt, .dbg_empty_stmt => {}, |
| 497 | .dbg_inline_block => { |
| 470 | 498 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 471 | 499 | const extra = l.extraData(Air.DbgInlineBlock, ty_pl.payload); |
| 472 | 500 | try l.legalizeBody(extra.end, extra.data.body_len); |
| ... | ... | @@ -484,14 +512,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 484 | 512 | .is_non_err_ptr, |
| 485 | 513 | .bool_and, |
| 486 | 514 | .bool_or, |
| 487 | | .load, |
| 488 | | .ret, |
| 489 | | .ret_safe, |
| 490 | | .ret_load, |
| 491 | | .store, |
| 492 | | .store_safe, |
| 493 | | .unreach, |
| 494 | 515 | => {}, |
| 516 | .load => if (l.features.contains(.expand_packed_load)) { |
| 517 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op; |
| 518 | const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu); |
| 519 | if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst)); |
| 520 | }, |
| 521 | .ret, .ret_safe, .ret_load => {}, |
| 522 | .store, .store_safe => if (l.features.contains(.expand_packed_store)) { |
| 523 | const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 524 | const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu); |
| 525 | if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst)); |
| 526 | }, |
| 527 | .unreach, |
| 495 | 528 | .optional_payload, |
| 496 | 529 | .optional_payload_ptr, |
| 497 | 530 | .optional_payload_ptr_set, |
| ... | ... | @@ -508,7 +541,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 508 | 541 | .struct_field_ptr_index_1, |
| 509 | 542 | .struct_field_ptr_index_2, |
| 510 | 543 | .struct_field_ptr_index_3, |
| 511 | | .struct_field_val, |
| 544 | => {}, |
| 545 | .struct_field_val => if (l.features.contains(.expand_packed_struct_field_val)) { |
| 546 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 547 | const extra = l.extraData(Air.StructField, ty_pl.payload).data; |
| 548 | switch (l.typeOf(extra.struct_operand).containerLayout(zcu)) { |
| 549 | .auto, .@"extern" => {}, |
| 550 | .@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedStructFieldValBlockPayload(inst)), |
| 551 | } |
| 552 | }, |
| 512 | 553 | .set_union_tag, |
| 513 | 554 | .get_union_tag, |
| 514 | 555 | .slice, |
| ... | ... | @@ -523,9 +564,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 523 | 564 | .ptr_elem_ptr, |
| 524 | 565 | .array_to_slice, |
| 525 | 566 | => {}, |
| 526 | | .reduce, |
| 527 | | .reduce_optimized, |
| 528 | | => if (l.features.contains(.reduce_one_elem_to_bitcast)) done: { |
| 567 | .reduce, .reduce_optimized => if (l.features.contains(.reduce_one_elem_to_bitcast)) { |
| 529 | 568 | const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce; |
| 530 | 569 | const vector_ty = l.typeOf(reduce.operand); |
| 531 | 570 | switch (vector_ty.vectorLen(zcu)) { |
| ... | ... | @@ -534,11 +573,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 534 | 573 | .ty = Air.internedToRef(vector_ty.childType(zcu).toIntern()), |
| 535 | 574 | .operand = reduce.operand, |
| 536 | 575 | } }), |
| 537 | | else => break :done, |
| 576 | else => {}, |
| 538 | 577 | } |
| 539 | 578 | }, |
| 540 | | .splat, |
| 541 | | => {}, |
| 579 | .splat => {}, |
| 542 | 580 | .shuffle_one => if (l.features.contains(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one), |
| 543 | 581 | .shuffle_two => if (l.features.contains(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two), |
| 544 | 582 | .select => if (l.features.contains(.scalarize_select)) continue :inst try l.scalarize(inst, .select), |
| ... | ... | @@ -558,12 +596,20 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 558 | 596 | .tag_name, |
| 559 | 597 | .error_name, |
| 560 | 598 | .error_set_has_value, |
| 561 | | .aggregate_init, |
| 562 | | .union_init, |
| 563 | | .prefetch, |
| 564 | 599 | => {}, |
| 565 | | inline .mul_add, |
| 566 | | => |air_tag| if (l.features.contains(comptime .scalarize(air_tag))) { |
| 600 | .aggregate_init => if (l.features.contains(.expand_packed_aggregate_init)) { |
| 601 | const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 602 | const agg_ty = ty_pl.ty.toType(); |
| 603 | switch (agg_ty.zigTypeTag(zcu)) { |
| 604 | else => {}, |
| 605 | .@"struct", .@"union" => switch (agg_ty.containerLayout(zcu)) { |
| 606 | .auto, .@"extern" => {}, |
| 607 | .@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)), |
| 608 | }, |
| 609 | } |
| 610 | }, |
| 611 | .union_init, .prefetch => {}, |
| 612 | .mul_add => if (l.features.contains(.scalarize_mul_add)) { |
| 567 | 613 | const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op; |
| 568 | 614 | if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin); |
| 569 | 615 | }, |
| ... | ... | @@ -589,7 +635,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { |
| 589 | 635 | } |
| 590 | 636 | } |
| 591 | 637 | |
| 592 | | const ScalarizeForm = enum { un_op, ty_op, bin_op, ty_pl_vector_cmp, pl_op_bin, shuffle_one, shuffle_two, select }; |
| 638 | const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select }; |
| 593 | 639 | inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag { |
| 594 | 640 | return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form)); |
| 595 | 641 | } |
| ... | ... | @@ -602,8 +648,8 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: |
| 602 | 648 | const res_len = res_ty.vectorLen(zcu); |
| 603 | 649 | |
| 604 | 650 | const extra_insts = switch (form) { |
| 605 | | .un_op, .ty_op => 1, |
| 606 | | .bin_op, .ty_pl_vector_cmp => 2, |
| 651 | .un_op, .ty_op, .bitcast => 1, |
| 652 | .bin_op, .cmp_vector => 2, |
| 607 | 653 | .pl_op_bin => 3, |
| 608 | 654 | .shuffle_one, .shuffle_two => 13, |
| 609 | 655 | .select => 6, |
| ... | ... | @@ -688,32 +734,6 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: |
| 688 | 734 | }).toRef(), |
| 689 | 735 | } }, |
| 690 | 736 | }).toRef(), |
| 691 | | .ty_pl_vector_cmp => { |
| 692 | | const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data; |
| 693 | | break :res_elem (try loop.block.addCmp( |
| 694 | | l, |
| 695 | | extra.compareOperator(), |
| 696 | | loop.block.add(l, .{ |
| 697 | | .tag = .array_elem_val, |
| 698 | | .data = .{ .bin_op = .{ |
| 699 | | .lhs = extra.lhs, |
| 700 | | .rhs = cur_index_inst.toRef(), |
| 701 | | } }, |
| 702 | | }).toRef(), |
| 703 | | loop.block.add(l, .{ |
| 704 | | .tag = .array_elem_val, |
| 705 | | .data = .{ .bin_op = .{ |
| 706 | | .lhs = extra.rhs, |
| 707 | | .rhs = cur_index_inst.toRef(), |
| 708 | | } }, |
| 709 | | }).toRef(), |
| 710 | | .{ .optimized = switch (orig.tag) { |
| 711 | | else => unreachable, |
| 712 | | .cmp_vector => false, |
| 713 | | .cmp_vector_optimized => true, |
| 714 | | } }, |
| 715 | | )).toRef(); |
| 716 | | }, |
| 717 | 737 | .pl_op_bin => { |
| 718 | 738 | const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data; |
| 719 | 739 | break :res_elem loop.block.add(l, .{ |
| ... | ... | @@ -745,6 +765,39 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: |
| 745 | 765 | } }, |
| 746 | 766 | }).toRef(); |
| 747 | 767 | }, |
| 768 | .bitcast => loop.block.addBitCast(l, res_ty.childType(zcu), loop.block.add(l, .{ |
| 769 | .tag = .array_elem_val, |
| 770 | .data = .{ .bin_op = .{ |
| 771 | .lhs = orig.data.ty_op.operand, |
| 772 | .rhs = cur_index_inst.toRef(), |
| 773 | } }, |
| 774 | }).toRef()), |
| 775 | .cmp_vector => { |
| 776 | const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data; |
| 777 | break :res_elem (try loop.block.addCmp( |
| 778 | l, |
| 779 | extra.compareOperator(), |
| 780 | loop.block.add(l, .{ |
| 781 | .tag = .array_elem_val, |
| 782 | .data = .{ .bin_op = .{ |
| 783 | .lhs = extra.lhs, |
| 784 | .rhs = cur_index_inst.toRef(), |
| 785 | } }, |
| 786 | }).toRef(), |
| 787 | loop.block.add(l, .{ |
| 788 | .tag = .array_elem_val, |
| 789 | .data = .{ .bin_op = .{ |
| 790 | .lhs = extra.rhs, |
| 791 | .rhs = cur_index_inst.toRef(), |
| 792 | } }, |
| 793 | }).toRef(), |
| 794 | .{ .optimized = switch (orig.tag) { |
| 795 | else => unreachable, |
| 796 | .cmp_vector => false, |
| 797 | .cmp_vector_optimized => true, |
| 798 | } }, |
| 799 | )).toRef(); |
| 800 | }, |
| 748 | 801 | .shuffle_one, .shuffle_two => { |
| 749 | 802 | const ip = &zcu.intern_pool; |
| 750 | 803 | const unwrapped = switch (form) { |
| ... | ... | @@ -1056,17 +1109,16 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: |
| 1056 | 1109 | .payload = try l.addBlockBody(res_block.body()), |
| 1057 | 1110 | } }; |
| 1058 | 1111 | } |
| 1059 | | fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1112 | fn scalarizeBitcastToArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1060 | 1113 | const pt = l.pt; |
| 1061 | 1114 | const zcu = pt.zcu; |
| 1062 | 1115 | |
| 1063 | | const orig = l.air_instructions.get(@intFromEnum(orig_inst)); |
| 1064 | | const res_ty = l.typeOfIndex(orig_inst); |
| 1065 | | const wrapped_res_ty = res_ty.fieldType(0, zcu); |
| 1066 | | const wrapped_res_scalar_ty = wrapped_res_ty.childType(zcu); |
| 1067 | | const res_len = wrapped_res_ty.vectorLen(zcu); |
| 1116 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1117 | const res_ty = orig_ty_op.ty.toType(); |
| 1118 | const res_elem_ty = res_ty.childType(zcu); |
| 1119 | const res_len = res_ty.arrayLen(zcu); |
| 1068 | 1120 | |
| 1069 | | var inst_buf: [21]Air.Inst.Index = undefined; |
| 1121 | var inst_buf: [16]Air.Inst.Index = undefined; |
| 1070 | 1122 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 1071 | 1123 | |
| 1072 | 1124 | var res_block: Block = .init(&inst_buf); |
| ... | ... | @@ -1075,20 +1127,6 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! |
| 1075 | 1127 | .tag = .alloc, |
| 1076 | 1128 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, |
| 1077 | 1129 | }); |
| 1078 | | const ptr_wrapped_res_inst = res_block.add(l, .{ |
| 1079 | | .tag = .struct_field_ptr_index_0, |
| 1080 | | .data = .{ .ty_op = .{ |
| 1081 | | .ty = Air.internedToRef((try pt.singleMutPtrType(wrapped_res_ty)).toIntern()), |
| 1082 | | .operand = res_alloc_inst.toRef(), |
| 1083 | | } }, |
| 1084 | | }); |
| 1085 | | const ptr_overflow_res_inst = res_block.add(l, .{ |
| 1086 | | .tag = .struct_field_ptr_index_1, |
| 1087 | | .data = .{ .ty_op = .{ |
| 1088 | | .ty = Air.internedToRef((try pt.singleMutPtrType(res_ty.fieldType(1, zcu))).toIntern()), |
| 1089 | | .operand = res_alloc_inst.toRef(), |
| 1090 | | } }, |
| 1091 | | }); |
| 1092 | 1130 | const index_alloc_inst = res_block.add(l, .{ |
| 1093 | 1131 | .tag = .alloc, |
| 1094 | 1132 | .data = .{ .ty = .ptr_usize }, |
| ... | ... | @@ -1111,68 +1149,26 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! |
| 1111 | 1149 | .operand = index_alloc_inst.toRef(), |
| 1112 | 1150 | } }, |
| 1113 | 1151 | }); |
| 1114 | | const extra = l.extraData(Air.Bin, orig.data.ty_pl.payload).data; |
| 1115 | | const res_elem = loop.block.add(l, .{ |
| 1116 | | .tag = orig.tag, |
| 1117 | | .data = .{ .ty_pl = .{ |
| 1118 | | .ty = Air.internedToRef(try zcu.intern_pool.getTupleType(zcu.gpa, pt.tid, .{ |
| 1119 | | .types = &.{ wrapped_res_scalar_ty.toIntern(), .u1_type }, |
| 1120 | | .values = &(.{.none} ** 2), |
| 1121 | | })), |
| 1122 | | .payload = try l.addExtra(Air.Bin, .{ |
| 1123 | | .lhs = loop.block.add(l, .{ |
| 1124 | | .tag = .array_elem_val, |
| 1125 | | .data = .{ .bin_op = .{ |
| 1126 | | .lhs = extra.lhs, |
| 1127 | | .rhs = cur_index_inst.toRef(), |
| 1128 | | } }, |
| 1129 | | }).toRef(), |
| 1130 | | .rhs = loop.block.add(l, .{ |
| 1131 | | .tag = .array_elem_val, |
| 1132 | | .data = .{ .bin_op = .{ |
| 1133 | | .lhs = extra.rhs, |
| 1134 | | .rhs = cur_index_inst.toRef(), |
| 1135 | | } }, |
| 1136 | | }).toRef(), |
| 1137 | | }), |
| 1138 | | } }, |
| 1139 | | }); |
| 1140 | 1152 | _ = loop.block.add(l, .{ |
| 1141 | | .tag = .vector_store_elem, |
| 1142 | | .data = .{ .vector_store_elem = .{ |
| 1143 | | .vector_ptr = ptr_overflow_res_inst.toRef(), |
| 1144 | | .payload = try l.addExtra(Air.Bin, .{ |
| 1145 | | .lhs = cur_index_inst.toRef(), |
| 1146 | | .rhs = loop.block.add(l, .{ |
| 1147 | | .tag = .struct_field_val, |
| 1148 | | .data = .{ .ty_pl = .{ |
| 1149 | | .ty = .u1_type, |
| 1150 | | .payload = try l.addExtra(Air.StructField, .{ |
| 1151 | | .struct_operand = res_elem.toRef(), |
| 1152 | | .field_index = 1, |
| 1153 | | }), |
| 1154 | | } }, |
| 1155 | | }).toRef(), |
| 1156 | | }), |
| 1157 | | } }, |
| 1158 | | }); |
| 1159 | | _ = loop.block.add(l, .{ |
| 1160 | | .tag = .vector_store_elem, |
| 1161 | | .data = .{ .vector_store_elem = .{ |
| 1162 | | .vector_ptr = ptr_wrapped_res_inst.toRef(), |
| 1163 | | .payload = try l.addExtra(Air.Bin, .{ |
| 1164 | | .lhs = cur_index_inst.toRef(), |
| 1165 | | .rhs = loop.block.add(l, .{ |
| 1166 | | .tag = .struct_field_val, |
| 1167 | | .data = .{ .ty_pl = .{ |
| 1168 | | .ty = Air.internedToRef(wrapped_res_scalar_ty.toIntern()), |
| 1169 | | .payload = try l.addExtra(Air.StructField, .{ |
| 1170 | | .struct_operand = res_elem.toRef(), |
| 1171 | | .field_index = 0, |
| 1172 | | }), |
| 1173 | | } }, |
| 1174 | | }).toRef(), |
| 1175 | | }), |
| 1153 | .tag = .store, |
| 1154 | .data = .{ .bin_op = .{ |
| 1155 | .lhs = loop.block.add(l, .{ |
| 1156 | .tag = .ptr_elem_ptr, |
| 1157 | .data = .{ .ty_pl = .{ |
| 1158 | .ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()), |
| 1159 | .payload = try l.addExtra(Air.Bin, .{ |
| 1160 | .lhs = res_alloc_inst.toRef(), |
| 1161 | .rhs = cur_index_inst.toRef(), |
| 1162 | }), |
| 1163 | } }, |
| 1164 | }).toRef(), |
| 1165 | .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{ |
| 1166 | .tag = .array_elem_val, |
| 1167 | .data = .{ .bin_op = .{ |
| 1168 | .lhs = orig_ty_op.operand, |
| 1169 | .rhs = cur_index_inst.toRef(), |
| 1170 | } }, |
| 1171 | }).toRef()), |
| 1176 | 1172 | } }, |
| 1177 | 1173 | }); |
| 1178 | 1174 | |
| ... | ... | @@ -1226,48 +1222,630 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error! |
| 1226 | 1222 | .payload = try l.addBlockBody(res_block.body()), |
| 1227 | 1223 | } }; |
| 1228 | 1224 | } |
| 1229 | | |
| 1230 | | fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1225 | fn scalarizeBitcastOperandBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1231 | 1226 | const pt = l.pt; |
| 1232 | 1227 | const zcu = pt.zcu; |
| 1233 | | const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1234 | | |
| 1235 | | const operand_ref = ty_op.operand; |
| 1236 | | const operand_ty = l.typeOf(operand_ref); |
| 1237 | | const dest_ty = ty_op.ty.toType(); |
| 1238 | 1228 | |
| 1239 | | const is_vector = operand_ty.zigTypeTag(zcu) == .vector; |
| 1240 | | const operand_scalar_ty = operand_ty.scalarType(zcu); |
| 1241 | | const dest_scalar_ty = dest_ty.scalarType(zcu); |
| 1229 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1230 | const res_ty = orig_ty_op.ty.toType(); |
| 1231 | const operand_ty = l.typeOf(orig_ty_op.operand); |
| 1232 | const int_bits: u16 = @intCast(operand_ty.bitSize(zcu)); |
| 1233 | const int_ty = try pt.intType(.unsigned, int_bits); |
| 1234 | const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits)); |
| 1235 | const elem_bits: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu)); |
| 1236 | const elem_int_ty = try pt.intType(.unsigned, elem_bits); |
| 1242 | 1237 | |
| 1243 | | assert(operand_scalar_ty.zigTypeTag(zcu) == .int); |
| 1244 | | const dest_is_enum = switch (dest_scalar_ty.zigTypeTag(zcu)) { |
| 1245 | | .int => false, |
| 1246 | | .@"enum" => true, |
| 1247 | | else => unreachable, |
| 1248 | | }; |
| 1238 | var inst_buf: [22]Air.Inst.Index = undefined; |
| 1239 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 1249 | 1240 | |
| 1250 | | const operand_info = operand_scalar_ty.intInfo(zcu); |
| 1251 | | const dest_info = dest_scalar_ty.intInfo(zcu); |
| 1241 | var res_block: Block = .init(&inst_buf); |
| 1242 | { |
| 1243 | const int_alloc_inst = res_block.add(l, .{ |
| 1244 | .tag = .alloc, |
| 1245 | .data = .{ .ty = try pt.singleMutPtrType(int_ty) }, |
| 1246 | }); |
| 1247 | _ = res_block.add(l, .{ |
| 1248 | .tag = .store, |
| 1249 | .data = .{ .bin_op = .{ |
| 1250 | .lhs = int_alloc_inst.toRef(), |
| 1251 | .rhs = try pt.intRef(int_ty, 0), |
| 1252 | } }, |
| 1253 | }); |
| 1254 | const index_alloc_inst = res_block.add(l, .{ |
| 1255 | .tag = .alloc, |
| 1256 | .data = .{ .ty = .ptr_usize }, |
| 1257 | }); |
| 1258 | _ = res_block.add(l, .{ |
| 1259 | .tag = .store, |
| 1260 | .data = .{ .bin_op = .{ |
| 1261 | .lhs = index_alloc_inst.toRef(), |
| 1262 | .rhs = .zero_usize, |
| 1263 | } }, |
| 1264 | }); |
| 1252 | 1265 | |
| 1253 | | const have_min_check, const have_max_check = c: { |
| 1254 | | const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed); |
| 1255 | | const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed); |
| 1256 | | const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0; |
| 1257 | | const operand_allows_neg = operand_info.signedness == .signed and operand_info.bits > 0; |
| 1258 | | break :c .{ |
| 1259 | | operand_allows_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits), |
| 1260 | | dest_pos_bits < operand_pos_bits, |
| 1261 | | }; |
| 1262 | | }; |
| 1266 | var loop: Loop = .init(l, &res_block); |
| 1267 | loop.block = .init(res_block.stealRemainingCapacity()); |
| 1268 | { |
| 1269 | const cur_index_inst = loop.block.add(l, .{ |
| 1270 | .tag = .load, |
| 1271 | .data = .{ .ty_op = .{ |
| 1272 | .ty = .usize_type, |
| 1273 | .operand = index_alloc_inst.toRef(), |
| 1274 | } }, |
| 1275 | }); |
| 1276 | const cur_int_inst = loop.block.add(l, .{ |
| 1277 | .tag = .bit_or, |
| 1278 | .data = .{ .bin_op = .{ |
| 1279 | .lhs = loop.block.add(l, .{ |
| 1280 | .tag = .shl_exact, |
| 1281 | .data = .{ .bin_op = .{ |
| 1282 | .lhs = loop.block.add(l, .{ |
| 1283 | .tag = .intcast, |
| 1284 | .data = .{ .ty_op = .{ |
| 1285 | .ty = Air.internedToRef(int_ty.toIntern()), |
| 1286 | .operand = loop.block.addBitCast(l, elem_int_ty, loop.block.add(l, .{ |
| 1287 | .tag = .array_elem_val, |
| 1288 | .data = .{ .bin_op = .{ |
| 1289 | .lhs = orig_ty_op.operand, |
| 1290 | .rhs = cur_index_inst.toRef(), |
| 1291 | } }, |
| 1292 | }).toRef()), |
| 1293 | } }, |
| 1294 | }).toRef(), |
| 1295 | .rhs = loop.block.add(l, .{ |
| 1296 | .tag = .mul, |
| 1297 | .data = .{ .bin_op = .{ |
| 1298 | .lhs = loop.block.add(l, .{ |
| 1299 | .tag = .intcast, |
| 1300 | .data = .{ .ty_op = .{ |
| 1301 | .ty = Air.internedToRef(shift_ty.toIntern()), |
| 1302 | .operand = cur_index_inst.toRef(), |
| 1303 | } }, |
| 1304 | }).toRef(), |
| 1305 | .rhs = try pt.intRef(shift_ty, elem_bits), |
| 1306 | } }, |
| 1307 | }).toRef(), |
| 1308 | } }, |
| 1309 | }).toRef(), |
| 1310 | .rhs = loop.block.add(l, .{ |
| 1311 | .tag = .load, |
| 1312 | .data = .{ .ty_op = .{ |
| 1313 | .ty = Air.internedToRef(int_ty.toIntern()), |
| 1314 | .operand = int_alloc_inst.toRef(), |
| 1315 | } }, |
| 1316 | }).toRef(), |
| 1317 | } }, |
| 1318 | }); |
| 1263 | 1319 | |
| 1264 | | // The worst-case scenario in terms of total instructions and total condbrs is the case where |
| 1265 | | // the result type is an exhaustive enum whose tag type is smaller than the operand type: |
| 1266 | | // |
| 1267 | | // %x = block({ |
| 1268 | | // %1 = cmp_lt(%y, @min_allowed_int) |
| 1269 | | // %2 = cmp_gt(%y, @max_allowed_int) |
| 1270 | | // %3 = bool_or(%1, %2) |
| 1320 | var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp( |
| 1321 | l, |
| 1322 | .lt, |
| 1323 | cur_index_inst.toRef(), |
| 1324 | try pt.intRef(.usize, operand_ty.arrayLen(zcu) - 1), |
| 1325 | .{}, |
| 1326 | )).toRef(), &loop.block, .{}); |
| 1327 | loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity()); |
| 1328 | { |
| 1329 | _ = loop_cond_br.then_block.add(l, .{ |
| 1330 | .tag = .store, |
| 1331 | .data = .{ .bin_op = .{ |
| 1332 | .lhs = int_alloc_inst.toRef(), |
| 1333 | .rhs = cur_int_inst.toRef(), |
| 1334 | } }, |
| 1335 | }); |
| 1336 | _ = loop_cond_br.then_block.add(l, .{ |
| 1337 | .tag = .store, |
| 1338 | .data = .{ .bin_op = .{ |
| 1339 | .lhs = index_alloc_inst.toRef(), |
| 1340 | .rhs = loop_cond_br.then_block.add(l, .{ |
| 1341 | .tag = .add, |
| 1342 | .data = .{ .bin_op = .{ |
| 1343 | .lhs = cur_index_inst.toRef(), |
| 1344 | .rhs = .one_usize, |
| 1345 | } }, |
| 1346 | }).toRef(), |
| 1347 | } }, |
| 1348 | }); |
| 1349 | _ = loop_cond_br.then_block.add(l, .{ |
| 1350 | .tag = .repeat, |
| 1351 | .data = .{ .repeat = .{ .loop_inst = loop.inst } }, |
| 1352 | }); |
| 1353 | } |
| 1354 | loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity()); |
| 1355 | _ = loop_cond_br.else_block.add(l, .{ |
| 1356 | .tag = .br, |
| 1357 | .data = .{ .br = .{ |
| 1358 | .block_inst = orig_inst, |
| 1359 | .operand = loop_cond_br.else_block.addBitCast(l, res_ty, cur_int_inst.toRef()), |
| 1360 | } }, |
| 1361 | }); |
| 1362 | try loop_cond_br.finish(l); |
| 1363 | } |
| 1364 | try loop.finish(l); |
| 1365 | } |
| 1366 | return .{ .ty_pl = .{ |
| 1367 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 1368 | .payload = try l.addBlockBody(res_block.body()), |
| 1369 | } }; |
| 1370 | } |
| 1371 | fn scalarizeBitcastResultArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1372 | const pt = l.pt; |
| 1373 | const zcu = pt.zcu; |
| 1374 | |
| 1375 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1376 | const res_ty = orig_ty_op.ty.toType(); |
| 1377 | const int_bits: u16 = @intCast(res_ty.bitSize(zcu)); |
| 1378 | const int_ty = try pt.intType(.unsigned, int_bits); |
| 1379 | const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits)); |
| 1380 | const res_elem_ty = res_ty.childType(zcu); |
| 1381 | const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu)); |
| 1382 | const elem_int_ty = try pt.intType(.unsigned, elem_bits); |
| 1383 | |
| 1384 | var inst_buf: [20]Air.Inst.Index = undefined; |
| 1385 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 1386 | |
| 1387 | var res_block: Block = .init(&inst_buf); |
| 1388 | { |
| 1389 | const res_alloc_inst = res_block.add(l, .{ |
| 1390 | .tag = .alloc, |
| 1391 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, |
| 1392 | }); |
| 1393 | const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand); |
| 1394 | const index_alloc_inst = res_block.add(l, .{ |
| 1395 | .tag = .alloc, |
| 1396 | .data = .{ .ty = .ptr_usize }, |
| 1397 | }); |
| 1398 | _ = res_block.add(l, .{ |
| 1399 | .tag = .store, |
| 1400 | .data = .{ .bin_op = .{ |
| 1401 | .lhs = index_alloc_inst.toRef(), |
| 1402 | .rhs = .zero_usize, |
| 1403 | } }, |
| 1404 | }); |
| 1405 | |
| 1406 | var loop: Loop = .init(l, &res_block); |
| 1407 | loop.block = .init(res_block.stealRemainingCapacity()); |
| 1408 | { |
| 1409 | const cur_index_inst = loop.block.add(l, .{ |
| 1410 | .tag = .load, |
| 1411 | .data = .{ .ty_op = .{ |
| 1412 | .ty = .usize_type, |
| 1413 | .operand = index_alloc_inst.toRef(), |
| 1414 | } }, |
| 1415 | }); |
| 1416 | _ = loop.block.add(l, .{ |
| 1417 | .tag = .store, |
| 1418 | .data = .{ .bin_op = .{ |
| 1419 | .lhs = loop.block.add(l, .{ |
| 1420 | .tag = .ptr_elem_ptr, |
| 1421 | .data = .{ .ty_pl = .{ |
| 1422 | .ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()), |
| 1423 | .payload = try l.addExtra(Air.Bin, .{ |
| 1424 | .lhs = res_alloc_inst.toRef(), |
| 1425 | .rhs = cur_index_inst.toRef(), |
| 1426 | }), |
| 1427 | } }, |
| 1428 | }).toRef(), |
| 1429 | .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{ |
| 1430 | .tag = .trunc, |
| 1431 | .data = .{ .ty_op = .{ |
| 1432 | .ty = Air.internedToRef(elem_int_ty.toIntern()), |
| 1433 | .operand = loop.block.add(l, .{ |
| 1434 | .tag = .shr, |
| 1435 | .data = .{ .bin_op = .{ |
| 1436 | .lhs = int_ref, |
| 1437 | .rhs = loop.block.add(l, .{ |
| 1438 | .tag = .mul, |
| 1439 | .data = .{ .bin_op = .{ |
| 1440 | .lhs = loop.block.add(l, .{ |
| 1441 | .tag = .intcast, |
| 1442 | .data = .{ .ty_op = .{ |
| 1443 | .ty = Air.internedToRef(shift_ty.toIntern()), |
| 1444 | .operand = cur_index_inst.toRef(), |
| 1445 | } }, |
| 1446 | }).toRef(), |
| 1447 | .rhs = try pt.intRef(shift_ty, elem_bits), |
| 1448 | } }, |
| 1449 | }).toRef(), |
| 1450 | } }, |
| 1451 | }).toRef(), |
| 1452 | } }, |
| 1453 | }).toRef()), |
| 1454 | } }, |
| 1455 | }); |
| 1456 | |
| 1457 | var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp( |
| 1458 | l, |
| 1459 | .lt, |
| 1460 | cur_index_inst.toRef(), |
| 1461 | try pt.intRef(.usize, res_ty.arrayLen(zcu) - 1), |
| 1462 | .{}, |
| 1463 | )).toRef(), &loop.block, .{}); |
| 1464 | loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity()); |
| 1465 | { |
| 1466 | _ = loop_cond_br.then_block.add(l, .{ |
| 1467 | .tag = .store, |
| 1468 | .data = .{ .bin_op = .{ |
| 1469 | .lhs = index_alloc_inst.toRef(), |
| 1470 | .rhs = loop_cond_br.then_block.add(l, .{ |
| 1471 | .tag = .add, |
| 1472 | .data = .{ .bin_op = .{ |
| 1473 | .lhs = cur_index_inst.toRef(), |
| 1474 | .rhs = .one_usize, |
| 1475 | } }, |
| 1476 | }).toRef(), |
| 1477 | } }, |
| 1478 | }); |
| 1479 | _ = loop_cond_br.then_block.add(l, .{ |
| 1480 | .tag = .repeat, |
| 1481 | .data = .{ .repeat = .{ .loop_inst = loop.inst } }, |
| 1482 | }); |
| 1483 | } |
| 1484 | loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity()); |
| 1485 | _ = loop_cond_br.else_block.add(l, .{ |
| 1486 | .tag = .br, |
| 1487 | .data = .{ .br = .{ |
| 1488 | .block_inst = orig_inst, |
| 1489 | .operand = loop_cond_br.else_block.add(l, .{ |
| 1490 | .tag = .load, |
| 1491 | .data = .{ .ty_op = .{ |
| 1492 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 1493 | .operand = res_alloc_inst.toRef(), |
| 1494 | } }, |
| 1495 | }).toRef(), |
| 1496 | } }, |
| 1497 | }); |
| 1498 | try loop_cond_br.finish(l); |
| 1499 | } |
| 1500 | try loop.finish(l); |
| 1501 | } |
| 1502 | return .{ .ty_pl = .{ |
| 1503 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 1504 | .payload = try l.addBlockBody(res_block.body()), |
| 1505 | } }; |
| 1506 | } |
| 1507 | fn scalarizeBitcastResultVectorBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1508 | const pt = l.pt; |
| 1509 | const zcu = pt.zcu; |
| 1510 | |
| 1511 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1512 | const res_ty = orig_ty_op.ty.toType(); |
| 1513 | const int_bits: u16 = @intCast(res_ty.bitSize(zcu)); |
| 1514 | const int_ty = try pt.intType(.unsigned, int_bits); |
| 1515 | const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits)); |
| 1516 | const res_elem_ty = res_ty.childType(zcu); |
| 1517 | const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu)); |
| 1518 | const elem_int_ty = try pt.intType(.unsigned, elem_bits); |
| 1519 | |
| 1520 | var inst_buf: [19]Air.Inst.Index = undefined; |
| 1521 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 1522 | |
| 1523 | var res_block: Block = .init(&inst_buf); |
| 1524 | { |
| 1525 | const res_alloc_inst = res_block.add(l, .{ |
| 1526 | .tag = .alloc, |
| 1527 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, |
| 1528 | }); |
| 1529 | const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand); |
| 1530 | const index_alloc_inst = res_block.add(l, .{ |
| 1531 | .tag = .alloc, |
| 1532 | .data = .{ .ty = .ptr_usize }, |
| 1533 | }); |
| 1534 | _ = res_block.add(l, .{ |
| 1535 | .tag = .store, |
| 1536 | .data = .{ .bin_op = .{ |
| 1537 | .lhs = index_alloc_inst.toRef(), |
| 1538 | .rhs = .zero_usize, |
| 1539 | } }, |
| 1540 | }); |
| 1541 | |
| 1542 | var loop: Loop = .init(l, &res_block); |
| 1543 | loop.block = .init(res_block.stealRemainingCapacity()); |
| 1544 | { |
| 1545 | const cur_index_inst = loop.block.add(l, .{ |
| 1546 | .tag = .load, |
| 1547 | .data = .{ .ty_op = .{ |
| 1548 | .ty = .usize_type, |
| 1549 | .operand = index_alloc_inst.toRef(), |
| 1550 | } }, |
| 1551 | }); |
| 1552 | _ = loop.block.add(l, .{ |
| 1553 | .tag = .vector_store_elem, |
| 1554 | .data = .{ .vector_store_elem = .{ |
| 1555 | .vector_ptr = res_alloc_inst.toRef(), |
| 1556 | .payload = try l.addExtra(Air.Bin, .{ |
| 1557 | .lhs = cur_index_inst.toRef(), |
| 1558 | .rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{ |
| 1559 | .tag = .trunc, |
| 1560 | .data = .{ .ty_op = .{ |
| 1561 | .ty = Air.internedToRef(elem_int_ty.toIntern()), |
| 1562 | .operand = loop.block.add(l, .{ |
| 1563 | .tag = .shr, |
| 1564 | .data = .{ .bin_op = .{ |
| 1565 | .lhs = int_ref, |
| 1566 | .rhs = loop.block.add(l, .{ |
| 1567 | .tag = .mul, |
| 1568 | .data = .{ .bin_op = .{ |
| 1569 | .lhs = loop.block.add(l, .{ |
| 1570 | .tag = .intcast, |
| 1571 | .data = .{ .ty_op = .{ |
| 1572 | .ty = Air.internedToRef(shift_ty.toIntern()), |
| 1573 | .operand = cur_index_inst.toRef(), |
| 1574 | } }, |
| 1575 | }).toRef(), |
| 1576 | .rhs = try pt.intRef(shift_ty, elem_bits), |
| 1577 | } }, |
| 1578 | }).toRef(), |
| 1579 | } }, |
| 1580 | }).toRef(), |
| 1581 | } }, |
| 1582 | }).toRef()), |
| 1583 | }), |
| 1584 | } }, |
| 1585 | }); |
| 1586 | |
| 1587 | var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp( |
| 1588 | l, |
| 1589 | .lt, |
| 1590 | cur_index_inst.toRef(), |
| 1591 | try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1), |
| 1592 | .{}, |
| 1593 | )).toRef(), &loop.block, .{}); |
| 1594 | loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity()); |
| 1595 | { |
| 1596 | _ = loop_cond_br.then_block.add(l, .{ |
| 1597 | .tag = .store, |
| 1598 | .data = .{ .bin_op = .{ |
| 1599 | .lhs = index_alloc_inst.toRef(), |
| 1600 | .rhs = loop_cond_br.then_block.add(l, .{ |
| 1601 | .tag = .add, |
| 1602 | .data = .{ .bin_op = .{ |
| 1603 | .lhs = cur_index_inst.toRef(), |
| 1604 | .rhs = .one_usize, |
| 1605 | } }, |
| 1606 | }).toRef(), |
| 1607 | } }, |
| 1608 | }); |
| 1609 | _ = loop_cond_br.then_block.add(l, .{ |
| 1610 | .tag = .repeat, |
| 1611 | .data = .{ .repeat = .{ .loop_inst = loop.inst } }, |
| 1612 | }); |
| 1613 | } |
| 1614 | loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity()); |
| 1615 | _ = loop_cond_br.else_block.add(l, .{ |
| 1616 | .tag = .br, |
| 1617 | .data = .{ .br = .{ |
| 1618 | .block_inst = orig_inst, |
| 1619 | .operand = loop_cond_br.else_block.add(l, .{ |
| 1620 | .tag = .load, |
| 1621 | .data = .{ .ty_op = .{ |
| 1622 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 1623 | .operand = res_alloc_inst.toRef(), |
| 1624 | } }, |
| 1625 | }).toRef(), |
| 1626 | } }, |
| 1627 | }); |
| 1628 | try loop_cond_br.finish(l); |
| 1629 | } |
| 1630 | try loop.finish(l); |
| 1631 | } |
| 1632 | return .{ .ty_pl = .{ |
| 1633 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 1634 | .payload = try l.addBlockBody(res_block.body()), |
| 1635 | } }; |
| 1636 | } |
| 1637 | fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1638 | const pt = l.pt; |
| 1639 | const zcu = pt.zcu; |
| 1640 | |
| 1641 | const orig = l.air_instructions.get(@intFromEnum(orig_inst)); |
| 1642 | const res_ty = l.typeOfIndex(orig_inst); |
| 1643 | const wrapped_res_ty = res_ty.fieldType(0, zcu); |
| 1644 | const wrapped_res_scalar_ty = wrapped_res_ty.childType(zcu); |
| 1645 | const res_len = wrapped_res_ty.vectorLen(zcu); |
| 1646 | |
| 1647 | var inst_buf: [21]Air.Inst.Index = undefined; |
| 1648 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 1649 | |
| 1650 | var res_block: Block = .init(&inst_buf); |
| 1651 | { |
| 1652 | const res_alloc_inst = res_block.add(l, .{ |
| 1653 | .tag = .alloc, |
| 1654 | .data = .{ .ty = try pt.singleMutPtrType(res_ty) }, |
| 1655 | }); |
| 1656 | const ptr_wrapped_res_inst = res_block.add(l, .{ |
| 1657 | .tag = .struct_field_ptr_index_0, |
| 1658 | .data = .{ .ty_op = .{ |
| 1659 | .ty = Air.internedToRef((try pt.singleMutPtrType(wrapped_res_ty)).toIntern()), |
| 1660 | .operand = res_alloc_inst.toRef(), |
| 1661 | } }, |
| 1662 | }); |
| 1663 | const ptr_overflow_res_inst = res_block.add(l, .{ |
| 1664 | .tag = .struct_field_ptr_index_1, |
| 1665 | .data = .{ .ty_op = .{ |
| 1666 | .ty = Air.internedToRef((try pt.singleMutPtrType(res_ty.fieldType(1, zcu))).toIntern()), |
| 1667 | .operand = res_alloc_inst.toRef(), |
| 1668 | } }, |
| 1669 | }); |
| 1670 | const index_alloc_inst = res_block.add(l, .{ |
| 1671 | .tag = .alloc, |
| 1672 | .data = .{ .ty = .ptr_usize }, |
| 1673 | }); |
| 1674 | _ = res_block.add(l, .{ |
| 1675 | .tag = .store, |
| 1676 | .data = .{ .bin_op = .{ |
| 1677 | .lhs = index_alloc_inst.toRef(), |
| 1678 | .rhs = .zero_usize, |
| 1679 | } }, |
| 1680 | }); |
| 1681 | |
| 1682 | var loop: Loop = .init(l, &res_block); |
| 1683 | loop.block = .init(res_block.stealRemainingCapacity()); |
| 1684 | { |
| 1685 | const cur_index_inst = loop.block.add(l, .{ |
| 1686 | .tag = .load, |
| 1687 | .data = .{ .ty_op = .{ |
| 1688 | .ty = .usize_type, |
| 1689 | .operand = index_alloc_inst.toRef(), |
| 1690 | } }, |
| 1691 | }); |
| 1692 | const extra = l.extraData(Air.Bin, orig.data.ty_pl.payload).data; |
| 1693 | const res_elem = loop.block.add(l, .{ |
| 1694 | .tag = orig.tag, |
| 1695 | .data = .{ .ty_pl = .{ |
| 1696 | .ty = Air.internedToRef(try zcu.intern_pool.getTupleType(zcu.gpa, pt.tid, .{ |
| 1697 | .types = &.{ wrapped_res_scalar_ty.toIntern(), .u1_type }, |
| 1698 | .values = &(.{.none} ** 2), |
| 1699 | })), |
| 1700 | .payload = try l.addExtra(Air.Bin, .{ |
| 1701 | .lhs = loop.block.add(l, .{ |
| 1702 | .tag = .array_elem_val, |
| 1703 | .data = .{ .bin_op = .{ |
| 1704 | .lhs = extra.lhs, |
| 1705 | .rhs = cur_index_inst.toRef(), |
| 1706 | } }, |
| 1707 | }).toRef(), |
| 1708 | .rhs = loop.block.add(l, .{ |
| 1709 | .tag = .array_elem_val, |
| 1710 | .data = .{ .bin_op = .{ |
| 1711 | .lhs = extra.rhs, |
| 1712 | .rhs = cur_index_inst.toRef(), |
| 1713 | } }, |
| 1714 | }).toRef(), |
| 1715 | }), |
| 1716 | } }, |
| 1717 | }); |
| 1718 | _ = loop.block.add(l, .{ |
| 1719 | .tag = .vector_store_elem, |
| 1720 | .data = .{ .vector_store_elem = .{ |
| 1721 | .vector_ptr = ptr_overflow_res_inst.toRef(), |
| 1722 | .payload = try l.addExtra(Air.Bin, .{ |
| 1723 | .lhs = cur_index_inst.toRef(), |
| 1724 | .rhs = loop.block.add(l, .{ |
| 1725 | .tag = .struct_field_val, |
| 1726 | .data = .{ .ty_pl = .{ |
| 1727 | .ty = .u1_type, |
| 1728 | .payload = try l.addExtra(Air.StructField, .{ |
| 1729 | .struct_operand = res_elem.toRef(), |
| 1730 | .field_index = 1, |
| 1731 | }), |
| 1732 | } }, |
| 1733 | }).toRef(), |
| 1734 | }), |
| 1735 | } }, |
| 1736 | }); |
| 1737 | _ = loop.block.add(l, .{ |
| 1738 | .tag = .vector_store_elem, |
| 1739 | .data = .{ .vector_store_elem = .{ |
| 1740 | .vector_ptr = ptr_wrapped_res_inst.toRef(), |
| 1741 | .payload = try l.addExtra(Air.Bin, .{ |
| 1742 | .lhs = cur_index_inst.toRef(), |
| 1743 | .rhs = loop.block.add(l, .{ |
| 1744 | .tag = .struct_field_val, |
| 1745 | .data = .{ .ty_pl = .{ |
| 1746 | .ty = Air.internedToRef(wrapped_res_scalar_ty.toIntern()), |
| 1747 | .payload = try l.addExtra(Air.StructField, .{ |
| 1748 | .struct_operand = res_elem.toRef(), |
| 1749 | .field_index = 0, |
| 1750 | }), |
| 1751 | } }, |
| 1752 | }).toRef(), |
| 1753 | }), |
| 1754 | } }, |
| 1755 | }); |
| 1756 | |
| 1757 | var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp( |
| 1758 | l, |
| 1759 | .lt, |
| 1760 | cur_index_inst.toRef(), |
| 1761 | try pt.intRef(.usize, res_len - 1), |
| 1762 | .{}, |
| 1763 | )).toRef(), &loop.block, .{}); |
| 1764 | loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity()); |
| 1765 | { |
| 1766 | _ = loop_cond_br.then_block.add(l, .{ |
| 1767 | .tag = .store, |
| 1768 | .data = .{ .bin_op = .{ |
| 1769 | .lhs = index_alloc_inst.toRef(), |
| 1770 | .rhs = loop_cond_br.then_block.add(l, .{ |
| 1771 | .tag = .add, |
| 1772 | .data = .{ .bin_op = .{ |
| 1773 | .lhs = cur_index_inst.toRef(), |
| 1774 | .rhs = .one_usize, |
| 1775 | } }, |
| 1776 | }).toRef(), |
| 1777 | } }, |
| 1778 | }); |
| 1779 | _ = loop_cond_br.then_block.add(l, .{ |
| 1780 | .tag = .repeat, |
| 1781 | .data = .{ .repeat = .{ .loop_inst = loop.inst } }, |
| 1782 | }); |
| 1783 | } |
| 1784 | loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity()); |
| 1785 | _ = loop_cond_br.else_block.add(l, .{ |
| 1786 | .tag = .br, |
| 1787 | .data = .{ .br = .{ |
| 1788 | .block_inst = orig_inst, |
| 1789 | .operand = loop_cond_br.else_block.add(l, .{ |
| 1790 | .tag = .load, |
| 1791 | .data = .{ .ty_op = .{ |
| 1792 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 1793 | .operand = res_alloc_inst.toRef(), |
| 1794 | } }, |
| 1795 | }).toRef(), |
| 1796 | } }, |
| 1797 | }); |
| 1798 | try loop_cond_br.finish(l); |
| 1799 | } |
| 1800 | try loop.finish(l); |
| 1801 | } |
| 1802 | return .{ .ty_pl = .{ |
| 1803 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 1804 | .payload = try l.addBlockBody(res_block.body()), |
| 1805 | } }; |
| 1806 | } |
| 1807 | |
| 1808 | fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 1809 | const pt = l.pt; |
| 1810 | const zcu = pt.zcu; |
| 1811 | const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 1812 | |
| 1813 | const operand_ref = ty_op.operand; |
| 1814 | const operand_ty = l.typeOf(operand_ref); |
| 1815 | const dest_ty = ty_op.ty.toType(); |
| 1816 | |
| 1817 | const is_vector = operand_ty.zigTypeTag(zcu) == .vector; |
| 1818 | const operand_scalar_ty = operand_ty.scalarType(zcu); |
| 1819 | const dest_scalar_ty = dest_ty.scalarType(zcu); |
| 1820 | |
| 1821 | assert(operand_scalar_ty.zigTypeTag(zcu) == .int); |
| 1822 | const dest_is_enum = switch (dest_scalar_ty.zigTypeTag(zcu)) { |
| 1823 | .int => false, |
| 1824 | .@"enum" => true, |
| 1825 | else => unreachable, |
| 1826 | }; |
| 1827 | |
| 1828 | const operand_info = operand_scalar_ty.intInfo(zcu); |
| 1829 | const dest_info = dest_scalar_ty.intInfo(zcu); |
| 1830 | |
| 1831 | const have_min_check, const have_max_check = c: { |
| 1832 | const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed); |
| 1833 | const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed); |
| 1834 | const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0; |
| 1835 | const operand_allows_neg = operand_info.signedness == .signed and operand_info.bits > 0; |
| 1836 | break :c .{ |
| 1837 | operand_allows_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits), |
| 1838 | dest_pos_bits < operand_pos_bits, |
| 1839 | }; |
| 1840 | }; |
| 1841 | |
| 1842 | // The worst-case scenario in terms of total instructions and total condbrs is the case where |
| 1843 | // the result type is an exhaustive enum whose tag type is smaller than the operand type: |
| 1844 | // |
| 1845 | // %x = block({ |
| 1846 | // %1 = cmp_lt(%y, @min_allowed_int) |
| 1847 | // %2 = cmp_gt(%y, @max_allowed_int) |
| 1848 | // %3 = bool_or(%1, %2) |
| 1271 | 1849 | // %4 = cond_br(%3, { |
| 1272 | 1850 | // %5 = call(@panic.invalidEnumValue, []) |
| 1273 | 1851 | // %6 = unreach() |
| ... | ... | @@ -1485,6 +2063,292 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_ |
| 1485 | 2063 | } }; |
| 1486 | 2064 | } |
| 1487 | 2065 | |
| 2066 | fn expandBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 2067 | const pt = l.pt; |
| 2068 | const zcu = pt.zcu; |
| 2069 | const ip = &zcu.intern_pool; |
| 2070 | |
| 2071 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 2072 | const res_ty = orig_ty_op.ty.toType(); |
| 2073 | const res_ty_key = ip.indexToKey(res_ty.toIntern()); |
| 2074 | const operand_ty = l.typeOf(orig_ty_op.operand); |
| 2075 | const operand_ty_key = ip.indexToKey(operand_ty.toIntern()); |
| 2076 | _ = res_ty_key; |
| 2077 | _ = operand_ty_key; |
| 2078 | |
| 2079 | var inst_buf: [1]Air.Inst.Index = undefined; |
| 2080 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 2081 | |
| 2082 | var res_block: Block = .init(&inst_buf); |
| 2083 | { |
| 2084 | _ = res_block.add(l, .{ |
| 2085 | .tag = .br, |
| 2086 | .data = .{ .br = .{ |
| 2087 | .block_inst = orig_inst, |
| 2088 | .operand = try pt.undefRef(res_ty), |
| 2089 | } }, |
| 2090 | }); |
| 2091 | } |
| 2092 | return .{ .ty_pl = .{ |
| 2093 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 2094 | .payload = try l.addBlockBody(res_block.body()), |
| 2095 | } }; |
| 2096 | } |
| 2097 | fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 2098 | const pt = l.pt; |
| 2099 | const zcu = pt.zcu; |
| 2100 | |
| 2101 | const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op; |
| 2102 | const res_ty = orig_ty_op.ty.toType(); |
| 2103 | const res_int_ty = try pt.intType(.unsigned, @intCast(res_ty.bitSize(zcu))); |
| 2104 | const ptr_ty = l.typeOf(orig_ty_op.operand); |
| 2105 | const ptr_info = ptr_ty.ptrInfo(zcu); |
| 2106 | // This relies on a heap of possibly invalid assumptions to work around not knowing the actual backing type. |
| 2107 | const load_bits = 8 * ptr_info.packed_offset.host_size; |
| 2108 | const load_ty = try pt.intType(.unsigned, load_bits); |
| 2109 | |
| 2110 | var inst_buf: [6]Air.Inst.Index = undefined; |
| 2111 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 2112 | |
| 2113 | var res_block: Block = .init(&inst_buf); |
| 2114 | _ = res_block.add(l, .{ |
| 2115 | .tag = .br, |
| 2116 | .data = .{ .br = .{ |
| 2117 | .block_inst = orig_inst, |
| 2118 | .operand = res_block.addBitCast(l, res_ty, res_block.add(l, .{ |
| 2119 | .tag = .trunc, |
| 2120 | .data = .{ .ty_op = .{ |
| 2121 | .ty = Air.internedToRef(res_int_ty.toIntern()), |
| 2122 | .operand = res_block.add(l, .{ |
| 2123 | .tag = .shr, |
| 2124 | .data = .{ .bin_op = .{ |
| 2125 | .lhs = res_block.add(l, .{ |
| 2126 | .tag = .load, |
| 2127 | .data = .{ .ty_op = .{ |
| 2128 | .ty = Air.internedToRef(load_ty.toIntern()), |
| 2129 | .operand = res_block.addBitCast(l, load_ptr_ty: { |
| 2130 | var load_ptr_info = ptr_info; |
| 2131 | load_ptr_info.child = load_ty.toIntern(); |
| 2132 | load_ptr_info.flags.vector_index = .none; |
| 2133 | load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 }; |
| 2134 | break :load_ptr_ty try pt.ptrType(load_ptr_info); |
| 2135 | }, orig_ty_op.operand), |
| 2136 | } }, |
| 2137 | }).toRef(), |
| 2138 | .rhs = try pt.intRef( |
| 2139 | try pt.intType(.unsigned, std.math.log2_int_ceil(u16, load_bits)), |
| 2140 | ptr_info.packed_offset.bit_offset, |
| 2141 | ), |
| 2142 | } }, |
| 2143 | }).toRef(), |
| 2144 | } }, |
| 2145 | }).toRef()), |
| 2146 | } }, |
| 2147 | }); |
| 2148 | return .{ .ty_pl = .{ |
| 2149 | .ty = Air.internedToRef(res_ty.toIntern()), |
| 2150 | .payload = try l.addBlockBody(res_block.body()), |
| 2151 | } }; |
| 2152 | } |
| 2153 | fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 2154 | const pt = l.pt; |
| 2155 | const zcu = pt.zcu; |
| 2156 | |
| 2157 | const orig_bin_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].bin_op; |
| 2158 | const ptr_ty = l.typeOf(orig_bin_op.lhs); |
| 2159 | const ptr_info = ptr_ty.ptrInfo(zcu); |
| 2160 | const operand_ty = l.typeOf(orig_bin_op.rhs); |
| 2161 | const operand_bits: u16 = @intCast(operand_ty.bitSize(zcu)); |
| 2162 | const operand_int_ty = try pt.intType(.unsigned, operand_bits); |
| 2163 | // This relies on a heap of possibly invalid assumptions to work around not knowing the actual backing type. |
| 2164 | const load_store_bits = 8 * ptr_info.packed_offset.host_size; |
| 2165 | const load_store_ty = try pt.intType(.unsigned, load_store_bits); |
| 2166 | |
| 2167 | var inst_buf: [9]Air.Inst.Index = undefined; |
| 2168 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 2169 | |
| 2170 | var res_block: Block = .init(&inst_buf); |
| 2171 | { |
| 2172 | const backing_ptr_inst = res_block.add(l, .{ |
| 2173 | .tag = .bitcast, |
| 2174 | .data = .{ .ty_op = .{ |
| 2175 | .ty = Air.internedToRef((load_store_ptr_ty: { |
| 2176 | var load_ptr_info = ptr_info; |
| 2177 | load_ptr_info.child = load_store_ty.toIntern(); |
| 2178 | load_ptr_info.flags.vector_index = .none; |
| 2179 | load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 }; |
| 2180 | break :load_store_ptr_ty try pt.ptrType(load_ptr_info); |
| 2181 | }).toIntern()), |
| 2182 | .operand = orig_bin_op.lhs, |
| 2183 | } }, |
| 2184 | }); |
| 2185 | _ = res_block.add(l, .{ |
| 2186 | .tag = .store, |
| 2187 | .data = .{ .bin_op = .{ |
| 2188 | .lhs = backing_ptr_inst.toRef(), |
| 2189 | .rhs = res_block.add(l, .{ |
| 2190 | .tag = .bit_or, |
| 2191 | .data = .{ .bin_op = .{ |
| 2192 | .lhs = res_block.add(l, .{ |
| 2193 | .tag = .bit_and, |
| 2194 | .data = .{ .bin_op = .{ |
| 2195 | .lhs = res_block.add(l, .{ |
| 2196 | .tag = .load, |
| 2197 | .data = .{ .ty_op = .{ |
| 2198 | .ty = Air.internedToRef(load_store_ty.toIntern()), |
| 2199 | .operand = backing_ptr_inst.toRef(), |
| 2200 | } }, |
| 2201 | }).toRef(), |
| 2202 | .rhs = Air.internedToRef((keep_mask: { |
| 2203 | const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; |
| 2204 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| 2205 | std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa); |
| 2206 | const gpa = stack.get(); |
| 2207 | |
| 2208 | var mask_big_int: std.math.big.int.Mutable = .{ |
| 2209 | .limbs = try gpa.alloc( |
| 2210 | std.math.big.Limb, |
| 2211 | std.math.big.int.calcTwosCompLimbCount(load_store_bits), |
| 2212 | ), |
| 2213 | .len = undefined, |
| 2214 | .positive = undefined, |
| 2215 | }; |
| 2216 | defer gpa.free(mask_big_int.limbs); |
| 2217 | mask_big_int.setTwosCompIntLimit(.max, .unsigned, operand_bits); |
| 2218 | mask_big_int.shiftLeft(mask_big_int.toConst(), ptr_info.packed_offset.bit_offset); |
| 2219 | mask_big_int.bitNotWrap(mask_big_int.toConst(), .unsigned, load_store_bits); |
| 2220 | break :keep_mask try pt.intValue_big(load_store_ty, mask_big_int.toConst()); |
| 2221 | }).toIntern()), |
| 2222 | } }, |
| 2223 | }).toRef(), |
| 2224 | .rhs = res_block.add(l, .{ |
| 2225 | .tag = .shl_exact, |
| 2226 | .data = .{ .bin_op = .{ |
| 2227 | .lhs = res_block.add(l, .{ |
| 2228 | .tag = .intcast, |
| 2229 | .data = .{ .ty_op = .{ |
| 2230 | .ty = Air.internedToRef(load_store_ty.toIntern()), |
| 2231 | .operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs), |
| 2232 | } }, |
| 2233 | }).toRef(), |
| 2234 | .rhs = try pt.intRef( |
| 2235 | try pt.intType(.unsigned, std.math.log2_int_ceil(u16, load_store_bits)), |
| 2236 | ptr_info.packed_offset.bit_offset, |
| 2237 | ), |
| 2238 | } }, |
| 2239 | }).toRef(), |
| 2240 | } }, |
| 2241 | }).toRef(), |
| 2242 | } }, |
| 2243 | }); |
| 2244 | _ = res_block.add(l, .{ |
| 2245 | .tag = .br, |
| 2246 | .data = .{ .br = .{ |
| 2247 | .block_inst = orig_inst, |
| 2248 | .operand = .void_value, |
| 2249 | } }, |
| 2250 | }); |
| 2251 | } |
| 2252 | return .{ .ty_pl = .{ |
| 2253 | .ty = .void_type, |
| 2254 | .payload = try l.addBlockBody(res_block.body()), |
| 2255 | } }; |
| 2256 | } |
| 2257 | fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 2258 | const pt = l.pt; |
| 2259 | const zcu = pt.zcu; |
| 2260 | |
| 2261 | const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl; |
| 2262 | const orig_extra = l.extraData(Air.StructField, orig_ty_pl.payload).data; |
| 2263 | const field_ty = orig_ty_pl.ty.toType(); |
| 2264 | const agg_ty = l.typeOf(orig_extra.struct_operand); |
| 2265 | |
| 2266 | var inst_buf: [5]Air.Inst.Index = undefined; |
| 2267 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 2268 | |
| 2269 | var res_block: Block = .init(&inst_buf); |
| 2270 | { |
| 2271 | const agg_alloc_inst = res_block.add(l, .{ |
| 2272 | .tag = .alloc, |
| 2273 | .data = .{ .ty = try pt.singleMutPtrType(agg_ty) }, |
| 2274 | }); |
| 2275 | _ = res_block.add(l, .{ |
| 2276 | .tag = .store, |
| 2277 | .data = .{ .bin_op = .{ |
| 2278 | .lhs = agg_alloc_inst.toRef(), |
| 2279 | .rhs = orig_extra.struct_operand, |
| 2280 | } }, |
| 2281 | }); |
| 2282 | _ = res_block.add(l, .{ |
| 2283 | .tag = .br, |
| 2284 | .data = .{ .br = .{ |
| 2285 | .block_inst = orig_inst, |
| 2286 | .operand = res_block.add(l, .{ |
| 2287 | .tag = .load, |
| 2288 | .data = .{ .ty_op = .{ |
| 2289 | .ty = Air.internedToRef(field_ty.toIntern()), |
| 2290 | .operand = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), orig_extra.field_index)).toRef(), |
| 2291 | } }, |
| 2292 | }).toRef(), |
| 2293 | } }, |
| 2294 | }); |
| 2295 | } |
| 2296 | return .{ .ty_pl = .{ |
| 2297 | .ty = Air.internedToRef(field_ty.toIntern()), |
| 2298 | .payload = try l.addBlockBody(res_block.body()), |
| 2299 | } }; |
| 2300 | } |
| 2301 | fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data { |
| 2302 | const pt = l.pt; |
| 2303 | const zcu = pt.zcu; |
| 2304 | |
| 2305 | const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl; |
| 2306 | const field_ty = orig_ty_pl.ty.toType(); |
| 2307 | const agg_ty = orig_ty_pl.ty.toType(); |
| 2308 | const agg_field_count = agg_ty.structFieldCount(zcu); |
| 2309 | |
| 2310 | const ExpectedContents = [1 + 2 * 32 + 2]Air.Inst.Index; |
| 2311 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| 2312 | std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa); |
| 2313 | const gpa = stack.get(); |
| 2314 | |
| 2315 | const inst_buf = try gpa.alloc(Air.Inst.Index, 1 + 2 * agg_field_count + 2); |
| 2316 | defer gpa.free(inst_buf); |
| 2317 | try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len); |
| 2318 | |
| 2319 | var res_block: Block = .init(inst_buf); |
| 2320 | { |
| 2321 | const agg_alloc_inst = res_block.add(l, .{ |
| 2322 | .tag = .alloc, |
| 2323 | .data = .{ .ty = try pt.singleMutPtrType(agg_ty) }, |
| 2324 | }); |
| 2325 | for (0..agg_field_count, orig_ty_pl.payload..) |field_index, extra_index| _ = res_block.add(l, .{ |
| 2326 | .tag = .store, |
| 2327 | .data = .{ .bin_op = .{ |
| 2328 | .lhs = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), field_index)).toRef(), |
| 2329 | .rhs = @enumFromInt(l.air_extra.items[extra_index]), |
| 2330 | } }, |
| 2331 | }); |
| 2332 | _ = res_block.add(l, .{ |
| 2333 | .tag = .br, |
| 2334 | .data = .{ .br = .{ |
| 2335 | .block_inst = orig_inst, |
| 2336 | .operand = res_block.add(l, .{ |
| 2337 | .tag = .load, |
| 2338 | .data = .{ .ty_op = .{ |
| 2339 | .ty = Air.internedToRef(field_ty.toIntern()), |
| 2340 | .operand = agg_alloc_inst.toRef(), |
| 2341 | } }, |
| 2342 | }).toRef(), |
| 2343 | } }, |
| 2344 | }); |
| 2345 | } |
| 2346 | return .{ .ty_pl = .{ |
| 2347 | .ty = Air.internedToRef(field_ty.toIntern()), |
| 2348 | .payload = try l.addBlockBody(res_block.body()), |
| 2349 | } }; |
| 2350 | } |
| 2351 | |
| 1488 | 2352 | const Block = struct { |
| 1489 | 2353 | instructions: []Air.Inst.Index, |
| 1490 | 2354 | len: usize, |
| ... | ... | @@ -1576,6 +2440,114 @@ const Block = struct { |
| 1576 | 2440 | }); |
| 1577 | 2441 | } |
| 1578 | 2442 | |
| 2443 | /// Adds a `struct_field_ptr*` instruction to `b`. This is a fairly thin wrapper around `add` |
| 2444 | /// that selects the optimized instruction encoding to use, although it does compute the |
| 2445 | /// proper field pointer type. |
| 2446 | fn addStructFieldPtr( |
| 2447 | b: *Block, |
| 2448 | l: *Legalize, |
| 2449 | struct_operand: Air.Inst.Ref, |
| 2450 | field_index: usize, |
| 2451 | ) Error!Air.Inst.Index { |
| 2452 | const pt = l.pt; |
| 2453 | const zcu = pt.zcu; |
| 2454 | |
| 2455 | const agg_ptr_ty = l.typeOf(struct_operand); |
| 2456 | const agg_ptr_info = agg_ptr_ty.ptrInfo(zcu); |
| 2457 | const agg_ty: Type = .fromInterned(agg_ptr_info.child); |
| 2458 | const agg_ptr_align = switch (agg_ptr_info.flags.alignment) { |
| 2459 | .none => agg_ty.abiAlignment(zcu), |
| 2460 | else => |agg_ptr_align| agg_ptr_align, |
| 2461 | }; |
| 2462 | const agg_layout = agg_ty.containerLayout(zcu); |
| 2463 | const field_ty = agg_ty.fieldType(field_index, zcu); |
| 2464 | var field_ptr_info: InternPool.Key.PtrType = .{ |
| 2465 | .child = field_ty.toIntern(), |
| 2466 | .flags = .{ |
| 2467 | .is_const = agg_ptr_info.flags.is_const, |
| 2468 | .is_volatile = agg_ptr_info.flags.is_volatile, |
| 2469 | .address_space = agg_ptr_info.flags.address_space, |
| 2470 | }, |
| 2471 | }; |
| 2472 | field_ptr_info.flags.alignment = field_ptr_align: switch (agg_layout) { |
| 2473 | .auto => agg_ty.fieldAlignment(field_index, zcu).min(agg_ptr_align), |
| 2474 | .@"extern" => switch (agg_ty.zigTypeTag(zcu)) { |
| 2475 | else => unreachable, |
| 2476 | .@"struct" => .fromLog2Units(@min( |
| 2477 | agg_ptr_align.toLog2Units(), |
| 2478 | @ctz(agg_ty.structFieldOffset(field_index, zcu)), |
| 2479 | )), |
| 2480 | .@"union" => agg_ptr_align, |
| 2481 | }, |
| 2482 | .@"packed" => switch (agg_ty.zigTypeTag(zcu)) { |
| 2483 | else => unreachable, |
| 2484 | .@"struct" => switch (agg_ty.packedStructFieldPtrInfo(agg_ptr_ty, @intCast(field_index), pt)) { |
| 2485 | .bit_ptr => |packed_offset| { |
| 2486 | field_ptr_info.packed_offset = packed_offset; |
| 2487 | break :field_ptr_align agg_ptr_align; |
| 2488 | }, |
| 2489 | .byte_ptr => |ptr_info| ptr_info.alignment, |
| 2490 | }, |
| 2491 | .@"union" => { |
| 2492 | field_ptr_info.packed_offset = .{ |
| 2493 | .host_size = switch (agg_ptr_info.packed_offset.host_size) { |
| 2494 | 0 => @intCast(agg_ty.abiSize(zcu)), |
| 2495 | else => |host_size| host_size, |
| 2496 | }, |
| 2497 | .bit_offset = agg_ptr_info.packed_offset.bit_offset, |
| 2498 | }; |
| 2499 | break :field_ptr_align agg_ptr_align; |
| 2500 | }, |
| 2501 | }, |
| 2502 | }; |
| 2503 | const field_ptr_ty = try pt.ptrType(field_ptr_info); |
| 2504 | const field_ptr_ty_ref = Air.internedToRef(field_ptr_ty.toIntern()); |
| 2505 | return switch (field_index) { |
| 2506 | inline 0...3 => |ct_field_index| b.add(l, .{ |
| 2507 | .tag = switch (ct_field_index) { |
| 2508 | 0 => .struct_field_ptr_index_0, |
| 2509 | 1 => .struct_field_ptr_index_1, |
| 2510 | 2 => .struct_field_ptr_index_2, |
| 2511 | 3 => .struct_field_ptr_index_3, |
| 2512 | else => comptime unreachable, |
| 2513 | }, |
| 2514 | .data = .{ .ty_op = .{ |
| 2515 | .ty = field_ptr_ty_ref, |
| 2516 | .operand = struct_operand, |
| 2517 | } }, |
| 2518 | }), |
| 2519 | else => b.add(l, .{ |
| 2520 | .tag = .struct_field_ptr, |
| 2521 | .data = .{ .ty_pl = .{ |
| 2522 | .ty = field_ptr_ty_ref, |
| 2523 | .payload = try l.addExtra(Air.StructField, .{ |
| 2524 | .struct_operand = struct_operand, |
| 2525 | .field_index = @intCast(field_index), |
| 2526 | }), |
| 2527 | } }, |
| 2528 | }), |
| 2529 | }; |
| 2530 | } |
| 2531 | |
| 2532 | /// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for |
| 2533 | /// no-op casts. |
| 2534 | fn addBitCast( |
| 2535 | b: *Block, |
| 2536 | l: *Legalize, |
| 2537 | ty: Type, |
| 2538 | operand: Air.Inst.Ref, |
| 2539 | ) Air.Inst.Ref { |
| 2540 | if (ty.toIntern() != l.typeOf(operand).toIntern()) return b.add(l, .{ |
| 2541 | .tag = .bitcast, |
| 2542 | .data = .{ .ty_op = .{ |
| 2543 | .ty = Air.internedToRef(ty.toIntern()), |
| 2544 | .operand = operand, |
| 2545 | } }, |
| 2546 | }).toRef(); |
| 2547 | _ = b.stealCapacity(1); |
| 2548 | return operand; |
| 2549 | } |
| 2550 | |
| 1579 | 2551 | /// Returns the unused capacity of `b.instructions`, and shrinks `b.instructions` down to `b.len`. |
| 1580 | 2552 | /// This is useful when you've provided a buffer big enough for all your instructions, but you are |
| 1581 | 2553 | /// now starting a new block and some of them need to live there instead. |
Jacob Young <jacobly@ziglang.org> 2025-06-03 00:47:33-04:00
Andrew Kelley <andrew@ziglang.org> 2025-06-03 15:04:43-04:00
src