| author |  David Rubin <david@vortan.dev> 2024-05-29 17:36:53-07:00 |
| committer | David Rubin <david@vortan.dev> 2024-07-14 23:02:32-07:00 |
| log | 0460572899482f7aa7ec6d9d177ed48984802c80 |
| tree | 027b9132b140fbfde4f0d7b3344a84c02b8699ab |
| parent | ea084e9519a8e6f14d1bcb5f1fb2cddf842333b6 |
| signature |  Commit is signed but in an unrecognized format. |
riscv: `@atomicRmw`
Now we generate debug undefined constants when the user asks for them to dedup across the function decl. This takes 2 instructions instead of 7 in the RISC-V backend.
TODO, we need to dedupe across function decl boundaries.8 files changed, 342 insertions(+), 71 deletions(-)
src- codegen.zig
- link
- test
- behavior
src/arch/riscv64/CodeGen.zig+125-21
| ... | ... | @@ -117,8 +117,9 @@ const MCValue = union(enum) { |
| 117 | 117 | /// No more references to this value remain. |
| 118 | 118 | /// The payload is the value of scope_generation at the point where the death occurred |
| 119 | 119 | dead: u32, |
| 120 | /// The value is undefined. | |
| 121 | undef, | |
| 120 | /// The value is undefined. Contains a symbol index to an undefined constant. Null means | |
| 121 | /// set the undefined value via immediate instead of a load. | |
| 122 | undef: ?u32, | |
| 122 | 123 | /// A pointer-sized integer that fits in a register. |
| 123 | 124 | /// If the type is a pointer, this is the pointer address in virtual address space. |
| 124 | 125 | immediate: u64, |
| ... | ... | @@ -1045,6 +1046,7 @@ pub fn addExtraAssumeCapacity(func: *Func, extra: anytype) u32 { |
| 1045 | 1046 | const required_features = [_]Target.riscv.Feature{ |
| 1046 | 1047 | .d, |
| 1047 | 1048 | .m, |
| 1049 | .a, | |
| 1048 | 1050 | }; |
| 1049 | 1051 | |
| 1050 | 1052 | fn gen(func: *Func) !void { |
| ... | ... | @@ -1631,7 +1633,7 @@ fn computeFrameLayout(func: *Func) !FrameLayout { |
| 1631 | 1633 | |
| 1632 | 1634 | // The total frame size is calculated by the amount of s registers you need to save * 8, as each |
| 1633 | 1635 | // register is 8 bytes, the total allocation sizes, and 16 more register for the spilled ra and s0 |
| 1634 | // register. Finally we align the frame size to the align of the base pointer. | |
| 1636 | // register. Finally we align the frame size to the alignment of the base pointer. | |
| 1635 | 1637 | const args_frame_size = frame_size[@intFromEnum(FrameIndex.args_frame)]; |
| 1636 | 1638 | const spill_frame_size = frame_size[@intFromEnum(FrameIndex.spill_frame)]; |
| 1637 | 1639 | const call_frame_size = frame_size[@intFromEnum(FrameIndex.call_frame)]; |
| ... | ... | @@ -2110,7 +2112,7 @@ fn airNot(func: *Func, inst: Air.Inst.Index) !void { |
| 2110 | 2112 | }); |
| 2111 | 2113 | }, |
| 2112 | 2114 | .Int => { |
| 2113 | const size = ty.bitSize(zcu); | |
| 2115 | const size = ty.bitSize(pt); | |
| 2114 | 2116 | if (!math.isPowerOfTwo(size)) |
| 2115 | 2117 | return func.fail("TODO: airNot non-pow 2 int size", .{}); |
| 2116 | 2118 | |
| ... | ... | @@ -3249,7 +3251,7 @@ fn airWrapErrUnionErr(func: *Func, inst: Air.Inst.Index) !void { |
| 3249 | 3251 | const frame_index = try func.allocFrameIndex(FrameAlloc.initSpill(eu_ty, pt)); |
| 3250 | 3252 | const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, pt)); |
| 3251 | 3253 | const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, pt)); |
| 3252 | try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef); | |
| 3254 | try func.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .{ .undef = null }); | |
| 3253 | 3255 | const operand = try func.resolveInst(ty_op.operand); |
| 3254 | 3256 | try func.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand); |
| 3255 | 3257 | break :result .{ .load_frame = .{ .index = frame_index } }; |
| ... | ... | @@ -5627,10 +5629,14 @@ fn genSetReg(func: *Func, ty: Type, reg: Register, src_mcv: MCValue) InnerError! |
| 5627 | 5629 | .none, |
| 5628 | 5630 | .dead, |
| 5629 | 5631 | => unreachable, |
| 5630 | .undef => { | |
| 5632 | .undef => |sym_index| { | |
| 5631 | 5633 | if (!func.wantSafety()) |
| 5632 | 5634 | return; |
| 5633 | 5635 | |
| 5636 | if (sym_index) |index| { | |
| 5637 | return func.genSetReg(ty, reg, .{ .load_symbol = .{ .sym = index } }); | |
| 5638 | } | |
| 5639 | ||
| 5634 | 5640 | switch (abi_size) { |
| 5635 | 5641 | 1 => return func.genSetReg(ty, reg, .{ .immediate = 0xAA }), |
| 5636 | 5642 | 2 => return func.genSetReg(ty, reg, .{ .immediate = 0xAAAA }), |
| ... | ... | @@ -5865,11 +5871,17 @@ fn genSetMem( |
| 5865 | 5871 | .dead, |
| 5866 | 5872 | .reserved_frame, |
| 5867 | 5873 | => unreachable, |
| 5868 | .undef => try func.genInlineMemset( | |
| 5869 | dst_ptr_mcv, | |
| 5870 | src_mcv, | |
| 5871 | .{ .immediate = abi_size }, | |
| 5872 | ), | |
| 5874 | .undef => |sym_index| { | |
| 5875 | if (sym_index) |index| { | |
| 5876 | return func.genSetMem(base, disp, ty, .{ .load_symbol = .{ .sym = index } }); | |
| 5877 | } | |
| 5878 | ||
| 5879 | try func.genInlineMemset( | |
| 5880 | dst_ptr_mcv, | |
| 5881 | src_mcv, | |
| 5882 | .{ .immediate = abi_size }, | |
| 5883 | ); | |
| 5884 | }, | |
| 5873 | 5885 | .register_offset, |
| 5874 | 5886 | .memory, |
| 5875 | 5887 | .indirect, |
| ... | ... | @@ -6069,12 +6081,82 @@ fn airCmpxchg(func: *Func, inst: Air.Inst.Index) !void { |
| 6069 | 6081 | } |
| 6070 | 6082 | |
| 6071 | 6083 | fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void { |
| 6072 | _ = inst; | |
| 6073 | return func.fail("TODO implement airCmpxchg for {}", .{func.target.cpu.arch}); | |
| 6084 | const zcu = func.pt.zcu; | |
| 6085 | const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; | |
| 6086 | const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data; | |
| 6087 | ||
| 6088 | const op = extra.op(); | |
| 6089 | const order = extra.ordering(); | |
| 6090 | ||
| 6091 | const ptr_ty = func.typeOf(pl_op.operand); | |
| 6092 | const ptr_mcv = try func.resolveInst(pl_op.operand); | |
| 6093 | ||
| 6094 | const val_ty = func.typeOf(extra.operand); | |
| 6095 | const val_size = val_ty.abiSize(func.pt); | |
| 6096 | const val_mcv = try func.resolveInst(extra.operand); | |
| 6097 | ||
| 6098 | if (!math.isPowerOfTwo(val_size)) | |
| 6099 | return func.fail("TODO: airAtomicRmw non-pow 2", .{}); | |
| 6100 | ||
| 6101 | switch (val_ty.zigTypeTag(zcu)) { | |
| 6102 | .Int => {}, | |
| 6103 | inline .Bool, .Float, .Enum, .Pointer => |ty| return func.fail("TODO: airAtomicRmw {s}", .{@tagName(ty)}), | |
| 6104 | else => unreachable, | |
| 6105 | } | |
| 6106 | ||
| 6107 | switch (val_size) { | |
| 6108 | 1, 2 => return func.fail("TODO: airAtomicRmw Int {}", .{val_size}), | |
| 6109 | 4, 8 => {}, | |
| 6110 | else => unreachable, | |
| 6111 | } | |
| 6112 | ||
| 6113 | const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv); | |
| 6114 | defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock); | |
| 6115 | ||
| 6116 | const val_register, const val_lock = try func.promoteReg(val_ty, val_mcv); | |
| 6117 | defer if (val_lock) |lock| func.register_manager.unlockReg(lock); | |
| 6118 | ||
| 6119 | const result_mcv = try func.allocRegOrMem(val_ty, inst, true); | |
| 6120 | assert(result_mcv == .register); // should fit into 8 bytes | |
| 6121 | ||
| 6122 | const aq, const rl = switch (order) { | |
| 6123 | .unordered => unreachable, | |
| 6124 | .monotonic => .{ false, false }, | |
| 6125 | .acquire => .{ true, false }, | |
| 6126 | .release => .{ false, true }, | |
| 6127 | .acq_rel => .{ true, true }, | |
| 6128 | .seq_cst => .{ true, true }, | |
| 6129 | }; | |
| 6130 | ||
| 6131 | _ = try func.addInst(.{ | |
| 6132 | .tag = .pseudo, | |
| 6133 | .ops = .pseudo_amo, | |
| 6134 | .data = .{ .amo = .{ | |
| 6135 | .rd = result_mcv.register, | |
| 6136 | .rs1 = ptr_register, | |
| 6137 | .rs2 = val_register, | |
| 6138 | .aq = if (aq) .aq else .none, | |
| 6139 | .rl = if (rl) .rl else .none, | |
| 6140 | .op = switch (op) { | |
| 6141 | .Xchg => .SWAP, | |
| 6142 | .Add => .ADD, | |
| 6143 | .Sub => return func.fail("TODO: airAtomicRmw SUB", .{}), | |
| 6144 | .And => .AND, | |
| 6145 | .Nand => return func.fail("TODO: airAtomicRmw NAND", .{}), | |
| 6146 | .Or => .OR, | |
| 6147 | .Xor => .XOR, | |
| 6148 | .Max => .MAX, | |
| 6149 | .Min => .MIN, | |
| 6150 | }, | |
| 6151 | .ty = val_ty, | |
| 6152 | } }, | |
| 6153 | }); | |
| 6154 | ||
| 6155 | return func.finishAir(inst, result_mcv, .{ pl_op.operand, extra.operand, .none }); | |
| 6074 | 6156 | } |
| 6075 | 6157 | |
| 6076 | 6158 | fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { |
| 6077 | const zcu = func.bin_file.comp.module.?; | |
| 6159 | const zcu = func.pt.zcu; | |
| 6078 | 6160 | const atomic_load = func.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; |
| 6079 | 6161 | const order: std.builtin.AtomicOrder = atomic_load.order; |
| 6080 | 6162 | |
| ... | ... | @@ -6083,6 +6165,7 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void { |
| 6083 | 6165 | const ptr_mcv = try func.resolveInst(atomic_load.ptr); |
| 6084 | 6166 | |
| 6085 | 6167 | const result_mcv = try func.allocRegOrMem(elem_ty, inst, true); |
| 6168 | assert(result_mcv == .register); // should be less than 8 bytes | |
| 6086 | 6169 | |
| 6087 | 6170 | if (order == .seq_cst) { |
| 6088 | 6171 | _ = try func.addInst(.{ |
| ... | ... | @@ -6535,19 +6618,40 @@ fn getResolvedInstValue(func: *Func, inst: Air.Inst.Index) *InstTracking { |
| 6535 | 6618 | } |
| 6536 | 6619 | |
| 6537 | 6620 | fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { |
| 6538 | const pt = func.pt; | |
| 6539 | const zcu = pt.zcu; | |
| 6621 | const zcu = func.pt.zcu; | |
| 6622 | const gpa = func.gpa; | |
| 6623 | ||
| 6624 | const owner_decl_index = zcu.funcOwnerDeclIndex(func.func_index); | |
| 6625 | const lf = func.bin_file; | |
| 6626 | const src_loc = func.src_loc; | |
| 6627 | ||
| 6628 | if (val.isUndef(zcu)) { | |
| 6629 | const local_sym_index = lf.lowerUnnamedConst(func.pt, val, owner_decl_index) catch |err| { | |
| 6630 | const msg = try ErrorMsg.create(gpa, src_loc, "lowering unnamed undefined constant failed: {s}", .{@errorName(err)}); | |
| 6631 | func.err_msg = msg; | |
| 6632 | return error.CodegenFail; | |
| 6633 | }; | |
| 6634 | switch (lf.tag) { | |
| 6635 | .elf => { | |
| 6636 | const elf_file = lf.cast(link.File.Elf).?; | |
| 6637 | const local = elf_file.symbol(local_sym_index); | |
| 6638 | return MCValue{ .undef = local.esym_index }; | |
| 6639 | }, | |
| 6640 | else => unreachable, | |
| 6641 | } | |
| 6642 | } | |
| 6643 | ||
| 6540 | 6644 | const result = try codegen.genTypedValue( |
| 6541 | func.bin_file, | |
| 6542 | pt, | |
| 6543 | func.src_loc, | |
| 6645 | lf, | |
| 6646 | func.pt, | |
| 6647 | src_loc, | |
| 6544 | 6648 | val, |
| 6545 | zcu.funcOwnerDeclIndex(func.func_index), | |
| 6649 | owner_decl_index, | |
| 6546 | 6650 | ); |
| 6547 | 6651 | const mcv: MCValue = switch (result) { |
| 6548 | 6652 | .mcv => |mcv| switch (mcv) { |
| 6549 | 6653 | .none => .none, |
| 6550 | .undef => .undef, | |
| 6654 | .undef => unreachable, | |
| 6551 | 6655 | .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, |
| 6552 | 6656 | .immediate => |imm| .{ .immediate = imm }, |
| 6553 | 6657 | .memory => |addr| .{ .memory = addr }, |
src/arch/riscv64/Encoding.zig+139-21
| ... | ... | @@ -28,7 +28,7 @@ const OpCode = enum(u7) { |
| 28 | 28 | NONE = 0b00000000, |
| 29 | 29 | }; |
| 30 | 30 | |
| 31 | const Fmt = enum(u2) { | |
| 31 | const FpFmt = enum(u2) { | |
| 32 | 32 | /// 32-bit single-precision |
| 33 | 33 | S = 0b00, |
| 34 | 34 | /// 64-bit double-precision |
| ... | ... | @@ -40,6 +40,11 @@ const Fmt = enum(u2) { |
| 40 | 40 | Q = 0b11, |
| 41 | 41 | }; |
| 42 | 42 | |
| 43 | const AmoWidth = enum(u3) { | |
| 44 | W = 0b010, | |
| 45 | D = 0b011, | |
| 46 | }; | |
| 47 | ||
| 43 | 48 | const Enc = struct { |
| 44 | 49 | opcode: OpCode, |
| 45 | 50 | |
| ... | ... | @@ -49,11 +54,15 @@ const Enc = struct { |
| 49 | 54 | funct3: u3, |
| 50 | 55 | funct7: u7, |
| 51 | 56 | }, |
| 57 | amo: struct { | |
| 58 | funct5: u5, | |
| 59 | width: AmoWidth, | |
| 60 | }, | |
| 52 | 61 | /// funct5 + rm + fmt |
| 53 | 62 | fmt: struct { |
| 54 | 63 | funct5: u5, |
| 55 | 64 | rm: u3, |
| 56 | fmt: Fmt, | |
| 65 | fmt: FpFmt, | |
| 57 | 66 | }, |
| 58 | 67 | /// funct3 |
| 59 | 68 | f: struct { |
| ... | ... | @@ -202,6 +211,27 @@ pub const Mnemonic = enum { |
| 202 | 211 | // MISC |
| 203 | 212 | fence, |
| 204 | 213 | |
| 214 | // AMO | |
| 215 | amoswapw, | |
| 216 | amoaddw, | |
| 217 | amoandw, | |
| 218 | amoorw, | |
| 219 | amoxorw, | |
| 220 | amomaxw, | |
| 221 | amominw, | |
| 222 | amomaxuw, | |
| 223 | amominuw, | |
| 224 | ||
| 225 | amoswapd, | |
| 226 | amoaddd, | |
| 227 | amoandd, | |
| 228 | amoord, | |
| 229 | amoxord, | |
| 230 | amomaxd, | |
| 231 | amomind, | |
| 232 | amomaxud, | |
| 233 | amominud, | |
| 234 | ||
| 205 | 235 | pub fn encoding(mnem: Mnemonic) Enc { |
| 206 | 236 | return switch (mnem) { |
| 207 | 237 | // zig fmt: off |
| ... | ... | @@ -379,7 +409,34 @@ pub const Mnemonic = enum { |
| 379 | 409 | // MISC_MEM |
| 380 | 410 | |
| 381 | 411 | .fence => .{ .opcode = .MISC_MEM, .data = .{ .f = .{ .funct3 = 0b000 } } }, |
| 382 | ||
| 412 | ||
| 413 | // AMO | |
| 414 | ||
| 415 | .amoaddw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00000 } } }, | |
| 416 | .amoswapw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00001 } } }, | |
| 417 | // LR.W | |
| 418 | // SC.W | |
| 419 | .amoxorw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00100 } } }, | |
| 420 | .amoandw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01100 } } }, | |
| 421 | .amoorw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01000 } } }, | |
| 422 | .amominw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10000 } } }, | |
| 423 | .amomaxw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b10100 } } }, | |
| 424 | .amominuw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11000 } } }, | |
| 425 | .amomaxuw => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11100 } } }, | |
| 426 | ||
| 427 | .amoaddd => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00000 } } }, | |
| 428 | .amoswapd => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00001 } } }, | |
| 429 | // LR.D | |
| 430 | // SC.D | |
| 431 | .amoxord => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00100 } } }, | |
| 432 | .amoandd => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01100 } } }, | |
| 433 | .amoord => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01000 } } }, | |
| 434 | .amomind => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10000 } } }, | |
| 435 | .amomaxd => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b10100 } } }, | |
| 436 | .amominud => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11000 } } }, | |
| 437 | .amomaxud => .{ .opcode = .AMO, .data = .{ .amo = .{ .width = .D, .funct5 = 0b11100 } } }, | |
| 438 | ||
| 439 | ||
| 383 | 440 | |
| 384 | 441 | // zig fmt: on |
| 385 | 442 | }; |
| ... | ... | @@ -395,6 +452,7 @@ pub const InstEnc = enum { |
| 395 | 452 | U, |
| 396 | 453 | J, |
| 397 | 454 | fence, |
| 455 | amo, | |
| 398 | 456 | /// extras that have unusual op counts |
| 399 | 457 | system, |
| 400 | 458 | |
| ... | ... | @@ -526,21 +584,43 @@ pub const InstEnc = enum { |
| 526 | 584 | |
| 527 | 585 | .fence, |
| 528 | 586 | => .fence, |
| 587 | ||
| 588 | .amoswapw, | |
| 589 | .amoaddw, | |
| 590 | .amoandw, | |
| 591 | .amoorw, | |
| 592 | .amoxorw, | |
| 593 | .amomaxw, | |
| 594 | .amominw, | |
| 595 | .amomaxuw, | |
| 596 | .amominuw, | |
| 597 | ||
| 598 | .amoswapd, | |
| 599 | .amoaddd, | |
| 600 | .amoandd, | |
| 601 | .amoord, | |
| 602 | .amoxord, | |
| 603 | .amomaxd, | |
| 604 | .amomind, | |
| 605 | .amomaxud, | |
| 606 | .amominud, | |
| 607 | => .amo, | |
| 529 | 608 | }; |
| 530 | 609 | } |
| 531 | 610 | |
| 532 | pub fn opsList(enc: InstEnc) [4]std.meta.FieldEnum(Operand) { | |
| 611 | pub fn opsList(enc: InstEnc) [5]std.meta.FieldEnum(Operand) { | |
| 533 | 612 | return switch (enc) { |
| 534 | 613 | // zig fmt: off |
| 535 | .R => .{ .reg, .reg, .reg, .none }, | |
| 536 | .R4 => .{ .reg, .reg, .reg, .reg }, | |
| 537 | .I => .{ .reg, .reg, .imm, .none }, | |
| 538 | .S => .{ .reg, .reg, .imm, .none }, | |
| 539 | .B => .{ .reg, .reg, .imm, .none }, | |
| 540 | .U => .{ .reg, .imm, .none, .none }, | |
| 541 | .J => .{ .reg, .imm, .none, .none }, | |
| 542 | .system => .{ .none, .none, .none, .none }, | |
| 543 | .fence => .{ .barrier, .barrier, .none, .none }, | |
| 614 | .R => .{ .reg, .reg, .reg, .none, .none, }, | |
| 615 | .R4 => .{ .reg, .reg, .reg, .reg, .none, }, | |
| 616 | .I => .{ .reg, .reg, .imm, .none, .none, }, | |
| 617 | .S => .{ .reg, .reg, .imm, .none, .none, }, | |
| 618 | .B => .{ .reg, .reg, .imm, .none, .none, }, | |
| 619 | .U => .{ .reg, .imm, .none, .none, .none, }, | |
| 620 | .J => .{ .reg, .imm, .none, .none, .none, }, | |
| 621 | .system => .{ .none, .none, .none, .none, .none, }, | |
| 622 | .fence => .{ .barrier, .barrier, .none, .none, .none, }, | |
| 623 | .amo => .{ .reg, .reg, .reg, .barrier, .barrier }, | |
| 544 | 624 | // zig fmt: on |
| 545 | 625 | }; |
| 546 | 626 | } |
| ... | ... | @@ -611,19 +691,29 @@ pub const Data = union(InstEnc) { |
| 611 | 691 | pred: u4, |
| 612 | 692 | _ignored: u4 = 0, |
| 613 | 693 | }, |
| 614 | system: void, | |
| 694 | amo: packed struct { | |
| 695 | opcode: u7, | |
| 696 | rd: u5, | |
| 697 | funct3: u3, | |
| 698 | rs1: u5, | |
| 699 | rs2: u5, | |
| 700 | rl: bool, | |
| 701 | aq: bool, | |
| 702 | funct5: u5, | |
| 703 | }, | |
| 704 | system: u32, | |
| 705 | ||
| 706 | comptime { | |
| 707 | for (std.meta.fields(Data)) |field| { | |
| 708 | assert(@bitSizeOf(field.type) == 32); | |
| 709 | } | |
| 710 | } | |
| 615 | 711 | |
| 616 | 712 | pub fn toU32(self: Data) u32 { |
| 617 | 713 | return switch (self) { |
| 618 | 714 | // zig fmt: off |
| 619 | .R => |v| @bitCast(v), | |
| 620 | .R4 => |v| @bitCast(v), | |
| 621 | .I => |v| @bitCast(v), | |
| 622 | .S => |v| @bitCast(v), | |
| 623 | 715 | .B => |v| @as(u32, @intCast(v.opcode)) + (@as(u32, @intCast(v.imm11)) << 7) + (@as(u32, @intCast(v.imm1_4)) << 8) + (@as(u32, @intCast(v.funct3)) << 12) + (@as(u32, @intCast(v.rs1)) << 15) + (@as(u32, @intCast(v.rs2)) << 20) + (@as(u32, @intCast(v.imm5_10)) << 25) + (@as(u32, @intCast(v.imm12)) << 31), |
| 624 | .U => |v| @bitCast(v), | |
| 625 | .J => |v| @bitCast(v), | |
| 626 | .fence => |v| @bitCast(v), | |
| 716 | inline else => |v| @bitCast(v), | |
| 627 | 717 | .system => unreachable, |
| 628 | 718 | // zig fmt: on |
| 629 | 719 | }; |
| ... | ... | @@ -792,6 +882,34 @@ pub const Data = union(InstEnc) { |
| 792 | 882 | }, |
| 793 | 883 | }; |
| 794 | 884 | }, |
| 885 | .amo => { | |
| 886 | assert(ops.len == 5); | |
| 887 | ||
| 888 | const rd = ops[0]; | |
| 889 | const rs1 = ops[1]; | |
| 890 | const rs2 = ops[2]; | |
| 891 | const rl = ops[3]; | |
| 892 | const aq = ops[4]; | |
| 893 | ||
| 894 | const ret: Data = .{ | |
| 895 | .amo = .{ | |
| 896 | .rd = rd.reg.encodeId(), | |
| 897 | .rs1 = rs1.reg.encodeId(), | |
| 898 | .rs2 = rs2.reg.encodeId(), | |
| 899 | ||
| 900 | // TODO: https://github.com/ziglang/zig/issues/20113 | |
| 901 | .rl = if (rl.barrier == .rl) true else false, | |
| 902 | .aq = if (aq.barrier == .aq) true else false, | |
| 903 | ||
| 904 | .opcode = @intFromEnum(enc.opcode), | |
| 905 | .funct3 = @intFromEnum(enc.data.amo.width), | |
| 906 | .funct5 = enc.data.amo.funct5, | |
| 907 | }, | |
| 908 | }; | |
| 909 | ||
| 910 | std.debug.print("ret: {}, {}", .{ ret.amo.rl, rl.barrier == .rl }); | |
| 911 | return ret; | |
| 912 | }, | |
| 795 | 913 | |
| 796 | 914 | else => std.debug.panic("TODO: construct {s}", .{@tagName(inst_enc)}), |
| 797 | 915 | } |
src/arch/riscv64/Lower.zig+26
| ... | ... | @@ -412,6 +412,32 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index) Error!struct { |
| 412 | 412 | }); |
| 413 | 413 | }, |
| 414 | 414 | |
| 415 | .pseudo_amo => { | |
| 416 | const amo = inst.data.amo; | |
| 417 | const is_d = amo.ty.abiSize(pt) == 8; | |
| 418 | const is_un = amo.ty.isUnsignedInt(pt.zcu); | |
| 419 | ||
| 420 | const mnem: Encoding.Mnemonic = switch (amo.op) { | |
| 421 | // zig fmt: off | |
| 422 | .SWAP => if (is_d) .amoswapd else .amoswapw, | |
| 423 | .ADD => if (is_d) .amoaddd else .amoaddw, | |
| 424 | .AND => if (is_d) .amoandd else .amoandw, | |
| 425 | .OR => if (is_d) .amoord else .amoorw, | |
| 426 | .XOR => if (is_d) .amoxord else .amoxorw, | |
| 427 | .MAX => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw, | |
| 428 | .MIN => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw, | |
| 429 | // zig fmt: on | |
| 430 | }; | |
| 431 | ||
| 432 | try lower.emit(mnem, &.{ | |
| 433 | .{ .reg = inst.data.amo.rd }, | |
| 434 | .{ .reg = inst.data.amo.rs1 }, | |
| 435 | .{ .reg = inst.data.amo.rs2 }, | |
| 436 | .{ .barrier = inst.data.amo.rl }, | |
| 437 | .{ .barrier = inst.data.amo.aq }, | |
| 438 | }); | |
| 439 | }, | |
| 440 | ||
| 415 | 441 | else => return lower.fail("TODO lower: psuedo {s}", .{@tagName(inst.ops)}), |
| 416 | 442 | }, |
| 417 | 443 | } |
src/arch/riscv64/Mir.zig+34-2
| ... | ... | @@ -39,8 +39,6 @@ pub const Inst = struct { |
| 39 | 39 | ecall, |
| 40 | 40 | unimp, |
| 41 | 41 | |
| 42 | fence, | |
| 43 | ||
| 44 | 42 | add, |
| 45 | 43 | addw, |
| 46 | 44 | sub, |
| ... | ... | @@ -82,6 +80,8 @@ pub const Inst = struct { |
| 82 | 80 | sh, |
| 83 | 81 | sb, |
| 84 | 82 | |
| 83 | fence, | |
| 84 | ||
| 85 | 85 | // M extension |
| 86 | 86 | mul, |
| 87 | 87 | mulw, |
| ... | ... | @@ -136,6 +136,9 @@ pub const Inst = struct { |
| 136 | 136 | fltd, |
| 137 | 137 | fled, |
| 138 | 138 | |
| 139 | /// A Extension Instructions | |
| 140 | amo, | |
| 141 | ||
| 139 | 142 | /// A pseudo-instruction. Used for anything that isn't 1:1 with an |
| 140 | 143 | /// assembly instruction. |
| 141 | 144 | pseudo, |
| ... | ... | @@ -254,6 +257,16 @@ pub const Inst = struct { |
| 254 | 257 | pred: Barrier, |
| 255 | 258 | succ: Barrier, |
| 256 | 259 | }, |
| 260 | ||
| 261 | amo: struct { | |
| 262 | rd: Register, | |
| 263 | rs1: Register, | |
| 264 | rs2: Register, | |
| 265 | aq: Barrier, | |
| 266 | rl: Barrier, | |
| 267 | op: AmoOp, | |
| 268 | ty: Type, | |
| 269 | }, | |
| 257 | 270 | }; |
| 258 | 271 | |
| 259 | 272 | pub const Ops = enum { |
| ... | ... | @@ -343,6 +356,9 @@ pub const Inst = struct { |
| 343 | 356 | |
| 344 | 357 | /// IORW, IORW |
| 345 | 358 | fence, |
| 359 | ||
| 360 | /// Ordering, Src, Addr, Dest | |
| 361 | pseudo_amo, | |
| 346 | 362 | }; |
| 347 | 363 | |
| 348 | 364 | // Make sure we don't accidentally make instructions bigger than expected. |
| ... | ... | @@ -379,9 +395,25 @@ pub const FrameLoc = struct { |
| 379 | 395 | }; |
| 380 | 396 | |
| 381 | 397 | pub const Barrier = enum(u4) { |
| 398 | // Fence | |
| 382 | 399 | r = 0b0001, |
| 383 | 400 | w = 0b0010, |
| 384 | 401 | rw = 0b0011, |
| 402 | ||
| 403 | // Amo | |
| 404 | none, | |
| 405 | aq, | |
| 406 | rl, | |
| 407 | }; | |
| 408 | ||
| 409 | pub const AmoOp = enum(u5) { | |
| 410 | SWAP, | |
| 411 | ADD, | |
| 412 | AND, | |
| 413 | OR, | |
| 414 | XOR, | |
| 415 | MAX, | |
| 416 | MIN, | |
| 385 | 417 | }; |
| 386 | 418 | |
| 387 | 419 | /// Returns the requested data, as well as the new index which is at the start of the |
src/arch/riscv64/encoder.zig+5-3
| ... | ... | @@ -1,6 +1,6 @@ |
| 1 | 1 | pub const Instruction = struct { |
| 2 | 2 | encoding: Encoding, |
| 3 | ops: [4]Operand = .{.none} ** 4, | |
| 3 | ops: [5]Operand = .{.none} ** 5, | |
| 4 | 4 | |
| 5 | 5 | pub const Operand = union(enum) { |
| 6 | 6 | none, |
| ... | ... | @@ -12,16 +12,18 @@ pub const Instruction = struct { |
| 12 | 12 | |
| 13 | 13 | pub fn new(mnemonic: Encoding.Mnemonic, ops: []const Operand) !Instruction { |
| 14 | 14 | const encoding = (try Encoding.findByMnemonic(mnemonic, ops)) orelse { |
| 15 | std.log.err("no encoding found for: {s} [{s} {s} {s}]", .{ | |
| 15 | std.log.err("no encoding found for: {s} [{s} {s} {s} {s} {s}]", .{ | |
| 16 | 16 | @tagName(mnemonic), |
| 17 | 17 | @tagName(if (ops.len > 0) ops[0] else .none), |
| 18 | 18 | @tagName(if (ops.len > 1) ops[1] else .none), |
| 19 | 19 | @tagName(if (ops.len > 2) ops[2] else .none), |
| 20 | @tagName(if (ops.len > 3) ops[3] else .none), | |
| 21 | @tagName(if (ops.len > 4) ops[4] else .none), | |
| 20 | 22 | }); |
| 21 | 23 | return error.InvalidInstruction; |
| 22 | 24 | }; |
| 23 | 25 | |
| 24 | var result_ops: [4]Operand = .{.none} ** 4; | |
| 26 | var result_ops: [5]Operand = .{.none} ** 5; | |
| 25 | 27 | @memcpy(result_ops[0..ops.len], ops); |
| 26 | 28 | |
| 27 | 29 | return .{ |
src/codegen.zig+2-1
| ... | ... | @@ -987,8 +987,9 @@ pub fn genTypedValue( |
| 987 | 987 | |
| 988 | 988 | log.debug("genTypedValue: val = {}", .{val.fmtValue(pt, null)}); |
| 989 | 989 | |
| 990 | if (val.isUndef(zcu)) | |
| 990 | if (val.isUndef(zcu)) { | |
| 991 | 991 | return GenResult.mcv(.undef); |
| 992 | } | |
| 992 | 993 | |
| 993 | 994 | const owner_decl = zcu.declPtr(owner_decl_index); |
| 994 | 995 | const namespace = zcu.namespacePtr(owner_decl.src_namespace); |
src/link/Elf/ZigObject.zig+11-7
| ... | ... | @@ -540,8 +540,8 @@ inline fn isGlobal(index: Symbol.Index) bool { |
| 540 | 540 | |
| 541 | 541 | pub fn symbol(self: ZigObject, index: Symbol.Index) Symbol.Index { |
| 542 | 542 | const actual_index = index & symbol_mask; |
| 543 | if (isGlobal(index)) return self.global_symbols.items[actual_index]; | |
| 544 | return self.local_symbols.items[actual_index]; | |
| 543 | if (isGlobal(index)) return self.globals()[actual_index]; | |
| 544 | return self.locals()[actual_index]; | |
| 545 | 545 | } |
| 546 | 546 | |
| 547 | 547 | pub fn elfSym(self: *ZigObject, index: Symbol.Index) *elf.Elf64_Sym { |
| ... | ... | @@ -1334,11 +1334,15 @@ fn lowerConst( |
| 1334 | 1334 | |
| 1335 | 1335 | const sym_index = try self.addAtom(elf_file); |
| 1336 | 1336 | |
| 1337 | const res = try codegen.generateSymbol(&elf_file.base, pt, src_loc, val, &code_buffer, .{ | |
| 1338 | .none = {}, | |
| 1339 | }, .{ | |
| 1340 | .parent_atom_index = sym_index, | |
| 1341 | }); | |
| 1337 | const res = try codegen.generateSymbol( | |
| 1338 | &elf_file.base, | |
| 1339 | pt, | |
| 1340 | src_loc, | |
| 1341 | val, | |
| 1342 | &code_buffer, | |
| 1343 | .{ .none = {} }, | |
| 1344 | .{ .parent_atom_index = sym_index }, | |
| 1345 | ); | |
| 1342 | 1346 | const code = switch (res) { |
| 1343 | 1347 | .ok => code_buffer.items, |
| 1344 | 1348 | .fail => |em| return .{ .fail = em }, |
test/behavior/atomics.zig-16
| ... | ... | @@ -188,21 +188,6 @@ test "atomic store" { |
| 188 | 188 | if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO |
| 189 | 189 | if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO |
| 190 | 190 | if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; |
| 191 | if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; | |
| 192 | ||
| 193 | var x: u32 = 0; | |
| 194 | @atomicStore(u32, &x, 1, .seq_cst); | |
| 195 | try expect(@atomicLoad(u32, &x, .seq_cst) == 1); | |
| 196 | @atomicStore(u32, &x, 12345678, .seq_cst); | |
| 197 | try expect(@atomicLoad(u32, &x, .seq_cst) == 12345678); | |
| 198 | } | |
| 199 | ||
| 200 | test "atomic store comptime" { | |
| 201 | if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO | |
| 202 | if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO | |
| 203 | if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO | |
| 204 | if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; | |
| 205 | if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; | |
| 206 | 191 | |
| 207 | 192 | try comptime testAtomicStore(); |
| 208 | 193 | try testAtomicStore(); |
| ... | ... | @@ -451,7 +436,6 @@ test "return @atomicStore, using it as a void value" { |
| 451 | 436 | if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO |
| 452 | 437 | if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO |
| 453 | 438 | if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; |
| 454 | if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; | |
| 455 | 439 | |
| 456 | 440 | const S = struct { |
| 457 | 441 | const A = struct { |