authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-07-28 17:06:57-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-07-31 12:59:19-07:00
log075f93fa108030fb0dd12faa6e389ace302cfb4c
tree73cc539ff6dc66dc67cb35ba30af0c9c2a263098
parent1ab15b6c9c529c9acc85f4b0bf3fcaea97a5a48e

stage2 LLVM: Pass inline assembly outputs directly when not targeting memory

This change provides a basic implementation of #2349 for stage2. There's still quite a lot of work before this logic is as complete as what's in Clang (https://github.com/llvm/llvm-project/blob/b3645353041818f61e2580635409ddb81ff5a272/clang/lib/CodeGen/CGStmt.cpp#L2304-L2795), particularly considering the diversity of constraints across targets. It's probably not worth doing the complete work until there's a clearer picture for constraints in Zig's future dedicated ASM syntax, but at least this gives us a small improvement for now. As a bonus, this also fixes a bug with how we were handling `_` identifiers.

1 files changed, 109 insertions(+), 32 deletions(-)

src/codegen/llvm.zig+109-32
......@@ -5491,22 +5491,26 @@ pub const FuncGen = struct {
54915491 defer arena_allocator.deinit();
54925492 const arena = arena_allocator.allocator();
54935493
5494 const return_count: u8 = for (outputs) |output| {
5495 if (output == .none) break 1;
5496 } else 0;
5497 const llvm_params_len = inputs.len + outputs.len - return_count;
5498 const llvm_param_types = try arena.alloc(*const llvm.Type, llvm_params_len);
5499 const llvm_param_values = try arena.alloc(*const llvm.Value, llvm_params_len);
5500 const llvm_param_attrs = try arena.alloc(bool, llvm_params_len);
5494 // The exact number of return / parameter values depends on which output values
5495 // are passed by reference as indirect outputs (determined below).
5496 const max_return_count = outputs.len;
5497 const llvm_ret_types = try arena.alloc(*const llvm.Type, max_return_count);
5498 const llvm_ret_indirect = try arena.alloc(bool, max_return_count);
5499
5500 const max_param_count = inputs.len + outputs.len;
5501 const llvm_param_types = try arena.alloc(*const llvm.Type, max_param_count);
5502 const llvm_param_values = try arena.alloc(*const llvm.Value, max_param_count);
5503 const llvm_param_attrs = try arena.alloc(bool, max_param_count);
55015504 const target = self.dg.module.getTarget();
55025505
5506 var llvm_ret_i: usize = 0;
55035507 var llvm_param_i: usize = 0;
5504 var total_i: usize = 0;
5508 var total_i: u16 = 0;
55055509
5506 var name_map: std.StringArrayHashMapUnmanaged(void) = .{};
5507 try name_map.ensureUnusedCapacity(arena, outputs.len + inputs.len);
5510 var name_map: std.StringArrayHashMapUnmanaged(u16) = .{};
5511 try name_map.ensureUnusedCapacity(arena, max_param_count);
55085512
5509 for (outputs) |output| {
5513 for (outputs) |output, i| {
55105514 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
55115515 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
55125516 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
......@@ -5519,15 +5523,30 @@ pub const FuncGen = struct {
55195523 llvm_constraints.appendAssumeCapacity(',');
55205524 }
55215525 llvm_constraints.appendAssumeCapacity('=');
5526
5527 // Pass any non-return outputs indirectly, if the constraint accepts a memory location
5528 llvm_ret_indirect[i] = (output != .none) and constraintAllowsMemory(constraint);
55225529 if (output != .none) {
55235530 try llvm_constraints.ensureUnusedCapacity(self.gpa, llvm_constraints.capacity + 1);
5524 llvm_constraints.appendAssumeCapacity('*');
5525
55265531 const output_inst = try self.resolveInst(output);
5527 llvm_param_values[llvm_param_i] = output_inst;
5528 llvm_param_types[llvm_param_i] = output_inst.typeOf();
5529 llvm_param_attrs[llvm_param_i] = true;
5530 llvm_param_i += 1;
5532
5533 if (llvm_ret_indirect[i]) {
5534 // Pass the result by reference as an indirect output (e.g. "=*m")
5535 llvm_constraints.appendAssumeCapacity('*');
5536
5537 llvm_param_values[llvm_param_i] = output_inst;
5538 llvm_param_types[llvm_param_i] = output_inst.typeOf();
5539 llvm_param_attrs[llvm_param_i] = true;
5540 llvm_param_i += 1;
5541 } else {
5542 // Pass the result directly (e.g. "=r")
5543 llvm_ret_types[llvm_ret_i] = output_inst.typeOf().getElementType();
5544 llvm_ret_i += 1;
5545 }
5546 } else {
5547 const ret_ty = self.air.typeOfIndex(inst);
5548 llvm_ret_types[llvm_ret_i] = try self.dg.lowerType(ret_ty);
5549 llvm_ret_i += 1;
55315550 }
55325551
55335552 // LLVM uses commas internally to separate different constraints,
......@@ -5536,13 +5555,16 @@ pub const FuncGen = struct {
55365555 // to GCC's inline assembly.
55375556 // http://llvm.org/docs/LangRef.html#constraint-codes
55385557 for (constraint[1..]) |byte| {
5539 llvm_constraints.appendAssumeCapacity(switch (byte) {
5540 ',' => '|',
5541 else => byte,
5542 });
5558 switch (byte) {
5559 ',' => llvm_constraints.appendAssumeCapacity('|'),
5560 '*' => {}, // Indirect outputs are handled above
5561 else => llvm_constraints.appendAssumeCapacity(byte),
5562 }
55435563 }
55445564
5545 name_map.putAssumeCapacityNoClobber(name, {});
5565 if (!std.mem.eql(u8, name, "_")) {
5566 name_map.putAssumeCapacityNoClobber(name, total_i);
5567 }
55465568 total_i += 1;
55475569 }
55485570
......@@ -5594,7 +5616,7 @@ pub const FuncGen = struct {
55945616 }
55955617
55965618 if (!std.mem.eql(u8, name, "_")) {
5597 name_map.putAssumeCapacityNoClobber(name, {});
5619 name_map.putAssumeCapacityNoClobber(name, total_i);
55985620 }
55995621
56005622 // In the case of indirect inputs, LLVM requires the callsite to have
......@@ -5625,6 +5647,11 @@ pub const FuncGen = struct {
56255647 }
56265648 }
56275649
5650 // We have finished scanning through all inputs/outputs, so the number of
5651 // parameters and return values is known.
5652 const param_count = llvm_param_i;
5653 const return_count = llvm_ret_i;
5654
56285655 // For some targets, Clang unconditionally adds some clobbers to all inline assembly.
56295656 // While this is probably not strictly necessary, if we don't follow Clang's lead
56305657 // here then we may risk tripping LLVM bugs since anything not used by Clang tends
......@@ -5682,7 +5709,7 @@ pub const FuncGen = struct {
56825709 const name = asm_source[name_start..i];
56835710 state = .start;
56845711
5685 const index = name_map.getIndex(name) orelse {
5712 const index = name_map.get(name) orelse {
56865713 // we should validate the assembly in Sema; by now it is too late
56875714 return self.todo("unknown input or output name: '{s}'", .{name});
56885715 };
......@@ -5693,12 +5720,20 @@ pub const FuncGen = struct {
56935720 }
56945721 }
56955722
5696 const ret_ty = self.air.typeOfIndex(inst);
5697 const ret_llvm_ty = try self.dg.lowerType(ret_ty);
5723 const ret_llvm_ty = switch (return_count) {
5724 0 => self.context.voidType(),
5725 1 => llvm_ret_types[0],
5726 else => self.context.structType(
5727 llvm_ret_types.ptr,
5728 @intCast(c_uint, return_count),
5729 .False,
5730 ),
5731 };
5732
56985733 const llvm_fn_ty = llvm.functionType(
56995734 ret_llvm_ty,
57005735 llvm_param_types.ptr,
5701 @intCast(c_uint, llvm_param_types.len),
5736 @intCast(c_uint, param_count),
57025737 .False,
57035738 );
57045739 const asm_fn = llvm.getInlineAsm(
......@@ -5715,18 +5750,40 @@ pub const FuncGen = struct {
57155750 const call = self.builder.buildCall(
57165751 asm_fn,
57175752 llvm_param_values.ptr,
5718 @intCast(c_uint, llvm_param_values.len),
5753 @intCast(c_uint, param_count),
57195754 .C,
57205755 .Auto,
57215756 "",
57225757 );
5723 for (llvm_param_attrs) |need_elem_ty, i| {
5758 for (llvm_param_attrs[0..param_count]) |need_elem_ty, i| {
57245759 if (need_elem_ty) {
57255760 const elem_ty = llvm_param_types[i].getElementType();
57265761 llvm.setCallElemTypeAttr(call, i, elem_ty);
57275762 }
57285763 }
5729 return call;
5764
5765 var ret_val = call;
5766 llvm_ret_i = 0;
5767 for (outputs) |output, i| {
5768 if (llvm_ret_indirect[i]) continue;
5769
5770 const output_value = if (return_count > 1) b: {
5771 break :b self.builder.buildExtractValue(call, @intCast(c_uint, llvm_ret_i), "");
5772 } else call;
5773
5774 if (output != .none) {
5775 const output_ptr = try self.resolveInst(output);
5776 const output_ptr_ty = self.air.typeOf(output);
5777
5778 const store_inst = self.builder.buildStore(output_value, output_ptr);
5779 store_inst.setAlignment(output_ptr_ty.ptrAlignment(target));
5780 } else {
5781 ret_val = output_value;
5782 }
5783 llvm_ret_i += 1;
5784 }
5785
5786 return ret_val;
57305787 }
57315788
57325789 fn airIsNonNull(
......@@ -9709,10 +9766,30 @@ fn errUnionErrorOffset(payload_ty: Type, target: std.Target) u1 {
97099766 return @boolToInt(Type.anyerror.abiAlignment(target) <= payload_ty.abiAlignment(target));
97109767}
97119768
9769/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location
9770///
9771/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
97129772fn constraintAllowsMemory(constraint: []const u8) bool {
9713 return constraint[0] == 'm';
9773 // TODO: This implementation is woefully incomplete.
9774 for (constraint) |byte| {
9775 switch (byte) {
9776 '=', '*', ',', '&' => {},
9777 'm', 'o', 'X', 'g' => return true,
9778 else => {},
9779 }
9780 } else return false;
97149781}
97159782
9783/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register
9784///
9785/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
97169786fn constraintAllowsRegister(constraint: []const u8) bool {
9717 return constraint[0] != 'm';
9787 // TODO: This implementation is woefully incomplete.
9788 for (constraint) |byte| {
9789 switch (byte) {
9790 '=', '*', ',', '&' => {},
9791 'm', 'o' => {},
9792 else => return true,
9793 }
9794 } else return false;
97189795}