authorgravatar for shawn@git.icuShawn Landden <shawn@git.icu> 2019-06-18 17:28:49-05:00
committergravatar for shawn@git.icuShawn Landden <shawn@git.icu> 2019-06-19 12:07:02-05:00
logfce2d2d18be279359dcd75254506d46085c59aaf
tree217e320590614581b201ba720b7236bdf7c6afa7
parentbbfb53d52411cc5b1f560293c757bff252e1e06f

stage1: add support for @mulAdd fused-multiply-add for floats and vectors of floats

Not all of the softfloat library is being built.... Vector support is very buggy at the moment, but should work when the bugs are fixed. (as I had the same code working with another vector function, that hasn't been merged yet).

8 files changed, 292 insertions(+), 7 deletions(-)

CMakeLists.txt+2
...@@ -389,6 +389,8 @@ set(EMBEDDED_SOFTFLOAT_SOURCES...@@ -389,6 +389,8 @@ set(EMBEDDED_SOFTFLOAT_SOURCES
389 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/s_subMagsF32.c"389 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/s_subMagsF32.c"
390 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/s_subMagsF64.c"390 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/s_subMagsF64.c"
391 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/s_tryPropagateNaNF128M.c"391 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/s_tryPropagateNaNF128M.c"
392 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/f16_mulAdd.c"
393 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/f128M_mulAdd.c"
392 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/softfloat_state.c"394 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/softfloat_state.c"
393 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/ui32_to_f128M.c"395 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/ui32_to_f128M.c"
394 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/ui64_to_f128M.c"396 "${CMAKE_SOURCE_DIR}/deps/SoftFloat-3e/source/ui64_to_f128M.c"
doc/langref.html.in+7
...@@ -6259,6 +6259,13 @@ comptime {...@@ -6259,6 +6259,13 @@ comptime {
6259 This function is only valid within function scope.6259 This function is only valid within function scope.
6260 </p>6260 </p>
62616261
6262 {#header_close#}
6263 {#header_open|@mulAdd#}
6264 <pre>{#syntax#}@mulAdd(comptime T: type, a: T, b: T, c: T) T{#endsyntax#}</pre>
6265 <p>
6266 Fused multiply add (for floats), similar to {#syntax#}(a * b) + c{#endsyntax#}, except
6267 only rounds once, and is thus more accurate.
6268 </p>
6262 {#header_close#}6269 {#header_close#}
62636270
6264 {#header_open|@byteSwap#}6271 {#header_open|@byteSwap#}
src/all_types.hpp+13
...@@ -1406,6 +1406,7 @@ enum BuiltinFnId {...@@ -1406,6 +1406,7 @@ enum BuiltinFnId {
1406 BuiltinFnIdSubWithOverflow,1406 BuiltinFnIdSubWithOverflow,
1407 BuiltinFnIdMulWithOverflow,1407 BuiltinFnIdMulWithOverflow,
1408 BuiltinFnIdShlWithOverflow,1408 BuiltinFnIdShlWithOverflow,
1409 BuiltinFnIdMulAdd,
1409 BuiltinFnIdCInclude,1410 BuiltinFnIdCInclude,
1410 BuiltinFnIdCDefine,1411 BuiltinFnIdCDefine,
1411 BuiltinFnIdCUndef,1412 BuiltinFnIdCUndef,
...@@ -1554,6 +1555,7 @@ enum ZigLLVMFnId {...@@ -1554,6 +1555,7 @@ enum ZigLLVMFnId {
1554 ZigLLVMFnIdClz,1555 ZigLLVMFnIdClz,
1555 ZigLLVMFnIdPopCount,1556 ZigLLVMFnIdPopCount,
1556 ZigLLVMFnIdOverflowArithmetic,1557 ZigLLVMFnIdOverflowArithmetic,
1558 ZigLLVMFnIdFMA,
1557 ZigLLVMFnIdFloor,1559 ZigLLVMFnIdFloor,
1558 ZigLLVMFnIdCeil,1560 ZigLLVMFnIdCeil,
1559 ZigLLVMFnIdSqrt,1561 ZigLLVMFnIdSqrt,
...@@ -1584,6 +1586,7 @@ struct ZigLLVMFnKey {...@@ -1584,6 +1586,7 @@ struct ZigLLVMFnKey {
1584 } pop_count;1586 } pop_count;
1585 struct {1587 struct {
1586 uint32_t bit_count;1588 uint32_t bit_count;
1589 uint32_t vector_len; // 0 means not a vector
1587 } floating;1590 } floating;
1588 struct {1591 struct {
1589 AddSubMul add_sub_mul;1592 AddSubMul add_sub_mul;
...@@ -2235,6 +2238,7 @@ enum IrInstructionId {...@@ -2235,6 +2238,7 @@ enum IrInstructionId {
2235 IrInstructionIdHandle,2238 IrInstructionIdHandle,
2236 IrInstructionIdAlignOf,2239 IrInstructionIdAlignOf,
2237 IrInstructionIdOverflowOp,2240 IrInstructionIdOverflowOp,
2241 IrInstructionIdMulAdd,
2238 IrInstructionIdTestErr,2242 IrInstructionIdTestErr,
2239 IrInstructionIdUnwrapErrCode,2243 IrInstructionIdUnwrapErrCode,
2240 IrInstructionIdUnwrapErrPayload,2244 IrInstructionIdUnwrapErrPayload,
...@@ -3038,6 +3042,15 @@ struct IrInstructionOverflowOp {...@@ -3038,6 +3042,15 @@ struct IrInstructionOverflowOp {
3038 ZigType *result_ptr_type;3042 ZigType *result_ptr_type;
3039};3043};
30403044
3045struct IrInstructionMulAdd {
3046 IrInstruction base;
3047
3048 IrInstruction *type_value;
3049 IrInstruction *op1;
3050 IrInstruction *op2;
3051 IrInstruction *op3;
3052};
3053
3041struct IrInstructionAlignOf {3054struct IrInstructionAlignOf {
3042 IrInstruction base;3055 IrInstruction base;
30433056
src/analyze.cpp+4-3
...@@ -5737,11 +5737,11 @@ uint32_t zig_llvm_fn_key_hash(ZigLLVMFnKey x) {...@@ -5737,11 +5737,11 @@ uint32_t zig_llvm_fn_key_hash(ZigLLVMFnKey x) {
5737 case ZigLLVMFnIdPopCount:5737 case ZigLLVMFnIdPopCount:
5738 return (uint32_t)(x.data.clz.bit_count) * (uint32_t)101195049;5738 return (uint32_t)(x.data.clz.bit_count) * (uint32_t)101195049;
5739 case ZigLLVMFnIdFloor:5739 case ZigLLVMFnIdFloor:
5740 return (uint32_t)(x.data.floating.bit_count) * (uint32_t)1899859168;
5741 case ZigLLVMFnIdCeil:5740 case ZigLLVMFnIdCeil:
5742 return (uint32_t)(x.data.floating.bit_count) * (uint32_t)1953839089;
5743 case ZigLLVMFnIdSqrt:5741 case ZigLLVMFnIdSqrt:
5744 return (uint32_t)(x.data.floating.bit_count) * (uint32_t)2225366385;5742 case ZigLLVMFnIdFMA:
5743 return (uint32_t)(x.data.floating.bit_count) * ((uint32_t)x.id + 1025) +
5744 (uint32_t)(x.data.floating.vector_len) * (((uint32_t)x.id << 5) + 1025);
5745 case ZigLLVMFnIdBswap:5745 case ZigLLVMFnIdBswap:
5746 return (uint32_t)(x.data.bswap.bit_count) * (uint32_t)3661994335;5746 return (uint32_t)(x.data.bswap.bit_count) * (uint32_t)3661994335;
5747 case ZigLLVMFnIdBitReverse:5747 case ZigLLVMFnIdBitReverse:
...@@ -5772,6 +5772,7 @@ bool zig_llvm_fn_key_eql(ZigLLVMFnKey a, ZigLLVMFnKey b) {...@@ -5772,6 +5772,7 @@ bool zig_llvm_fn_key_eql(ZigLLVMFnKey a, ZigLLVMFnKey b) {
5772 case ZigLLVMFnIdFloor:5772 case ZigLLVMFnIdFloor:
5773 case ZigLLVMFnIdCeil:5773 case ZigLLVMFnIdCeil:
5774 case ZigLLVMFnIdSqrt:5774 case ZigLLVMFnIdSqrt:
5775 case ZigLLVMFnIdFMA:
5775 return a.data.floating.bit_count == b.data.floating.bit_count;5776 return a.data.floating.bit_count == b.data.floating.bit_count;
5776 case ZigLLVMFnIdOverflowArithmetic:5777 case ZigLLVMFnIdOverflowArithmetic:
5777 return (a.data.overflow_arithmetic.bit_count == b.data.overflow_arithmetic.bit_count) &&5778 return (a.data.overflow_arithmetic.bit_count == b.data.overflow_arithmetic.bit_count) &&
src/codegen.cpp+42-4
...@@ -807,31 +807,51 @@ static LLVMValueRef get_int_overflow_fn(CodeGen *g, ZigType *operand_type, AddSu...@@ -807,31 +807,51 @@ static LLVMValueRef get_int_overflow_fn(CodeGen *g, ZigType *operand_type, AddSu
807}807}
808808
809static LLVMValueRef get_float_fn(CodeGen *g, ZigType *type_entry, ZigLLVMFnId fn_id) {809static LLVMValueRef get_float_fn(CodeGen *g, ZigType *type_entry, ZigLLVMFnId fn_id) {
810 assert(type_entry->id == ZigTypeIdFloat);810 assert(type_entry->id == ZigTypeIdFloat ||
811 type_entry->id == ZigTypeIdVector);
812
813 bool is_vector = (type_entry->id == ZigTypeIdVector);
814 ZigType *float_type = is_vector ? type_entry->data.vector.elem_type : type_entry;
811815
812 ZigLLVMFnKey key = {};816 ZigLLVMFnKey key = {};
813 key.id = fn_id;817 key.id = fn_id;
814 key.data.floating.bit_count = (uint32_t)type_entry->data.floating.bit_count;818 key.data.floating.bit_count = (uint32_t)float_type->data.floating.bit_count;
819 key.data.floating.vector_len = is_vector ? (uint32_t)type_entry->data.vector.len : 0;
815820
816 auto existing_entry = g->llvm_fn_table.maybe_get(key);821 auto existing_entry = g->llvm_fn_table.maybe_get(key);
817 if (existing_entry)822 if (existing_entry)
818 return existing_entry->value;823 return existing_entry->value;
819824
820 const char *name;825 const char *name;
826 uint32_t num_args;
821 if (fn_id == ZigLLVMFnIdFloor) {827 if (fn_id == ZigLLVMFnIdFloor) {
822 name = "floor";828 name = "floor";
829 num_args = 1;
823 } else if (fn_id == ZigLLVMFnIdCeil) {830 } else if (fn_id == ZigLLVMFnIdCeil) {
824 name = "ceil";831 name = "ceil";
832 num_args = 1;
825 } else if (fn_id == ZigLLVMFnIdSqrt) {833 } else if (fn_id == ZigLLVMFnIdSqrt) {
826 name = "sqrt";834 name = "sqrt";
835 num_args = 1;
836 } else if (fn_id == ZigLLVMFnIdFMA) {
837 name = "fma";
838 num_args = 3;
827 } else {839 } else {
828 zig_unreachable();840 zig_unreachable();
829 }841 }
830842
831 char fn_name[64];843 char fn_name[64];
832 sprintf(fn_name, "llvm.%s.f%" ZIG_PRI_usize "", name, type_entry->data.floating.bit_count);844 if (is_vector)
845 sprintf(fn_name, "llvm.%s.v%" PRIu32 "f%" PRIu32, name, key.data.floating.vector_len, key.data.floating.bit_count);
846 else
847 sprintf(fn_name, "llvm.%s.f%" PRIu32, name, key.data.floating.bit_count);
833 LLVMTypeRef float_type_ref = get_llvm_type(g, type_entry);848 LLVMTypeRef float_type_ref = get_llvm_type(g, type_entry);
834 LLVMTypeRef fn_type = LLVMFunctionType(float_type_ref, &float_type_ref, 1, false);849 LLVMTypeRef return_elem_types[3] = {
850 float_type_ref,
851 float_type_ref,
852 float_type_ref,
853 };
854 LLVMTypeRef fn_type = LLVMFunctionType(float_type_ref, return_elem_types, num_args, false);
835 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);855 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);
836 assert(LLVMGetIntrinsicID(fn_val));856 assert(LLVMGetIntrinsicID(fn_val));
837857
...@@ -5437,6 +5457,21 @@ static LLVMValueRef ir_render_sqrt(CodeGen *g, IrExecutable *executable, IrInstr...@@ -5437,6 +5457,21 @@ static LLVMValueRef ir_render_sqrt(CodeGen *g, IrExecutable *executable, IrInstr
5437 return LLVMBuildCall(g->builder, fn_val, &op, 1, "");5457 return LLVMBuildCall(g->builder, fn_val, &op, 1, "");
5438}5458}
54395459
5460static LLVMValueRef ir_render_mul_add(CodeGen *g, IrExecutable *executable, IrInstructionMulAdd *instruction) {
5461 LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
5462 LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);
5463 LLVMValueRef op3 = ir_llvm_value(g, instruction->op3);
5464 assert(instruction->base.value.type->id == ZigTypeIdFloat ||
5465 instruction->base.value.type->id == ZigTypeIdVector);
5466 LLVMValueRef fn_val = get_float_fn(g, instruction->base.value.type, ZigLLVMFnIdFMA);
5467 LLVMValueRef args[3] = {
5468 op1,
5469 op2,
5470 op3,
5471 };
5472 return LLVMBuildCall(g->builder, fn_val, args, 3, "");
5473}
5474
5440static LLVMValueRef ir_render_bswap(CodeGen *g, IrExecutable *executable, IrInstructionBswap *instruction) {5475static LLVMValueRef ir_render_bswap(CodeGen *g, IrExecutable *executable, IrInstructionBswap *instruction) {
5441 LLVMValueRef op = ir_llvm_value(g, instruction->op);5476 LLVMValueRef op = ir_llvm_value(g, instruction->op);
5442 ZigType *int_type = instruction->base.value.type;5477 ZigType *int_type = instruction->base.value.type;
...@@ -5781,6 +5816,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable,...@@ -5781,6 +5816,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable,
5781 return ir_render_mark_err_ret_trace_ptr(g, executable, (IrInstructionMarkErrRetTracePtr *)instruction);5816 return ir_render_mark_err_ret_trace_ptr(g, executable, (IrInstructionMarkErrRetTracePtr *)instruction);
5782 case IrInstructionIdSqrt:5817 case IrInstructionIdSqrt:
5783 return ir_render_sqrt(g, executable, (IrInstructionSqrt *)instruction);5818 return ir_render_sqrt(g, executable, (IrInstructionSqrt *)instruction);
5819 case IrInstructionIdMulAdd:
5820 return ir_render_mul_add(g, executable, (IrInstructionMulAdd *)instruction);
5784 case IrInstructionIdArrayToVector:5821 case IrInstructionIdArrayToVector:
5785 return ir_render_array_to_vector(g, executable, (IrInstructionArrayToVector *)instruction);5822 return ir_render_array_to_vector(g, executable, (IrInstructionArrayToVector *)instruction);
5786 case IrInstructionIdVectorToArray:5823 case IrInstructionIdVectorToArray:
...@@ -7398,6 +7435,7 @@ static void define_builtin_fns(CodeGen *g) {...@@ -7398,6 +7435,7 @@ static void define_builtin_fns(CodeGen *g) {
7398 create_builtin_fn(g, BuiltinFnIdRem, "rem", 2);7435 create_builtin_fn(g, BuiltinFnIdRem, "rem", 2);
7399 create_builtin_fn(g, BuiltinFnIdMod, "mod", 2);7436 create_builtin_fn(g, BuiltinFnIdMod, "mod", 2);
7400 create_builtin_fn(g, BuiltinFnIdSqrt, "sqrt", 2);7437 create_builtin_fn(g, BuiltinFnIdSqrt, "sqrt", 2);
7438 create_builtin_fn(g, BuiltinFnIdMulAdd, "mulAdd", 4);
7401 create_builtin_fn(g, BuiltinFnIdInlineCall, "inlineCall", SIZE_MAX);7439 create_builtin_fn(g, BuiltinFnIdInlineCall, "inlineCall", SIZE_MAX);
7402 create_builtin_fn(g, BuiltinFnIdNoInlineCall, "noInlineCall", SIZE_MAX);7440 create_builtin_fn(g, BuiltinFnIdNoInlineCall, "noInlineCall", SIZE_MAX);
7403 create_builtin_fn(g, BuiltinFnIdNewStackCall, "newStackCall", SIZE_MAX);7441 create_builtin_fn(g, BuiltinFnIdNewStackCall, "newStackCall", SIZE_MAX);
src/ir.cpp+171
...@@ -747,6 +747,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionTestErr *) {...@@ -747,6 +747,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionTestErr *) {
747 return IrInstructionIdTestErr;747 return IrInstructionIdTestErr;
748}748}
749749
750static constexpr IrInstructionId ir_instruction_id(IrInstructionMulAdd *) {
751 return IrInstructionIdMulAdd;
752}
753
750static constexpr IrInstructionId ir_instruction_id(IrInstructionUnwrapErrCode *) {754static constexpr IrInstructionId ir_instruction_id(IrInstructionUnwrapErrCode *) {
751 return IrInstructionIdUnwrapErrCode;755 return IrInstructionIdUnwrapErrCode;
752}756}
...@@ -2308,6 +2312,22 @@ static IrInstruction *ir_build_overflow_op(IrBuilder *irb, Scope *scope, AstNode...@@ -2308,6 +2312,22 @@ static IrInstruction *ir_build_overflow_op(IrBuilder *irb, Scope *scope, AstNode
2308 return &instruction->base;2312 return &instruction->base;
2309}2313}
23102314
2315static IrInstruction *ir_build_mul_add(IrBuilder *irb, Scope *scope, AstNode *source_node,
2316 IrInstruction *type_value, IrInstruction *op1, IrInstruction *op2, IrInstruction *op3) {
2317 IrInstructionMulAdd *instruction = ir_build_instruction<IrInstructionMulAdd>(irb, scope, source_node);
2318 instruction->type_value = type_value;
2319 instruction->op1 = op1;
2320 instruction->op2 = op2;
2321 instruction->op3 = op3;
2322
2323 ir_ref_instruction(type_value, irb->current_basic_block);
2324 ir_ref_instruction(op1, irb->current_basic_block);
2325 ir_ref_instruction(op2, irb->current_basic_block);
2326 ir_ref_instruction(op3, irb->current_basic_block);
2327
2328 return &instruction->base;
2329}
2330
2311static IrInstruction *ir_build_align_of(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *type_value) {2331static IrInstruction *ir_build_align_of(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *type_value) {
2312 IrInstructionAlignOf *instruction = ir_build_instruction<IrInstructionAlignOf>(irb, scope, source_node);2332 IrInstructionAlignOf *instruction = ir_build_instruction<IrInstructionAlignOf>(irb, scope, source_node);
2313 instruction->type_value = type_value;2333 instruction->type_value = type_value;
...@@ -4028,6 +4048,33 @@ static IrInstruction *ir_gen_overflow_op(IrBuilder *irb, Scope *scope, AstNode *...@@ -4028,6 +4048,33 @@ static IrInstruction *ir_gen_overflow_op(IrBuilder *irb, Scope *scope, AstNode *
4028 return ir_build_overflow_op(irb, scope, node, op, type_value, op1, op2, result_ptr, nullptr);4048 return ir_build_overflow_op(irb, scope, node, op, type_value, op1, op2, result_ptr, nullptr);
4029}4049}
40304050
4051static IrInstruction *ir_gen_mul_add(IrBuilder *irb, Scope *scope, AstNode *node) {
4052 assert(node->type == NodeTypeFnCallExpr);
4053
4054 AstNode *type_node = node->data.fn_call_expr.params.at(0);
4055 AstNode *op1_node = node->data.fn_call_expr.params.at(1);
4056 AstNode *op2_node = node->data.fn_call_expr.params.at(2);
4057 AstNode *op3_node = node->data.fn_call_expr.params.at(3);
4058
4059 IrInstruction *type_value = ir_gen_node(irb, type_node, scope);
4060 if (type_value == irb->codegen->invalid_instruction)
4061 return irb->codegen->invalid_instruction;
4062
4063 IrInstruction *op1 = ir_gen_node(irb, op1_node, scope);
4064 if (op1 == irb->codegen->invalid_instruction)
4065 return irb->codegen->invalid_instruction;
4066
4067 IrInstruction *op2 = ir_gen_node(irb, op2_node, scope);
4068 if (op2 == irb->codegen->invalid_instruction)
4069 return irb->codegen->invalid_instruction;
4070
4071 IrInstruction *op3 = ir_gen_node(irb, op3_node, scope);
4072 if (op3 == irb->codegen->invalid_instruction)
4073 return irb->codegen->invalid_instruction;
4074
4075 return ir_build_mul_add(irb, scope, node, type_value, op1, op2, op3);
4076}
4077
4031static IrInstruction *ir_gen_this(IrBuilder *irb, Scope *orig_scope, AstNode *node) {4078static IrInstruction *ir_gen_this(IrBuilder *irb, Scope *orig_scope, AstNode *node) {
4032 for (Scope *it_scope = orig_scope; it_scope != nullptr; it_scope = it_scope->parent) {4079 for (Scope *it_scope = orig_scope; it_scope != nullptr; it_scope = it_scope->parent) {
4033 if (it_scope->id == ScopeIdDecls) {4080 if (it_scope->id == ScopeIdDecls) {
...@@ -4687,6 +4734,8 @@ static IrInstruction *ir_gen_builtin_fn_call(IrBuilder *irb, Scope *scope, AstNo...@@ -4687,6 +4734,8 @@ static IrInstruction *ir_gen_builtin_fn_call(IrBuilder *irb, Scope *scope, AstNo
4687 return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpMul), lval);4734 return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpMul), lval);
4688 case BuiltinFnIdShlWithOverflow:4735 case BuiltinFnIdShlWithOverflow:
4689 return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpShl), lval);4736 return ir_lval_wrap(irb, scope, ir_gen_overflow_op(irb, scope, node, IrOverflowOpShl), lval);
4737 case BuiltinFnIdMulAdd:
4738 return ir_lval_wrap(irb, scope, ir_gen_mul_add(irb, scope, node), lval);
4690 case BuiltinFnIdTypeName:4739 case BuiltinFnIdTypeName:
4691 {4740 {
4692 AstNode *arg0_node = node->data.fn_call_expr.params.at(0);4741 AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
...@@ -21185,6 +21234,125 @@ static IrInstruction *ir_analyze_instruction_overflow_op(IrAnalyze *ira, IrInstr...@@ -21185,6 +21234,125 @@ static IrInstruction *ir_analyze_instruction_overflow_op(IrAnalyze *ira, IrInstr
21185 return result;21234 return result;
21186}21235}
2118721236
21237static void ir_eval_mul_add(IrAnalyze *ira, IrInstructionMulAdd *source_instr, ZigType *float_type,
21238 ConstExprValue *op1, ConstExprValue *op2, ConstExprValue *op3, ConstExprValue *out_val) {
21239 if (float_type->id == ZigTypeIdComptimeFloat) {
21240 f128M_mulAdd(&out_val->data.x_bigfloat.value, &op1->data.x_bigfloat.value, &op2->data.x_bigfloat.value,
21241 &op3->data.x_bigfloat.value);
21242 } else if (float_type->id == ZigTypeIdFloat) {
21243 switch (float_type->data.floating.bit_count) {
21244 case 16:
21245 out_val->data.x_f16 = f16_mulAdd(op1->data.x_f16, op2->data.x_f16, op3->data.x_f16);
21246 break;
21247 case 32:
21248 out_val->data.x_f32 = fmaf(op1->data.x_f32, op2->data.x_f32, op3->data.x_f32);
21249 break;
21250 case 64:
21251 out_val->data.x_f64 = fma(op1->data.x_f64, op2->data.x_f64, op3->data.x_f64);
21252 break;
21253 case 128:
21254 f128M_mulAdd(&op1->data.x_f128, &op2->data.x_f128, &op3->data.x_f128, &out_val->data.x_f128);
21255 break;
21256 default:
21257 zig_unreachable();
21258 }
21259 } else {
21260 zig_unreachable();
21261 }
21262}
21263
21264static IrInstruction *ir_analyze_instruction_mul_add(IrAnalyze *ira, IrInstructionMulAdd *instruction) {
21265 IrInstruction *type_value = instruction->type_value->child;
21266 if (type_is_invalid(type_value->value.type))
21267 return ira->codegen->invalid_instruction;
21268
21269 ZigType *expr_type = ir_resolve_type(ira, type_value);
21270 if (type_is_invalid(expr_type))
21271 return ira->codegen->invalid_instruction;
21272
21273 // Only allow float types, and vectors of floats.
21274 ZigType *float_type = (expr_type->id == ZigTypeIdVector) ? expr_type->data.vector.elem_type : expr_type;
21275 if (float_type->id != ZigTypeIdFloat) {
21276 ir_add_error(ira, type_value,
21277 buf_sprintf("expected float or vector of float type, found '%s'", buf_ptr(&float_type->name)));
21278 return ira->codegen->invalid_instruction;
21279 }
21280
21281 IrInstruction *op1 = instruction->op1->child;
21282 if (type_is_invalid(op1->value.type))
21283 return ira->codegen->invalid_instruction;
21284
21285 IrInstruction *casted_op1 = ir_implicit_cast(ira, op1, expr_type);
21286 if (type_is_invalid(casted_op1->value.type))
21287 return ira->codegen->invalid_instruction;
21288
21289 IrInstruction *op2 = instruction->op2->child;
21290 if (type_is_invalid(op2->value.type))
21291 return ira->codegen->invalid_instruction;
21292
21293 IrInstruction *casted_op2 = ir_implicit_cast(ira, op2, expr_type);
21294 if (type_is_invalid(casted_op2->value.type))
21295 return ira->codegen->invalid_instruction;
21296
21297 IrInstruction *op3 = instruction->op3->child;
21298 if (type_is_invalid(op3->value.type))
21299 return ira->codegen->invalid_instruction;
21300
21301 IrInstruction *casted_op3 = ir_implicit_cast(ira, op3, expr_type);
21302 if (type_is_invalid(casted_op3->value.type))
21303 return ira->codegen->invalid_instruction;
21304
21305 if (instr_is_comptime(casted_op1) &&
21306 instr_is_comptime(casted_op2) &&
21307 instr_is_comptime(casted_op3)) {
21308 ConstExprValue *op1_const = ir_resolve_const(ira, casted_op1, UndefBad);
21309 if (!op1_const)
21310 return ira->codegen->invalid_instruction;
21311 ConstExprValue *op2_const = ir_resolve_const(ira, casted_op2, UndefBad);
21312 if (!op2_const)
21313 return ira->codegen->invalid_instruction;
21314 ConstExprValue *op3_const = ir_resolve_const(ira, casted_op3, UndefBad);
21315 if (!op3_const)
21316 return ira->codegen->invalid_instruction;
21317
21318 IrInstruction *result = ir_const(ira, &instruction->base, expr_type);
21319 ConstExprValue *out_val = &result->value;
21320
21321 if (expr_type->id == ZigTypeIdVector) {
21322 expand_undef_array(ira->codegen, op1_const);
21323 expand_undef_array(ira->codegen, op2_const);
21324 expand_undef_array(ira->codegen, op3_const);
21325 out_val->special = ConstValSpecialUndef;
21326 expand_undef_array(ira->codegen, out_val);
21327 size_t len = expr_type->data.vector.len;
21328 for (size_t i = 0; i < len; i += 1) {
21329 ConstExprValue *float_operand_op1 = &op1_const->data.x_array.data.s_none.elements[i];
21330 ConstExprValue *float_operand_op2 = &op2_const->data.x_array.data.s_none.elements[i];
21331 ConstExprValue *float_operand_op3 = &op3_const->data.x_array.data.s_none.elements[i];
21332 ConstExprValue *float_out_val = &out_val->data.x_array.data.s_none.elements[i];
21333 assert(float_operand_op1->type == float_type);
21334 assert(float_operand_op2->type == float_type);
21335 assert(float_operand_op3->type == float_type);
21336 assert(float_out_val->type == float_type);
21337 ir_eval_mul_add(ira, instruction, float_type,
21338 op1_const, op2_const, op3_const, float_out_val);
21339 float_out_val->type = float_type;
21340 }
21341 out_val->type = expr_type;
21342 out_val->special = ConstValSpecialStatic;
21343 } else {
21344 ir_eval_mul_add(ira, instruction, float_type, op1_const, op2_const, op3_const, out_val);
21345 }
21346 return result;
21347 }
21348
21349 IrInstruction *result = ir_build_mul_add(&ira->new_irb,
21350 instruction->base.scope, instruction->base.source_node,
21351 type_value, casted_op1, casted_op2, casted_op3);
21352 result->value.type = expr_type;
21353 return result;
21354}
21355
21188static IrInstruction *ir_analyze_instruction_test_err(IrAnalyze *ira, IrInstructionTestErr *instruction) {21356static IrInstruction *ir_analyze_instruction_test_err(IrAnalyze *ira, IrInstructionTestErr *instruction) {
21189 IrInstruction *value = instruction->value->child;21357 IrInstruction *value = instruction->value->child;
21190 if (type_is_invalid(value->value.type))21358 if (type_is_invalid(value->value.type))
...@@ -23596,6 +23764,8 @@ static IrInstruction *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructio...@@ -23596,6 +23764,8 @@ static IrInstruction *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructio
23596 return ir_analyze_instruction_mark_err_ret_trace_ptr(ira, (IrInstructionMarkErrRetTracePtr *)instruction);23764 return ir_analyze_instruction_mark_err_ret_trace_ptr(ira, (IrInstructionMarkErrRetTracePtr *)instruction);
23597 case IrInstructionIdSqrt:23765 case IrInstructionIdSqrt:
23598 return ir_analyze_instruction_sqrt(ira, (IrInstructionSqrt *)instruction);23766 return ir_analyze_instruction_sqrt(ira, (IrInstructionSqrt *)instruction);
23767 case IrInstructionIdMulAdd:
23768 return ir_analyze_instruction_mul_add(ira, (IrInstructionMulAdd *)instruction);
23599 case IrInstructionIdIntToErr:23769 case IrInstructionIdIntToErr:
23600 return ir_analyze_instruction_int_to_err(ira, (IrInstructionIntToErr *)instruction);23770 return ir_analyze_instruction_int_to_err(ira, (IrInstructionIntToErr *)instruction);
23601 case IrInstructionIdErrToInt:23771 case IrInstructionIdErrToInt:
...@@ -23835,6 +24005,7 @@ bool ir_has_side_effects(IrInstruction *instruction) {...@@ -23835,6 +24005,7 @@ bool ir_has_side_effects(IrInstruction *instruction) {
23835 case IrInstructionIdCoroPromise:24005 case IrInstructionIdCoroPromise:
23836 case IrInstructionIdPromiseResultType:24006 case IrInstructionIdPromiseResultType:
23837 case IrInstructionIdSqrt:24007 case IrInstructionIdSqrt:
24008 case IrInstructionIdMulAdd:
23838 case IrInstructionIdAtomicLoad:24009 case IrInstructionIdAtomicLoad:
23839 case IrInstructionIdIntCast:24010 case IrInstructionIdIntCast:
23840 case IrInstructionIdFloatCast:24011 case IrInstructionIdFloatCast:
src/ir_print.cpp+19
...@@ -1439,6 +1439,22 @@ static void ir_print_sqrt(IrPrint *irp, IrInstructionSqrt *instruction) {...@@ -1439,6 +1439,22 @@ static void ir_print_sqrt(IrPrint *irp, IrInstructionSqrt *instruction) {
1439 fprintf(irp->f, ")");1439 fprintf(irp->f, ")");
1440}1440}
14411441
1442static void ir_print_mul_add(IrPrint *irp, IrInstructionMulAdd *instruction) {
1443 fprintf(irp->f, "@mulAdd(");
1444 if (instruction->type_value != nullptr) {
1445 ir_print_other_instruction(irp, instruction->type_value);
1446 } else {
1447 fprintf(irp->f, "null");
1448 }
1449 fprintf(irp->f, ",");
1450 ir_print_other_instruction(irp, instruction->op1);
1451 fprintf(irp->f, ",");
1452 ir_print_other_instruction(irp, instruction->op2);
1453 fprintf(irp->f, ",");
1454 ir_print_other_instruction(irp, instruction->op3);
1455 fprintf(irp->f, ")");
1456}
1457
1442static void ir_print_decl_var_gen(IrPrint *irp, IrInstructionDeclVarGen *decl_var_instruction) {1458static void ir_print_decl_var_gen(IrPrint *irp, IrInstructionDeclVarGen *decl_var_instruction) {
1443 ZigVar *var = decl_var_instruction->var;1459 ZigVar *var = decl_var_instruction->var;
1444 const char *var_or_const = decl_var_instruction->var->gen_is_const ? "const" : "var";1460 const char *var_or_const = decl_var_instruction->var->gen_is_const ? "const" : "var";
...@@ -1905,6 +1921,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) {...@@ -1905,6 +1921,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) {
1905 case IrInstructionIdSqrt:1921 case IrInstructionIdSqrt:
1906 ir_print_sqrt(irp, (IrInstructionSqrt *)instruction);1922 ir_print_sqrt(irp, (IrInstructionSqrt *)instruction);
1907 break;1923 break;
1924 case IrInstructionIdMulAdd:
1925 ir_print_mul_add(irp, (IrInstructionMulAdd *)instruction);
1926 break;
1908 case IrInstructionIdAtomicLoad:1927 case IrInstructionIdAtomicLoad:
1909 ir_print_atomic_load(irp, (IrInstructionAtomicLoad *)instruction);1928 ir_print_atomic_load(irp, (IrInstructionAtomicLoad *)instruction);
1910 break;1929 break;
test/stage1/behavior/muladd.zig created+34
...@@ -0,0 +1,34 @@
1const expect = @import("std").testing.expect;
2
3test "@mulAdd" {
4 comptime testMulAdd();
5 testMulAdd();
6}
7
8fn testMulAdd() void {
9 {
10 var a: f16 = 5.5;
11 var b: f16 = 2.5;
12 var c: f16 = 6.25;
13 expect(@mulAdd(f16, a, b, c) == 20);
14 }
15 {
16 var a: f32 = 5.5;
17 var b: f32 = 2.5;
18 var c: f32 = 6.25;
19 expect(@mulAdd(f32, a, b, c) == 20);
20 }
21 {
22 var a: f64 = 5.5;
23 var b: f64 = 2.5;
24 var c: f64 = 6.25;
25 expect(@mulAdd(f64, a, b, c) == 20);
26 }
27 // Awaits implementation in libm.zig
28 //{
29 // var a: f16 = 5.5;
30 // var b: f128 = 2.5;
31 // var c: f128 = 6.25;
32 // expect(@mulAdd(f128, a, b, c) == 20);
33 //}
34}
\ No newline at end of file