authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-08-06 16:37:25-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-08-06 16:53:22-04:00
log400500a3afafca8178f13a7e4e1cd0ae7808aff2
treef6ee011721206db3cda4952dd2eccaeb71c7e632
parent20f63e588e62c4a7250bc96c9e5b54c8106ad1af
signature Commit is signed but in an unrecognized format.

improve async function semantics

* add safety panic for resuming a function which is returning, pending an await * remove IrInstructionResultPtr * add IrInstructionReturnBegin. This does the early return in async functions; does nothing in normal functions. * `await` gets a result location * `analyze_fn_async` will call `analyze_fn_body` if necessary. * async function frames have a result pointer field for themselves to access and one for the awaiter to supply before the atomic rmw. when returning, async functions copy the result to the awaiter result pointer, if it is non-null. * async function frames have a stack trace pointer which is supplied by the awaiter before the atomicrmw. Later in the frame is a stack trace struct and addresses, which is used for its own calls and awaits. * when awaiting an async function, if an early return occurred, the awaiter tail resumes the frame. * when an async function returns, early return does a suspend (in IrInstructionReturnBegin) before copying the error return trace data, result, and running the defers. After the last defer runs, the frame will no longer be accessed. * proper acquire/release atomic ordering attributes in async functions.

7 files changed, 491 insertions(+), 323 deletions(-)

BRANCH_TODO+3
......@@ -33,3 +33,6 @@
3333 - anyframe, anyframe->T
3434 * safety for double await
3535 * call graph analysis to have fewer stack trace frames
36 * grep for "coroutine" and "coro" and replace all that nomenclature with "async functions"
37 * when there are multiple calls to async functions in a function, reuse the same frame buffer, so that the
38 needed bytes is equal to the largest callee's frame
src/all_types.hpp+23-4
......@@ -1557,6 +1557,7 @@ enum PanicMsgId {
15571557 PanicMsgIdBadReturn,
15581558 PanicMsgIdResumedAnAwaitingFn,
15591559 PanicMsgIdFrameTooSmall,
1560 PanicMsgIdResumedFnPendingAwait,
15601561
15611562 PanicMsgIdCount,
15621563};
......@@ -1717,10 +1718,12 @@ struct CodeGen {
17171718 LLVMTargetMachineRef target_machine;
17181719 ZigLLVMDIFile *dummy_di_file;
17191720 LLVMValueRef cur_ret_ptr;
1721 LLVMValueRef cur_ret_ptr_ptr;
17201722 LLVMValueRef cur_fn_val;
17211723 LLVMValueRef cur_async_switch_instr;
17221724 LLVMValueRef cur_async_resume_index_ptr;
17231725 LLVMValueRef cur_async_awaiter_ptr;
1726 LLVMValueRef cur_async_prev_val;
17241727 LLVMBasicBlockRef cur_preamble_llvm_block;
17251728 size_t cur_resume_block_count;
17261729 LLVMValueRef cur_err_ret_trace_val_arg;
......@@ -2223,6 +2226,7 @@ enum IrInstructionId {
22232226 IrInstructionIdCallGen,
22242227 IrInstructionIdConst,
22252228 IrInstructionIdReturn,
2229 IrInstructionIdReturnBegin,
22262230 IrInstructionIdCast,
22272231 IrInstructionIdResizeSlice,
22282232 IrInstructionIdContainerInitList,
......@@ -2326,7 +2330,6 @@ enum IrInstructionId {
23262330 IrInstructionIdImplicitCast,
23272331 IrInstructionIdResolveResult,
23282332 IrInstructionIdResetResult,
2329 IrInstructionIdResultPtr,
23302333 IrInstructionIdOpaqueType,
23312334 IrInstructionIdSetAlignStack,
23322335 IrInstructionIdArgType,
......@@ -2355,7 +2358,8 @@ enum IrInstructionId {
23552358 IrInstructionIdUnionInitNamedField,
23562359 IrInstructionIdSuspendBegin,
23572360 IrInstructionIdSuspendFinish,
2358 IrInstructionIdAwait,
2361 IrInstructionIdAwaitSrc,
2362 IrInstructionIdAwaitGen,
23592363 IrInstructionIdCoroResume,
23602364};
23612365
......@@ -2630,7 +2634,13 @@ struct IrInstructionConst {
26302634struct IrInstructionReturn {
26312635 IrInstruction base;
26322636
2633 IrInstruction *value;
2637 IrInstruction *operand;
2638};
2639
2640struct IrInstructionReturnBegin {
2641 IrInstruction base;
2642
2643 IrInstruction *operand;
26342644};
26352645
26362646enum CastOp {
......@@ -3136,6 +3146,7 @@ struct IrInstructionTestErrSrc {
31363146 IrInstruction base;
31373147
31383148 bool resolve_err_set;
3149 bool base_ptr_is_payload;
31393150 IrInstruction *base_ptr;
31403151};
31413152
......@@ -3603,10 +3614,18 @@ struct IrInstructionSuspendFinish {
36033614 IrInstructionSuspendBegin *begin;
36043615};
36053616
3606struct IrInstructionAwait {
3617struct IrInstructionAwaitSrc {
36073618 IrInstruction base;
36083619
36093620 IrInstruction *frame;
3621 ResultLoc *result_loc;
3622};
3623
3624struct IrInstructionAwaitGen {
3625 IrInstruction base;
3626
3627 IrInstruction *frame;
3628 IrInstruction *result_loc;
36103629};
36113630
36123631struct IrInstructionCoroResume {
src/analyze.cpp+28-23
......@@ -3848,6 +3848,13 @@ static void analyze_fn_async(CodeGen *g, ZigFn *fn) {
38483848
38493849 if (callee->type_entry->data.fn.fn_type_id.cc != CallingConventionUnspecified)
38503850 continue;
3851 if (callee->anal_state == FnAnalStateReady) {
3852 analyze_fn_body(g, callee);
3853 if (callee->anal_state == FnAnalStateInvalid) {
3854 fn->anal_state = FnAnalStateInvalid;
3855 return;
3856 }
3857 }
38513858 assert(callee->anal_state == FnAnalStateComplete);
38523859 analyze_fn_async(g, callee);
38533860 if (callee->anal_state == FnAnalStateInvalid) {
......@@ -5224,20 +5231,18 @@ static Error resolve_coro_frame(CodeGen *g, ZigType *frame_type) {
52245231
52255232 FnTypeId *fn_type_id = &fn_type->data.fn.fn_type_id;
52265233 ZigType *ptr_return_type = get_pointer_to_type(g, fn_type_id->return_type, false);
5227 field_names.append("@ptr_result");
5234 field_names.append("@result_ptr_callee");
5235 field_types.append(ptr_return_type);
5236
5237 field_names.append("@result_ptr_awaiter");
52285238 field_types.append(ptr_return_type);
52295239
52305240 field_names.append("@result");
52315241 field_types.append(fn_type_id->return_type);
52325242
52335243 if (codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type)) {
5234 (void)get_ptr_to_stack_trace_type(g); // populate g->stack_trace_type
5235
5236 field_names.append("@stack_trace");
5237 field_types.append(g->stack_trace_type);
5238
5239 field_names.append("@instruction_addresses");
5240 field_types.append(get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count));
5244 field_names.append("@ptr_stack_trace");
5245 field_types.append(get_ptr_to_stack_trace_type(g));
52415246 }
52425247
52435248 for (size_t arg_i = 0; arg_i < fn_type_id->param_count; arg_i += 1) {
......@@ -5255,7 +5260,7 @@ static Error resolve_coro_frame(CodeGen *g, ZigType *frame_type) {
52555260 field_types.append(param_type);
52565261 }
52575262
5258 if (codegen_fn_has_err_ret_tracing_stack(g, fn)) {
5263 if (codegen_fn_has_err_ret_tracing_stack(g, fn, true)) {
52595264 (void)get_ptr_to_stack_trace_type(g); // populate g->stack_trace_type
52605265
52615266 field_names.append("@stack_trace");
......@@ -7570,11 +7575,11 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
75707575
75717576 bool have_result_type = result_type != nullptr && type_has_bits(result_type);
75727577 if (have_result_type) {
7573 field_types.append(get_llvm_type(g, ptr_result_type)); // ptr_result
7578 field_types.append(get_llvm_type(g, ptr_result_type)); // result_ptr_callee
7579 field_types.append(get_llvm_type(g, ptr_result_type)); // result_ptr_awaiter
75747580 field_types.append(get_llvm_type(g, result_type)); // result
75757581 if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
7576 field_types.append(get_llvm_type(g, g->stack_trace_type)); // stack_trace
7577 field_types.append(get_llvm_type(g, get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count))); // instruction_addresses
7582 field_types.append(get_llvm_type(g, get_ptr_to_stack_trace_type(g))); // ptr_stack_trace
75787583 }
75797584 }
75807585 LLVMStructSetBody(frame_header_type, field_types.items, field_types.length, false);
......@@ -7607,7 +7612,15 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
76077612 if (have_result_type) {
76087613 di_element_types.append(
76097614 ZigLLVMCreateDebugMemberType(g->dbuilder,
7610 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "ptr_result",
7615 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result_ptr_callee",
7616 di_file, line,
7617 8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
7618 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
7619 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
7620 ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, ptr_result_type)));
7621 di_element_types.append(
7622 ZigLLVMCreateDebugMemberType(g->dbuilder,
7623 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "result_ptr_awaiter",
76117624 di_file, line,
76127625 8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
76137626 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
......@@ -7625,20 +7638,12 @@ static void resolve_llvm_types_any_frame(CodeGen *g, ZigType *any_frame_type, Re
76257638 if (codegen_fn_has_err_ret_tracing_arg(g, result_type)) {
76267639 di_element_types.append(
76277640 ZigLLVMCreateDebugMemberType(g->dbuilder,
7628 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "stack_trace",
7629 di_file, line,
7630 8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
7631 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
7632 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
7633 ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, g->stack_trace_type)));
7634 di_element_types.append(
7635 ZigLLVMCreateDebugMemberType(g->dbuilder,
7636 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "instruction_addresses",
7641 ZigLLVMTypeToScope(any_frame_type->llvm_di_type), "ptr_stack_trace",
76377642 di_file, line,
76387643 8*LLVMABISizeOfType(g->target_data_ref, field_types.at(di_element_types.length)),
76397644 8*LLVMABIAlignmentOfType(g->target_data_ref, field_types.at(di_element_types.length)),
76407645 8*LLVMOffsetOfElement(g->target_data_ref, frame_header_type, di_element_types.length),
7641 ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count))));
7646 ZigLLVM_DIFlags_Zero, get_llvm_di_type(g, get_ptr_to_stack_trace_type(g))));
76427647 }
76437648 };
76447649
src/codegen.cpp+270-171
......@@ -24,6 +24,14 @@
2424#include <stdio.h>
2525#include <errno.h>
2626
27enum ResumeId {
28 ResumeIdManual,
29 ResumeIdReturn,
30 ResumeIdCall,
31
32 ResumeIdAwaitEarlyReturn // must be last
33};
34
2735static void init_darwin_native(CodeGen *g) {
2836 char *osx_target = getenv("MACOSX_DEPLOYMENT_TARGET");
2937 char *ios_target = getenv("IPHONEOS_DEPLOYMENT_TARGET");
......@@ -298,25 +306,25 @@ static LLVMLinkage to_llvm_linkage(GlobalLinkageId id) {
298306}
299307
300308// label (grep this): [coro_frame_struct_layout]
301static uint32_t frame_index_trace_arg(CodeGen *g, FnTypeId *fn_type_id) {
302 // [0] *ReturnType
303 // [1] ReturnType
304 uint32_t return_field_count = type_has_bits(fn_type_id->return_type) ? 2 : 0;
309static uint32_t frame_index_trace_arg(CodeGen *g, ZigType *return_type) {
310 // [0] *ReturnType (callee's)
311 // [1] *ReturnType (awaiter's)
312 // [2] ReturnType
313 uint32_t return_field_count = type_has_bits(return_type) ? 3 : 0;
305314 return coro_ret_start + return_field_count;
306315}
307316
308317// label (grep this): [coro_frame_struct_layout]
309static uint32_t frame_index_arg(CodeGen *g, FnTypeId *fn_type_id) {
310 bool have_stack_trace = codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type);
311 // [0] StackTrace
312 // [1] [stack_trace_ptr_count]usize
313 uint32_t trace_field_count = have_stack_trace ? 2 : 0;
314 return frame_index_trace_arg(g, fn_type_id) + trace_field_count;
318static uint32_t frame_index_arg(CodeGen *g, ZigType *return_type) {
319 bool have_stack_trace = codegen_fn_has_err_ret_tracing_arg(g, return_type);
320 // [0] *StackTrace
321 uint32_t trace_field_count = have_stack_trace ? 1 : 0;
322 return frame_index_trace_arg(g, return_type) + trace_field_count;
315323}
316324
317325// label (grep this): [coro_frame_struct_layout]
318326static uint32_t frame_index_trace_stack(CodeGen *g, FnTypeId *fn_type_id) {
319 uint32_t result = frame_index_arg(g, fn_type_id);
327 uint32_t result = frame_index_arg(g, fn_type_id->return_type);
320328 for (size_t i = 0; i < fn_type_id->param_count; i += 1) {
321329 if (type_has_bits(fn_type_id->param_info->type)) {
322330 result += 1;
......@@ -901,7 +909,7 @@ static Buf *panic_msg_buf(PanicMsgId msg_id) {
901909 case PanicMsgIdPtrCastNull:
902910 return buf_create_from_str("cast causes pointer to be null");
903911 case PanicMsgIdBadResume:
904 return buf_create_from_str("invalid resume of async function");
912 return buf_create_from_str("resumed an async function which already returned");
905913 case PanicMsgIdBadAwait:
906914 return buf_create_from_str("async function awaited twice");
907915 case PanicMsgIdBadReturn:
......@@ -910,6 +918,8 @@ static Buf *panic_msg_buf(PanicMsgId msg_id) {
910918 return buf_create_from_str("awaiting function resumed");
911919 case PanicMsgIdFrameTooSmall:
912920 return buf_create_from_str("frame too small");
921 case PanicMsgIdResumedFnPendingAwait:
922 return buf_create_from_str("resumed an async function which can only be awaited");
913923 }
914924 zig_unreachable();
915925}
......@@ -1301,7 +1311,14 @@ static LLVMValueRef get_cur_err_ret_trace_val(CodeGen *g, Scope *scope) {
13011311 if (g->cur_err_ret_trace_val_stack != nullptr) {
13021312 return g->cur_err_ret_trace_val_stack;
13031313 }
1304 return g->cur_err_ret_trace_val_arg;
1314 if (g->cur_err_ret_trace_val_arg != nullptr) {
1315 if (fn_is_async(g->cur_fn)) {
1316 return LLVMBuildLoad(g->builder, g->cur_err_ret_trace_val_arg, "");
1317 } else {
1318 return g->cur_err_ret_trace_val_arg;
1319 }
1320 }
1321 return nullptr;
13051322}
13061323
13071324static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val, Scope *scope) {
......@@ -2023,99 +2040,191 @@ static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *execut
20232040 return call_instruction;
20242041}
20252042
2026static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable,
2027 IrInstructionReturn *return_instruction)
2043static void gen_assert_resume_id(CodeGen *g, IrInstruction *source_instr, ResumeId resume_id, PanicMsgId msg_id,
2044 LLVMBasicBlockRef end_bb)
2045{
2046 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2047 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
2048 if (end_bb == nullptr) end_bb = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume");
2049 LLVMValueRef ok_bit;
2050 if (resume_id == ResumeIdAwaitEarlyReturn) {
2051 LLVMValueRef last_value = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref),
2052 LLVMConstInt(usize_type_ref, ResumeIdAwaitEarlyReturn, false), "");
2053 ok_bit = LLVMBuildICmp(g->builder, LLVMIntULT, LLVMGetParam(g->cur_fn_val, 1), last_value, "");
2054 } else {
2055 LLVMValueRef expected_value = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref),
2056 LLVMConstInt(usize_type_ref, resume_id, false), "");
2057 ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, LLVMGetParam(g->cur_fn_val, 1), expected_value, "");
2058 }
2059 LLVMBuildCondBr(g->builder, ok_bit, end_bb, bad_resume_block);
2060
2061 LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
2062 gen_assertion(g, msg_id, source_instr);
2063
2064 LLVMPositionBuilderAtEnd(g->builder, end_bb);
2065}
2066
2067static LLVMValueRef gen_resume(CodeGen *g, LLVMValueRef fn_val, LLVMValueRef target_frame_ptr,
2068 ResumeId resume_id, LLVMValueRef arg_val)
2069{
2070 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2071 if (fn_val == nullptr) {
2072 if (g->anyframe_fn_type == nullptr) {
2073 (void)get_llvm_type(g, get_any_frame_type(g, nullptr));
2074 }
2075 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_fn_ptr_index, "");
2076 fn_val = LLVMBuildLoad(g->builder, fn_ptr_ptr, "");
2077 }
2078 if (arg_val == nullptr) {
2079 arg_val = LLVMBuildSub(g->builder, LLVMConstAllOnes(usize_type_ref),
2080 LLVMConstInt(usize_type_ref, resume_id, false), "");
2081 } else {
2082 assert(resume_id == ResumeIdAwaitEarlyReturn);
2083 }
2084 LLVMValueRef args[] = {target_frame_ptr, arg_val};
2085 return ZigLLVMBuildCall(g->builder, fn_val, args, 2, LLVMFastCallConv, ZigLLVM_FnInlineAuto, "");
2086}
2087
2088static LLVMValueRef ir_render_return_begin(CodeGen *g, IrExecutable *executable,
2089 IrInstructionReturnBegin *instruction)
20282090{
2091 if (!fn_is_async(g->cur_fn)) return nullptr;
2092
2093 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2094
2095 bool ret_type_has_bits = instruction->operand != nullptr &&
2096 type_has_bits(instruction->operand->value.type);
2097 ZigType *ret_type = ret_type_has_bits ? instruction->operand->value.type : nullptr;
2098 if (ret_type_has_bits && !handle_is_ptr(ret_type)) {
2099 // It's a scalar, so it didn't get written to the result ptr. Do that before the atomic rmw.
2100 LLVMValueRef result_ptr = LLVMBuildLoad(g->builder, g->cur_ret_ptr_ptr, "");
2101 LLVMBuildStore(g->builder, ir_llvm_value(g, instruction->operand), result_ptr);
2102 }
2103
2104 // Prepare to be suspended. We might end up not having to suspend though.
2105 LLVMBasicBlockRef resume_bb = LLVMAppendBasicBlock(g->cur_fn_val, "ReturnResume");
2106 size_t new_block_index = g->cur_resume_block_count;
2107 g->cur_resume_block_count += 1;
2108 LLVMValueRef new_block_index_val = LLVMConstInt(usize_type_ref, new_block_index, false);
2109 LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, resume_bb);
2110 LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr);
2111
2112 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
2113 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
2114 LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, g->cur_async_awaiter_ptr,
2115 all_ones, LLVMAtomicOrderingAcquire, g->is_single_threaded);
2116
2117 LLVMBasicBlockRef bad_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadReturn");
2118 LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
2119 LLVMBasicBlockRef resume_them_block = LLVMAppendBasicBlock(g->cur_fn_val, "ResumeThem");
2120
2121 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, resume_them_block, 2);
2122 LLVMBasicBlockRef switch_bb = LLVMGetInsertBlock(g->builder);
2123
2124 LLVMAddCase(switch_instr, zero, early_return_block);
2125 LLVMAddCase(switch_instr, all_ones, bad_return_block);
2126
2127 // Something has gone horribly wrong, and this is an invalid second return.
2128 LLVMPositionBuilderAtEnd(g->builder, bad_return_block);
2129 gen_assertion(g, PanicMsgIdBadReturn, &instruction->base);
2130
2131 // The caller has not done an await yet. So we suspend at the return instruction, until a
2132 // cancel or await is performed.
2133 LLVMPositionBuilderAtEnd(g->builder, early_return_block);
2134 LLVMBuildRetVoid(g->builder);
2135
2136 // Add a safety check for when getting resumed by the awaiter.
2137 LLVMPositionBuilderAtEnd(g->builder, resume_bb);
2138 LLVMBasicBlockRef after_resume_block = LLVMGetInsertBlock(g->builder);
2139 gen_assert_resume_id(g, &instruction->base, ResumeIdAwaitEarlyReturn, PanicMsgIdResumedFnPendingAwait,
2140 resume_them_block);
2141
2142 // We need to resume the caller by tail calling them.
2143 // That will happen when rendering IrInstructionReturn after running the defers/errdefers.
2144 // We either got here from Entry (function call) or from the switch above
2145 g->cur_async_prev_val = LLVMBuildPhi(g->builder, usize_type_ref, "");
2146 LLVMValueRef incoming_values[] = { LLVMGetParam(g->cur_fn_val, 1), prev_val };
2147 LLVMBasicBlockRef incoming_blocks[] = { after_resume_block, switch_bb };
2148 LLVMAddIncoming(g->cur_async_prev_val, incoming_values, incoming_blocks, 2);
2149
2150 return nullptr;
2151}
2152
2153static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrInstructionReturn *instruction) {
20292154 if (fn_is_async(g->cur_fn)) {
20302155 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
2031 bool ret_type_has_bits = return_instruction->value != nullptr &&
2032 type_has_bits(return_instruction->value->value.type);
2033 ZigType *ret_type = ret_type_has_bits ? return_instruction->value->value.type : nullptr;
2156 bool ret_type_has_bits = instruction->operand != nullptr &&
2157 type_has_bits(instruction->operand->value.type);
2158 ZigType *ret_type = ret_type_has_bits ? instruction->operand->value.type : nullptr;
20342159
2035 if (ir_want_runtime_safety(g, &return_instruction->base)) {
2160 if (ir_want_runtime_safety(g, &instruction->base)) {
20362161 LLVMValueRef new_resume_index = LLVMConstAllOnes(usize_type_ref);
20372162 LLVMBuildStore(g->builder, new_resume_index, g->cur_async_resume_index_ptr);
20382163 }
20392164
2040 LLVMValueRef result_ptr_as_usize;
20412165 if (ret_type_has_bits) {
2042 LLVMValueRef result_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_ret_start, "");
2043 LLVMValueRef result_ptr = LLVMBuildLoad(g->builder, result_ptr_ptr, "");
2044 if (!handle_is_ptr(ret_type)) {
2045 // It's a scalar, so it didn't get written to the result ptr. Do that now.
2046 LLVMBuildStore(g->builder, ir_llvm_value(g, return_instruction->value), result_ptr);
2047 }
2048 result_ptr_as_usize = LLVMBuildPtrToInt(g->builder, result_ptr, usize_type_ref, "");
2049 } else {
2050 // For debug safety, this value has to be anything other than all 1's, which signals
2051 // that it is being resumed. 0 is a bad choice since null pointers are special.
2052 result_ptr_as_usize = ir_want_runtime_safety(g, &return_instruction->base) ?
2053 LLVMConstInt(usize_type_ref, 1, false) : LLVMGetUndef(usize_type_ref);
2166 // If the awaiter result pointer is non-null, we need to copy the result to there.
2167 LLVMBasicBlockRef copy_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResult");
2168 LLVMBasicBlockRef copy_end_block = LLVMAppendBasicBlock(g->cur_fn_val, "CopyResultEnd");
2169 LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_ret_start + 1, "");
2170 LLVMValueRef awaiter_ret_ptr = LLVMBuildLoad(g->builder, awaiter_ret_ptr_ptr, "");
2171 LLVMValueRef zero_ptr = LLVMConstNull(LLVMTypeOf(awaiter_ret_ptr));
2172 LLVMValueRef need_copy_bit = LLVMBuildICmp(g->builder, LLVMIntNE, awaiter_ret_ptr, zero_ptr, "");
2173 LLVMBuildCondBr(g->builder, need_copy_bit, copy_block, copy_end_block);
2174
2175 LLVMPositionBuilderAtEnd(g->builder, copy_block);
2176 LLVMValueRef ret_ptr = LLVMBuildLoad(g->builder, g->cur_ret_ptr_ptr, "");
2177 LLVMTypeRef ptr_u8 = LLVMPointerType(LLVMInt8Type(), 0);
2178 LLVMValueRef dest_ptr_casted = LLVMBuildBitCast(g->builder, awaiter_ret_ptr, ptr_u8, "");
2179 LLVMValueRef src_ptr_casted = LLVMBuildBitCast(g->builder, ret_ptr, ptr_u8, "");
2180 bool is_volatile = false;
2181 uint32_t abi_align = get_abi_alignment(g, ret_type);
2182 LLVMValueRef byte_count_val = LLVMConstInt(usize_type_ref, type_size(g, ret_type), false);
2183 ZigLLVMBuildMemCpy(g->builder,
2184 dest_ptr_casted, abi_align,
2185 src_ptr_casted, abi_align, byte_count_val, is_volatile);
2186 LLVMBuildBr(g->builder, copy_end_block);
2187
2188 LLVMPositionBuilderAtEnd(g->builder, copy_end_block);
20542189 }
2055 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
2056 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
2057 LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, g->cur_async_awaiter_ptr,
2058 all_ones, LLVMAtomicOrderingMonotonic, g->is_single_threaded);
2059
2060 LLVMBasicBlockRef bad_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadReturn");
2061 LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
2062 LLVMBasicBlockRef resume_them_block = LLVMAppendBasicBlock(g->cur_fn_val, "ResumeThem");
2063
2064 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, resume_them_block, 2);
2065
2066 LLVMAddCase(switch_instr, zero, early_return_block);
2067 LLVMAddCase(switch_instr, all_ones, bad_return_block);
2068
2069 // Something has gone horribly wrong, and this is an invalid second return.
2070 LLVMPositionBuilderAtEnd(g->builder, bad_return_block);
2071 gen_assertion(g, PanicMsgIdBadReturn, &return_instruction->base);
2072
2073 // The caller will deal with fetching the result - we're done.
2074 LLVMPositionBuilderAtEnd(g->builder, early_return_block);
2075 LLVMBuildRetVoid(g->builder);
20762190
20772191 // We need to resume the caller by tail calling them.
2078 LLVMPositionBuilderAtEnd(g->builder, resume_them_block);
20792192 ZigType *any_frame_type = get_any_frame_type(g, ret_type);
2080 LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, prev_val,
2193 LLVMValueRef their_frame_ptr = LLVMBuildIntToPtr(g->builder, g->cur_async_prev_val,
20812194 get_llvm_type(g, any_frame_type), "");
2082 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, their_frame_ptr, coro_fn_ptr_index, "");
2083 LLVMValueRef awaiter_fn = LLVMBuildLoad(g->builder, fn_ptr_ptr, "");
2084 LLVMValueRef args[] = {their_frame_ptr, result_ptr_as_usize};
2085 LLVMValueRef call_inst = ZigLLVMBuildCall(g->builder, awaiter_fn, args, 2, LLVMFastCallConv,
2086 ZigLLVM_FnInlineAuto, "");
2195 LLVMValueRef call_inst = gen_resume(g, nullptr, their_frame_ptr, ResumeIdReturn, nullptr);
20872196 ZigLLVMSetTailCall(call_inst);
20882197 LLVMBuildRetVoid(g->builder);
20892198
20902199 return nullptr;
20912200 }
20922201 if (want_first_arg_sret(g, &g->cur_fn->type_entry->data.fn.fn_type_id)) {
2093 if (return_instruction->value == nullptr) {
2202 if (instruction->operand == nullptr) {
20942203 LLVMBuildRetVoid(g->builder);
20952204 return nullptr;
20962205 }
20972206 assert(g->cur_ret_ptr);
2098 src_assert(return_instruction->value->value.special != ConstValSpecialRuntime,
2099 return_instruction->base.source_node);
2100 LLVMValueRef value = ir_llvm_value(g, return_instruction->value);
2101 ZigType *return_type = return_instruction->value->value.type;
2207 src_assert(instruction->operand->value.special != ConstValSpecialRuntime,
2208 instruction->base.source_node);
2209 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
2210 ZigType *return_type = instruction->operand->value.type;
21022211 gen_assign_raw(g, g->cur_ret_ptr, get_pointer_to_type(g, return_type, false), value);
21032212 LLVMBuildRetVoid(g->builder);
21042213 } else if (g->cur_fn->type_entry->data.fn.fn_type_id.cc != CallingConventionAsync &&
21052214 handle_is_ptr(g->cur_fn->type_entry->data.fn.fn_type_id.return_type))
21062215 {
2107 if (return_instruction->value == nullptr) {
2216 if (instruction->operand == nullptr) {
21082217 LLVMValueRef by_val_value = gen_load_untyped(g, g->cur_ret_ptr, 0, false, "");
21092218 LLVMBuildRet(g->builder, by_val_value);
21102219 } else {
2111 LLVMValueRef value = ir_llvm_value(g, return_instruction->value);
2220 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
21122221 LLVMValueRef by_val_value = gen_load_untyped(g, value, 0, false, "");
21132222 LLVMBuildRet(g->builder, by_val_value);
21142223 }
2115 } else if (return_instruction->value == nullptr) {
2224 } else if (instruction->operand == nullptr) {
21162225 LLVMBuildRetVoid(g->builder);
21172226 } else {
2118 LLVMValueRef value = ir_llvm_value(g, return_instruction->value);
2227 LLVMValueRef value = ir_llvm_value(g, instruction->operand);
21192228 LLVMBuildRet(g->builder, value);
21202229 }
21212230 return nullptr;
......@@ -3417,7 +3526,7 @@ static void set_call_instr_sret(CodeGen *g, LLVMValueRef call_instr) {
34173526static void render_async_spills(CodeGen *g) {
34183527 ZigType *fn_type = g->cur_fn->type_entry;
34193528 ZigType *import = get_scope_import(&g->cur_fn->fndef_scope->base);
3420 uint32_t async_var_index = frame_index_arg(g, &fn_type->data.fn.fn_type_id);
3529 uint32_t async_var_index = frame_index_arg(g, fn_type->data.fn.fn_type_id.return_type);
34213530 for (size_t var_i = 0; var_i < g->cur_fn->variable_list.length; var_i += 1) {
34223531 ZigVar *var = g->cur_fn->variable_list.at(var_i);
34233532
......@@ -3450,7 +3559,7 @@ static void render_async_spills(CodeGen *g) {
34503559 }
34513560 }
34523561 // label (grep this): [coro_frame_struct_layout]
3453 if (codegen_fn_has_err_ret_tracing_stack(g, g->cur_fn)) {
3562 if (codegen_fn_has_err_ret_tracing_stack(g, g->cur_fn, true)) {
34543563 async_var_index += 2;
34553564 }
34563565 for (size_t alloca_i = 0; alloca_i < g->cur_fn->alloca_gen_list.length; alloca_i += 1) {
......@@ -3553,7 +3662,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr
35533662
35543663 if (ret_has_bits) {
35553664 // Use the result location which is inside the frame if this is an async call.
3556 ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, coro_ret_start + 1, "");
3665 ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, coro_ret_start + 2, "");
35573666 }
35583667 } else {
35593668 LLVMValueRef frame_slice_ptr = ir_llvm_value(g, instruction->new_stack);
......@@ -3590,17 +3699,26 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr
35903699 frame_result_loc = ir_llvm_value(g, instruction->frame_result_loc);
35913700 awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_ret_ptr, usize_type_ref, ""); // caller's own frame pointer
35923701 if (ret_has_bits) {
3593 if (result_loc != nullptr) {
3702 if (result_loc == nullptr) {
3703 // return type is a scalar, but we still need a pointer to it. Use the async fn frame.
3704 ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, coro_ret_start + 2, "");
3705 } else {
35943706 // Use the call instruction's result location.
35953707 ret_ptr = result_loc;
3596 } else {
3597 // return type is a scalar, but we still need a pointer to it. Use the async fn frame.
3598 ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, coro_ret_start + 1, "");
35993708 }
3709
3710 // Store a zero in the awaiter's result ptr to indicate we do not need a copy made.
3711 LLVMValueRef awaiter_ret_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, coro_ret_start + 1, "");
3712 LLVMValueRef zero_ptr = LLVMConstNull(LLVMGetElementType(LLVMTypeOf(awaiter_ret_ptr)));
3713 LLVMBuildStore(g->builder, zero_ptr, awaiter_ret_ptr);
36003714 }
36013715
3602 // even if prefix_arg_err_ret_stack is true, let the async function do its
3603 // error return tracing normally, and then we'll invoke merge_error_return_traces like normal.
3716 if (prefix_arg_err_ret_stack) {
3717 LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc,
3718 frame_index_trace_arg(g, src_return_type), "");
3719 LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
3720 LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr);
3721 }
36043722 }
36053723 if (instruction->is_async || callee_is_async) {
36063724 assert(frame_result_loc != nullptr);
......@@ -3652,7 +3770,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr
36523770 LLVMValueRef result;
36533771
36543772 if (instruction->is_async || callee_is_async) {
3655 uint32_t arg_start_i = frame_index_arg(g, &fn_type->data.fn.fn_type_id);
3773 uint32_t arg_start_i = frame_index_arg(g, fn_type->data.fn.fn_type_id.return_type);
36563774
36573775 LLVMValueRef casted_frame;
36583776 if (instruction->new_stack != nullptr) {
......@@ -3678,8 +3796,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr
36783796 }
36793797 }
36803798 if (instruction->is_async) {
3681 LLVMValueRef args[] = {frame_result_loc, LLVMGetUndef(usize_type_ref)};
3682 ZigLLVMBuildCall(g->builder, fn_val, args, 2, llvm_cc, fn_inline, "");
3799 gen_resume(g, fn_val, frame_result_loc, ResumeIdCall, nullptr);
36833800 if (instruction->new_stack != nullptr) {
36843801 return frame_result_loc;
36853802 }
......@@ -3694,36 +3811,23 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr
36943811 LLVMAddCase(g->cur_async_switch_instr, new_block_index_val, call_bb);
36953812
36963813 LLVMBuildStore(g->builder, new_block_index_val, g->cur_async_resume_index_ptr);
3697 LLVMValueRef args[] = {frame_result_loc, LLVMGetUndef(usize_type_ref)};
3698 LLVMValueRef call_inst = ZigLLVMBuildCall(g->builder, fn_val, args, 2, llvm_cc, fn_inline, "");
3814
3815 LLVMValueRef call_inst = gen_resume(g, fn_val, frame_result_loc, ResumeIdCall, nullptr);
36993816 ZigLLVMSetTailCall(call_inst);
37003817 LLVMBuildRetVoid(g->builder);
37013818
37023819 LLVMPositionBuilderAtEnd(g->builder, call_bb);
3703 if (ir_want_runtime_safety(g, &instruction->base)) {
3704 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
3705 LLVMBasicBlockRef ok_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume");
3706 LLVMValueRef arg_val = LLVMGetParam(g->cur_fn_val, 1);
3707 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
3708 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntNE, arg_val, all_ones, "");
3709 LLVMBuildCondBr(g->builder, ok_bit, ok_resume_block, bad_resume_block);
3710
3711 LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
3712 gen_safety_crash(g, PanicMsgIdResumedAnAwaitingFn);
3713
3714 LLVMPositionBuilderAtEnd(g->builder, ok_resume_block);
3715 }
3716
3820 gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr);
37173821 render_async_var_decls(g, instruction->base.scope);
37183822
3719 if (type_has_bits(src_return_type)) {
3720 LLVMValueRef spilled_result_ptr = LLVMGetParam(g->cur_fn_val, 1);
3721 LLVMValueRef casted_spilled_result_ptr = LLVMBuildIntToPtr(g->builder, spilled_result_ptr,
3722 get_llvm_type(g, ptr_result_type), "");
3723 return get_handle_value(g, casted_spilled_result_ptr, src_return_type, ptr_result_type);
3724 } else {
3823 if (!type_has_bits(src_return_type))
37253824 return nullptr;
3726 }
3825
3826 if (result_loc != nullptr)
3827 return get_handle_value(g, result_loc, src_return_type, ptr_result_type);
3828
3829 LLVMValueRef result_ptr = LLVMBuildStructGEP(g->builder, frame_result_loc, coro_ret_start + 2, "");
3830 return LLVMBuildLoad(g->builder, result_ptr, "");
37273831 }
37283832
37293833 if (instruction->new_stack == nullptr) {
......@@ -5191,8 +5295,9 @@ static LLVMValueRef ir_render_suspend_finish(CodeGen *g, IrExecutable *executabl
51915295 return nullptr;
51925296}
51935297
5194static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwait *instruction) {
5298static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInstructionAwaitGen *instruction) {
51955299 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
5300 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
51965301 LLVMValueRef target_frame_ptr = ir_llvm_value(g, instruction->frame);
51975302 ZigType *result_type = instruction->base.value.type;
51985303 ZigType *ptr_result_type = get_pointer_to_type(g, result_type, true);
......@@ -5208,86 +5313,75 @@ static LLVMValueRef ir_render_await(CodeGen *g, IrExecutable *executable, IrInst
52085313 // At this point resuming the function will do the correct thing.
52095314 // This code is as if it is running inside the suspend block.
52105315
5316 // supply the awaiter return pointer
5317 LLVMValueRef result_loc = (instruction->result_loc == nullptr) ?
5318 nullptr : ir_llvm_value(g, instruction->result_loc);
5319 if (type_has_bits(result_type)) {
5320 LLVMValueRef awaiter_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_ret_start + 1, "");
5321 if (result_loc == nullptr) {
5322 // no copy needed
5323 LLVMBuildStore(g->builder, zero, awaiter_ret_ptr_ptr);
5324 } else {
5325 LLVMBuildStore(g->builder, result_loc, awaiter_ret_ptr_ptr);
5326 }
5327 }
5328
5329 // supply the error return trace pointer
5330 LLVMValueRef my_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope);
5331 if (my_err_ret_trace_val != nullptr) {
5332 LLVMValueRef err_ret_trace_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr,
5333 frame_index_trace_arg(g, result_type), "");
5334 LLVMBuildStore(g->builder, my_err_ret_trace_val, err_ret_trace_ptr_ptr);
5335 }
5336
52115337 // caller's own frame pointer
52125338 LLVMValueRef awaiter_init_val = LLVMBuildPtrToInt(g->builder, g->cur_ret_ptr, usize_type_ref, "");
52135339 LLVMValueRef awaiter_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_awaiter_index, "");
5214 LLVMValueRef result_ptr_as_usize;
5215 if (type_has_bits(result_type)) {
5216 LLVMValueRef result_ptr_ptr = LLVMBuildStructGEP(g->builder, target_frame_ptr, coro_ret_start, "");
5217 LLVMValueRef result_ptr = LLVMBuildLoad(g->builder, result_ptr_ptr, "");
5218 result_ptr_as_usize = LLVMBuildPtrToInt(g->builder, result_ptr, usize_type_ref, "");
5219 } else {
5220 result_ptr_as_usize = LLVMGetUndef(usize_type_ref);
5221 }
52225340 LLVMValueRef prev_val = LLVMBuildAtomicRMW(g->builder, LLVMAtomicRMWBinOpXchg, awaiter_ptr, awaiter_init_val,
5223 LLVMAtomicOrderingMonotonic, g->is_single_threaded);
5341 LLVMAtomicOrderingRelease, g->is_single_threaded);
52245342
52255343 LLVMBasicBlockRef bad_await_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadAwait");
52265344 LLVMBasicBlockRef complete_suspend_block = LLVMAppendBasicBlock(g->cur_fn_val, "CompleteSuspend");
5345 LLVMBasicBlockRef early_return_block = LLVMAppendBasicBlock(g->cur_fn_val, "EarlyReturn");
52275346
5228 LLVMValueRef zero = LLVMConstNull(usize_type_ref);
52295347 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
52305348 LLVMValueRef switch_instr = LLVMBuildSwitch(g->builder, prev_val, bad_await_block, 2);
5231 LLVMBasicBlockRef predecessor_bb = LLVMGetInsertBlock(g->builder);
52325349
52335350 LLVMAddCase(switch_instr, zero, complete_suspend_block);
5234
5235 // Early return: The async function has already completed. No need to suspend.
5236 LLVMAddCase(switch_instr, all_ones, resume_bb);
5351 LLVMAddCase(switch_instr, all_ones, early_return_block);
52375352
52385353 // We discovered that another awaiter was already here.
52395354 LLVMPositionBuilderAtEnd(g->builder, bad_await_block);
52405355 gen_assertion(g, PanicMsgIdBadAwait, &instruction->base);
52415356
5357 // Early return: The async function has already completed, but it is suspending before setting the result,
5358 // populating the error return trace if applicable, and running the defers.
5359 // Tail resume it now, so that it can complete.
5360 LLVMPositionBuilderAtEnd(g->builder, early_return_block);
5361 LLVMValueRef call_inst = gen_resume(g, nullptr, target_frame_ptr, ResumeIdAwaitEarlyReturn, awaiter_init_val);
5362 ZigLLVMSetTailCall(call_inst);
5363 LLVMBuildRetVoid(g->builder);
5364
52425365 // Rely on the target to resume us from suspension.
52435366 LLVMPositionBuilderAtEnd(g->builder, complete_suspend_block);
52445367 LLVMBuildRetVoid(g->builder);
52455368
52465369 LLVMPositionBuilderAtEnd(g->builder, resume_bb);
5247 // We either got here from Entry (function call) or from the switch above
5248 LLVMValueRef spilled_result_ptr = LLVMBuildPhi(g->builder, usize_type_ref, "");
5249 LLVMValueRef incoming_values[] = { LLVMGetParam(g->cur_fn_val, 1), result_ptr_as_usize };
5250 LLVMBasicBlockRef incoming_blocks[] = { g->cur_preamble_llvm_block, predecessor_bb };
5251 LLVMAddIncoming(spilled_result_ptr, incoming_values, incoming_blocks, 2);
5252
5253 if (ir_want_runtime_safety(g, &instruction->base)) {
5254 LLVMBasicBlockRef bad_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "BadResume");
5255 LLVMBasicBlockRef ok_resume_block = LLVMAppendBasicBlock(g->cur_fn_val, "OkResume");
5256 LLVMValueRef all_ones = LLVMConstAllOnes(usize_type_ref);
5257 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntNE, spilled_result_ptr, all_ones, "");
5258 LLVMBuildCondBr(g->builder, ok_bit, ok_resume_block, bad_resume_block);
5259
5260 LLVMPositionBuilderAtEnd(g->builder, bad_resume_block);
5261 gen_safety_crash(g, PanicMsgIdResumedAnAwaitingFn);
5262
5263 LLVMPositionBuilderAtEnd(g->builder, ok_resume_block);
5264 }
5265
5266 render_async_var_decls(g, instruction->base.scope);
5267
5268 if (type_has_bits(result_type)) {
5269 LLVMValueRef casted_spilled_result_ptr = LLVMBuildIntToPtr(g->builder, spilled_result_ptr,
5270 get_llvm_type(g, ptr_result_type), "");
5271 return get_handle_value(g, casted_spilled_result_ptr, result_type, ptr_result_type);
5272 } else {
5273 return nullptr;
5370 gen_assert_resume_id(g, &instruction->base, ResumeIdReturn, PanicMsgIdResumedAnAwaitingFn, nullptr);
5371 if (type_has_bits(result_type) && result_loc != nullptr) {
5372 return get_handle_value(g, result_loc, result_type, ptr_result_type);
52745373 }
5374 return nullptr;
52755375}
52765376
52775377static LLVMValueRef ir_render_coro_resume(CodeGen *g, IrExecutable *executable,
52785378 IrInstructionCoroResume *instruction)
52795379{
5280 LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->llvm_type;
52815380 LLVMValueRef frame = ir_llvm_value(g, instruction->frame);
52825381 ZigType *frame_type = instruction->frame->value.type;
52835382 assert(frame_type->id == ZigTypeIdAnyFrame);
5284 LLVMValueRef fn_ptr_ptr = LLVMBuildStructGEP(g->builder, frame, coro_fn_ptr_index, "");
5285 LLVMValueRef uncasted_fn_val = LLVMBuildLoad(g->builder, fn_ptr_ptr, "");
5286 LLVMValueRef fn_val = LLVMBuildIntToPtr(g->builder, uncasted_fn_val, g->anyframe_fn_type, "");
5287 LLVMValueRef arg_val = ir_want_runtime_safety(g, &instruction->base) ?
5288 LLVMConstAllOnes(usize_type_ref) : LLVMGetUndef(usize_type_ref);
5289 LLVMValueRef args[] = {frame, arg_val};
5290 ZigLLVMBuildCall(g->builder, fn_val, args, 2, LLVMFastCallConv, ZigLLVM_FnInlineAuto, "");
5383
5384 gen_resume(g, nullptr, frame, ResumeIdManual, nullptr);
52915385 return nullptr;
52925386}
52935387
......@@ -5383,7 +5477,6 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable,
53835477 case IrInstructionIdImplicitCast:
53845478 case IrInstructionIdResolveResult:
53855479 case IrInstructionIdResetResult:
5386 case IrInstructionIdResultPtr:
53875480 case IrInstructionIdContainerInitList:
53885481 case IrInstructionIdSliceSrc:
53895482 case IrInstructionIdRef:
......@@ -5393,10 +5486,13 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable,
53935486 case IrInstructionIdFrameType:
53945487 case IrInstructionIdFrameSizeSrc:
53955488 case IrInstructionIdAllocaGen:
5489 case IrInstructionIdAwaitSrc:
53965490 zig_unreachable();
53975491
53985492 case IrInstructionIdDeclVarGen:
53995493 return ir_render_decl_var(g, executable, (IrInstructionDeclVarGen *)instruction);
5494 case IrInstructionIdReturnBegin:
5495 return ir_render_return_begin(g, executable, (IrInstructionReturnBegin *)instruction);
54005496 case IrInstructionIdReturn:
54015497 return ir_render_return(g, executable, (IrInstructionReturn *)instruction);
54025498 case IrInstructionIdBinOp:
......@@ -5547,8 +5643,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable,
55475643 return ir_render_coro_resume(g, executable, (IrInstructionCoroResume *)instruction);
55485644 case IrInstructionIdFrameSizeGen:
55495645 return ir_render_frame_size(g, executable, (IrInstructionFrameSizeGen *)instruction);
5550 case IrInstructionIdAwait:
5551 return ir_render_await(g, executable, (IrInstructionAwait *)instruction);
5646 case IrInstructionIdAwaitGen:
5647 return ir_render_await(g, executable, (IrInstructionAwaitGen *)instruction);
55525648 }
55535649 zig_unreachable();
55545650}
......@@ -6777,16 +6873,19 @@ static void do_code_gen(CodeGen *g) {
67776873 g->cur_async_awaiter_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_awaiter_index, "");
67786874 LLVMValueRef resume_index_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_resume_index, "");
67796875 g->cur_async_resume_index_ptr = resume_index_ptr;
6780 LLVMValueRef err_ret_trace_val = nullptr;
6781 uint32_t trace_field_index;
6876
6877 if (type_has_bits(fn_type_id->return_type)) {
6878 g->cur_ret_ptr_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, coro_ret_start, "");
6879 }
67826880 if (codegen_fn_has_err_ret_tracing_arg(g, fn_type_id->return_type)) {
6783 trace_field_index = frame_index_trace_arg(g, fn_type_id);
6784 err_ret_trace_val = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, trace_field_index, "");
6785 g->cur_err_ret_trace_val_arg = err_ret_trace_val;
6786 } else if (codegen_fn_has_err_ret_tracing_stack(g, fn_table_entry)) {
6787 trace_field_index = frame_index_trace_stack(g, fn_type_id);
6788 err_ret_trace_val = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, trace_field_index, "");
6789 g->cur_err_ret_trace_val_stack = err_ret_trace_val;
6881 uint32_t trace_field_index = frame_index_trace_arg(g, fn_type_id->return_type);
6882 g->cur_err_ret_trace_val_arg = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr, trace_field_index, "");
6883 }
6884 uint32_t trace_field_index_stack = UINT32_MAX;
6885 if (codegen_fn_has_err_ret_tracing_stack(g, fn_table_entry, true)) {
6886 trace_field_index_stack = frame_index_trace_stack(g, fn_type_id);
6887 g->cur_err_ret_trace_val_stack = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr,
6888 trace_field_index_stack, "");
67906889 }
67916890
67926891 LLVMValueRef resume_index = LLVMBuildLoad(g->builder, resume_index_ptr, "");
......@@ -6798,11 +6897,11 @@ static void do_code_gen(CodeGen *g) {
67986897 LLVMAddCase(switch_instr, zero, entry_block->llvm_block);
67996898 g->cur_resume_block_count += 1;
68006899 LLVMPositionBuilderAtEnd(g->builder, entry_block->llvm_block);
6801 if (err_ret_trace_val != nullptr) {
6900 if (trace_field_index_stack != UINT32_MAX) {
68026901 LLVMValueRef trace_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr,
6803 trace_field_index, "");
6902 trace_field_index_stack, "");
68046903 LLVMValueRef trace_field_addrs = LLVMBuildStructGEP(g->builder, g->cur_ret_ptr,
6805 trace_field_index + 1, "");
6904 trace_field_index_stack + 1, "");
68066905
68076906 LLVMValueRef index_ptr = LLVMBuildStructGEP(g->builder, trace_field_ptr, 0, "");
68086907 LLVMBuildStore(g->builder, zero, index_ptr);
......@@ -9725,7 +9824,7 @@ bool codegen_fn_has_err_ret_tracing_arg(CodeGen *g, ZigType *return_type) {
97259824 return_type->id == ZigTypeIdErrorSet);
97269825}
97279826
9728bool codegen_fn_has_err_ret_tracing_stack(CodeGen *g, ZigFn *fn) {
9827bool codegen_fn_has_err_ret_tracing_stack(CodeGen *g, ZigFn *fn, bool is_async) {
97299828 return g->have_err_ret_tracing && fn->calls_or_awaits_errorable_fn &&
9730 !codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type);
9829 (is_async || !codegen_fn_has_err_ret_tracing_arg(g, fn->type_entry->data.fn.fn_type_id.return_type));
97319830}
src/codegen.hpp+1-1
......@@ -62,6 +62,6 @@ TargetSubsystem detect_subsystem(CodeGen *g);
6262
6363void codegen_release_caches(CodeGen *codegen);
6464bool codegen_fn_has_err_ret_tracing_arg(CodeGen *g, ZigType *return_type);
65bool codegen_fn_has_err_ret_tracing_stack(CodeGen *g, ZigFn *fn);
65bool codegen_fn_has_err_ret_tracing_stack(CodeGen *g, ZigFn *fn, bool is_async);
6666
6767#endif
src/ir.cpp+139-106
......@@ -525,6 +525,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionReturn *) {
525525 return IrInstructionIdReturn;
526526}
527527
528static constexpr IrInstructionId ir_instruction_id(IrInstructionReturnBegin *) {
529 return IrInstructionIdReturnBegin;
530}
531
528532static constexpr IrInstructionId ir_instruction_id(IrInstructionCast *) {
529533 return IrInstructionIdCast;
530534}
......@@ -945,10 +949,6 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionResetResult *) {
945949 return IrInstructionIdResetResult;
946950}
947951
948static constexpr IrInstructionId ir_instruction_id(IrInstructionResultPtr *) {
949 return IrInstructionIdResultPtr;
950}
951
952952static constexpr IrInstructionId ir_instruction_id(IrInstructionPtrOfArrayToSlice *) {
953953 return IrInstructionIdPtrOfArrayToSlice;
954954}
......@@ -1049,8 +1049,12 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSuspendFinish *)
10491049 return IrInstructionIdSuspendFinish;
10501050}
10511051
1052static constexpr IrInstructionId ir_instruction_id(IrInstructionAwait *) {
1053 return IrInstructionIdAwait;
1052static constexpr IrInstructionId ir_instruction_id(IrInstructionAwaitSrc *) {
1053 return IrInstructionIdAwaitSrc;
1054}
1055
1056static constexpr IrInstructionId ir_instruction_id(IrInstructionAwaitGen *) {
1057 return IrInstructionIdAwaitGen;
10541058}
10551059
10561060static constexpr IrInstructionId ir_instruction_id(IrInstructionCoroResume *) {
......@@ -1109,18 +1113,32 @@ static IrInstruction *ir_build_cond_br(IrBuilder *irb, Scope *scope, AstNode *so
11091113}
11101114
11111115static IrInstruction *ir_build_return(IrBuilder *irb, Scope *scope, AstNode *source_node,
1112 IrInstruction *return_value)
1116 IrInstruction *operand)
11131117{
11141118 IrInstructionReturn *return_instruction = ir_build_instruction<IrInstructionReturn>(irb, scope, source_node);
11151119 return_instruction->base.value.type = irb->codegen->builtin_types.entry_unreachable;
11161120 return_instruction->base.value.special = ConstValSpecialStatic;
1117 return_instruction->value = return_value;
1121 return_instruction->operand = operand;
1122
1123 if (operand != nullptr) ir_ref_instruction(operand, irb->current_basic_block);
1124
1125 return &return_instruction->base;
1126}
1127
1128static IrInstruction *ir_build_return_begin(IrBuilder *irb, Scope *scope, AstNode *source_node,
1129 IrInstruction *operand)
1130{
1131 IrInstructionReturnBegin *return_instruction = ir_build_instruction<IrInstructionReturnBegin>(irb, scope, source_node);
1132 return_instruction->base.value.type = irb->codegen->builtin_types.entry_void;
1133 return_instruction->base.value.special = ConstValSpecialStatic;
1134 return_instruction->operand = operand;
11181135
1119 if (return_value != nullptr) ir_ref_instruction(return_value, irb->current_basic_block);
1136 ir_ref_instruction(operand, irb->current_basic_block);
11201137
11211138 return &return_instruction->base;
11221139}
11231140
1141
11241142static IrInstruction *ir_build_const_void(IrBuilder *irb, Scope *scope, AstNode *source_node) {
11251143 IrInstructionConst *const_instruction = ir_build_instruction<IrInstructionConst>(irb, scope, source_node);
11261144 const_instruction->base.value.type = irb->codegen->builtin_types.entry_void;
......@@ -2525,11 +2543,12 @@ static IrInstruction *ir_build_align_of(IrBuilder *irb, Scope *scope, AstNode *s
25252543}
25262544
25272545static IrInstruction *ir_build_test_err_src(IrBuilder *irb, Scope *scope, AstNode *source_node,
2528 IrInstruction *base_ptr, bool resolve_err_set)
2546 IrInstruction *base_ptr, bool resolve_err_set, bool base_ptr_is_payload)
25292547{
25302548 IrInstructionTestErrSrc *instruction = ir_build_instruction<IrInstructionTestErrSrc>(irb, scope, source_node);
25312549 instruction->base_ptr = base_ptr;
25322550 instruction->resolve_err_set = resolve_err_set;
2551 instruction->base_ptr_is_payload = base_ptr_is_payload;
25332552
25342553 ir_ref_instruction(base_ptr, irb->current_basic_block);
25352554
......@@ -2971,18 +2990,6 @@ static IrInstruction *ir_build_reset_result(IrBuilder *irb, Scope *scope, AstNod
29712990 return &instruction->base;
29722991}
29732992
2974static IrInstruction *ir_build_result_ptr(IrBuilder *irb, Scope *scope, AstNode *source_node,
2975 ResultLoc *result_loc, IrInstruction *result)
2976{
2977 IrInstructionResultPtr *instruction = ir_build_instruction<IrInstructionResultPtr>(irb, scope, source_node);
2978 instruction->result_loc = result_loc;
2979 instruction->result = result;
2980
2981 ir_ref_instruction(result, irb->current_basic_block);
2982
2983 return &instruction->base;
2984}
2985
29862993static IrInstruction *ir_build_opaque_type(IrBuilder *irb, Scope *scope, AstNode *source_node) {
29872994 IrInstructionOpaqueType *instruction = ir_build_instruction<IrInstructionOpaqueType>(irb, scope, source_node);
29882995
......@@ -3266,17 +3273,33 @@ static IrInstruction *ir_build_suspend_finish(IrBuilder *irb, Scope *scope, AstN
32663273 return &instruction->base;
32673274}
32683275
3269static IrInstruction *ir_build_await(IrBuilder *irb, Scope *scope, AstNode *source_node,
3270 IrInstruction *frame)
3276static IrInstruction *ir_build_await_src(IrBuilder *irb, Scope *scope, AstNode *source_node,
3277 IrInstruction *frame, ResultLoc *result_loc)
32713278{
3272 IrInstructionAwait *instruction = ir_build_instruction<IrInstructionAwait>(irb, scope, source_node);
3279 IrInstructionAwaitSrc *instruction = ir_build_instruction<IrInstructionAwaitSrc>(irb, scope, source_node);
32733280 instruction->frame = frame;
3281 instruction->result_loc = result_loc;
32743282
32753283 ir_ref_instruction(frame, irb->current_basic_block);
32763284
32773285 return &instruction->base;
32783286}
32793287
3288static IrInstruction *ir_build_await_gen(IrAnalyze *ira, IrInstruction *source_instruction,
3289 IrInstruction *frame, ZigType *result_type, IrInstruction *result_loc)
3290{
3291 IrInstructionAwaitGen *instruction = ir_build_instruction<IrInstructionAwaitGen>(&ira->new_irb,
3292 source_instruction->scope, source_instruction->source_node);
3293 instruction->base.value.type = result_type;
3294 instruction->frame = frame;
3295 instruction->result_loc = result_loc;
3296
3297 ir_ref_instruction(frame, ira->new_irb.current_basic_block);
3298 if (result_loc != nullptr) ir_ref_instruction(result_loc, ira->new_irb.current_basic_block);
3299
3300 return &instruction->base;
3301}
3302
32803303static IrInstruction *ir_build_coro_resume(IrBuilder *irb, Scope *scope, AstNode *source_node,
32813304 IrInstruction *frame)
32823305{
......@@ -3416,16 +3439,6 @@ static ScopeDeferExpr *get_scope_defer_expr(Scope *scope) {
34163439 return nullptr;
34173440}
34183441
3419static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *return_value,
3420 bool is_generated_code)
3421{
3422 ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value));
3423
3424 IrInstruction *return_inst = ir_build_return(irb, scope, node, return_value);
3425 return_inst->is_gen = is_generated_code;
3426 return return_inst;
3427}
3428
34293442static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, LVal lval, ResultLoc *result_loc) {
34303443 assert(node->type == NodeTypeReturnExpr);
34313444
......@@ -3467,19 +3480,16 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
34673480 return_value = ir_build_const_void(irb, scope, node);
34683481 }
34693482
3483 ir_build_return_begin(irb, scope, node, return_value);
3484
34703485 size_t defer_counts[2];
34713486 ir_count_defers(irb, scope, outer_scope, defer_counts);
34723487 bool have_err_defers = defer_counts[ReturnKindError] > 0;
34733488 if (have_err_defers || irb->codegen->have_err_ret_tracing) {
34743489 IrBasicBlock *err_block = ir_create_basic_block(irb, scope, "ErrRetErr");
34753490 IrBasicBlock *ok_block = ir_create_basic_block(irb, scope, "ErrRetOk");
3476 if (!have_err_defers) {
3477 ir_gen_defers_for_block(irb, scope, outer_scope, false);
3478 }
34793491
3480 IrInstruction *ret_ptr = ir_build_result_ptr(irb, scope, node, &result_loc_ret->base,
3481 return_value);
3482 IrInstruction *is_err = ir_build_test_err_src(irb, scope, node, ret_ptr, false);
3492 IrInstruction *is_err = ir_build_test_err_src(irb, scope, node, return_value, false, true);
34833493
34843494 bool should_inline = ir_should_inline(irb->exec, scope);
34853495 IrInstruction *is_comptime;
......@@ -3493,28 +3503,26 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
34933503 IrBasicBlock *ret_stmt_block = ir_create_basic_block(irb, scope, "RetStmt");
34943504
34953505 ir_set_cursor_at_end_and_append_block(irb, err_block);
3496 if (have_err_defers) {
3497 ir_gen_defers_for_block(irb, scope, outer_scope, true);
3498 }
34993506 if (irb->codegen->have_err_ret_tracing && !should_inline) {
35003507 ir_build_save_err_ret_addr(irb, scope, node);
35013508 }
3509 ir_gen_defers_for_block(irb, scope, outer_scope, true);
35023510 ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
35033511
35043512 ir_set_cursor_at_end_and_append_block(irb, ok_block);
3505 if (have_err_defers) {
3506 ir_gen_defers_for_block(irb, scope, outer_scope, false);
3507 }
3513 ir_gen_defers_for_block(irb, scope, outer_scope, false);
35083514 ir_build_br(irb, scope, node, ret_stmt_block, is_comptime);
35093515
35103516 ir_set_cursor_at_end_and_append_block(irb, ret_stmt_block);
3511 IrInstruction *result = ir_gen_async_return(irb, scope, node, return_value, false);
3517 ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value));
3518 IrInstruction *result = ir_build_return(irb, scope, node, return_value);
35123519 result_loc_ret->base.source_instruction = result;
35133520 return result;
35143521 } else {
35153522 // generate unconditional defers
35163523 ir_gen_defers_for_block(irb, scope, outer_scope, false);
3517 IrInstruction *result = ir_gen_async_return(irb, scope, node, return_value, false);
3524 ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value));
3525 IrInstruction *result = ir_build_return(irb, scope, node, return_value);
35183526 result_loc_ret->base.source_instruction = result;
35193527 return result;
35203528 }
......@@ -3525,7 +3533,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
35253533 IrInstruction *err_union_ptr = ir_gen_node_extra(irb, expr_node, scope, LValPtr, nullptr);
35263534 if (err_union_ptr == irb->codegen->invalid_instruction)
35273535 return irb->codegen->invalid_instruction;
3528 IrInstruction *is_err_val = ir_build_test_err_src(irb, scope, node, err_union_ptr, true);
3536 IrInstruction *is_err_val = ir_build_test_err_src(irb, scope, node, err_union_ptr, true, false);
35293537
35303538 IrBasicBlock *return_block = ir_create_basic_block(irb, scope, "ErrRetReturn");
35313539 IrBasicBlock *continue_block = ir_create_basic_block(irb, scope, "ErrRetContinue");
......@@ -3539,10 +3547,10 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
35393547 ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err_val, return_block, continue_block, is_comptime));
35403548
35413549 ir_set_cursor_at_end_and_append_block(irb, return_block);
3550 IrInstruction *err_val_ptr = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr);
3551 IrInstruction *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
3552 ir_build_return_begin(irb, scope, node, err_val);
35423553 if (!ir_gen_defers_for_block(irb, scope, outer_scope, true)) {
3543 IrInstruction *err_val_ptr = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr);
3544 IrInstruction *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
3545
35463554 ResultLocReturn *result_loc_ret = allocate<ResultLocReturn>(1);
35473555 result_loc_ret->base.id = ResultLocIdReturn;
35483556 ir_build_reset_result(irb, scope, node, &result_loc_ret->base);
......@@ -3551,7 +3559,8 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node,
35513559 if (irb->codegen->have_err_ret_tracing && !should_inline) {
35523560 ir_build_save_err_ret_addr(irb, scope, node);
35533561 }
3554 IrInstruction *ret_inst = ir_gen_async_return(irb, scope, node, err_val, false);
3562 ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val));
3563 IrInstruction *ret_inst = ir_build_return(irb, scope, node, err_val);
35553564 result_loc_ret->base.source_instruction = ret_inst;
35563565 }
35573566
......@@ -6081,7 +6090,8 @@ static IrInstruction *ir_gen_while_expr(IrBuilder *irb, Scope *scope, AstNode *n
60816090 LValPtr, nullptr);
60826091 if (err_val_ptr == irb->codegen->invalid_instruction)
60836092 return err_val_ptr;
6084 IrInstruction *is_err = ir_build_test_err_src(irb, scope, node->data.while_expr.condition, err_val_ptr, true);
6093 IrInstruction *is_err = ir_build_test_err_src(irb, scope, node->data.while_expr.condition, err_val_ptr,
6094 true, false);
60856095 IrBasicBlock *after_cond_block = irb->current_basic_block;
60866096 IrInstruction *void_else_result = else_node ? nullptr : ir_mark_gen(ir_build_const_void(irb, scope, node));
60876097 IrInstruction *cond_br_inst;
......@@ -6897,7 +6907,7 @@ static IrInstruction *ir_gen_if_err_expr(IrBuilder *irb, Scope *scope, AstNode *
68976907 return err_val_ptr;
68986908
68996909 IrInstruction *err_val = ir_build_load_ptr(irb, scope, node, err_val_ptr);
6900 IrInstruction *is_err = ir_build_test_err_src(irb, scope, node, err_val_ptr, true);
6910 IrInstruction *is_err = ir_build_test_err_src(irb, scope, node, err_val_ptr, true, false);
69016911
69026912 IrBasicBlock *ok_block = ir_create_basic_block(irb, scope, "TryOk");
69036913 IrBasicBlock *else_block = ir_create_basic_block(irb, scope, "TryElse");
......@@ -7513,7 +7523,7 @@ static IrInstruction *ir_gen_catch(IrBuilder *irb, Scope *parent_scope, AstNode
75137523 if (err_union_ptr == irb->codegen->invalid_instruction)
75147524 return irb->codegen->invalid_instruction;
75157525
7516 IrInstruction *is_err = ir_build_test_err_src(irb, parent_scope, node, err_union_ptr, true);
7526 IrInstruction *is_err = ir_build_test_err_src(irb, parent_scope, node, err_union_ptr, true, false);
75177527
75187528 IrInstruction *is_comptime;
75197529 if (ir_should_inline(irb->exec, parent_scope)) {
......@@ -7830,7 +7840,9 @@ static IrInstruction *ir_gen_resume(IrBuilder *irb, Scope *scope, AstNode *node)
78307840 return ir_build_coro_resume(irb, scope, node, target_inst);
78317841}
78327842
7833static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *node) {
7843static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *node, LVal lval,
7844 ResultLoc *result_loc)
7845{
78347846 assert(node->type == NodeTypeAwaitExpr);
78357847
78367848 ZigFn *fn_entry = exec_fn_entry(irb->exec);
......@@ -7852,7 +7864,8 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n
78527864 if (target_inst == irb->codegen->invalid_instruction)
78537865 return irb->codegen->invalid_instruction;
78547866
7855 return ir_build_await(irb, scope, node, target_inst);
7867 IrInstruction *await_inst = ir_build_await_src(irb, scope, node, target_inst, result_loc);
7868 return ir_lval_wrap(irb, scope, await_inst, lval, result_loc);
78567869}
78577870
78587871static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNode *node) {
......@@ -8016,7 +8029,7 @@ static IrInstruction *ir_gen_node_raw(IrBuilder *irb, AstNode *node, Scope *scop
80168029 case NodeTypeResume:
80178030 return ir_lval_wrap(irb, scope, ir_gen_resume(irb, scope, node), lval, result_loc);
80188031 case NodeTypeAwaitExpr:
8019 return ir_lval_wrap(irb, scope, ir_gen_await_expr(irb, scope, node), lval, result_loc);
8032 return ir_gen_await_expr(irb, scope, node, lval, result_loc);
80208033 case NodeTypeSuspend:
80218034 return ir_lval_wrap(irb, scope, ir_gen_suspend(irb, scope, node), lval, result_loc);
80228035 case NodeTypeEnumLiteral:
......@@ -8088,8 +8101,10 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec
80888101 return false;
80898102
80908103 if (!instr_is_unreachable(result)) {
8104 ir_mark_gen(ir_build_return_begin(irb, scope, node, result));
80918105 // no need for save_err_ret_addr because this cannot return error
8092 ir_gen_async_return(irb, scope, result->source_node, result, true);
8106 ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, result->source_node, result));
8107 ir_mark_gen(ir_build_return(irb, scope, result->source_node, result));
80938108 }
80948109
80958110 return true;
......@@ -8181,18 +8196,19 @@ static ConstExprValue *ir_exec_const_result(CodeGen *codegen, IrExecutable *exec
81818196 IrInstruction *instruction = bb->instruction_list.at(i);
81828197 if (instruction->id == IrInstructionIdReturn) {
81838198 IrInstructionReturn *ret_inst = (IrInstructionReturn *)instruction;
8184 IrInstruction *value = ret_inst->value;
8185 if (value->value.special == ConstValSpecialRuntime) {
8186 exec_add_error_node(codegen, exec, value->source_node,
8199 IrInstruction *operand = ret_inst->operand;
8200 if (operand->value.special == ConstValSpecialRuntime) {
8201 exec_add_error_node(codegen, exec, operand->source_node,
81878202 buf_sprintf("unable to evaluate constant expression"));
81888203 return &codegen->invalid_instruction->value;
81898204 }
8190 return &value->value;
8205 return &operand->value;
81918206 } else if (ir_has_side_effects(instruction)) {
81928207 if (instr_is_comptime(instruction)) {
81938208 switch (instruction->id) {
81948209 case IrInstructionIdUnwrapErrPayload:
81958210 case IrInstructionIdUnionFieldPtr:
8211 case IrInstructionIdReturnBegin:
81968212 continue;
81978213 default:
81988214 break;
......@@ -12593,12 +12609,32 @@ static IrInstruction *ir_analyze_instruction_add_implicit_return_type(IrAnalyze
1259312609 return ir_const_void(ira, &instruction->base);
1259412610}
1259512611
12612static IrInstruction *ir_analyze_instruction_return_begin(IrAnalyze *ira, IrInstructionReturnBegin *instruction) {
12613 IrInstruction *operand = instruction->operand->child;
12614 if (type_is_invalid(operand->value.type))
12615 return ira->codegen->invalid_instruction;
12616
12617 IrInstruction *casted_operand = ir_implicit_cast(ira, operand, ira->explicit_return_type);
12618 if (type_is_invalid(casted_operand->value.type)) {
12619 AstNode *source_node = ira->explicit_return_type_source_node;
12620 if (source_node != nullptr) {
12621 ErrorMsg *msg = ira->codegen->errors.last();
12622 add_error_note(ira->codegen, msg, source_node,
12623 buf_sprintf("return type declared here"));
12624 }
12625 return ir_unreach_error(ira);
12626 }
12627
12628 return ir_build_return_begin(&ira->new_irb, instruction->base.scope, instruction->base.source_node,
12629 casted_operand);
12630}
12631
1259612632static IrInstruction *ir_analyze_instruction_return(IrAnalyze *ira, IrInstructionReturn *instruction) {
12597 IrInstruction *value = instruction->value->child;
12598 if (type_is_invalid(value->value.type))
12633 IrInstruction *operand = instruction->operand->child;
12634 if (type_is_invalid(operand->value.type))
1259912635 return ir_unreach_error(ira);
1260012636
12601 if (!instr_is_comptime(value) && handle_is_ptr(ira->explicit_return_type)) {
12637 if (!instr_is_comptime(operand) && handle_is_ptr(ira->explicit_return_type)) {
1260212638 // result location mechanism took care of it.
1260312639 IrInstruction *result = ir_build_return(&ira->new_irb, instruction->base.scope,
1260412640 instruction->base.source_node, nullptr);
......@@ -12606,26 +12642,21 @@ static IrInstruction *ir_analyze_instruction_return(IrAnalyze *ira, IrInstructio
1260612642 return ir_finish_anal(ira, result);
1260712643 }
1260812644
12609 IrInstruction *casted_value = ir_implicit_cast(ira, value, ira->explicit_return_type);
12610 if (type_is_invalid(casted_value->value.type)) {
12611 AstNode *source_node = ira->explicit_return_type_source_node;
12612 if (source_node != nullptr) {
12613 ErrorMsg *msg = ira->codegen->errors.last();
12614 add_error_note(ira->codegen, msg, source_node,
12615 buf_sprintf("return type declared here"));
12616 }
12645 IrInstruction *casted_operand = ir_implicit_cast(ira, operand, ira->explicit_return_type);
12646 if (type_is_invalid(casted_operand->value.type)) {
12647 // error already reported by IrInstructionReturnBegin
1261712648 return ir_unreach_error(ira);
1261812649 }
1261912650
12620 if (casted_value->value.special == ConstValSpecialRuntime &&
12621 casted_value->value.type->id == ZigTypeIdPointer &&
12622 casted_value->value.data.rh_ptr == RuntimeHintPtrStack)
12651 if (casted_operand->value.special == ConstValSpecialRuntime &&
12652 casted_operand->value.type->id == ZigTypeIdPointer &&
12653 casted_operand->value.data.rh_ptr == RuntimeHintPtrStack)
1262312654 {
12624 ir_add_error(ira, casted_value, buf_sprintf("function returns address of local variable"));
12655 ir_add_error(ira, casted_operand, buf_sprintf("function returns address of local variable"));
1262512656 return ir_unreach_error(ira);
1262612657 }
1262712658 IrInstruction *result = ir_build_return(&ira->new_irb, instruction->base.scope,
12628 instruction->base.source_node, casted_value);
12659 instruction->base.source_node, casted_operand);
1262912660 result->value.type = ira->codegen->builtin_types.entry_unreachable;
1263012661 return ir_finish_anal(ira, result);
1263112662}
......@@ -22176,19 +22207,6 @@ static IrInstruction *ir_analyze_instruction_overflow_op(IrAnalyze *ira, IrInstr
2217622207 return result;
2217722208}
2217822209
22179static IrInstruction *ir_analyze_instruction_result_ptr(IrAnalyze *ira, IrInstructionResultPtr *instruction) {
22180 IrInstruction *result = instruction->result->child;
22181 if (type_is_invalid(result->value.type))
22182 return result;
22183
22184 if (instruction->result_loc->written && instruction->result_loc->resolved_loc != nullptr &&
22185 !instr_is_comptime(result))
22186 {
22187 return instruction->result_loc->resolved_loc;
22188 }
22189 return ir_get_ref(ira, &instruction->base, result, true, false);
22190}
22191
2219222210static void ir_eval_mul_add(IrAnalyze *ira, IrInstructionMulAdd *source_instr, ZigType *float_type,
2219322211 ConstExprValue *op1, ConstExprValue *op2, ConstExprValue *op3, ConstExprValue *out_val) {
2219422212 if (float_type->id == ZigTypeIdComptimeFloat) {
......@@ -22313,11 +22331,16 @@ static IrInstruction *ir_analyze_instruction_test_err(IrAnalyze *ira, IrInstruct
2231322331 if (type_is_invalid(base_ptr->value.type))
2231422332 return ira->codegen->invalid_instruction;
2231522333
22316 IrInstruction *value = ir_get_deref(ira, &instruction->base, base_ptr, nullptr);
22334 IrInstruction *value;
22335 if (instruction->base_ptr_is_payload) {
22336 value = base_ptr;
22337 } else {
22338 value = ir_get_deref(ira, &instruction->base, base_ptr, nullptr);
22339 }
22340
2231722341 ZigType *type_entry = value->value.type;
2231822342 if (type_is_invalid(type_entry))
2231922343 return ira->codegen->invalid_instruction;
22320
2232122344 if (type_entry->id == ZigTypeIdErrorUnion) {
2232222345 if (instr_is_comptime(value)) {
2232322346 ConstExprValue *err_union_val = ir_resolve_const(ira, value, UndefBad);
......@@ -24443,7 +24466,7 @@ static IrInstruction *ir_analyze_instruction_suspend_finish(IrAnalyze *ira,
2444324466 return ir_build_suspend_finish(&ira->new_irb, instruction->base.scope, instruction->base.source_node, begin);
2444424467}
2444524468
24446static IrInstruction *ir_analyze_instruction_await(IrAnalyze *ira, IrInstructionAwait *instruction) {
24469static IrInstruction *ir_analyze_instruction_await(IrAnalyze *ira, IrInstructionAwaitSrc *instruction) {
2444724470 IrInstruction *frame_ptr = instruction->frame->child;
2444824471 if (type_is_invalid(frame_ptr->value.type))
2444924472 return ira->codegen->invalid_instruction;
......@@ -24484,9 +24507,17 @@ static IrInstruction *ir_analyze_instruction_await(IrAnalyze *ira, IrInstruction
2448424507 fn_entry->calls_or_awaits_errorable_fn = true;
2448524508 }
2448624509
24487 IrInstruction *result = ir_build_await(&ira->new_irb,
24488 instruction->base.scope, instruction->base.source_node, frame);
24489 result->value.type = result_type;
24510 IrInstruction *result_loc;
24511 if (type_has_bits(result_type)) {
24512 result_loc = ir_resolve_result(ira, &instruction->base, instruction->result_loc,
24513 result_type, nullptr, true, false, true);
24514 if (result_loc != nullptr && (type_is_invalid(result_loc->value.type) || instr_is_unreachable(result_loc)))
24515 return result_loc;
24516 } else {
24517 result_loc = nullptr;
24518 }
24519
24520 IrInstruction *result = ir_build_await_gen(ira, &instruction->base, frame, result_type, result_loc);
2449024521 return ir_finish_anal(ira, result);
2449124522}
2449224523
......@@ -24541,8 +24572,11 @@ static IrInstruction *ir_analyze_instruction_base(IrAnalyze *ira, IrInstruction
2454124572 case IrInstructionIdRefGen:
2454224573 case IrInstructionIdTestErrGen:
2454324574 case IrInstructionIdFrameSizeGen:
24575 case IrInstructionIdAwaitGen:
2454424576 zig_unreachable();
2454524577
24578 case IrInstructionIdReturnBegin:
24579 return ir_analyze_instruction_return_begin(ira, (IrInstructionReturnBegin *)instruction);
2454624580 case IrInstructionIdReturn:
2454724581 return ir_analyze_instruction_return(ira, (IrInstructionReturn *)instruction);
2454824582 case IrInstructionIdConst:
......@@ -24749,8 +24783,6 @@ static IrInstruction *ir_analyze_instruction_base(IrAnalyze *ira, IrInstruction
2474924783 return ir_analyze_instruction_resolve_result(ira, (IrInstructionResolveResult *)instruction);
2475024784 case IrInstructionIdResetResult:
2475124785 return ir_analyze_instruction_reset_result(ira, (IrInstructionResetResult *)instruction);
24752 case IrInstructionIdResultPtr:
24753 return ir_analyze_instruction_result_ptr(ira, (IrInstructionResultPtr *)instruction);
2475424786 case IrInstructionIdOpaqueType:
2475524787 return ir_analyze_instruction_opaque_type(ira, (IrInstructionOpaqueType *)instruction);
2475624788 case IrInstructionIdSetAlignStack:
......@@ -24807,8 +24839,8 @@ static IrInstruction *ir_analyze_instruction_base(IrAnalyze *ira, IrInstruction
2480724839 return ir_analyze_instruction_suspend_finish(ira, (IrInstructionSuspendFinish *)instruction);
2480824840 case IrInstructionIdCoroResume:
2480924841 return ir_analyze_instruction_coro_resume(ira, (IrInstructionCoroResume *)instruction);
24810 case IrInstructionIdAwait:
24811 return ir_analyze_instruction_await(ira, (IrInstructionAwait *)instruction);
24842 case IrInstructionIdAwaitSrc:
24843 return ir_analyze_instruction_await(ira, (IrInstructionAwaitSrc *)instruction);
2481224844 }
2481324845 zig_unreachable();
2481424846}
......@@ -24898,6 +24930,7 @@ bool ir_has_side_effects(IrInstruction *instruction) {
2489824930 case IrInstructionIdStorePtr:
2489924931 case IrInstructionIdCallSrc:
2490024932 case IrInstructionIdCallGen:
24933 case IrInstructionIdReturnBegin:
2490124934 case IrInstructionIdReturn:
2490224935 case IrInstructionIdUnreachable:
2490324936 case IrInstructionIdSetCold:
......@@ -24943,7 +24976,8 @@ bool ir_has_side_effects(IrInstruction *instruction) {
2494324976 case IrInstructionIdSuspendBegin:
2494424977 case IrInstructionIdSuspendFinish:
2494524978 case IrInstructionIdCoroResume:
24946 case IrInstructionIdAwait:
24979 case IrInstructionIdAwaitSrc:
24980 case IrInstructionIdAwaitGen:
2494724981 return true;
2494824982
2494924983 case IrInstructionIdPhi:
......@@ -25041,7 +25075,6 @@ bool ir_has_side_effects(IrInstruction *instruction) {
2504125075 case IrInstructionIdHasDecl:
2504225076 case IrInstructionIdAllocaSrc:
2504325077 case IrInstructionIdAllocaGen:
25044 case IrInstructionIdResultPtr:
2504525078 return false;
2504625079
2504725080 case IrInstructionIdAsm:
src/ir_print.cpp+27-18
......@@ -64,11 +64,15 @@ static void ir_print_other_block(IrPrint *irp, IrBasicBlock *bb) {
6464 }
6565}
6666
67static void ir_print_return(IrPrint *irp, IrInstructionReturn *return_instruction) {
67static void ir_print_return_begin(IrPrint *irp, IrInstructionReturnBegin *instruction) {
68 fprintf(irp->f, "@returnBegin(");
69 ir_print_other_instruction(irp, instruction->operand);
70 fprintf(irp->f, ")");
71}
72
73static void ir_print_return(IrPrint *irp, IrInstructionReturn *instruction) {
6874 fprintf(irp->f, "return ");
69 if (return_instruction->value != nullptr) {
70 ir_print_other_instruction(irp, return_instruction->value);
71 }
75 ir_print_other_instruction(irp, instruction->operand);
7276}
7377
7478static void ir_print_const(IrPrint *irp, IrInstructionConst *const_instruction) {
......@@ -1329,14 +1333,6 @@ static void ir_print_reset_result(IrPrint *irp, IrInstructionResetResult *instru
13291333 fprintf(irp->f, ")");
13301334}
13311335
1332static void ir_print_result_ptr(IrPrint *irp, IrInstructionResultPtr *instruction) {
1333 fprintf(irp->f, "ResultPtr(");
1334 ir_print_result_loc(irp, instruction->result_loc);
1335 fprintf(irp->f, ",");
1336 ir_print_other_instruction(irp, instruction->result);
1337 fprintf(irp->f, ")");
1338}
1339
13401336static void ir_print_opaque_type(IrPrint *irp, IrInstructionOpaqueType *instruction) {
13411337 fprintf(irp->f, "@OpaqueType()");
13421338}
......@@ -1538,9 +1534,19 @@ static void ir_print_coro_resume(IrPrint *irp, IrInstructionCoroResume *instruct
15381534 fprintf(irp->f, ")");
15391535}
15401536
1541static void ir_print_await(IrPrint *irp, IrInstructionAwait *instruction) {
1537static void ir_print_await_src(IrPrint *irp, IrInstructionAwaitSrc *instruction) {
15421538 fprintf(irp->f, "@await(");
15431539 ir_print_other_instruction(irp, instruction->frame);
1540 fprintf(irp->f, ",");
1541 ir_print_result_loc(irp, instruction->result_loc);
1542 fprintf(irp->f, ")");
1543}
1544
1545static void ir_print_await_gen(IrPrint *irp, IrInstructionAwaitGen *instruction) {
1546 fprintf(irp->f, "@await(");
1547 ir_print_other_instruction(irp, instruction->frame);
1548 fprintf(irp->f, ",");
1549 ir_print_other_instruction(irp, instruction->result_loc);
15441550 fprintf(irp->f, ")");
15451551}
15461552
......@@ -1549,6 +1555,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) {
15491555 switch (instruction->id) {
15501556 case IrInstructionIdInvalid:
15511557 zig_unreachable();
1558 case IrInstructionIdReturnBegin:
1559 ir_print_return_begin(irp, (IrInstructionReturnBegin *)instruction);
1560 break;
15521561 case IrInstructionIdReturn:
15531562 ir_print_return(irp, (IrInstructionReturn *)instruction);
15541563 break;
......@@ -1921,9 +1930,6 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) {
19211930 case IrInstructionIdResetResult:
19221931 ir_print_reset_result(irp, (IrInstructionResetResult *)instruction);
19231932 break;
1924 case IrInstructionIdResultPtr:
1925 ir_print_result_ptr(irp, (IrInstructionResultPtr *)instruction);
1926 break;
19271933 case IrInstructionIdOpaqueType:
19281934 ir_print_opaque_type(irp, (IrInstructionOpaqueType *)instruction);
19291935 break;
......@@ -2020,8 +2026,11 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) {
20202026 case IrInstructionIdCoroResume:
20212027 ir_print_coro_resume(irp, (IrInstructionCoroResume *)instruction);
20222028 break;
2023 case IrInstructionIdAwait:
2024 ir_print_await(irp, (IrInstructionAwait *)instruction);
2029 case IrInstructionIdAwaitSrc:
2030 ir_print_await_src(irp, (IrInstructionAwaitSrc *)instruction);
2031 break;
2032 case IrInstructionIdAwaitGen:
2033 ir_print_await_gen(irp, (IrInstructionAwaitGen *)instruction);
20252034 break;
20262035 }
20272036 fprintf(irp->f, "\n");