authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-07-26 22:26:00-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-07-26 22:26:00-04:00
logb3f4182ca1756ccf84fe5bbc88594a91ead617b5
tree705dce8f230ae984dd5305bf9fb60252b50e5cca
parent2cbad364c1d23b64ae064f8547590c133b4f070a

coroutines have 3 more bits of atomic state

3 files changed, 66 insertions(+), 29 deletions(-)

src/all_types.hpp+1-1
...@@ -3245,7 +3245,7 @@ static const size_t stack_trace_ptr_count = 30;...@@ -3245,7 +3245,7 @@ static const size_t stack_trace_ptr_count = 30;
3245#define RESULT_FIELD_NAME "result"3245#define RESULT_FIELD_NAME "result"
3246#define ASYNC_ALLOC_FIELD_NAME "allocFn"3246#define ASYNC_ALLOC_FIELD_NAME "allocFn"
3247#define ASYNC_FREE_FIELD_NAME "freeFn"3247#define ASYNC_FREE_FIELD_NAME "freeFn"
3248#define AWAITER_HANDLE_FIELD_NAME "awaiter_handle"3248#define ATOMIC_STATE_FIELD_NAME "atomic_state"
3249// these point to data belonging to the awaiter3249// these point to data belonging to the awaiter
3250#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr"3250#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr"
3251#define RESULT_PTR_FIELD_NAME "result_ptr"3251#define RESULT_PTR_FIELD_NAME "result_ptr"
src/analyze.cpp+9-4
...@@ -519,11 +519,11 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type)...@@ -519,11 +519,11 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type)
519 return return_type->promise_frame_parent;519 return return_type->promise_frame_parent;
520 }520 }
521521
522 TypeTableEntry *awaiter_handle_type = get_optional_type(g, g->builtin_types.entry_promise);522 TypeTableEntry *atomic_state_type = g->builtin_types.entry_usize;
523 TypeTableEntry *result_ptr_type = get_pointer_to_type(g, return_type, false);523 TypeTableEntry *result_ptr_type = get_pointer_to_type(g, return_type, false);
524524
525 ZigList<const char *> field_names = {};525 ZigList<const char *> field_names = {};
526 field_names.append(AWAITER_HANDLE_FIELD_NAME);526 field_names.append(ATOMIC_STATE_FIELD_NAME);
527 field_names.append(RESULT_FIELD_NAME);527 field_names.append(RESULT_FIELD_NAME);
528 field_names.append(RESULT_PTR_FIELD_NAME);528 field_names.append(RESULT_PTR_FIELD_NAME);
529 if (g->have_err_ret_tracing) {529 if (g->have_err_ret_tracing) {
...@@ -533,7 +533,7 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type)...@@ -533,7 +533,7 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type)
533 }533 }
534534
535 ZigList<TypeTableEntry *> field_types = {};535 ZigList<TypeTableEntry *> field_types = {};
536 field_types.append(awaiter_handle_type);536 field_types.append(atomic_state_type);
537 field_types.append(return_type);537 field_types.append(return_type);
538 field_types.append(result_ptr_type);538 field_types.append(result_ptr_type);
539 if (g->have_err_ret_tracing) {539 if (g->have_err_ret_tracing) {
...@@ -6228,7 +6228,12 @@ uint32_t get_abi_alignment(CodeGen *g, TypeTableEntry *type_entry) {...@@ -6228,7 +6228,12 @@ uint32_t get_abi_alignment(CodeGen *g, TypeTableEntry *type_entry) {
6228 } else if (type_entry->id == TypeTableEntryIdOpaque) {6228 } else if (type_entry->id == TypeTableEntryIdOpaque) {
6229 return 1;6229 return 1;
6230 } else {6230 } else {
6231 return LLVMABIAlignmentOfType(g->target_data_ref, type_entry->type_ref);6231 uint32_t llvm_alignment = LLVMABIAlignmentOfType(g->target_data_ref, type_entry->type_ref);
6232 // promises have at least alignment 8 so that we can have 3 extra bits when doing atomicrmw
6233 if (type_entry->id == TypeTableEntryIdPromise && llvm_alignment < 8) {
6234 return 8;
6235 }
6236 return llvm_alignment;
6232 }6237 }
6233}6238}
62346239
src/ir.cpp+56-24
...@@ -3097,19 +3097,47 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode...@@ -3097,19 +3097,47 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode
3097 return return_inst;3097 return return_inst;
3098 }3098 }
30993099
3100 IrBasicBlock *canceled_block = ir_create_basic_block(irb, scope, "Canceled");
3101 IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled");
3102 IrBasicBlock *suspended_block = ir_create_basic_block(irb, scope, "Suspended");
3103 IrBasicBlock *not_suspended_block = ir_create_basic_block(irb, scope, "NotSuspended");
3104
3100 ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_field_ptr, return_value);3105 ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_field_ptr, return_value);
3101 IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node,3106 IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize);
3102 get_optional_type(irb->codegen, irb->codegen->builtin_types.entry_promise));3107 IrInstruction *replacement_value = ir_build_const_usize(irb, scope, node, 0xa); // 0b1010
3103 // TODO replace replacement_value with @intToPtr(?promise, 0x1) when it doesn't crash zig3108 IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node,
3104 IrInstruction *replacement_value = irb->exec->coro_handle;3109 usize_type_val, irb->exec->coro_awaiter_field_ptr, nullptr, replacement_value, nullptr,
3105 IrInstruction *maybe_await_handle = ir_build_atomic_rmw(irb, scope, node,3110 AtomicRmwOp_or, AtomicOrderSeqCst);
3106 promise_type_val, irb->exec->coro_awaiter_field_ptr, nullptr, replacement_value, nullptr,3111
3107 AtomicRmwOp_xchg, AtomicOrderSeqCst);3112 IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
3108 ir_build_store_ptr(irb, scope, node, irb->exec->await_handle_var_ptr, maybe_await_handle);
3109 IrInstruction *is_non_null = ir_build_test_nonnull(irb, scope, node, maybe_await_handle);
3110 IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false);3113 IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false);
3111 return ir_build_cond_br(irb, scope, node, is_non_null, irb->exec->coro_normal_final, irb->exec->coro_early_final,3114 IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001
3112 is_comptime);3115 IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false);
3116 IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false);
3117 ir_build_cond_br(irb, scope, node, is_canceled_bool, canceled_block, not_canceled_block, is_comptime);
3118
3119 ir_set_cursor_at_end_and_append_block(irb, canceled_block);
3120 ir_mark_gen(ir_build_br(irb, scope, node, irb->exec->coro_final_cleanup_block, is_comptime));
3121
3122 ir_set_cursor_at_end_and_append_block(irb, not_canceled_block);
3123 IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, scope, node, 0x7); // 0b111
3124 IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, inverted_ptr_mask, false);
3125 IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false);
3126 ir_build_cond_br(irb, scope, node, is_suspended_bool, suspended_block, not_suspended_block, is_comptime);
3127
3128 ir_set_cursor_at_end_and_append_block(irb, suspended_block);
3129 ir_build_unreachable(irb, scope, node);
3130
3131 ir_set_cursor_at_end_and_append_block(irb, not_suspended_block);
3132 IrInstruction *ptr_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000
3133 IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false);
3134 IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_promise);
3135 // if we ever add null checking safety to the ptrtoint instruction, it needs to be disabled here
3136 IrInstruction *await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, await_handle_addr);
3137 ir_build_store_ptr(irb, scope, node, irb->exec->await_handle_var_ptr, await_handle);
3138 IrInstruction *is_non_null = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false);
3139 return ir_build_cond_br(irb, scope, node, is_non_null, irb->exec->coro_normal_final,
3140 irb->exec->coro_early_final, is_comptime);
3113 // the above blocks are rendered by ir_gen after the rest of codegen3141 // the above blocks are rendered by ir_gen after the rest of codegen
3114}3142}
31153143
...@@ -6708,9 +6736,9 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast...@@ -6708,9 +6736,9 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast
6708 ir_build_store_ptr(irb, parent_scope, node, err_ret_trace_ptr_field_ptr, err_ret_trace_ptr);6736 ir_build_store_ptr(irb, parent_scope, node, err_ret_trace_ptr_field_ptr, err_ret_trace_ptr);
6709 }6737 }
67106738
6711 Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME);6739 Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME);
6712 IrInstruction *awaiter_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr,6740 IrInstruction *atomic_state_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr,
6713 awaiter_handle_field_name);6741 atomic_state_field_name);
67146742
6715 IrInstruction *const_bool_false = ir_build_const_bool(irb, parent_scope, node, false);6743 IrInstruction *const_bool_false = ir_build_const_bool(irb, parent_scope, node, false);
6716 VariableTableEntry *result_var = ir_create_var(irb, node, parent_scope, nullptr,6744 VariableTableEntry *result_var = ir_create_var(irb, node, parent_scope, nullptr,
...@@ -6723,12 +6751,16 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast...@@ -6723,12 +6751,16 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast
6723 IrInstruction *my_result_var_ptr = ir_build_var_ptr(irb, parent_scope, node, result_var);6751 IrInstruction *my_result_var_ptr = ir_build_var_ptr(irb, parent_scope, node, result_var);
6724 ir_build_store_ptr(irb, parent_scope, node, result_ptr_field_ptr, my_result_var_ptr);6752 ir_build_store_ptr(irb, parent_scope, node, result_ptr_field_ptr, my_result_var_ptr);
6725 IrInstruction *save_token = ir_build_coro_save(irb, parent_scope, node, irb->exec->coro_handle);6753 IrInstruction *save_token = ir_build_coro_save(irb, parent_scope, node, irb->exec->coro_handle);
6726 IrInstruction *promise_type_val = ir_build_const_type(irb, parent_scope, node,6754 IrInstruction *usize_type_val = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_usize);
6727 get_optional_type(irb->codegen, irb->codegen->builtin_types.entry_promise));6755 IrInstruction *coro_handle_addr = ir_build_ptr_to_int(irb, parent_scope, node, irb->exec->coro_handle);
6728 IrInstruction *maybe_await_handle = ir_build_atomic_rmw(irb, parent_scope, node, 6756 IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, parent_scope, node,
6729 promise_type_val, awaiter_field_ptr, nullptr, irb->exec->coro_handle, nullptr,6757 usize_type_val, atomic_state_ptr, nullptr, coro_handle_addr, nullptr,
6730 AtomicRmwOp_xchg, AtomicOrderSeqCst);6758 AtomicRmwOp_or, AtomicOrderSeqCst);
6731 IrInstruction *is_non_null = ir_build_test_nonnull(irb, parent_scope, node, maybe_await_handle);6759 IrInstruction *zero = ir_build_const_usize(irb, parent_scope, node, 0);
6760 IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, parent_scope, node, 0x7); // 0b111
6761 IrInstruction *ptr_mask = ir_build_un_op(irb, parent_scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000
6762 IrInstruction *await_handle_addr = ir_build_bin_op(irb, parent_scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false);
6763 IrInstruction *is_non_null = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false);
6732 IrBasicBlock *yes_suspend_block = ir_create_basic_block(irb, parent_scope, "YesSuspend");6764 IrBasicBlock *yes_suspend_block = ir_create_basic_block(irb, parent_scope, "YesSuspend");
6733 IrBasicBlock *no_suspend_block = ir_create_basic_block(irb, parent_scope, "NoSuspend");6765 IrBasicBlock *no_suspend_block = ir_create_basic_block(irb, parent_scope, "NoSuspend");
6734 IrBasicBlock *merge_block = ir_create_basic_block(irb, parent_scope, "MergeSuspend");6766 IrBasicBlock *merge_block = ir_create_basic_block(irb, parent_scope, "MergeSuspend");
...@@ -7087,10 +7119,11 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec...@@ -7087,10 +7119,11 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec
7087 IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, maybe_coro_mem_ptr);7119 IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, maybe_coro_mem_ptr);
7088 irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr);7120 irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr);
70897121
7090 Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME);7122 Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME);
7091 irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr,7123 irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr,
7092 awaiter_handle_field_name);7124 atomic_state_field_name);
7093 ir_build_store_ptr(irb, scope, node, irb->exec->coro_awaiter_field_ptr, null_value);7125 IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
7126 ir_build_store_ptr(irb, scope, node, irb->exec->coro_awaiter_field_ptr, zero);
7094 Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME);7127 Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME);
7095 irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name);7128 irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name);
7096 result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME);7129 result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME);
...@@ -7108,7 +7141,6 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec...@@ -7108,7 +7141,6 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec
7108 // coordinate with builtin.zig7141 // coordinate with builtin.zig
7109 Buf *index_name = buf_create_from_str("index");7142 Buf *index_name = buf_create_from_str("index");
7110 IrInstruction *index_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, index_name);7143 IrInstruction *index_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, index_name);
7111 IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
7112 ir_build_store_ptr(irb, scope, node, index_ptr, zero);7144 ir_build_store_ptr(irb, scope, node, index_ptr, zero);
71137145
7114 Buf *instruction_addresses_name = buf_create_from_str("instruction_addresses");7146 Buf *instruction_addresses_name = buf_create_from_str("instruction_addresses");