authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-07-26 22:26:00-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-07-26 22:26:00-04:00
logb3f4182ca1756ccf84fe5bbc88594a91ead617b5
tree705dce8f230ae984dd5305bf9fb60252b50e5cca
parent2cbad364c1d23b64ae064f8547590c133b4f070a

coroutines have 3 more bits of atomic state

3 files changed, 66 insertions(+), 29 deletions(-)

src/all_types.hpp+1-1
......@@ -3245,7 +3245,7 @@ static const size_t stack_trace_ptr_count = 30;
32453245#define RESULT_FIELD_NAME "result"
32463246#define ASYNC_ALLOC_FIELD_NAME "allocFn"
32473247#define ASYNC_FREE_FIELD_NAME "freeFn"
3248#define AWAITER_HANDLE_FIELD_NAME "awaiter_handle"
3248#define ATOMIC_STATE_FIELD_NAME "atomic_state"
32493249// these point to data belonging to the awaiter
32503250#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr"
32513251#define RESULT_PTR_FIELD_NAME "result_ptr"
src/analyze.cpp+9-4
......@@ -519,11 +519,11 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type)
519519 return return_type->promise_frame_parent;
520520 }
521521
522 TypeTableEntry *awaiter_handle_type = get_optional_type(g, g->builtin_types.entry_promise);
522 TypeTableEntry *atomic_state_type = g->builtin_types.entry_usize;
523523 TypeTableEntry *result_ptr_type = get_pointer_to_type(g, return_type, false);
524524
525525 ZigList<const char *> field_names = {};
526 field_names.append(AWAITER_HANDLE_FIELD_NAME);
526 field_names.append(ATOMIC_STATE_FIELD_NAME);
527527 field_names.append(RESULT_FIELD_NAME);
528528 field_names.append(RESULT_PTR_FIELD_NAME);
529529 if (g->have_err_ret_tracing) {
......@@ -533,7 +533,7 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type)
533533 }
534534
535535 ZigList<TypeTableEntry *> field_types = {};
536 field_types.append(awaiter_handle_type);
536 field_types.append(atomic_state_type);
537537 field_types.append(return_type);
538538 field_types.append(result_ptr_type);
539539 if (g->have_err_ret_tracing) {
......@@ -6228,7 +6228,12 @@ uint32_t get_abi_alignment(CodeGen *g, TypeTableEntry *type_entry) {
62286228 } else if (type_entry->id == TypeTableEntryIdOpaque) {
62296229 return 1;
62306230 } else {
6231 return LLVMABIAlignmentOfType(g->target_data_ref, type_entry->type_ref);
6231 uint32_t llvm_alignment = LLVMABIAlignmentOfType(g->target_data_ref, type_entry->type_ref);
6232 // promises have at least alignment 8 so that we can have 3 extra bits when doing atomicrmw
6233 if (type_entry->id == TypeTableEntryIdPromise && llvm_alignment < 8) {
6234 return 8;
6235 }
6236 return llvm_alignment;
62326237 }
62336238}
62346239
src/ir.cpp+56-24
......@@ -3097,19 +3097,47 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode
30973097 return return_inst;
30983098 }
30993099
3100 IrBasicBlock *canceled_block = ir_create_basic_block(irb, scope, "Canceled");
3101 IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled");
3102 IrBasicBlock *suspended_block = ir_create_basic_block(irb, scope, "Suspended");
3103 IrBasicBlock *not_suspended_block = ir_create_basic_block(irb, scope, "NotSuspended");
3104
31003105 ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_field_ptr, return_value);
3101 IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node,
3102 get_optional_type(irb->codegen, irb->codegen->builtin_types.entry_promise));
3103 // TODO replace replacement_value with @intToPtr(?promise, 0x1) when it doesn't crash zig
3104 IrInstruction *replacement_value = irb->exec->coro_handle;
3105 IrInstruction *maybe_await_handle = ir_build_atomic_rmw(irb, scope, node,
3106 promise_type_val, irb->exec->coro_awaiter_field_ptr, nullptr, replacement_value, nullptr,
3107 AtomicRmwOp_xchg, AtomicOrderSeqCst);
3108 ir_build_store_ptr(irb, scope, node, irb->exec->await_handle_var_ptr, maybe_await_handle);
3109 IrInstruction *is_non_null = ir_build_test_nonnull(irb, scope, node, maybe_await_handle);
3106 IrInstruction *usize_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_usize);
3107 IrInstruction *replacement_value = ir_build_const_usize(irb, scope, node, 0xa); // 0b1010
3108 IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node,
3109 usize_type_val, irb->exec->coro_awaiter_field_ptr, nullptr, replacement_value, nullptr,
3110 AtomicRmwOp_or, AtomicOrderSeqCst);
3111
3112 IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
31103113 IrInstruction *is_comptime = ir_build_const_bool(irb, scope, node, false);
3111 return ir_build_cond_br(irb, scope, node, is_non_null, irb->exec->coro_normal_final, irb->exec->coro_early_final,
3112 is_comptime);
3114 IrInstruction *is_canceled_mask = ir_build_const_usize(irb, scope, node, 0x1); // 0b001
3115 IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false);
3116 IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false);
3117 ir_build_cond_br(irb, scope, node, is_canceled_bool, canceled_block, not_canceled_block, is_comptime);
3118
3119 ir_set_cursor_at_end_and_append_block(irb, canceled_block);
3120 ir_mark_gen(ir_build_br(irb, scope, node, irb->exec->coro_final_cleanup_block, is_comptime));
3121
3122 ir_set_cursor_at_end_and_append_block(irb, not_canceled_block);
3123 IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, scope, node, 0x7); // 0b111
3124 IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, inverted_ptr_mask, false);
3125 IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false);
3126 ir_build_cond_br(irb, scope, node, is_suspended_bool, suspended_block, not_suspended_block, is_comptime);
3127
3128 ir_set_cursor_at_end_and_append_block(irb, suspended_block);
3129 ir_build_unreachable(irb, scope, node);
3130
3131 ir_set_cursor_at_end_and_append_block(irb, not_suspended_block);
3132 IrInstruction *ptr_mask = ir_build_un_op(irb, scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000
3133 IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false);
3134 IrInstruction *promise_type_val = ir_build_const_type(irb, scope, node, irb->codegen->builtin_types.entry_promise);
3135 // if we ever add null checking safety to the ptrtoint instruction, it needs to be disabled here
3136 IrInstruction *await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, await_handle_addr);
3137 ir_build_store_ptr(irb, scope, node, irb->exec->await_handle_var_ptr, await_handle);
3138 IrInstruction *is_non_null = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false);
3139 return ir_build_cond_br(irb, scope, node, is_non_null, irb->exec->coro_normal_final,
3140 irb->exec->coro_early_final, is_comptime);
31133141 // the above blocks are rendered by ir_gen after the rest of codegen
31143142}
31153143
......@@ -6708,9 +6736,9 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast
67086736 ir_build_store_ptr(irb, parent_scope, node, err_ret_trace_ptr_field_ptr, err_ret_trace_ptr);
67096737 }
67106738
6711 Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME);
6712 IrInstruction *awaiter_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr,
6713 awaiter_handle_field_name);
6739 Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME);
6740 IrInstruction *atomic_state_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr,
6741 atomic_state_field_name);
67146742
67156743 IrInstruction *const_bool_false = ir_build_const_bool(irb, parent_scope, node, false);
67166744 VariableTableEntry *result_var = ir_create_var(irb, node, parent_scope, nullptr,
......@@ -6723,12 +6751,16 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast
67236751 IrInstruction *my_result_var_ptr = ir_build_var_ptr(irb, parent_scope, node, result_var);
67246752 ir_build_store_ptr(irb, parent_scope, node, result_ptr_field_ptr, my_result_var_ptr);
67256753 IrInstruction *save_token = ir_build_coro_save(irb, parent_scope, node, irb->exec->coro_handle);
6726 IrInstruction *promise_type_val = ir_build_const_type(irb, parent_scope, node,
6727 get_optional_type(irb->codegen, irb->codegen->builtin_types.entry_promise));
6728 IrInstruction *maybe_await_handle = ir_build_atomic_rmw(irb, parent_scope, node,
6729 promise_type_val, awaiter_field_ptr, nullptr, irb->exec->coro_handle, nullptr,
6730 AtomicRmwOp_xchg, AtomicOrderSeqCst);
6731 IrInstruction *is_non_null = ir_build_test_nonnull(irb, parent_scope, node, maybe_await_handle);
6754 IrInstruction *usize_type_val = ir_build_const_type(irb, parent_scope, node, irb->codegen->builtin_types.entry_usize);
6755 IrInstruction *coro_handle_addr = ir_build_ptr_to_int(irb, parent_scope, node, irb->exec->coro_handle);
6756 IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, parent_scope, node,
6757 usize_type_val, atomic_state_ptr, nullptr, coro_handle_addr, nullptr,
6758 AtomicRmwOp_or, AtomicOrderSeqCst);
6759 IrInstruction *zero = ir_build_const_usize(irb, parent_scope, node, 0);
6760 IrInstruction *inverted_ptr_mask = ir_build_const_usize(irb, parent_scope, node, 0x7); // 0b111
6761 IrInstruction *ptr_mask = ir_build_un_op(irb, parent_scope, node, IrUnOpBinNot, inverted_ptr_mask); // 0b111...000
6762 IrInstruction *await_handle_addr = ir_build_bin_op(irb, parent_scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false);
6763 IrInstruction *is_non_null = ir_build_bin_op(irb, parent_scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false);
67326764 IrBasicBlock *yes_suspend_block = ir_create_basic_block(irb, parent_scope, "YesSuspend");
67336765 IrBasicBlock *no_suspend_block = ir_create_basic_block(irb, parent_scope, "NoSuspend");
67346766 IrBasicBlock *merge_block = ir_create_basic_block(irb, parent_scope, "MergeSuspend");
......@@ -7087,10 +7119,11 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec
70877119 IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, maybe_coro_mem_ptr);
70887120 irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr);
70897121
7090 Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME);
7122 Buf *atomic_state_field_name = buf_create_from_str(ATOMIC_STATE_FIELD_NAME);
70917123 irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr,
7092 awaiter_handle_field_name);
7093 ir_build_store_ptr(irb, scope, node, irb->exec->coro_awaiter_field_ptr, null_value);
7124 atomic_state_field_name);
7125 IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
7126 ir_build_store_ptr(irb, scope, node, irb->exec->coro_awaiter_field_ptr, zero);
70947127 Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME);
70957128 irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name);
70967129 result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME);
......@@ -7108,7 +7141,6 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec
71087141 // coordinate with builtin.zig
71097142 Buf *index_name = buf_create_from_str("index");
71107143 IrInstruction *index_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, index_name);
7111 IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0);
71127144 ir_build_store_ptr(irb, scope, node, index_ptr, zero);
71137145
71147146 Buf *instruction_addresses_name = buf_create_from_str("instruction_addresses");