| ... | @@ -6663,7 +6663,9 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo | ... | @@ -6663,7 +6663,9 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo |
| 6663 | async_allocator_type_value, is_var_args); | 6663 | async_allocator_type_value, is_var_args); |
| 6664 | } | 6664 | } |
| 6665 | | 6665 | |
| 6666 | static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *target_inst) { | 6666 | static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode *node, |
| | 6667 | IrInstruction *target_inst, bool cancel_non_suspended, bool cancel_awaited) |
| | 6668 | { |
| 6667 | IrBasicBlock *done_block = ir_create_basic_block(irb, scope, "CancelDone"); | 6669 | IrBasicBlock *done_block = ir_create_basic_block(irb, scope, "CancelDone"); |
| 6668 | IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled"); | 6670 | IrBasicBlock *not_canceled_block = ir_create_basic_block(irb, scope, "NotCanceled"); |
| 6669 | IrBasicBlock *pre_return_block = ir_create_basic_block(irb, scope, "PreReturn"); | 6671 | IrBasicBlock *pre_return_block = ir_create_basic_block(irb, scope, "PreReturn"); |
| ... | @@ -6691,7 +6693,7 @@ static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode | ... | @@ -6691,7 +6693,7 @@ static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode |
| 6691 | // set the is_canceled bit | 6693 | // set the is_canceled bit |
| 6692 | IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, | 6694 | IrInstruction *prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| 6693 | usize_type_val, atomic_state_ptr, nullptr, is_canceled_mask, nullptr, | 6695 | usize_type_val, atomic_state_ptr, nullptr, is_canceled_mask, nullptr, |
| 6694 | AtomicRmwOp_and, AtomicOrderSeqCst); | 6696 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| 6695 | | 6697 | |
| 6696 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); | 6698 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| 6697 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); | 6699 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| ... | @@ -6703,14 +6705,26 @@ static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode | ... | @@ -6703,14 +6705,26 @@ static IrInstruction *ir_gen_cancel_target(IrBuilder *irb, Scope *scope, AstNode |
| 6703 | ir_build_cond_br(irb, scope, node, is_returned_bool, post_return_block, pre_return_block, is_comptime); | 6705 | ir_build_cond_br(irb, scope, node, is_returned_bool, post_return_block, pre_return_block, is_comptime); |
| 6704 | | 6706 | |
| 6705 | ir_set_cursor_at_end_and_append_block(irb, post_return_block); | 6707 | ir_set_cursor_at_end_and_append_block(irb, post_return_block); |
| 6706 | IrInstruction *is_awaited_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, await_mask, false); | 6708 | if (cancel_awaited) { |
| 6707 | IrInstruction *is_awaited_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_awaited_value, zero, false); | 6709 | ir_build_br(irb, scope, node, do_cancel_block, is_comptime); |
| 6708 | ir_build_cond_br(irb, scope, node, is_awaited_bool, done_block, do_cancel_block, is_comptime); | 6710 | } else { |
| | 6711 | IrInstruction *is_awaited_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, await_mask, false); |
| | 6712 | IrInstruction *is_awaited_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_awaited_value, zero, false); |
| | 6713 | ir_build_cond_br(irb, scope, node, is_awaited_bool, done_block, do_cancel_block, is_comptime); |
| | 6714 | } |
| 6709 | | 6715 | |
| 6710 | ir_set_cursor_at_end_and_append_block(irb, pre_return_block); | 6716 | ir_set_cursor_at_end_and_append_block(irb, pre_return_block); |
| 6711 | IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false); | 6717 | if (cancel_awaited) { |
| 6712 | IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false); | 6718 | if (cancel_non_suspended) { |
| 6713 | ir_build_cond_br(irb, scope, node, is_suspended_bool, do_cancel_block, done_block, is_comptime); | 6719 | ir_build_br(irb, scope, node, do_cancel_block, is_comptime); |
| | 6720 | } else { |
| | 6721 | IrInstruction *is_suspended_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_suspended_mask, false); |
| | 6722 | IrInstruction *is_suspended_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_suspended_value, zero, false); |
| | 6723 | ir_build_cond_br(irb, scope, node, is_suspended_bool, do_cancel_block, done_block, is_comptime); |
| | 6724 | } |
| | 6725 | } else { |
| | 6726 | ir_build_br(irb, scope, node, done_block, is_comptime); |
| | 6727 | } |
| 6714 | | 6728 | |
| 6715 | ir_set_cursor_at_end_and_append_block(irb, do_cancel_block); | 6729 | ir_set_cursor_at_end_and_append_block(irb, do_cancel_block); |
| 6716 | ir_build_cancel(irb, scope, node, target_inst); | 6730 | ir_build_cancel(irb, scope, node, target_inst); |
| ... | @@ -6727,7 +6741,7 @@ static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *scope, AstNode *node) | ... | @@ -6727,7 +6741,7 @@ static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *scope, AstNode *node) |
| 6727 | if (target_inst == irb->codegen->invalid_instruction) | 6741 | if (target_inst == irb->codegen->invalid_instruction) |
| 6728 | return irb->codegen->invalid_instruction; | 6742 | return irb->codegen->invalid_instruction; |
| 6729 | | 6743 | |
| 6730 | return ir_gen_cancel_target(irb, scope, node, target_inst); | 6744 | return ir_gen_cancel_target(irb, scope, node, target_inst, false, true); |
| 6731 | } | 6745 | } |
| 6732 | | 6746 | |
| 6733 | static IrInstruction *ir_gen_resume(IrBuilder *irb, Scope *scope, AstNode *node) { | 6747 | static IrInstruction *ir_gen_resume(IrBuilder *irb, Scope *scope, AstNode *node) { |
| ... | @@ -6872,15 +6886,19 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n | ... | @@ -6872,15 +6886,19 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n |
| 6872 | ir_build_unreachable(irb, scope, node); | 6886 | ir_build_unreachable(irb, scope, node); |
| 6873 | | 6887 | |
| 6874 | ir_set_cursor_at_end_and_append_block(irb, not_awaited_block); | 6888 | ir_set_cursor_at_end_and_append_block(irb, not_awaited_block); |
| | 6889 | IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false); |
| | 6890 | IrInstruction *is_non_null = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false); |
| 6875 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); | 6891 | IrInstruction *is_canceled_value = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, is_canceled_mask, false); |
| 6876 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); | 6892 | IrInstruction *is_canceled_bool = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, is_canceled_value, zero, false); |
| 6877 | ir_build_cond_br(irb, scope, node, is_canceled_bool, cleanup_block, not_canceled_block, const_bool_false); | 6893 | ir_build_cond_br(irb, scope, node, is_canceled_bool, cancel_target_block, not_canceled_block, const_bool_false); |
| 6878 | | 6894 | |
| 6879 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); | 6895 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); |
| 6880 | IrInstruction *await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, prev_atomic_value, ptr_mask, false); | | |
| 6881 | IrInstruction *is_non_null = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, await_handle_addr, zero, false); | | |
| 6882 | ir_build_cond_br(irb, scope, node, is_non_null, no_suspend_block, yes_suspend_block, const_bool_false); | 6896 | ir_build_cond_br(irb, scope, node, is_non_null, no_suspend_block, yes_suspend_block, const_bool_false); |
| 6883 | | 6897 | |
| | 6898 | ir_set_cursor_at_end_and_append_block(irb, cancel_target_block); |
| | 6899 | ir_build_cancel(irb, scope, node, target_inst); |
| | 6900 | ir_mark_gen(ir_build_br(irb, scope, node, cleanup_block, const_bool_false)); |
| | 6901 | |
| 6884 | ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); | 6902 | ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); |
| 6885 | if (irb->codegen->have_err_ret_tracing) { | 6903 | if (irb->codegen->have_err_ret_tracing) { |
| 6886 | Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); | 6904 | Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); |
| ... | @@ -6897,6 +6915,7 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n | ... | @@ -6897,6 +6915,7 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n |
| 6897 | ir_build_cancel(irb, scope, node, target_inst); | 6915 | ir_build_cancel(irb, scope, node, target_inst); |
| 6898 | ir_build_br(irb, scope, node, merge_block, const_bool_false); | 6916 | ir_build_br(irb, scope, node, merge_block, const_bool_false); |
| 6899 | | 6917 | |
| | 6918 | |
| 6900 | ir_set_cursor_at_end_and_append_block(irb, yes_suspend_block); | 6919 | ir_set_cursor_at_end_and_append_block(irb, yes_suspend_block); |
| 6901 | IrInstruction *suspend_code = ir_build_coro_suspend(irb, scope, node, save_token, const_bool_false); | 6920 | IrInstruction *suspend_code = ir_build_coro_suspend(irb, scope, node, save_token, const_bool_false); |
| 6902 | | 6921 | |
| ... | @@ -6904,32 +6923,28 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n | ... | @@ -6904,32 +6923,28 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *scope, AstNode *n |
| 6904 | cases[0].value = ir_build_const_u8(irb, scope, node, 0); | 6923 | cases[0].value = ir_build_const_u8(irb, scope, node, 0); |
| 6905 | cases[0].block = resume_block; | 6924 | cases[0].block = resume_block; |
| 6906 | cases[1].value = ir_build_const_u8(irb, scope, node, 1); | 6925 | cases[1].value = ir_build_const_u8(irb, scope, node, 1); |
| 6907 | cases[1].block = cancel_target_block; | 6926 | cases[1].block = cleanup_block; |
| 6908 | ir_build_switch_br(irb, scope, node, suspend_code, irb->exec->coro_suspend_block, | 6927 | ir_build_switch_br(irb, scope, node, suspend_code, irb->exec->coro_suspend_block, |
| 6909 | 2, cases, const_bool_false, nullptr); | 6928 | 2, cases, const_bool_false, nullptr); |
| 6910 | | 6929 | |
| 6911 | ir_set_cursor_at_end_and_append_block(irb, cancel_target_block); | | |
| 6912 | IrInstruction *await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, await_handle_addr); | | |
| 6913 | ir_gen_cancel_target(irb, scope, node, await_handle); | | |
| 6914 | ir_mark_gen(ir_build_br(irb, scope, node, cleanup_block, const_bool_false)); | | |
| 6915 | | | |
| 6916 | ir_set_cursor_at_end_and_append_block(irb, cleanup_block); | 6930 | ir_set_cursor_at_end_and_append_block(irb, cleanup_block); |
| 6917 | IrInstruction *my_mask_bits = ir_build_bin_op(irb, scope, node, IrBinOpBinOr, ptr_mask, is_canceled_mask, false); | 6931 | IrInstruction *my_mask_bits = ir_build_bin_op(irb, scope, node, IrBinOpBinOr, ptr_mask, is_canceled_mask, false); |
| 6918 | IrInstruction *my_prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, | 6932 | IrInstruction *my_prev_atomic_value = ir_build_atomic_rmw(irb, scope, node, |
| 6919 | usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, my_mask_bits, nullptr, | 6933 | usize_type_val, irb->exec->atomic_state_field_ptr, nullptr, my_mask_bits, nullptr, |
| 6920 | AtomicRmwOp_or, AtomicOrderSeqCst); | 6934 | AtomicRmwOp_or, AtomicOrderSeqCst); |
| 6921 | IrInstruction *my_await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, my_prev_atomic_value, ptr_mask, false); | 6935 | IrInstruction *my_await_handle_addr = ir_build_bin_op(irb, scope, node, IrBinOpBinAnd, my_prev_atomic_value, ptr_mask, false); |
| 6922 | IrInstruction *have_my_await_handle = ir_build_bin_op(irb, scope, node, IrBinOpCmpNotEq, my_await_handle_addr, zero, false); | 6936 | IrInstruction *dont_have_my_await_handle = ir_build_bin_op(irb, scope, node, IrBinOpCmpEq, my_await_handle_addr, zero, false); |
| 6923 | ir_build_cond_br(irb, scope, node, have_my_await_handle, do_cancel_block, do_defers_block, const_bool_false); | 6937 | IrInstruction *dont_destroy_ourselves = ir_build_bin_op(irb, scope, node, IrBinOpBoolAnd, dont_have_my_await_handle, is_canceled_bool, false); |
| | 6938 | ir_build_cond_br(irb, scope, node, dont_have_my_await_handle, do_defers_block, do_cancel_block, const_bool_false); |
| 6924 | | 6939 | |
| 6925 | ir_set_cursor_at_end_and_append_block(irb, do_cancel_block); | 6940 | ir_set_cursor_at_end_and_append_block(irb, do_cancel_block); |
| 6926 | IrInstruction *my_await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, my_await_handle_addr); | 6941 | IrInstruction *my_await_handle = ir_build_int_to_ptr(irb, scope, node, promise_type_val, my_await_handle_addr); |
| 6927 | ir_gen_cancel_target(irb, scope, node, my_await_handle); | 6942 | ir_gen_cancel_target(irb, scope, node, my_await_handle, true, false); |
| 6928 | ir_mark_gen(ir_build_br(irb, scope, node, do_defers_block, const_bool_false)); | 6943 | ir_mark_gen(ir_build_br(irb, scope, node, do_defers_block, const_bool_false)); |
| 6929 | | 6944 | |
| 6930 | ir_set_cursor_at_end_and_append_block(irb, do_defers_block); | 6945 | ir_set_cursor_at_end_and_append_block(irb, do_defers_block); |
| 6931 | ir_gen_defers_for_block(irb, scope, outer_scope, true); | 6946 | ir_gen_defers_for_block(irb, scope, outer_scope, true); |
| 6932 | ir_mark_gen(ir_build_br(irb, scope, node, irb->exec->coro_final_cleanup_block, const_bool_false)); | 6947 | ir_mark_gen(ir_build_cond_br(irb, scope, node, dont_destroy_ourselves, irb->exec->coro_early_final, irb->exec->coro_final_cleanup_block, const_bool_false)); |
| 6933 | | 6948 | |
| 6934 | ir_set_cursor_at_end_and_append_block(irb, resume_block); | 6949 | ir_set_cursor_at_end_and_append_block(irb, resume_block); |
| 6935 | ir_build_br(irb, scope, node, merge_block, const_bool_false); | 6950 | ir_build_br(irb, scope, node, merge_block, const_bool_false); |
| ... | @@ -7004,7 +7019,7 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod | ... | @@ -7004,7 +7019,7 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod |
| 7004 | | 7019 | |
| 7005 | ir_set_cursor_at_end_and_append_block(irb, cancel_awaiter_block); | 7020 | ir_set_cursor_at_end_and_append_block(irb, cancel_awaiter_block); |
| 7006 | IrInstruction *await_handle = ir_build_int_to_ptr(irb, parent_scope, node, promise_type_val, await_handle_addr); | 7021 | IrInstruction *await_handle = ir_build_int_to_ptr(irb, parent_scope, node, promise_type_val, await_handle_addr); |
| 7007 | ir_gen_cancel_target(irb, parent_scope, node, await_handle); | 7022 | ir_gen_cancel_target(irb, parent_scope, node, await_handle, true, false); |
| 7008 | ir_build_br(irb, parent_scope, node, cleanup_block, const_bool_false); | 7023 | ir_build_br(irb, parent_scope, node, cleanup_block, const_bool_false); |
| 7009 | | 7024 | |
| 7010 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); | 7025 | ir_set_cursor_at_end_and_append_block(irb, not_canceled_block); |