Commit 457c94d353b77b08786aa8794e1afc6a62d5c34a | zig

lib/std/builtin.zig+19

...	...	@@ -675,6 +675,25 @@ pub const ExternOptions = struct {
675	675	is_thread_local: bool = false,
676	676	};
677	677
	678	/// This data structure is used by the Zig language code generation and
	679	/// therefore must be kept in sync with the compiler implementation.
	680	pub const BranchHint = enum(u3) {
	681	/// Equivalent to no hint given.
	682	none,
	683	/// This branch of control flow is more likely to be reached than its peers.
	684	/// The optimizer should optimize for reaching it.
	685	likely,
	686	/// This branch of control flow is less likely to be reached than its peers.
	687	/// The optimizer should optimize for not reaching it.
	688	unlikely,
	689	/// This branch of control flow is unlikely to ever be reached.
	690	/// The optimizer may place it in a different page of memory to optimize other branches.
	691	cold,
	692	/// It is difficult to predict whether this branch of control flow will be reached.
	693	/// The optimizer should avoid branching behavior with expensive mispredictions.
	694	unpredictable,
	695	};
	696
678	697	/// This enum is set by the compiler and communicates which compiler backend is
679	698	/// used to produce machine code.
680	699	/// Think carefully before deciding to observe this value. Nearly all code should

lib/std/zig/AstGen.zig+91-53

...	...	@@ -811,18 +811,18 @@ fn expr(gz: GenZir, scope: Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE
811	811	.builtin_call_two, .builtin_call_two_comma => {
812	812	if (node_datas[node].lhs == 0) {
813	813	const params = [_]Ast.Node.Index{};
814		return builtinCall(gz, scope, ri, node, &params);
	814	return builtinCall(gz, scope, ri, node, &params, false);
815	815	} else if (node_datas[node].rhs == 0) {
816	816	const params = [_]Ast.Node.Index{node_datas[node].lhs};
817		return builtinCall(gz, scope, ri, node, &params);
	817	return builtinCall(gz, scope, ri, node, &params, false);
818	818	} else {
819	819	const params = [_]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
820		return builtinCall(gz, scope, ri, node, &params);
	820	return builtinCall(gz, scope, ri, node, &params, false);
821	821	}
822	822	},
823	823	.builtin_call, .builtin_call_comma => {
824	824	const params = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
825		return builtinCall(gz, scope, ri, node, params);
	825	return builtinCall(gz, scope, ri, node, params, false);
826	826	},
827	827
828	828	.call_one,
...	...	@@ -1017,16 +1017,16 @@ fn expr(gz: GenZir, scope: Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE
1017	1017	.block_two, .block_two_semicolon => {
1018	1018	const statements = [2]Ast.Node.Index{ node_datas[node].lhs, node_datas[node].rhs };
1019	1019	if (node_datas[node].lhs == 0) {
1020		return blockExpr(gz, scope, ri, node, statements[0..0]);
	1020	return blockExpr(gz, scope, ri, node, statements[0..0], .normal);
1021	1021	} else if (node_datas[node].rhs == 0) {
1022		return blockExpr(gz, scope, ri, node, statements[0..1]);
	1022	return blockExpr(gz, scope, ri, node, statements[0..1], .normal);
1023	1023	} else {
1024		return blockExpr(gz, scope, ri, node, statements[0..2]);
	1024	return blockExpr(gz, scope, ri, node, statements[0..2], .normal);
1025	1025	}
1026	1026	},
1027	1027	.block, .block_semicolon => {
1028	1028	const statements = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
1029		return blockExpr(gz, scope, ri, node, statements);
	1029	return blockExpr(gz, scope, ri, node, statements, .normal);
1030	1030	},
1031	1031	.enum_literal => return simpleStrTok(gz, ri, main_tokens[node], node, .enum_literal),
1032	1032	.error_value => return simpleStrTok(gz, ri, node_datas[node].rhs, node, .error_value),
...	...	@@ -1241,7 +1241,7 @@ fn suspendExpr(
1241	1241	suspend_scope.suspend_node = node;
1242	1242	defer suspend_scope.unstack();
1243	1243
1244		const body_result = try fullBodyExpr(&suspend_scope, &suspend_scope.base, .{ .rl = .none }, body_node);
	1244	const body_result = try fullBodyExpr(&suspend_scope, &suspend_scope.base, .{ .rl = .none }, body_node, .normal);
1245	1245	if (!gz.refIsNoReturn(body_result)) {
1246	1246	_ = try suspend_scope.addBreak(.break_inline, suspend_inst, .void_value);
1247	1247	}
...	...	@@ -1362,7 +1362,7 @@ fn fnProtoExpr(
1362	1362	assert(param_type_node != 0);
1363	1363	var param_gz = block_scope.makeSubBlock(scope);
1364	1364	defer param_gz.unstack();
1365		const param_type = try fullBodyExpr(&param_gz, scope, coerced_type_ri, param_type_node);
	1365	const param_type = try fullBodyExpr(&param_gz, scope, coerced_type_ri, param_type_node, .normal);
1366	1366	const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1);
1367	1367	_ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node);
1368	1368	const main_tokens = tree.nodes.items(.main_token);
...	...	@@ -2040,13 +2040,13 @@ fn comptimeExpr(
2040	2040	else
2041	2041	stmts[0..2];
2042	2042
2043		const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmt_slice, true);
	2043	const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmt_slice, true, .normal);
2044	2044	return rvalue(gz, ri, block_ref, node);
2045	2045	},
2046	2046	.block, .block_semicolon => {
2047	2047	const stmts = tree.extra_data[node_datas[node].lhs..node_datas[node].rhs];
2048	2048	// Replace result location and copy back later - see above.
2049		const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmts, true);
	2049	const block_ref = try labeledBlockExpr(gz, scope, ty_only_ri, node, stmts, true, .normal);
2050	2050	return rvalue(gz, ri, block_ref, node);
2051	2051	},
2052	2052	else => unreachable,
...	...	@@ -2071,7 +2071,7 @@ fn comptimeExpr(
2071	2071	else
2072	2072	.none,
2073	2073	};
2074		const block_result = try fullBodyExpr(&block_scope, scope, ty_only_ri, node);
	2074	const block_result = try fullBodyExpr(&block_scope, scope, ty_only_ri, node, .normal);
2075	2075	if (!gz.refIsNoReturn(block_result)) {
2076	2076	_ = try block_scope.addBreak(.@"break", block_inst, block_result);
2077	2077	}
...	...	@@ -2311,6 +2311,7 @@ fn fullBodyExpr(
2311	2311	scope: *Scope,
2312	2312	ri: ResultInfo,
2313	2313	node: Ast.Node.Index,
	2314	block_kind: BlockKind,
2314	2315	) InnerError!Zir.Inst.Ref {
2315	2316	const tree = gz.astgen.tree;
2316	2317	const node_tags = tree.nodes.items(.tag);
...	...	@@ -2340,21 +2341,24 @@ fn fullBodyExpr(
2340	2341	// Labeled blocks are tricky - forwarding result location information properly is non-trivial,
2341	2342	// plus if this block is exited with a `break_inline` we aren't allowed multiple breaks. This
2342	2343	// case is rare, so just treat it as a normal expression and create a nested block.
2343		return expr(gz, scope, ri, node);
	2344	return blockExpr(gz, scope, ri, node, statements, block_kind);
2344	2345	}
2345	2346
2346	2347	var sub_gz = gz.makeSubBlock(scope);
2347		try blockExprStmts(&sub_gz, &sub_gz.base, statements);
	2348	try blockExprStmts(&sub_gz, &sub_gz.base, statements, block_kind);
2348	2349
2349	2350	return rvalue(gz, ri, .void_value, node);
2350	2351	}
2351	2352
	2353	const BlockKind = enum { normal, allow_branch_hint };
	2354
2352	2355	fn blockExpr(
2353	2356	gz: *GenZir,
2354	2357	scope: *Scope,
2355	2358	ri: ResultInfo,
2356	2359	block_node: Ast.Node.Index,
2357	2360	statements: []const Ast.Node.Index,
	2361	kind: BlockKind,
2358	2362	) InnerError!Zir.Inst.Ref {
2359	2363	const astgen = gz.astgen;
2360	2364	const tree = astgen.tree;
...	...	@@ -2365,7 +2369,7 @@ fn blockExpr(
2365	2369	if (token_tags[lbrace - 1] == .colon and
2366	2370	token_tags[lbrace - 2] == .identifier)
2367	2371	{
2368		return labeledBlockExpr(gz, scope, ri, block_node, statements, false);
	2372	return labeledBlockExpr(gz, scope, ri, block_node, statements, false, kind);
2369	2373	}
2370	2374
2371	2375	if (!gz.is_comptime) {
...	...	@@ -2380,7 +2384,7 @@ fn blockExpr(
2380	2384	var block_scope = gz.makeSubBlock(scope);
2381	2385	defer block_scope.unstack();
2382	2386
2383		try blockExprStmts(&block_scope, &block_scope.base, statements);
	2387	try blockExprStmts(&block_scope, &block_scope.base, statements, kind);
2384	2388
2385	2389	if (!block_scope.endsWithNoReturn()) {
2386	2390	// As our last action before the break, "pop" the error trace if needed
...	...	@@ -2391,7 +2395,7 @@ fn blockExpr(
2391	2395	try block_scope.setBlockBody(block_inst);
2392	2396	} else {
2393	2397	var sub_gz = gz.makeSubBlock(scope);
2394		try blockExprStmts(&sub_gz, &sub_gz.base, statements);
	2398	try blockExprStmts(&sub_gz, &sub_gz.base, statements, kind);
2395	2399	}
2396	2400
2397	2401	return rvalue(gz, ri, .void_value, block_node);
...	...	@@ -2436,6 +2440,7 @@ fn labeledBlockExpr(
2436	2440	block_node: Ast.Node.Index,
2437	2441	statements: []const Ast.Node.Index,
2438	2442	force_comptime: bool,
	2443	block_kind: BlockKind,
2439	2444	) InnerError!Zir.Inst.Ref {
2440	2445	const astgen = gz.astgen;
2441	2446	const tree = astgen.tree;
...	...	@@ -2476,7 +2481,7 @@ fn labeledBlockExpr(
2476	2481	if (force_comptime) block_scope.is_comptime = true;
2477	2482	defer block_scope.unstack();
2478	2483
2479		try blockExprStmts(&block_scope, &block_scope.base, statements);
	2484	try blockExprStmts(&block_scope, &block_scope.base, statements, block_kind);
2480	2485	if (!block_scope.endsWithNoReturn()) {
2481	2486	// As our last action before the return, "pop" the error trace if needed
2482	2487	_ = try gz.addRestoreErrRetIndex(.{ .block = block_inst }, .always, block_node);
...	...	@@ -2495,7 +2500,7 @@ fn labeledBlockExpr(
2495	2500	}
2496	2501	}
2497	2502
2498		fn blockExprStmts(gz: GenZir, parent_scope: Scope, statements: []const Ast.Node.Index) !void {
	2503	fn blockExprStmts(gz: GenZir, parent_scope: Scope, statements: []const Ast.Node.Index, block_kind: BlockKind) !void {
2499	2504	const astgen = gz.astgen;
2500	2505	const tree = astgen.tree;
2501	2506	const node_tags = tree.nodes.items(.tag);
...	...	@@ -2509,7 +2514,7 @@ fn blockExprStmts(gz: GenZir, parent_scope: Scope, statements: []const Ast.Nod
2509	2514
2510	2515	var noreturn_src_node: Ast.Node.Index = 0;
2511	2516	var scope = parent_scope;
2512		for (statements) \|statement\| {
	2517	for (statements, 0..) \|statement, stmt_idx\| {
2513	2518	if (noreturn_src_node != 0) {
2514	2519	try astgen.appendErrorNodeNotes(
2515	2520	statement,
...	...	@@ -2524,6 +2529,10 @@ fn blockExprStmts(gz: GenZir, parent_scope: Scope, statements: []const Ast.Nod
2524	2529	},
2525	2530	);
2526	2531	}
	2532	const allow_branch_hint = switch (block_kind) {
	2533	.normal => false,
	2534	.allow_branch_hint => stmt_idx == 0,
	2535	};
2527	2536	var inner_node = statement;
2528	2537	while (true) {
2529	2538	switch (node_tags[inner_node]) {
...	...	@@ -2567,6 +2576,30 @@ fn blockExprStmts(gz: GenZir, parent_scope: Scope, statements: []const Ast.Nod
2567	2576	.for_simple,
2568	2577	.@"for", => _ = try forExpr(gz, scope, .{ .rl = .none }, inner_node, tree.fullFor(inner_node).?, true),
2569	2578
	2579	// These cases are here to allow branch hints.
	2580	.builtin_call_two, .builtin_call_two_comma => {
	2581	try emitDbgNode(gz, inner_node);
	2582	const ri: ResultInfo = .{ .rl = .none };
	2583	const result = if (node_data[inner_node].lhs == 0) r: {
	2584	break :r try builtinCall(gz, scope, ri, inner_node, &.{}, allow_branch_hint);
	2585	} else if (node_data[inner_node].rhs == 0) r: {
	2586	break :r try builtinCall(gz, scope, ri, inner_node, &.{node_data[inner_node].lhs}, allow_branch_hint);
	2587	} else r: {
	2588	break :r try builtinCall(gz, scope, ri, inner_node, &.{
	2589	node_data[inner_node].lhs,
	2590	node_data[inner_node].rhs,
	2591	}, allow_branch_hint);
	2592	};
	2593	noreturn_src_node = try addEnsureResult(gz, result, inner_node);
	2594	},
	2595	.builtin_call, .builtin_call_comma => {
	2596	try emitDbgNode(gz, inner_node);
	2597	const ri: ResultInfo = .{ .rl = .none };
	2598	const params = tree.extra_data[node_data[inner_node].lhs..node_data[inner_node].rhs];
	2599	const result = try builtinCall(gz, scope, ri, inner_node, params, allow_branch_hint);
	2600	noreturn_src_node = try addEnsureResult(gz, result, inner_node);
	2601	},
	2602
2570	2603	else => noreturn_src_node = try unusedResultExpr(gz, scope, inner_node),
2571	2604	// zig fmt: on
2572	2605	}
...	...	@@ -2827,7 +2860,7 @@ fn addEnsureResult(gz: *GenZir, maybe_unused_result: Zir.Inst.Ref, statement: As
2827	2860	.fence,
2828	2861	.set_float_mode,
2829	2862	.set_align_stack,
2830		.set_cold,
	2863	.branch_hint,
2831	2864	=> break :b true,
2832	2865	else => break :b false,
2833	2866	},
...	...	@@ -4154,7 +4187,7 @@ fn fnDecl(
4154	4187	assert(param_type_node != 0);
4155	4188	var param_gz = decl_gz.makeSubBlock(scope);
4156	4189	defer param_gz.unstack();
4157		const param_type = try fullBodyExpr(&param_gz, params_scope, coerced_type_ri, param_type_node);
	4190	const param_type = try fullBodyExpr(&param_gz, params_scope, coerced_type_ri, param_type_node, .normal);
4158	4191	const param_inst_expected: Zir.Inst.Index = @enumFromInt(astgen.instructions.len + 1);
4159	4192	_ = try param_gz.addBreakWithSrcNode(.break_inline, param_inst_expected, param_type, param_type_node);
4160	4193
...	...	@@ -4276,7 +4309,7 @@ fn fnDecl(
4276	4309	var ret_gz = decl_gz.makeSubBlock(params_scope);
4277	4310	defer ret_gz.unstack();
4278	4311	const ret_ref: Zir.Inst.Ref = inst: {
4279		const inst = try fullBodyExpr(&ret_gz, params_scope, coerced_type_ri, fn_proto.ast.return_type);
	4312	const inst = try fullBodyExpr(&ret_gz, params_scope, coerced_type_ri, fn_proto.ast.return_type, .normal);
4280	4313	if (ret_gz.instructionsSlice().len == 0) {
4281	4314	// In this case we will send a len=0 body which can be encoded more efficiently.
4282	4315	break :inst inst;
...	...	@@ -4351,7 +4384,7 @@ fn fnDecl(
4351	4384	const lbrace_line = astgen.source_line - decl_gz.decl_line;
4352	4385	const lbrace_column = astgen.source_column;
4353	4386
4354		_ = try fullBodyExpr(&fn_gz, params_scope, .{ .rl = .none }, body_node);
	4387	_ = try fullBodyExpr(&fn_gz, params_scope, .{ .rl = .none }, body_node, .allow_branch_hint);
4355	4388	try checkUsed(gz, &fn_gz.base, params_scope);
4356	4389
4357	4390	if (!fn_gz.endsWithNoReturn()) {
...	...	@@ -4552,20 +4585,20 @@ fn globalVarDecl(
4552	4585
4553	4586	var align_gz = block_scope.makeSubBlock(scope);
4554	4587	if (var_decl.ast.align_node != 0) {
4555		const align_inst = try fullBodyExpr(&align_gz, &align_gz.base, coerced_align_ri, var_decl.ast.align_node);
	4588	const align_inst = try fullBodyExpr(&align_gz, &align_gz.base, coerced_align_ri, var_decl.ast.align_node, .normal);
4556	4589	_ = try align_gz.addBreakWithSrcNode(.break_inline, decl_inst, align_inst, node);
4557	4590	}
4558	4591
4559	4592	var linksection_gz = align_gz.makeSubBlock(scope);
4560	4593	if (var_decl.ast.section_node != 0) {
4561		const linksection_inst = try fullBodyExpr(&linksection_gz, &linksection_gz.base, coerced_linksection_ri, var_decl.ast.section_node);
	4594	const linksection_inst = try fullBodyExpr(&linksection_gz, &linksection_gz.base, coerced_linksection_ri, var_decl.ast.section_node, .normal);
4562	4595	_ = try linksection_gz.addBreakWithSrcNode(.break_inline, decl_inst, linksection_inst, node);
4563	4596	}
4564	4597
4565	4598	var addrspace_gz = linksection_gz.makeSubBlock(scope);
4566	4599	if (var_decl.ast.addrspace_node != 0) {
4567	4600	const addrspace_ty = try addrspace_gz.addBuiltinValue(var_decl.ast.addrspace_node, .address_space);
4568		const addrspace_inst = try fullBodyExpr(&addrspace_gz, &addrspace_gz.base, .{ .rl = .{ .coerced_ty = addrspace_ty } }, var_decl.ast.addrspace_node);
	4601	const addrspace_inst = try fullBodyExpr(&addrspace_gz, &addrspace_gz.base, .{ .rl = .{ .coerced_ty = addrspace_ty } }, var_decl.ast.addrspace_node, .normal);
4569	4602	_ = try addrspace_gz.addBreakWithSrcNode(.break_inline, decl_inst, addrspace_inst, node);
4570	4603	}
4571	4604
...	...	@@ -4622,7 +4655,7 @@ fn comptimeDecl(
4622	4655	};
4623	4656	defer decl_block.unstack();
4624	4657
4625		const block_result = try fullBodyExpr(&decl_block, &decl_block.base, .{ .rl = .none }, body_node);
	4658	const block_result = try fullBodyExpr(&decl_block, &decl_block.base, .{ .rl = .none }, body_node, .normal);
4626	4659	if (decl_block.isEmpty() or !decl_block.refIsNoReturn(block_result)) {
4627	4660	_ = try decl_block.addBreak(.break_inline, decl_inst, .void_value);
4628	4661	}
...	...	@@ -4843,7 +4876,7 @@ fn testDecl(
4843	4876	const lbrace_line = astgen.source_line - decl_block.decl_line;
4844	4877	const lbrace_column = astgen.source_column;
4845	4878
4846		const block_result = try fullBodyExpr(&fn_block, &fn_block.base, .{ .rl = .none }, body_node);
	4879	const block_result = try fullBodyExpr(&fn_block, &fn_block.base, .{ .rl = .none }, body_node, .normal);
4847	4880	if (fn_block.isEmpty() or !fn_block.refIsNoReturn(block_result)) {
4848	4881
4849	4882	// As our last action before the return, "pop" the error trace if needed
...	...	@@ -6112,7 +6145,7 @@ fn orelseCatchExpr(
6112	6145	break :blk &err_val_scope.base;
6113	6146	};
6114	6147
6115		const else_result = try fullBodyExpr(&else_scope, else_sub_scope, block_scope.break_result_info, rhs);
	6148	const else_result = try fullBodyExpr(&else_scope, else_sub_scope, block_scope.break_result_info, rhs, .allow_branch_hint);
6116	6149	if (!else_scope.endsWithNoReturn()) {
6117	6150	// As our last action before the break, "pop" the error trace if needed
6118	6151	if (do_err_trace)
...	...	@@ -6280,7 +6313,7 @@ fn boolBinOp(
6280	6313
6281	6314	var rhs_scope = gz.makeSubBlock(scope);
6282	6315	defer rhs_scope.unstack();
6283		const rhs = try fullBodyExpr(&rhs_scope, &rhs_scope.base, coerced_bool_ri, node_datas[node].rhs);
	6316	const rhs = try fullBodyExpr(&rhs_scope, &rhs_scope.base, coerced_bool_ri, node_datas[node].rhs, .allow_branch_hint);
6284	6317	if (!gz.refIsNoReturn(rhs)) {
6285	6318	_ = try rhs_scope.addBreakWithSrcNode(.break_inline, bool_br, rhs, node_datas[node].rhs);
6286	6319	}
...	...	@@ -6424,7 +6457,7 @@ fn ifExpr(
6424	6457	}
6425	6458	};
6426	6459
6427		const then_result = try fullBodyExpr(&then_scope, then_sub_scope, block_scope.break_result_info, then_node);
	6460	const then_result = try fullBodyExpr(&then_scope, then_sub_scope, block_scope.break_result_info, then_node, .allow_branch_hint);
6428	6461	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
6429	6462	if (!then_scope.endsWithNoReturn()) {
6430	6463	_ = try then_scope.addBreakWithSrcNode(.@"break", block, then_result, then_node);
...	...	@@ -6466,7 +6499,7 @@ fn ifExpr(
6466	6499	break :s &else_scope.base;
6467	6500	}
6468	6501	};
6469		const else_result = try fullBodyExpr(&else_scope, sub_scope, block_scope.break_result_info, else_node);
	6502	const else_result = try fullBodyExpr(&else_scope, sub_scope, block_scope.break_result_info, else_node, .allow_branch_hint);
6470	6503	if (!else_scope.endsWithNoReturn()) {
6471	6504	// As our last action before the break, "pop" the error trace if needed
6472	6505	if (do_err_trace)
...	...	@@ -6575,7 +6608,7 @@ fn whileExpr(
6575	6608	} = c: {
6576	6609	if (while_full.error_token) \|_\| {
6577	6610	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
6578		const err_union = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr);
	6611	const err_union = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr, .normal);
6579	6612	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_err_ptr else .is_non_err;
6580	6613	break :c .{
6581	6614	.inst = err_union,
...	...	@@ -6583,14 +6616,14 @@ fn whileExpr(
6583	6616	};
6584	6617	} else if (while_full.payload_token) \|_\| {
6585	6618	const cond_ri: ResultInfo = .{ .rl = if (payload_is_ref) .ref else .none };
6586		const optional = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr);
	6619	const optional = try fullBodyExpr(&cond_scope, &cond_scope.base, cond_ri, while_full.ast.cond_expr, .normal);
6587	6620	const tag: Zir.Inst.Tag = if (payload_is_ref) .is_non_null_ptr else .is_non_null;
6588	6621	break :c .{
6589	6622	.inst = optional,
6590	6623	.bool_bit = try cond_scope.addUnNode(tag, optional, while_full.ast.cond_expr),
6591	6624	};
6592	6625	} else {
6593		const cond = try fullBodyExpr(&cond_scope, &cond_scope.base, coerced_bool_ri, while_full.ast.cond_expr);
	6626	const cond = try fullBodyExpr(&cond_scope, &cond_scope.base, coerced_bool_ri, while_full.ast.cond_expr, .normal);
6594	6627	break :c .{
6595	6628	.inst = cond,
6596	6629	.bool_bit = cond,
...	...	@@ -6715,7 +6748,7 @@ fn whileExpr(
6715	6748	continue_scope.instructions_top = continue_scope.instructions.items.len;
6716	6749	{
6717	6750	try emitDbgNode(&continue_scope, then_node);
6718		const unused_result = try fullBodyExpr(&continue_scope, &continue_scope.base, .{ .rl = .none }, then_node);
	6751	const unused_result = try fullBodyExpr(&continue_scope, &continue_scope.base, .{ .rl = .none }, then_node, .allow_branch_hint);
6719	6752	_ = try addEnsureResult(&continue_scope, unused_result, then_node);
6720	6753	}
6721	6754	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
...	...	@@ -6761,7 +6794,7 @@ fn whileExpr(
6761	6794	// control flow apply to outer loops; not this one.
6762	6795	loop_scope.continue_block = .none;
6763	6796	loop_scope.break_block = .none;
6764		const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node);
	6797	const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node, .allow_branch_hint);
6765	6798	if (is_statement) {
6766	6799	_ = try addEnsureResult(&else_scope, else_result, else_node);
6767	6800	}
...	...	@@ -7029,7 +7062,7 @@ fn forExpr(
7029	7062	break :blk capture_sub_scope;
7030	7063	};
7031	7064
7032		const then_result = try fullBodyExpr(&then_scope, then_sub_scope, .{ .rl = .none }, then_node);
	7065	const then_result = try fullBodyExpr(&then_scope, then_sub_scope, .{ .rl = .none }, then_node, .allow_branch_hint);
7033	7066	_ = try addEnsureResult(&then_scope, then_result, then_node);
7034	7067
7035	7068	try checkUsed(parent_gz, &then_scope.base, then_sub_scope);
...	...	@@ -7048,7 +7081,7 @@ fn forExpr(
7048	7081	// control flow apply to outer loops; not this one.
7049	7082	loop_scope.continue_block = .none;
7050	7083	loop_scope.break_block = .none;
7051		const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node);
	7084	const else_result = try fullBodyExpr(&else_scope, sub_scope, loop_scope.break_result_info, else_node, .allow_branch_hint);
7052	7085	if (is_statement) {
7053	7086	_ = try addEnsureResult(&else_scope, else_result, else_node);
7054	7087	}
...	...	@@ -7525,7 +7558,7 @@ fn switchExprErrUnion(
7525	7558	}
7526	7559
7527	7560	const target_expr_node = case.ast.target_expr;
7528		const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node);
	7561	const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node, .allow_branch_hint);
7529	7562	// check capture_scope, not err_scope to avoid false positive unused error capture
7530	7563	try checkUsed(parent_gz, &case_scope.base, err_scope.parent);
7531	7564	const uses_err = err_scope.used != 0 or err_scope.discarded != 0;
...	...	@@ -7986,7 +8019,7 @@ fn switchExpr(
7986	8019	try case_scope.addDbgVar(.dbg_var_val, dbg_var_tag_name, dbg_var_tag_inst);
7987	8020	}
7988	8021	const target_expr_node = case.ast.target_expr;
7989		const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node);
	8022	const case_result = try fullBodyExpr(&case_scope, sub_scope, block_scope.break_result_info, target_expr_node, .allow_branch_hint);
7990	8023	try checkUsed(parent_gz, &case_scope.base, sub_scope);
7991	8024	if (!parent_gz.refIsNoReturn(case_result)) {
7992	8025	_ = try case_scope.addBreakWithSrcNode(.@"break", switch_block, case_result, target_expr_node);
...	...	@@ -9154,6 +9187,7 @@ fn builtinCall(
9154	9187	ri: ResultInfo,
9155	9188	node: Ast.Node.Index,
9156	9189	params: []const Ast.Node.Index,
	9190	allow_branch_hint: bool,
9157	9191	) InnerError!Zir.Inst.Ref {
9158	9192	const astgen = gz.astgen;
9159	9193	const tree = astgen.tree;
...	...	@@ -9187,6 +9221,18 @@ fn builtinCall(
9187	9221	return astgen.failNode(node, "'{s}' outside function scope", .{builtin_name});
9188	9222
9189	9223	switch (info.tag) {
	9224	.branch_hint => {
	9225	if (!allow_branch_hint) {
	9226	return astgen.failNode(node, "'@branchHint' must appear as the first statement in a function or conditional branch", .{});
	9227	}
	9228	const hint_ty = try gz.addBuiltinValue(node, .branch_hint);
	9229	const hint_val = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = hint_ty } }, params[0]);
	9230	_ = try gz.addExtendedPayload(.branch_hint, Zir.Inst.UnNode{
	9231	.node = gz.nodeIndexToRelative(node),
	9232	.operand = hint_val,
	9233	});
	9234	return rvalue(gz, ri, .void_value, node);
	9235	},
9190	9236	.import => {
9191	9237	const node_tags = tree.nodes.items(.tag);
9192	9238	const operand_node = params[0];
...	...	@@ -9294,14 +9340,6 @@ fn builtinCall(
9294	9340	});
9295	9341	return rvalue(gz, ri, .void_value, node);
9296	9342	},
9297		.set_cold => {
9298		const order = try expr(gz, scope, ri, params[0]);
9299		_ = try gz.addExtendedPayload(.set_cold, Zir.Inst.UnNode{
9300		.node = gz.nodeIndexToRelative(node),
9301		.operand = order,
9302		});
9303		return rvalue(gz, ri, .void_value, node);
9304		},
9305	9343
9306	9344	.src => {
9307	9345	// Incorporate the source location into the source hash, so that
...	...	@@ -9963,7 +10001,7 @@ fn cImport(
9963	10001	defer block_scope.unstack();
9964	10002
9965	10003	const block_inst = try gz.makeBlockInst(.c_import, node);
9966		const block_result = try fullBodyExpr(&block_scope, &block_scope.base, .{ .rl = .none }, body_node);
	10004	const block_result = try fullBodyExpr(&block_scope, &block_scope.base, .{ .rl = .none }, body_node, .normal);
9967	10005	_ = try gz.addUnNode(.ensure_result_used, block_result, node);
9968	10006	if (!gz.refIsNoReturn(block_result)) {
9969	10007	_ = try block_scope.addBreak(.break_inline, block_inst, .void_value);
...	...	@@ -10046,7 +10084,7 @@ fn callExpr(
10046	10084	defer arg_block.unstack();
10047	10085
10048	10086	// `call_inst` is reused to provide the param type.
10049		const arg_ref = try fullBodyExpr(&arg_block, &arg_block.base, .{ .rl = .{ .coerced_ty = call_inst }, .ctx = .fn_arg }, param_node);
	10087	const arg_ref = try fullBodyExpr(&arg_block, &arg_block.base, .{ .rl = .{ .coerced_ty = call_inst }, .ctx = .fn_arg }, param_node, .normal);
10050	10088	_ = try arg_block.addBreakWithSrcNode(.break_inline, call_index, arg_ref, param_node);
10051	10089
10052	10090	const body = arg_block.instructionsSlice();

lib/std/zig/AstRlAnnotate.zig+4-1

...	...	@@ -829,6 +829,10 @@ fn builtinCall(astrl: AstRlAnnotate, block: ?Block, ri: ResultInfo, node: Ast.
829	829	}
830	830	switch (info.tag) {
831	831	.import => return false,
	832	.branch_hint => {
	833	_ = try astrl.expr(args[0], block, ResultInfo.type_only);
	834	return false;
	835	},
832	836	.compile_log, .TypeOf => {
833	837	for (args) \|arg_node\| {
834	838	_ = try astrl.expr(arg_node, block, ResultInfo.none);
...	...	@@ -907,7 +911,6 @@ fn builtinCall(astrl: AstRlAnnotate, block: ?Block, ri: ResultInfo, node: Ast.
907	911	.fence,
908	912	.set_float_mode,
909	913	.set_align_stack,
910		.set_cold,
911	914	.type_info,
912	915	.work_item_id,
913	916	.work_group_size,

lib/std/zig/BuiltinFn.zig+9-9

...	...	@@ -14,6 +14,7 @@ pub const Tag = enum {
14	14	bit_offset_of,
15	15	int_from_bool,
16	16	bit_size_of,
	17	branch_hint,
17	18	breakpoint,
18	19	disable_instrumentation,
19	20	mul_add,
...	...	@@ -82,7 +83,6 @@ pub const Tag = enum {
82	83	return_address,
83	84	select,
84	85	set_align_stack,
85		set_cold,
86	86	set_eval_branch_quota,
87	87	set_float_mode,
88	88	set_runtime_safety,
...	...	@@ -256,6 +256,14 @@ pub const list = list: {
256	256	.param_count = 1,
257	257	},
258	258	},
	259	.{
	260	"@branchHint",
	261	.{
	262	.tag = .branch_hint,
	263	.param_count = 1,
	264	.illegal_outside_function = true,
	265	},
	266	},
259	267	.{
260	268	"@breakpoint",
261	269	.{
...	...	@@ -744,14 +752,6 @@ pub const list = list: {
744	752	.illegal_outside_function = true,
745	753	},
746	754	},
747		.{
748		"@setCold",
749		.{
750		.tag = .set_cold,
751		.param_count = 1,
752		.illegal_outside_function = true,
753		},
754		},
755	755	.{
756	756	"@setEvalBranchQuota",
757	757	.{

lib/std/zig/Zir.zig+7-5

...	...	@@ -1546,7 +1546,7 @@ pub const Inst = struct {
1546	1546	=> false,
1547	1547
1548	1548	.extended => switch (data.extended.opcode) {
1549		.fence, .set_cold, .breakpoint, .disable_instrumentation => true,
	1549	.fence, .branch_hint, .breakpoint, .disable_instrumentation => true,
1550	1550	else => false,
1551	1551	},
1552	1552	};
...	...	@@ -1954,9 +1954,6 @@ pub const Inst = struct {
1954	1954	/// Implement builtin `@setAlignStack`.
1955	1955	/// `operand` is payload index to `UnNode`.
1956	1956	set_align_stack,
1957		/// Implements `@setCold`.
1958		/// `operand` is payload index to `UnNode`.
1959		set_cold,
1960	1957	/// Implements the `@errorCast` builtin.
1961	1958	/// `operand` is payload index to `BinNode`. `lhs` is dest type, `rhs` is operand.
1962	1959	error_cast,
...	...	@@ -2051,6 +2048,10 @@ pub const Inst = struct {
2051	2048	/// `operand` is `src_node: i32`.
2052	2049	/// `small` is an `Inst.BuiltinValue`.
2053	2050	builtin_value,
	2051	/// Provide a `@branchHint` for the current block.
	2052	/// `operand` is payload index to `UnNode`.
	2053	/// `small` is unused.
	2054	branch_hint,
2054	2055
2055	2056	pub const InstData = struct {
2056	2057	opcode: Extended,
...	...	@@ -3142,6 +3143,7 @@ pub const Inst = struct {
3142	3143	export_options,
3143	3144	extern_options,
3144	3145	type_info,
	3146	branch_hint,
3145	3147	// Values
3146	3148	calling_convention_c,
3147	3149	calling_convention_inline,
...	...	@@ -3962,7 +3964,6 @@ fn findDeclsInner(
3962	3964	.fence,
3963	3965	.set_float_mode,
3964	3966	.set_align_stack,
3965		.set_cold,
3966	3967	.error_cast,
3967	3968	.await_nosuspend,
3968	3969	.breakpoint,
...	...	@@ -3986,6 +3987,7 @@ fn findDeclsInner(
3986	3987	.closure_get,
3987	3988	.field_parent_ptr,
3988	3989	.builtin_value,
	3990	.branch_hint,
3989	3991	=> return,
3990	3992
3991	3993	// `@TypeOf` has a body.

src/Air.zig+109-6

...	...	@@ -433,13 +433,18 @@ pub const Inst = struct {
433	433	/// In the case of non-error, control flow proceeds to the next instruction
434	434	/// after the `try`, with the result of this instruction being the unwrapped
435	435	/// payload value, as if `unwrap_errunion_payload` was executed on the operand.
	436	/// The error branch is considered to have a branch hint of `.unlikely`.
436	437	/// Uses the `pl_op` field. Payload is `Try`.
437	438	@"try",
	439	/// Same as `try` except the error branch hint is `.cold`.
	440	try_cold,
438	441	/// Same as `try` except the operand is a pointer to an error union, and the
439	442	/// result is a pointer to the payload. Result is as if `unwrap_errunion_payload_ptr`
440	443	/// was executed on the operand.
441	444	/// Uses the `ty_pl` field. Payload is `TryPtr`.
442	445	try_ptr,
	446	/// Same as `try_ptr` except the error branch hint is `.cold`.
	447	try_ptr_cold,
443	448	/// Notes the beginning of a source code statement and marks the line and column.
444	449	/// Result type is always void.
445	450	/// Uses the `dbg_stmt` field.
...	...	@@ -1116,11 +1121,20 @@ pub const Call = struct {
1116	1121	pub const CondBr = struct {
1117	1122	then_body_len: u32,
1118	1123	else_body_len: u32,
	1124	branch_hints: BranchHints,
	1125	pub const BranchHints = packed struct(u32) {
	1126	true: std.builtin.BranchHint,
	1127	false: std.builtin.BranchHint,
	1128	_: u26 = 0,
	1129	};
1119	1130	};
1120	1131
1121	1132	/// Trailing:
1122		/// * 0. `Case` for each `cases_len`
1123		/// * 1. the else body, according to `else_body_len`.
	1133	/// * 0. `BranchHint` for each `cases_len + 1`. bit-packed into `u32`
	1134	/// elems such that each `u32` contains up to 10x `BranchHint`.
	1135	/// LSBs are first case. Final hint is `else`.
	1136	/// * 1. `Case` for each `cases_len`
	1137	/// * 2. the else body, according to `else_body_len`.
1124	1138	pub const SwitchBr = struct {
1125	1139	cases_len: u32,
1126	1140	else_body_len: u32,
...	...	@@ -1380,6 +1394,7 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1380	1394	.ptr_add,
1381	1395	.ptr_sub,
1382	1396	.try_ptr,
	1397	.try_ptr_cold,
1383	1398	=> return datas[@intFromEnum(inst)].ty_pl.ty.toType(),
1384	1399
1385	1400	.not,
...	...	@@ -1500,7 +1515,7 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1500	1515	return air.typeOf(extra.lhs, ip);
1501	1516	},
1502	1517
1503		.@"try" => {
	1518	.@"try", .try_cold => {
1504	1519	const err_union_ty = air.typeOf(datas[@intFromEnum(inst)].pl_op.operand, ip);
1505	1520	return Type.fromInterned(ip.indexToKey(err_union_ty.ip_index).error_union_type.payload_type);
1506	1521	},
...	...	@@ -1524,9 +1539,8 @@ pub fn extraData(air: Air, comptime T: type, index: usize) struct { data: T, end
1524	1539	inline for (fields) \|field\| {
1525	1540	@field(result, field.name) = switch (field.type) {
1526	1541	u32 => air.extra[i],
1527		Inst.Ref => @as(Inst.Ref, @enumFromInt(air.extra[i])),
1528		i32 => @as(i32, @bitCast(air.extra[i])),
1529		InternPool.Index => @as(InternPool.Index, @enumFromInt(air.extra[i])),
	1542	InternPool.Index, Inst.Ref => @enumFromInt(air.extra[i]),
	1543	i32, CondBr.BranchHints => @bitCast(air.extra[i]),
1530	1544	else => @compileError("bad field type: " ++ @typeName(field.type)),
1531	1545	};
1532	1546	i += 1;
...	...	@@ -1593,7 +1607,9 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1593	1607	.cond_br,
1594	1608	.switch_br,
1595	1609	.@"try",
	1610	.try_cold,
1596	1611	.try_ptr,
	1612	.try_ptr_cold,
1597	1613	.dbg_stmt,
1598	1614	.dbg_inline_block,
1599	1615	.dbg_var_ptr,
...	...	@@ -1796,4 +1812,91 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1796	1812	};
1797	1813	}
1798	1814
	1815	pub const UnwrappedSwitch = struct {
	1816	air: *const Air,
	1817	operand: Inst.Ref,
	1818	cases_len: u32,
	1819	else_body_len: u32,
	1820	branch_hints_start: u32,
	1821	cases_start: u32,
	1822
	1823	/// Asserts that `case_idx < us.cases_len`.
	1824	pub fn getHint(us: UnwrappedSwitch, case_idx: u32) std.builtin.BranchHint {
	1825	assert(case_idx < us.cases_len);
	1826	return us.getHintInner(case_idx);
	1827	}
	1828	pub fn getElseHint(us: UnwrappedSwitch) std.builtin.BranchHint {
	1829	return us.getHintInner(us.cases_len);
	1830	}
	1831	fn getHintInner(us: UnwrappedSwitch, idx: u32) std.builtin.BranchHint {
	1832	const bag = us.air.extra[us.branch_hints_start..][idx / 10];
	1833	const bits: u3 = @truncate(bag >> @intCast(3 * (idx % 10)));
	1834	return @enumFromInt(bits);
	1835	}
	1836
	1837	pub fn iterateCases(us: UnwrappedSwitch) CaseIterator {
	1838	return .{
	1839	.air = us.air,
	1840	.cases_len = us.cases_len,
	1841	.else_body_len = us.else_body_len,
	1842	.next_case = 0,
	1843	.extra_index = us.cases_start,
	1844	};
	1845	}
	1846	pub const CaseIterator = struct {
	1847	air: *const Air,
	1848	cases_len: u32,
	1849	else_body_len: u32,
	1850	next_case: u32,
	1851	extra_index: u32,
	1852
	1853	pub fn next(it: *CaseIterator) ?Case {
	1854	if (it.next_case == it.cases_len) return null;
	1855	const idx = it.next_case;
	1856	it.next_case += 1;
	1857
	1858	const extra = it.air.extraData(SwitchBr.Case, it.extra_index);
	1859	var extra_index = extra.end;
	1860	const items: []const Inst.Ref = @ptrCast(it.air.extra[extra_index..][0..extra.data.items_len]);
	1861	extra_index += items.len;
	1862	const body: []const Inst.Index = @ptrCast(it.air.extra[extra_index..][0..extra.data.body_len]);
	1863	extra_index += body.len;
	1864	it.extra_index = @intCast(extra_index);
	1865
	1866	return .{
	1867	.idx = idx,
	1868	.items = items,
	1869	.body = body,
	1870	};
	1871	}
	1872	/// Only valid to call once all cases have been iterated, i.e. `next` returns `null`.
	1873	/// Returns the body of the "default" (`else`) case.
	1874	pub fn elseBody(it: *CaseIterator) []const Inst.Index {
	1875	assert(it.next_case == it.cases_len);
	1876	return @ptrCast(it.air.extra[it.extra_index..][0..it.else_body_len]);
	1877	}
	1878	pub const Case = struct {
	1879	idx: u32,
	1880	items: []const Inst.Ref,
	1881	body: []const Inst.Index,
	1882	};
	1883	};
	1884	};
	1885
	1886	pub fn unwrapSwitch(air: *const Air, switch_inst: Inst.Index) UnwrappedSwitch {
	1887	const inst = air.instructions.get(@intFromEnum(switch_inst));
	1888	assert(inst.tag == .switch_br);
	1889	const pl_op = inst.data.pl_op;
	1890	const extra = air.extraData(SwitchBr, pl_op.payload);
	1891	const hint_bag_count = std.math.divCeil(usize, extra.data.cases_len + 1, 10) catch unreachable;
	1892	return .{
	1893	.air = air,
	1894	.operand = pl_op.operand,
	1895	.cases_len = extra.data.cases_len,
	1896	.else_body_len = extra.data.else_body_len,
	1897	.branch_hints_start = @intCast(extra.end),
	1898	.cases_start = @intCast(extra.end + hint_bag_count),
	1899	};
	1900	}
	1901
1799	1902	pub const typesFullyResolved = @import("Air/types_resolved.zig").typesFullyResolved;

src/Air/types_resolved.zig+9-22

...	...	@@ -344,7 +344,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
344	344	if (!checkRef(data.pl_op.operand, zcu)) return false;
345	345	},
346	346
347		.@"try" => {
	347	.@"try", .try_cold => {
348	348	const extra = air.extraData(Air.Try, data.pl_op.payload);
349	349	if (!checkRef(data.pl_op.operand, zcu)) return false;
350	350	if (!checkBody(
...	...	@@ -354,7 +354,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
354	354	)) return false;
355	355	},
356	356
357		.try_ptr => {
	357	.try_ptr, .try_ptr_cold => {
358	358	const extra = air.extraData(Air.TryPtr, data.ty_pl.payload);
359	359	if (!checkType(data.ty_pl.ty.toType(), zcu)) return false;
360	360	if (!checkRef(extra.data.ptr, zcu)) return false;
...	...	@@ -381,27 +381,14 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
381	381	},
382	382
383	383	.switch_br => {
384		const extra = air.extraData(Air.SwitchBr, data.pl_op.payload);
385		if (!checkRef(data.pl_op.operand, zcu)) return false;
386		var extra_index = extra.end;
387		for (0..extra.data.cases_len) \|_\| {
388		const case = air.extraData(Air.SwitchBr.Case, extra_index);
389		extra_index = case.end;
390		const items: []const Air.Inst.Ref = @ptrCast(air.extra[extra_index..][0..case.data.items_len]);
391		extra_index += case.data.items_len;
392		for (items) \|item\| if (!checkRef(item, zcu)) return false;
393		if (!checkBody(
394		air,
395		@ptrCast(air.extra[extra_index..][0..case.data.body_len]),
396		zcu,
397		)) return false;
398		extra_index += case.data.body_len;
	384	const switch_br = air.unwrapSwitch(inst);
	385	if (!checkRef(switch_br.operand, zcu)) return false;
	386	var it = switch_br.iterateCases();
	387	while (it.next()) \|case\| {
	388	for (case.items) \|item\| if (!checkRef(item, zcu)) return false;
	389	if (!checkBody(air, case.body, zcu)) return false;
399	390	}
400		if (!checkBody(
401		air,
402		@ptrCast(air.extra[extra_index..][0..extra.data.else_body_len]),
403		zcu,
404		)) return false;
	391	if (!checkBody(air, it.elseBody(), zcu)) return false;
405	392	},
406	393
407	394	.assembly => {

src/InternPool.zig+17-18

...	...	@@ -2121,6 +2121,17 @@ pub const Key = union(enum) {
2121	2121	@atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
2122	2122	}
2123	2123
	2124	pub fn setBranchHint(func: Func, ip: *InternPool, hint: std.builtin.BranchHint) void {
	2125	const extra_mutex = &ip.getLocal(func.tid).mutate.extra.mutex;
	2126	extra_mutex.lock();
	2127	defer extra_mutex.unlock();
	2128
	2129	const analysis_ptr = func.analysisPtr(ip);
	2130	var analysis = analysis_ptr.*;
	2131	analysis.branch_hint = hint;
	2132	@atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
	2133	}
	2134
2124	2135	/// Returns a pointer that becomes invalid after any additions to the `InternPool`.
2125	2136	fn zirBodyInstPtr(func: Func, ip: InternPool) TrackedInst.Index {
2126	2137	const extra = ip.getLocalShared(func.tid).extra.acquire();
...	...	@@ -5575,7 +5586,7 @@ pub const Tag = enum(u8) {
5575	5586	/// to be part of the type of the function.
5576	5587	pub const FuncAnalysis = packed struct(u32) {
5577	5588	state: State,
5578		is_cold: bool,
	5589	branch_hint: std.builtin.BranchHint,
5579	5590	is_noinline: bool,
5580	5591	calls_or_awaits_errorable_fn: bool,
5581	5592	stack_alignment: Alignment,
...	...	@@ -5583,7 +5594,7 @@ pub const FuncAnalysis = packed struct(u32) {
5583	5594	inferred_error_set: bool,
5584	5595	disable_instrumentation: bool,
5585	5596
5586		_: u19 = 0,
	5597	_: u17 = 0,
5587	5598
5588	5599	pub const State = enum(u2) {
5589	5600	/// The runtime function has never been referenced.
...	...	@@ -8636,7 +8647,7 @@ pub fn getFuncDecl(
8636	8647	const func_decl_extra_index = addExtraAssumeCapacity(extra, Tag.FuncDecl{
8637	8648	.analysis = .{
8638	8649	.state = .unreferenced,
8639		.is_cold = false,
	8650	.branch_hint = .none,
8640	8651	.is_noinline = key.is_noinline,
8641	8652	.calls_or_awaits_errorable_fn = false,
8642	8653	.stack_alignment = .none,
...	...	@@ -8740,7 +8751,7 @@ pub fn getFuncDeclIes(
8740	8751	const func_decl_extra_index = addExtraAssumeCapacity(extra, Tag.FuncDecl{
8741	8752	.analysis = .{
8742	8753	.state = .unreferenced,
8743		.is_cold = false,
	8754	.branch_hint = .none,
8744	8755	.is_noinline = key.is_noinline,
8745	8756	.calls_or_awaits_errorable_fn = false,
8746	8757	.stack_alignment = .none,
...	...	@@ -8932,7 +8943,7 @@ pub fn getFuncInstance(
8932	8943	const func_extra_index = addExtraAssumeCapacity(extra, Tag.FuncInstance{
8933	8944	.analysis = .{
8934	8945	.state = .unreferenced,
8935		.is_cold = false,
	8946	.branch_hint = .none,
8936	8947	.is_noinline = arg.is_noinline,
8937	8948	.calls_or_awaits_errorable_fn = false,
8938	8949	.stack_alignment = .none,
...	...	@@ -9032,7 +9043,7 @@ pub fn getFuncInstanceIes(
9032	9043	const func_extra_index = addExtraAssumeCapacity(extra, Tag.FuncInstance{
9033	9044	.analysis = .{
9034	9045	.state = .unreferenced,
9035		.is_cold = false,
	9046	.branch_hint = .none,
9036	9047	.is_noinline = arg.is_noinline,
9037	9048	.calls_or_awaits_errorable_fn = false,
9038	9049	.stack_alignment = .none,
...	...	@@ -11853,18 +11864,6 @@ pub fn funcSetDisableInstrumentation(ip: *InternPool, func: Index) void {
11853	11864	@atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
11854	11865	}
11855	11866
11856		pub fn funcSetCold(ip: *InternPool, func: Index, is_cold: bool) void {
11857		const unwrapped_func = func.unwrap(ip);
11858		const extra_mutex = &ip.getLocal(unwrapped_func.tid).mutate.extra.mutex;
11859		extra_mutex.lock();
11860		defer extra_mutex.unlock();
11861
11862		const analysis_ptr = ip.funcAnalysisPtr(func);
11863		var analysis = analysis_ptr.*;
11864		analysis.is_cold = is_cold;
11865		@atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
11866		}
11867
11868	11867	pub fn funcZirBodyInst(ip: *const InternPool, func: Index) TrackedInst.Index {
11869	11868	const unwrapped_func = func.unwrap(ip);
11870	11869	const item = unwrapped_func.getItem(ip);

src/Liveness.zig+15-21

...	...	@@ -658,10 +658,10 @@ pub fn categorizeOperand(
658	658
659	659	return .complex;
660	660	},
661		.@"try" => {
	661	.@"try", .try_cold => {
662	662	return .complex;
663	663	},
664		.try_ptr => {
	664	.try_ptr, .try_ptr_cold => {
665	665	return .complex;
666	666	},
667	667	.loop => {
...	...	@@ -1254,8 +1254,8 @@ fn analyzeInst(
1254	1254	},
1255	1255	.loop => return analyzeInstLoop(a, pass, data, inst),
1256	1256
1257		.@"try" => return analyzeInstCondBr(a, pass, data, inst, .@"try"),
1258		.try_ptr => return analyzeInstCondBr(a, pass, data, inst, .try_ptr),
	1257	.@"try", .try_cold => return analyzeInstCondBr(a, pass, data, inst, .@"try"),
	1258	.try_ptr, .try_ptr_cold => return analyzeInstCondBr(a, pass, data, inst, .try_ptr),
1259	1259	.cond_br => return analyzeInstCondBr(a, pass, data, inst, .cond_br),
1260	1260	.switch_br => return analyzeInstSwitchBr(a, pass, data, inst),
1261	1261
...	...	@@ -1674,21 +1674,18 @@ fn analyzeInstSwitchBr(
1674	1674	const inst_datas = a.air.instructions.items(.data);
1675	1675	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
1676	1676	const condition = pl_op.operand;
1677		const switch_br = a.air.extraData(Air.SwitchBr, pl_op.payload);
	1677	const switch_br = a.air.unwrapSwitch(inst);
1678	1678	const gpa = a.gpa;
1679		const ncases = switch_br.data.cases_len;
	1679	const ncases = switch_br.cases_len;
1680	1680
1681	1681	switch (pass) {
1682	1682	.loop_analysis => {
1683		var air_extra_index: usize = switch_br.end;
1684		for (0..ncases) \|_\| {
1685		const case = a.air.extraData(Air.SwitchBr.Case, air_extra_index);
1686		const case_body: []const Air.Inst.Index = @ptrCast(a.air.extra[case.end + case.data.items_len ..][0..case.data.body_len]);
1687		air_extra_index = case.end + case.data.items_len + case_body.len;
1688		try analyzeBody(a, pass, data, case_body);
	1683	var it = switch_br.iterateCases();
	1684	while (it.next()) \|case\| {
	1685	try analyzeBody(a, pass, data, case.body);
1689	1686	}
1690	1687	{ // else
1691		const else_body: []const Air.Inst.Index = @ptrCast(a.air.extra[air_extra_index..][0..switch_br.data.else_body_len]);
	1688	const else_body = it.elseBody();
1692	1689	try analyzeBody(a, pass, data, else_body);
1693	1690	}
1694	1691	},
...	...	@@ -1706,16 +1703,13 @@ fn analyzeInstSwitchBr(
1706	1703	@memset(case_live_sets, .{});
1707	1704	defer for (case_live_sets) \|*live_set\| live_set.deinit(gpa);
1708	1705
1709		var air_extra_index: usize = switch_br.end;
1710		for (case_live_sets[0..ncases]) \|*live_set\| {
1711		const case = a.air.extraData(Air.SwitchBr.Case, air_extra_index);
1712		const case_body: []const Air.Inst.Index = @ptrCast(a.air.extra[case.end + case.data.items_len ..][0..case.data.body_len]);
1713		air_extra_index = case.end + case.data.items_len + case_body.len;
1714		try analyzeBody(a, pass, data, case_body);
1715		live_set.* = data.live_set.move();
	1706	var case_it = switch_br.iterateCases();
	1707	while (case_it.next()) \|case\| {
	1708	try analyzeBody(a, pass, data, case.body);
	1709	case_live_sets[case.idx] = data.live_set.move();
1716	1710	}
1717	1711	{ // else
1718		const else_body: []const Air.Inst.Index = @ptrCast(a.air.extra[air_extra_index..][0..switch_br.data.else_body_len]);
	1712	const else_body = case_it.elseBody();
1719	1713	try analyzeBody(a, pass, data, else_body);
1720	1714	case_live_sets[ncases] = data.live_set.move();
1721	1715	}

src/Liveness/Verify.zig+11-22

...	...	@@ -374,7 +374,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
374	374	},
375	375
376	376	// control flow
377		.@"try" => {
	377	.@"try", .try_cold => {
378	378	const pl_op = data[@intFromEnum(inst)].pl_op;
379	379	const extra = self.air.extraData(Air.Try, pl_op.payload);
380	380	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
...	...	@@ -396,7 +396,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
396	396
397	397	try self.verifyInst(inst);
398	398	},
399		.try_ptr => {
	399	.try_ptr, .try_ptr_cold => {
400	400	const ty_pl = data[@intFromEnum(inst)].ty_pl;
401	401	const extra = self.air.extraData(Air.TryPtr, ty_pl.payload);
402	402	const try_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
...	...	@@ -509,44 +509,33 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
509	509	try self.verifyInst(inst);
510	510	},
511	511	.switch_br => {
512		const pl_op = data[@intFromEnum(inst)].pl_op;
513		const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
514		var extra_index = switch_br.end;
515		var case_i: u32 = 0;
	512	const switch_br = self.air.unwrapSwitch(inst);
516	513	const switch_br_liveness = try self.liveness.getSwitchBr(
517	514	self.gpa,
518	515	inst,
519		switch_br.data.cases_len + 1,
	516	switch_br.cases_len + 1,
520	517	);
521	518	defer self.gpa.free(switch_br_liveness.deaths);
522	519
523		try self.verifyOperand(inst, pl_op.operand, self.liveness.operandDies(inst, 0));
	520	try self.verifyOperand(inst, switch_br.operand, self.liveness.operandDies(inst, 0));
524	521
525	522	var live = self.live.move();
526	523	defer live.deinit(self.gpa);
527	524
528		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
529		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
530		const items = @as(
531		[]const Air.Inst.Ref,
532		@ptrCast(self.air.extra[case.end..][0..case.data.items_len]),
533		);
534		const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
535		extra_index = case.end + items.len + case_body.len;
536
	525	var it = switch_br.iterateCases();
	526	while (it.next()) \|case\| {
537	527	self.live.deinit(self.gpa);
538	528	self.live = try live.clone(self.gpa);
539	529
540		for (switch_br_liveness.deaths[case_i]) \|death\| try self.verifyDeath(inst, death);
541		try self.verifyBody(case_body);
	530	for (switch_br_liveness.deaths[case.idx]) \|death\| try self.verifyDeath(inst, death);
	531	try self.verifyBody(case.body);
542	532	}
543	533
544		const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	534	const else_body = it.elseBody();
545	535	if (else_body.len > 0) {
546	536	self.live.deinit(self.gpa);
547	537	self.live = try live.clone(self.gpa);
548
549		for (switch_br_liveness.deaths[case_i]) \|death\| try self.verifyDeath(inst, death);
	538	for (switch_br_liveness.deaths[switch_br.cases_len]) \|death\| try self.verifyDeath(inst, death);
550	539	try self.verifyBody(else_body);
551	540	}
552	541

src/Sema.zig+258-107

...	...	@@ -118,6 +118,10 @@ dependencies: std.AutoArrayHashMapUnmanaged(InternPool.Dependee, void) = .{},
118	118	/// by `analyzeCall`.
119	119	allow_memoize: bool = true,
120	120
	121	/// The `BranchHint` for the current branch of runtime control flow.
	122	/// This state is on `Sema` so that `cold` hints can be propagated up through blocks with less special handling.
	123	branch_hint: ?std.builtin.BranchHint = null,
	124
121	125	const MaybeComptimeAlloc = struct {
122	126	/// The runtime index of the `alloc` instruction.
123	127	runtime_index: Value.RuntimeIndex,
...	...	@@ -892,7 +896,12 @@ pub fn deinit(sema: *Sema) void {
892	896	/// Performs semantic analysis of a ZIR body which is behind a runtime condition. If comptime
893	897	/// control flow happens here, Sema will convert it to runtime control flow by introducing post-hoc
894	898	/// blocks where necessary.
895		fn analyzeBodyRuntimeBreak(sema: Sema, block: Block, body: []const Zir.Inst.Index) !void {
	899	/// Returns the branch hint for this branch.
	900	fn analyzeBodyRuntimeBreak(sema: Sema, block: Block, body: []const Zir.Inst.Index) !std.builtin.BranchHint {
	901	const parent_hint = sema.branch_hint;
	902	defer sema.branch_hint = parent_hint;
	903	sema.branch_hint = null;
	904
896	905	sema.analyzeBodyInner(block, body) catch \|err\| switch (err) {
897	906	error.ComptimeBreak => {
898	907	const zir_datas = sema.code.instructions.items(.data);
...	...	@@ -902,6 +911,8 @@ fn analyzeBodyRuntimeBreak(sema: Sema, block: Block, body: []const Zir.Inst.In
902	911	},
903	912	else => \|e\| return e,
904	913	};
	914
	915	return sema.branch_hint orelse .none;
905	916	}
906	917
907	918	/// Semantically analyze a ZIR function body. It is guranteed by AstGen that such a body cannot
...	...	@@ -1304,11 +1315,6 @@ fn analyzeBodyInner(
1304	1315	i += 1;
1305	1316	continue;
1306	1317	},
1307		.set_cold => {
1308		try sema.zirSetCold(block, extended);
1309		i += 1;
1310		continue;
1311		},
1312	1318	.breakpoint => {
1313	1319	if (!block.is_comptime) {
1314	1320	_ = try block.addNoOp(.breakpoint);
...	...	@@ -1326,6 +1332,11 @@ fn analyzeBodyInner(
1326	1332	i += 1;
1327	1333	continue;
1328	1334	},
	1335	.branch_hint => {
	1336	try sema.zirBranchHint(block, extended);
	1337	i += 1;
	1338	continue;
	1339	},
1329	1340	.value_placeholder => unreachable, // never appears in a body
1330	1341	.field_parent_ptr => try sema.zirFieldParentPtr(block, extended),
1331	1342	.builtin_value => try sema.zirBuiltinValue(extended),
...	...	@@ -5727,6 +5738,13 @@ fn zirPanic(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!void
5727	5738	if (block.is_comptime) {
5728	5739	return sema.fail(block, src, "encountered @panic at comptime", .{});
5729	5740	}
	5741
	5742	// We only apply the first hint in a branch.
	5743	// This allows user-provided hints to override implicit cold hints.
	5744	if (sema.branch_hint == null) {
	5745	sema.branch_hint = .cold;
	5746	}
	5747
5730	5748	try sema.panicWithMsg(block, src, coerced_msg, .@"@panic");
5731	5749	}
5732	5750
...	...	@@ -6418,25 +6436,6 @@ fn zirSetAlignStack(sema: Sema, block: Block, extended: Zir.Inst.Extended.Inst
6418	6436	sema.allow_memoize = false;
6419	6437	}
6420	6438
6421		fn zirSetCold(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData) CompileError!void {
6422		const pt = sema.pt;
6423		const zcu = pt.zcu;
6424		const ip = &zcu.intern_pool;
6425		const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data;
6426		const operand_src = block.builtinCallArgSrc(extra.node, 0);
6427		const is_cold = try sema.resolveConstBool(block, operand_src, extra.operand, .{
6428		.needed_comptime_reason = "operand to @setCold must be comptime-known",
6429		});
6430		// TODO: should `@setCold` apply to the parent in an inline call?
6431		// See also #20642 and friends.
6432		const func = switch (sema.owner.unwrap()) {
6433		.func => \|func\| func,
6434		.cau => return, // does nothing outside a function
6435		};
6436		ip.funcSetCold(func, is_cold);
6437		sema.allow_memoize = false;
6438		}
6439
6440	6439	fn zirDisableInstrumentation(sema: *Sema) CompileError!void {
6441	6440	const pt = sema.pt;
6442	6441	const zcu = pt.zcu;
...	...	@@ -6891,13 +6890,20 @@ fn popErrorReturnTrace(
6891	6890	@typeInfo(Air.Block).Struct.fields.len + 1); // +1 for the sole .cond_br instruction in the .block
6892	6891
6893	6892	const cond_br_inst: Air.Inst.Index = @enumFromInt(sema.air_instructions.len);
6894		try sema.air_instructions.append(gpa, .{ .tag = .cond_br, .data = .{ .pl_op = .{
6895		.operand = is_non_error_inst,
6896		.payload = sema.addExtraAssumeCapacity(Air.CondBr{
6897		.then_body_len = @intCast(then_block.instructions.items.len),
6898		.else_body_len = @intCast(else_block.instructions.items.len),
6899		}),
6900		} } });
	6893	try sema.air_instructions.append(gpa, .{
	6894	.tag = .cond_br,
	6895	.data = .{
	6896	.pl_op = .{
	6897	.operand = is_non_error_inst,
	6898	.payload = sema.addExtraAssumeCapacity(Air.CondBr{
	6899	.then_body_len = @intCast(then_block.instructions.items.len),
	6900	.else_body_len = @intCast(else_block.instructions.items.len),
	6901	// weight against error branch
	6902	.branch_hints = .{ .true = .likely, .false = .unlikely },
	6903	}),
	6904	},
	6905	},
	6906	});
6901	6907	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items));
6902	6908	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items));
6903	6909
...	...	@@ -10954,6 +10960,11 @@ const SwitchProngAnalysis = struct {
10954	10960	sema.code.instructions.items(.data)[@intFromEnum(spa.switch_block_inst)].pl_node.src_node,
10955	10961	);
10956	10962
	10963	// We can propagate `.cold` hints from this branch since it's comptime-known
	10964	// to be taken from the parent branch.
	10965	const parent_hint = sema.branch_hint;
	10966	defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null;
	10967
10957	10968	if (has_tag_capture) {
10958	10969	const tag_ref = try spa.analyzeTagCapture(child_block, capture_src, inline_case_capture);
10959	10970	sema.inst_map.putAssumeCapacity(spa.tag_capture_inst, tag_ref);
...	...	@@ -10990,6 +11001,7 @@ const SwitchProngAnalysis = struct {
10990	11001
10991	11002	/// Analyze a switch prong which may have peers at runtime.
10992	11003	/// Uses `analyzeBodyRuntimeBreak`. Sets up captures as needed.
	11004	/// Returns the `BranchHint` for the prong.
10993	11005	fn analyzeProngRuntime(
10994	11006	spa: SwitchProngAnalysis,
10995	11007	case_block: *Block,
...	...	@@ -11007,7 +11019,7 @@ const SwitchProngAnalysis = struct {
11007	11019	/// Whether this prong has an inline tag capture. If `true`, then
11008	11020	/// `inline_case_capture` cannot be `.none`.
11009	11021	has_tag_capture: bool,
11010		) CompileError!void {
	11022	) CompileError!std.builtin.BranchHint {
11011	11023	const sema = spa.sema;
11012	11024
11013	11025	if (has_tag_capture) {
...	...	@@ -11033,7 +11045,7 @@ const SwitchProngAnalysis = struct {
11033	11045
11034	11046	if (sema.typeOf(capture_ref).isNoReturn(sema.pt.zcu)) {
11035	11047	// No need to analyze any further, the prong is unreachable
11036		return;
	11048	return .none;
11037	11049	}
11038	11050
11039	11051	sema.inst_map.putAssumeCapacity(spa.switch_block_inst, capture_ref);
...	...	@@ -11302,10 +11314,17 @@ const SwitchProngAnalysis = struct {
11302	11314
11303	11315	const prong_count = field_indices.len - in_mem_coercible.count();
11304	11316
11305		const estimated_extra = prong_count * 6; // 2 for Case, 1 item, probably 3 insts
	11317	const estimated_extra = prong_count * 6 + (prong_count / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
11306	11318	var cases_extra = try std.ArrayList(u32).initCapacity(sema.gpa, estimated_extra);
11307	11319	defer cases_extra.deinit();
11308	11320
	11321	{
	11322	// All branch hints are `.none`, so just add zero elems.
	11323	comptime assert(@intFromEnum(std.builtin.BranchHint.none) == 0);
	11324	const need_elems = std.math.divCeil(usize, prong_count + 1, 10) catch unreachable;
	11325	try cases_extra.appendNTimes(0, need_elems);
	11326	}
	11327
11309	11328	{
11310	11329	// Non-bitcast cases
11311	11330	var it = in_mem_coercible.iterator(.{ .kind = .unset });
...	...	@@ -11728,7 +11747,7 @@ fn zirSwitchBlockErrUnion(sema: Sema, block: Block, inst: Zir.Inst.Index) Comp
11728	11747	sub_block.need_debug_scope = null; // this body is emitted regardless
11729	11748	defer sub_block.instructions.deinit(gpa);
11730	11749
11731		try sema.analyzeBodyRuntimeBreak(&sub_block, non_error_case.body);
	11750	const non_error_hint = try sema.analyzeBodyRuntimeBreak(&sub_block, non_error_case.body);
11732	11751	const true_instructions = try sub_block.instructions.toOwnedSlice(gpa);
11733	11752	defer gpa.free(true_instructions);
11734	11753
...	...	@@ -11782,6 +11801,7 @@ fn zirSwitchBlockErrUnion(sema: Sema, block: Block, inst: Zir.Inst.Index) Comp
11782	11801	.payload = sema.addExtraAssumeCapacity(Air.CondBr{
11783	11802	.then_body_len = @intCast(true_instructions.len),
11784	11803	.else_body_len = @intCast(sub_block.instructions.items.len),
	11804	.branch_hints = .{ .true = non_error_hint, .false = .none },
11785	11805	}),
11786	11806	} },
11787	11807	});
...	...	@@ -12486,6 +12506,9 @@ fn analyzeSwitchRuntimeBlock(
12486	12506	var cases_extra = try std.ArrayListUnmanaged(u32).initCapacity(gpa, estimated_cases_extra);
12487	12507	defer cases_extra.deinit(gpa);
12488	12508
	12509	var branch_hints = try std.ArrayListUnmanaged(std.builtin.BranchHint).initCapacity(gpa, scalar_cases_len);
	12510	defer branch_hints.deinit(gpa);
	12511
12489	12512	var case_block = child_block.makeSubBlock();
12490	12513	case_block.runtime_loop = null;
12491	12514	case_block.runtime_cond = operand_src;
...	...	@@ -12516,10 +12539,13 @@ fn analyzeSwitchRuntimeBlock(
12516	12539	break :blk field_ty.zigTypeTag(zcu) != .NoReturn;
12517	12540	} else true;
12518	12541
12519		if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) {
12520		// nothing to do here
12521		} else if (analyze_body) {
12522		try spa.analyzeProngRuntime(
	12542	const prong_hint: std.builtin.BranchHint = if (err_set and
	12543	try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap))
	12544	h: {
	12545	// nothing to do here. weight against error branch
	12546	break :h .unlikely;
	12547	} else if (analyze_body) h: {
	12548	break :h try spa.analyzeProngRuntime(
12523	12549	&case_block,
12524	12550	.normal,
12525	12551	body,
...	...	@@ -12532,10 +12558,12 @@ fn analyzeSwitchRuntimeBlock(
12532	12558	if (info.is_inline) item else .none,
12533	12559	info.has_tag_capture,
12534	12560	);
12535		} else {
	12561	} else h: {
12536	12562	_ = try case_block.addNoOp(.unreach);
12537		}
	12563	break :h .none;
	12564	};
12538	12565
	12566	try branch_hints.append(gpa, prong_hint);
12539	12567	try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
12540	12568	cases_extra.appendAssumeCapacity(1); // items_len
12541	12569	cases_extra.appendAssumeCapacity(@intCast(case_block.instructions.items.len));
...	...	@@ -12545,6 +12573,7 @@ fn analyzeSwitchRuntimeBlock(
12545	12573
12546	12574	var is_first = true;
12547	12575	var prev_cond_br: Air.Inst.Index = undefined;
	12576	var prev_hint: std.builtin.BranchHint = undefined;
12548	12577	var first_else_body: []const Air.Inst.Index = &.{};
12549	12578	defer gpa.free(first_else_body);
12550	12579	var prev_then_body: []const Air.Inst.Index = &.{};
...	...	@@ -12606,7 +12635,7 @@ fn analyzeSwitchRuntimeBlock(
12606	12635	} }));
12607	12636	emit_bb = true;
12608	12637
12609		try spa.analyzeProngRuntime(
	12638	const prong_hint = try spa.analyzeProngRuntime(
12610	12639	&case_block,
12611	12640	.normal,
12612	12641	body,
...	...	@@ -12619,6 +12648,7 @@ fn analyzeSwitchRuntimeBlock(
12619	12648	item_ref,
12620	12649	info.has_tag_capture,
12621	12650	);
	12651	try branch_hints.append(gpa, prong_hint);
12622	12652
12623	12653	try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
12624	12654	cases_extra.appendAssumeCapacity(1); // items_len
...	...	@@ -12649,8 +12679,8 @@ fn analyzeSwitchRuntimeBlock(
12649	12679	} }));
12650	12680	emit_bb = true;
12651	12681
12652		if (analyze_body) {
12653		try spa.analyzeProngRuntime(
	12682	const prong_hint: std.builtin.BranchHint = if (analyze_body) h: {
	12683	break :h try spa.analyzeProngRuntime(
12654	12684	&case_block,
12655	12685	.normal,
12656	12686	body,
...	...	@@ -12663,9 +12693,11 @@ fn analyzeSwitchRuntimeBlock(
12663	12693	item,
12664	12694	info.has_tag_capture,
12665	12695	);
12666		} else {
	12696	} else h: {
12667	12697	_ = try case_block.addNoOp(.unreach);
12668		}
	12698	break :h .none;
	12699	};
	12700	try branch_hints.append(gpa, prong_hint);
12669	12701
12670	12702	try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
12671	12703	cases_extra.appendAssumeCapacity(1); // items_len
...	...	@@ -12697,10 +12729,13 @@ fn analyzeSwitchRuntimeBlock(
12697	12729
12698	12730	const body = sema.code.bodySlice(extra_index, info.body_len);
12699	12731	extra_index += info.body_len;
12700		if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) {
12701		// nothing to do here
12702		} else if (analyze_body) {
12703		try spa.analyzeProngRuntime(
	12732	const prong_hint: std.builtin.BranchHint = if (err_set and
	12733	try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap))
	12734	h: {
	12735	// nothing to do here. weight against error branch
	12736	break :h .unlikely;
	12737	} else if (analyze_body) h: {
	12738	break :h try spa.analyzeProngRuntime(
12704	12739	&case_block,
12705	12740	.normal,
12706	12741	body,
...	...	@@ -12713,10 +12748,12 @@ fn analyzeSwitchRuntimeBlock(
12713	12748	.none,
12714	12749	false,
12715	12750	);
12716		} else {
	12751	} else h: {
12717	12752	_ = try case_block.addNoOp(.unreach);
12718		}
	12753	break :h .none;
	12754	};
12719	12755
	12756	try branch_hints.append(gpa, prong_hint);
12720	12757	try cases_extra.ensureUnusedCapacity(gpa, 2 + items.len +
12721	12758	case_block.instructions.items.len);
12722	12759
...	...	@@ -12784,23 +12821,24 @@ fn analyzeSwitchRuntimeBlock(
12784	12821
12785	12822	const body = sema.code.bodySlice(extra_index, info.body_len);
12786	12823	extra_index += info.body_len;
12787		if (err_set and try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap)) {
12788		// nothing to do here
12789		} else {
12790		try spa.analyzeProngRuntime(
12791		&case_block,
12792		.normal,
12793		body,
12794		info.capture,
12795		child_block.src(.{ .switch_capture = .{
12796		.switch_node_offset = switch_node_offset,
12797		.case_idx = .{ .kind = .multi, .index = @intCast(multi_i) },
12798		} }),
12799		items,
12800		.none,
12801		false,
12802		);
12803		}
	12824	const prong_hint: std.builtin.BranchHint = if (err_set and
	12825	try sema.maybeErrorUnwrap(&case_block, body, operand, operand_src, allow_err_code_unwrap))
	12826	h: {
	12827	// nothing to do here. weight against error branch
	12828	break :h .unlikely;
	12829	} else try spa.analyzeProngRuntime(
	12830	&case_block,
	12831	.normal,
	12832	body,
	12833	info.capture,
	12834	child_block.src(.{ .switch_capture = .{
	12835	.switch_node_offset = switch_node_offset,
	12836	.case_idx = .{ .kind = .multi, .index = @intCast(multi_i) },
	12837	} }),
	12838	items,
	12839	.none,
	12840	false,
	12841	);
12804	12842
12805	12843	if (is_first) {
12806	12844	is_first = false;
...	...	@@ -12812,10 +12850,10 @@ fn analyzeSwitchRuntimeBlock(
12812	12850	@typeInfo(Air.CondBr).Struct.fields.len + prev_then_body.len + cond_body.len,
12813	12851	);
12814	12852
12815		sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload =
12816		sema.addExtraAssumeCapacity(Air.CondBr{
	12853	sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload = sema.addExtraAssumeCapacity(Air.CondBr{
12817	12854	.then_body_len = @intCast(prev_then_body.len),
12818	12855	.else_body_len = @intCast(cond_body.len),
	12856	.branch_hints = .{ .true = prev_hint, .false = .none },
12819	12857	});
12820	12858	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(prev_then_body));
12821	12859	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(cond_body));
...	...	@@ -12823,6 +12861,7 @@ fn analyzeSwitchRuntimeBlock(
12823	12861	gpa.free(prev_then_body);
12824	12862	prev_then_body = try case_block.instructions.toOwnedSlice(gpa);
12825	12863	prev_cond_br = new_cond_br;
	12864	prev_hint = prong_hint;
12826	12865	}
12827	12866	}
12828	12867
...	...	@@ -12854,8 +12893,8 @@ fn analyzeSwitchRuntimeBlock(
12854	12893	if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
12855	12894	emit_bb = true;
12856	12895
12857		if (analyze_body) {
12858		try spa.analyzeProngRuntime(
	12896	const prong_hint: std.builtin.BranchHint = if (analyze_body) h: {
	12897	break :h try spa.analyzeProngRuntime(
12859	12898	&case_block,
12860	12899	.special,
12861	12900	special.body,
...	...	@@ -12868,9 +12907,11 @@ fn analyzeSwitchRuntimeBlock(
12868	12907	item_ref,
12869	12908	special.has_tag_capture,
12870	12909	);
12871		} else {
	12910	} else h: {
12872	12911	_ = try case_block.addNoOp(.unreach);
12873		}
	12912	break :h .none;
	12913	};
	12914	try branch_hints.append(gpa, prong_hint);
12874	12915
12875	12916	try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
12876	12917	cases_extra.appendAssumeCapacity(1); // items_len
...	...	@@ -12903,7 +12944,7 @@ fn analyzeSwitchRuntimeBlock(
12903	12944	if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
12904	12945	emit_bb = true;
12905	12946
12906		try spa.analyzeProngRuntime(
	12947	const prong_hint = try spa.analyzeProngRuntime(
12907	12948	&case_block,
12908	12949	.special,
12909	12950	special.body,
...	...	@@ -12916,6 +12957,7 @@ fn analyzeSwitchRuntimeBlock(
12916	12957	item_ref,
12917	12958	special.has_tag_capture,
12918	12959	);
	12960	try branch_hints.append(gpa, prong_hint);
12919	12961
12920	12962	try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
12921	12963	cases_extra.appendAssumeCapacity(1); // items_len
...	...	@@ -12937,7 +12979,7 @@ fn analyzeSwitchRuntimeBlock(
12937	12979	if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
12938	12980	emit_bb = true;
12939	12981
12940		try spa.analyzeProngRuntime(
	12982	const prong_hint = try spa.analyzeProngRuntime(
12941	12983	&case_block,
12942	12984	.special,
12943	12985	special.body,
...	...	@@ -12950,6 +12992,7 @@ fn analyzeSwitchRuntimeBlock(
12950	12992	item_ref,
12951	12993	special.has_tag_capture,
12952	12994	);
	12995	try branch_hints.append(gpa, prong_hint);
12953	12996
12954	12997	try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
12955	12998	cases_extra.appendAssumeCapacity(1); // items_len
...	...	@@ -12968,7 +13011,7 @@ fn analyzeSwitchRuntimeBlock(
12968	13011	if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
12969	13012	emit_bb = true;
12970	13013
12971		try spa.analyzeProngRuntime(
	13014	const prong_hint = try spa.analyzeProngRuntime(
12972	13015	&case_block,
12973	13016	.special,
12974	13017	special.body,
...	...	@@ -12981,6 +13024,7 @@ fn analyzeSwitchRuntimeBlock(
12981	13024	.bool_true,
12982	13025	special.has_tag_capture,
12983	13026	);
	13027	try branch_hints.append(gpa, prong_hint);
12984	13028
12985	13029	try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
12986	13030	cases_extra.appendAssumeCapacity(1); // items_len
...	...	@@ -12997,7 +13041,7 @@ fn analyzeSwitchRuntimeBlock(
12997	13041	if (emit_bb) try sema.emitBackwardBranch(block, special_prong_src);
12998	13042	emit_bb = true;
12999	13043
13000		try spa.analyzeProngRuntime(
	13044	const prong_hint = try spa.analyzeProngRuntime(
13001	13045	&case_block,
13002	13046	.special,
13003	13047	special.body,
...	...	@@ -13010,6 +13054,7 @@ fn analyzeSwitchRuntimeBlock(
13010	13054	.bool_false,
13011	13055	special.has_tag_capture,
13012	13056	);
	13057	try branch_hints.append(gpa, prong_hint);
13013	13058
13014	13059	try cases_extra.ensureUnusedCapacity(gpa, 3 + case_block.instructions.items.len);
13015	13060	cases_extra.appendAssumeCapacity(1); // items_len
...	...	@@ -13045,12 +13090,13 @@ fn analyzeSwitchRuntimeBlock(
13045	13090	} else false
13046	13091	else
13047	13092	true;
13048		if (special.body.len != 0 and err_set and
	13093	const else_hint: std.builtin.BranchHint = if (special.body.len != 0 and err_set and
13049	13094	try sema.maybeErrorUnwrap(&case_block, special.body, operand, operand_src, allow_err_code_unwrap))
13050		{
13051		// nothing to do here
13052		} else if (special.body.len != 0 and analyze_body and !special.is_inline) {
13053		try spa.analyzeProngRuntime(
	13095	h: {
	13096	// nothing to do here. weight against error branch
	13097	break :h .unlikely;
	13098	} else if (special.body.len != 0 and analyze_body and !special.is_inline) h: {
	13099	break :h try spa.analyzeProngRuntime(
13054	13100	&case_block,
13055	13101	.special,
13056	13102	special.body,
...	...	@@ -13063,7 +13109,7 @@ fn analyzeSwitchRuntimeBlock(
13063	13109	.none,
13064	13110	false,
13065	13111	);
13066		} else {
	13112	} else h: {
13067	13113	// We still need a terminator in this block, but we have proven
13068	13114	// that it is unreachable.
13069	13115	if (case_block.wantSafety()) {
...	...	@@ -13072,33 +13118,57 @@ fn analyzeSwitchRuntimeBlock(
13072	13118	} else {
13073	13119	_ = try case_block.addNoOp(.unreach);
13074	13120	}
13075		}
	13121	// Safety check / unreachable branches are cold.
	13122	break :h .cold;
	13123	};
13076	13124
13077	13125	if (is_first) {
	13126	try branch_hints.append(gpa, else_hint);
13078	13127	final_else_body = case_block.instructions.items;
13079	13128	} else {
	13129	try branch_hints.append(gpa, .none); // we have the range conditionals first
13080	13130	try sema.air_extra.ensureUnusedCapacity(gpa, prev_then_body.len +
13081	13131	@typeInfo(Air.CondBr).Struct.fields.len + case_block.instructions.items.len);
13082	13132
13083		sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload =
13084		sema.addExtraAssumeCapacity(Air.CondBr{
	13133	sema.air_instructions.items(.data)[@intFromEnum(prev_cond_br)].pl_op.payload = sema.addExtraAssumeCapacity(Air.CondBr{
13085	13134	.then_body_len = @intCast(prev_then_body.len),
13086	13135	.else_body_len = @intCast(case_block.instructions.items.len),
	13136	.branch_hints = .{ .true = prev_hint, .false = else_hint },
13087	13137	});
13088	13138	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(prev_then_body));
13089	13139	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(case_block.instructions.items));
13090	13140	final_else_body = first_else_body;
13091	13141	}
	13142	} else {
	13143	try branch_hints.append(gpa, .none);
13092	13144	}
13093	13145
	13146	assert(branch_hints.items.len == cases_len + 1);
	13147
13094	13148	try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).Struct.fields.len +
13095		cases_extra.items.len + final_else_body.len);
	13149	cases_extra.items.len + final_else_body.len +
	13150	(std.math.divCeil(usize, branch_hints.items.len, 10) catch unreachable)); // branch hints
13096	13151
13097	13152	const payload_index = sema.addExtraAssumeCapacity(Air.SwitchBr{
13098	13153	.cases_len = @intCast(cases_len),
13099	13154	.else_body_len = @intCast(final_else_body.len),
13100	13155	});
13101	13156
	13157	{
	13158	// Add branch hints.
	13159	var cur_bag: u32 = 0;
	13160	for (branch_hints.items, 0..) \|hint, idx\| {
	13161	const idx_in_bag = idx % 10;
	13162	cur_bag \|= @as(u32, @intFromEnum(hint)) << @intCast(idx_in_bag * 3);
	13163	if (idx_in_bag == 9) {
	13164	sema.air_extra.appendAssumeCapacity(cur_bag);
	13165	cur_bag = 0;
	13166	}
	13167	}
	13168	if (branch_hints.items.len % 10 != 0) {
	13169	sema.air_extra.appendAssumeCapacity(cur_bag);
	13170	}
	13171	}
13102	13172	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(cases_extra.items));
13103	13173	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(final_else_body));
13104	13174
...	...	@@ -19159,6 +19229,10 @@ fn zirBoolBr(
19159	19229	const lhs_result: Air.Inst.Ref = if (is_bool_or) .bool_true else .bool_false;
19160	19230	_ = try lhs_block.addBr(block_inst, lhs_result);
19161	19231
	19232	const parent_hint = sema.branch_hint;
	19233	defer sema.branch_hint = parent_hint;
	19234	sema.branch_hint = null;
	19235
19162	19236	const rhs_result = try sema.resolveInlineBody(rhs_block, body, inst);
19163	19237	const rhs_noret = sema.typeOf(rhs_result).isNoReturn(zcu);
19164	19238	const coerced_rhs_result = if (!rhs_noret) rhs: {
...	...	@@ -19167,7 +19241,17 @@ fn zirBoolBr(
19167	19241	break :rhs coerced_result;
19168	19242	} else rhs_result;
19169	19243
19170		const result = sema.finishCondBr(parent_block, &child_block, &then_block, &else_block, lhs, block_inst);
	19244	const rhs_hint = sema.branch_hint orelse .none;
	19245
	19246	const result = try sema.finishCondBr(
	19247	parent_block,
	19248	&child_block,
	19249	&then_block,
	19250	&else_block,
	19251	lhs,
	19252	block_inst,
	19253	if (is_bool_or) .{ .true = .none, .false = rhs_hint } else .{ .true = rhs_hint, .false = .none },
	19254	);
19171	19255	if (!rhs_noret) {
19172	19256	if (try sema.resolveDefinedValue(rhs_block, rhs_src, coerced_rhs_result)) \|rhs_val\| {
19173	19257	if (is_bool_or and rhs_val.toBool()) {
...	...	@@ -19189,6 +19273,7 @@ fn finishCondBr(
19189	19273	else_block: *Block,
19190	19274	cond: Air.Inst.Ref,
19191	19275	block_inst: Air.Inst.Index,
	19276	branch_hints: Air.CondBr.BranchHints,
19192	19277	) !Air.Inst.Ref {
19193	19278	const gpa = sema.gpa;
19194	19279
...	...	@@ -19199,6 +19284,7 @@ fn finishCondBr(
19199	19284	const cond_br_payload = sema.addExtraAssumeCapacity(Air.CondBr{
19200	19285	.then_body_len = @intCast(then_block.instructions.items.len),
19201	19286	.else_body_len = @intCast(else_block.instructions.items.len),
	19287	.branch_hints = branch_hints,
19202	19288	});
19203	19289	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items));
19204	19290	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items));
...	...	@@ -19333,6 +19419,11 @@ fn zirCondbr(
19333	19419	if (try sema.resolveDefinedValue(parent_block, cond_src, cond)) \|cond_val\| {
19334	19420	const body = if (cond_val.toBool()) then_body else else_body;
19335	19421
	19422	// We can propagate `.cold` hints from this branch since it's comptime-known
	19423	// to be taken from the parent branch.
	19424	const parent_hint = sema.branch_hint;
	19425	defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null;
	19426
19336	19427	try sema.maybeErrorUnwrapCondbr(parent_block, body, extra.data.condition, cond_src);
19337	19428	// We use `analyzeBodyInner` since we want to propagate any comptime control flow to the caller.
19338	19429	return sema.analyzeBodyInner(parent_block, body);
...	...	@@ -19349,7 +19440,7 @@ fn zirCondbr(
19349	19440	sub_block.need_debug_scope = null; // this body is emitted regardless
19350	19441	defer sub_block.instructions.deinit(gpa);
19351	19442
19352		try sema.analyzeBodyRuntimeBreak(&sub_block, then_body);
	19443	const true_hint = try sema.analyzeBodyRuntimeBreak(&sub_block, then_body);
19353	19444	const true_instructions = try sub_block.instructions.toOwnedSlice(gpa);
19354	19445	defer gpa.free(true_instructions);
19355	19446
...	...	@@ -19365,11 +19456,13 @@ fn zirCondbr(
19365	19456	break :blk try sub_block.addTyOp(.unwrap_errunion_err, result_ty, err_operand);
19366	19457	};
19367	19458
19368		if (err_cond != null and try sema.maybeErrorUnwrap(&sub_block, else_body, err_cond.?, cond_src, false)) {
19369		// nothing to do
19370		} else {
19371		try sema.analyzeBodyRuntimeBreak(&sub_block, else_body);
19372		}
	19459	const false_hint: std.builtin.BranchHint = if (err_cond != null and
	19460	try sema.maybeErrorUnwrap(&sub_block, else_body, err_cond.?, cond_src, false))
	19461	h: {
	19462	// nothing to do here. weight against error branch
	19463	break :h .unlikely;
	19464	} else try sema.analyzeBodyRuntimeBreak(&sub_block, else_body);
	19465
19373	19466	try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.CondBr).Struct.fields.len +
19374	19467	true_instructions.len + sub_block.instructions.items.len);
19375	19468	_ = try parent_block.addInst(.{
...	...	@@ -19379,6 +19472,7 @@ fn zirCondbr(
19379	19472	.payload = sema.addExtraAssumeCapacity(Air.CondBr{
19380	19473	.then_body_len = @intCast(true_instructions.len),
19381	19474	.else_body_len = @intCast(sub_block.instructions.items.len),
	19475	.branch_hints = .{ .true = true_hint, .false = false_hint },
19382	19476	}),
19383	19477	} },
19384	19478	});
...	...	@@ -19403,6 +19497,11 @@ fn zirTry(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileError!
19403	19497	}
19404	19498	const is_non_err = try sema.analyzeIsNonErrComptimeOnly(parent_block, operand_src, err_union);
19405	19499	if (is_non_err != .none) {
	19500	// We can propagate `.cold` hints from this branch since it's comptime-known
	19501	// to be taken from the parent branch.
	19502	const parent_hint = sema.branch_hint;
	19503	defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null;
	19504
19406	19505	const is_non_err_val = (try sema.resolveDefinedValue(parent_block, operand_src, is_non_err)).?;
19407	19506	if (is_non_err_val.toBool()) {
19408	19507	return sema.analyzeErrUnionPayload(parent_block, src, err_union_ty, err_union, operand_src, false);
...	...	@@ -19416,13 +19515,19 @@ fn zirTry(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileError!
19416	19515	var sub_block = parent_block.makeSubBlock();
19417	19516	defer sub_block.instructions.deinit(sema.gpa);
19418	19517
	19518	const parent_hint = sema.branch_hint;
	19519	defer sema.branch_hint = parent_hint;
	19520
19419	19521	// This body is guaranteed to end with noreturn and has no breaks.
19420	19522	try sema.analyzeBodyInner(&sub_block, body);
19421	19523
	19524	// The only interesting hint here is `.cold`, which can come from e.g. `errdefer @panic`.
	19525	const is_cold = sema.branch_hint == .cold;
	19526
19422	19527	try sema.air_extra.ensureUnusedCapacity(sema.gpa, @typeInfo(Air.Try).Struct.fields.len +
19423	19528	sub_block.instructions.items.len);
19424	19529	const try_inst = try parent_block.addInst(.{
19425		.tag = .@"try",
	19530	.tag = if (is_cold) .try_cold else .@"try",
19426	19531	.data = .{ .pl_op = .{
19427	19532	.operand = err_union,
19428	19533	.payload = sema.addExtraAssumeCapacity(Air.Try{
...	...	@@ -19452,6 +19557,11 @@ fn zirTryPtr(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileErr
19452	19557	}
19453	19558	const is_non_err = try sema.analyzeIsNonErrComptimeOnly(parent_block, operand_src, err_union);
19454	19559	if (is_non_err != .none) {
	19560	// We can propagate `.cold` hints from this branch since it's comptime-known
	19561	// to be taken from the parent branch.
	19562	const parent_hint = sema.branch_hint;
	19563	defer sema.branch_hint = parent_hint orelse if (sema.branch_hint == .cold) .cold else null;
	19564
19455	19565	const is_non_err_val = (try sema.resolveDefinedValue(parent_block, operand_src, is_non_err)).?;
19456	19566	if (is_non_err_val.toBool()) {
19457	19567	return sema.analyzeErrUnionPayloadPtr(parent_block, src, operand, false, false);
...	...	@@ -19465,9 +19575,15 @@ fn zirTryPtr(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileErr
19465	19575	var sub_block = parent_block.makeSubBlock();
19466	19576	defer sub_block.instructions.deinit(sema.gpa);
19467	19577
	19578	const parent_hint = sema.branch_hint;
	19579	defer sema.branch_hint = parent_hint;
	19580
19468	19581	// This body is guaranteed to end with noreturn and has no breaks.
19469	19582	try sema.analyzeBodyInner(&sub_block, body);
19470	19583
	19584	// The only interesting hint here is `.cold`, which can come from e.g. `errdefer @panic`.
	19585	const is_cold = sema.branch_hint == .cold;
	19586
19471	19587	const operand_ty = sema.typeOf(operand);
19472	19588	const ptr_info = operand_ty.ptrInfo(zcu);
19473	19589	const res_ty = try pt.ptrTypeSema(.{
...	...	@@ -19483,7 +19599,7 @@ fn zirTryPtr(sema: Sema, parent_block: Block, inst: Zir.Inst.Index) CompileErr
19483	19599	try sema.air_extra.ensureUnusedCapacity(sema.gpa, @typeInfo(Air.TryPtr).Struct.fields.len +
19484	19600	sub_block.instructions.items.len);
19485	19601	const try_inst = try parent_block.addInst(.{
19486		.tag = .try_ptr,
	19602	.tag = if (is_cold) .try_ptr_cold else .try_ptr,
19487	19603	.data = .{ .ty_pl = .{
19488	19604	.ty = res_ty_ref,
19489	19605	.payload = sema.addExtraAssumeCapacity(Air.TryPtr{
...	...	@@ -19735,6 +19851,8 @@ fn retWithErrTracing(
19735	19851	const cond_br_payload = sema.addExtraAssumeCapacity(Air.CondBr{
19736	19852	.then_body_len = @intCast(then_block.instructions.items.len),
19737	19853	.else_body_len = @intCast(else_block.instructions.items.len),
	19854	// weight against error branch
	19855	.branch_hints = .{ .true = .likely, .false = .unlikely },
19738	19856	});
19739	19857	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(then_block.instructions.items));
19740	19858	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(else_block.instructions.items));
...	...	@@ -26747,6 +26865,7 @@ fn zirBuiltinValue(sema: *Sema, extended: Zir.Inst.Extended.InstData) CompileErr
26747	26865	.export_options => "ExportOptions",
26748	26866	.extern_options => "ExternOptions",
26749	26867	.type_info => "Type",
	26868	.branch_hint => "BranchHint",
26750	26869
26751	26870	// Values are handled here.
26752	26871	.calling_convention_c => {
...	...	@@ -26772,6 +26891,27 @@ fn zirBuiltinValue(sema: *Sema, extended: Zir.Inst.Extended.InstData) CompileErr
26772	26891	return Air.internedToRef(ty.toIntern());
26773	26892	}
26774	26893
	26894	fn zirBranchHint(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData) CompileError!void {
	26895	const pt = sema.pt;
	26896	const zcu = pt.zcu;
	26897
	26898	const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data;
	26899	const uncoerced_hint = try sema.resolveInst(extra.operand);
	26900	const operand_src = block.builtinCallArgSrc(extra.node, 0);
	26901
	26902	const hint_ty = try pt.getBuiltinType("BranchHint");
	26903	const coerced_hint = try sema.coerce(block, hint_ty, uncoerced_hint, operand_src);
	26904	const hint_val = try sema.resolveConstDefinedValue(block, operand_src, coerced_hint, .{
	26905	.needed_comptime_reason = "operand to '@branchHint' must be comptime-known",
	26906	});
	26907
	26908	// We only apply the first hint in a branch.
	26909	// This allows user-provided hints to override implicit cold hints.
	26910	if (sema.branch_hint == null) {
	26911	sema.branch_hint = zcu.toEnum(std.builtin.BranchHint, hint_val);
	26912	}
	26913	}
	26914
26775	26915	fn requireRuntimeBlock(sema: Sema, block: Block, src: LazySrcLoc, runtime_src: ?LazySrcLoc) !void {
26776	26916	if (block.is_comptime) {
26777	26917	const msg = msg: {
...	...	@@ -27327,13 +27467,17 @@ fn addSafetyCheckExtra(
27327	27467
27328	27468	sema.air_instructions.appendAssumeCapacity(.{
27329	27469	.tag = .cond_br,
27330		.data = .{ .pl_op = .{
27331		.operand = ok,
27332		.payload = sema.addExtraAssumeCapacity(Air.CondBr{
27333		.then_body_len = 1,
27334		.else_body_len = @intCast(fail_block.instructions.items.len),
27335		}),
27336		} },
	27470	.data = .{
	27471	.pl_op = .{
	27472	.operand = ok,
	27473	.payload = sema.addExtraAssumeCapacity(Air.CondBr{
	27474	.then_body_len = 1,
	27475	.else_body_len = @intCast(fail_block.instructions.items.len),
	27476	// safety check failure branch is cold
	27477	.branch_hints = .{ .true = .likely, .false = .cold },
	27478	}),
	27479	},
	27480	},
27337	27481	});
27338	27482	sema.air_extra.appendAssumeCapacity(@intFromEnum(br_inst));
27339	27483	sema.air_extra.appendSliceAssumeCapacity(@ptrCast(fail_block.instructions.items));
...	...	@@ -27530,6 +27674,7 @@ fn safetyCheckFormatted(
27530	27674	try sema.addSafetyCheckExtra(parent_block, ok, &fail_block);
27531	27675	}
27532	27676
	27677	/// This does not set `sema.branch_hint`.
27533	27678	fn safetyPanic(sema: Sema, block: Block, src: LazySrcLoc, panic_id: Zcu.PanicId) CompileError!void {
27534	27679	const msg_nav_index = try sema.preparePanicId(block, src, panic_id);
27535	27680	const msg_inst = try sema.analyzeNavVal(block, src, msg_nav_index);
...	...	@@ -37179,7 +37324,7 @@ pub fn addExtraAssumeCapacity(sema: *Sema, extra: anytype) u32 {
37179	37324	inline for (fields) \|field\| {
37180	37325	sema.air_extra.appendAssumeCapacity(switch (field.type) {
37181	37326	u32 => @field(extra, field.name),
37182		i32 => @bitCast(@field(extra, field.name)),
	37327	i32, Air.CondBr.BranchHints => @bitCast(@field(extra, field.name)),
37183	37328	Air.Inst.Ref, InternPool.Index => @intFromEnum(@field(extra, field.name)),
37184	37329	else => @compileError("bad field type: " ++ @typeName(field.type)),
37185	37330	});
...	...	@@ -38247,6 +38392,12 @@ fn maybeDerefSliceAsArray(
38247	38392
38248	38393	fn analyzeUnreachable(sema: Sema, block: Block, src: LazySrcLoc, safety_check: bool) !void {
38249	38394	if (safety_check and block.wantSafety()) {
	38395	// We only apply the first hint in a branch.
	38396	// This allows user-provided hints to override implicit cold hints.
	38397	if (sema.branch_hint == null) {
	38398	sema.branch_hint = .cold;
	38399	}
	38400
38250	38401	try sema.safetyPanic(block, src, .unreach);
38251	38402	} else {
38252	38403	_ = try block.addNoOp(.unreach);

src/Zcu/PerThread.zig+2

...	...	@@ -2188,6 +2188,8 @@ fn analyzeFnBody(pt: Zcu.PerThread, func_index: InternPool.Index) Zcu.SemaError!
2188	2188	});
2189	2189	}
2190	2190
	2191	func.setBranchHint(ip, sema.branch_hint orelse .none);
	2192
2191	2193	// If we don't get an error return trace from a caller, create our own.
2192	2194	if (func.analysisUnordered(ip).calls_or_awaits_errorable_fn and
2193	2195	zcu.comp.config.any_error_tracing and

src/arch/aarch64/CodeGen.zig+16-22

...	...	@@ -795,7 +795,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
795	795	.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
796	796
797	797	.@"try" => try self.airTry(inst),
	798	.try_cold => try self.airTry(inst),
798	799	.try_ptr => try self.airTryPtr(inst),
	800	.try_ptr_cold => try self.airTryPtr(inst),
799	801
800	802	.dbg_stmt => try self.airDbgStmt(inst),
801	803	.dbg_inline_block => try self.airDbgInlineBlock(inst),
...	...	@@ -5092,25 +5094,17 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
5092	5094	}
5093	5095
5094	5096	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5095		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5096		const condition_ty = self.typeOf(pl_op.operand);
5097		const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
	5097	const switch_br = self.air.unwrapSwitch(inst);
	5098	const condition_ty = self.typeOf(switch_br.operand);
5098	5099	const liveness = try self.liveness.getSwitchBr(
5099	5100	self.gpa,
5100	5101	inst,
5101		switch_br.data.cases_len + 1,
	5102	switch_br.cases_len + 1,
5102	5103	);
5103	5104	defer self.gpa.free(liveness.deaths);
5104	5105
5105		var extra_index: usize = switch_br.end;
5106		var case_i: u32 = 0;
5107		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
5108		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
5109		const items = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[case.end..][0..case.data.items_len]));
5110		assert(items.len > 0);
5111		const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
5112		extra_index = case.end + items.len + case_body.len;
5113
	5106	var it = switch_br.iterateCases();
	5107	while (it.next()) \|case\| {
5114	5108	// For every item, we compare it to condition and branch into
5115	5109	// the prong if they are equal. After we compared to all
5116	5110	// items, we branch into the next prong (or if no other prongs
...	...	@@ -5126,11 +5120,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5126	5120	// prong: ...
5127	5121	// ...
5128	5122	// out: ...
5129		const branch_into_prong_relocs = try self.gpa.alloc(u32, items.len);
	5123	const branch_into_prong_relocs = try self.gpa.alloc(u32, case.items.len);
5130	5124	defer self.gpa.free(branch_into_prong_relocs);
5131	5125
5132		for (items, 0..) \|item, idx\| {
5133		const cmp_result = try self.cmp(.{ .inst = pl_op.operand }, .{ .inst = item }, condition_ty, .neq);
	5126	for (case.items, 0..) \|item, idx\| {
	5127	const cmp_result = try self.cmp(.{ .inst = switch_br.operand }, .{ .inst = item }, condition_ty, .neq);
5134	5128	branch_into_prong_relocs[idx] = try self.condBr(cmp_result);
5135	5129	}
5136	5130
...	...	@@ -5156,11 +5150,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5156	5150	_ = self.branch_stack.pop();
5157	5151	}
5158	5152
5159		try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len);
5160		for (liveness.deaths[case_i]) \|operand\| {
	5153	try self.ensureProcessDeathCapacity(liveness.deaths[case.idx].len);
	5154	for (liveness.deaths[case.idx]) \|operand\| {
5161	5155	self.processDeath(operand);
5162	5156	}
5163		try self.genBody(case_body);
	5157	try self.genBody(case.body);
5164	5158
5165	5159	// Revert to the previous register and stack allocation state.
5166	5160	var saved_case_branch = self.branch_stack.pop();
...	...	@@ -5178,8 +5172,8 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5178	5172	try self.performReloc(branch_away_from_prong_reloc);
5179	5173	}
5180	5174
5181		if (switch_br.data.else_body_len > 0) {
5182		const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	5175	if (switch_br.else_body_len > 0) {
	5176	const else_body = it.elseBody();
5183	5177
5184	5178	// Capture the state of register and stack allocation state so that we can revert to it.
5185	5179	const parent_next_stack_offset = self.next_stack_offset;
...	...	@@ -5218,7 +5212,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5218	5212	// in airCondBr.
5219	5213	}
5220	5214
5221		return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
	5215	return self.finishAir(inst, .unreach, .{ switch_br.operand, .none, .none });
5222	5216	}
5223	5217
5224	5218	fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {

src/arch/arm/CodeGen.zig+16-22

...	...	@@ -782,7 +782,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
782	782	.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
783	783
784	784	.@"try" => try self.airTry(inst),
	785	.try_cold => try self.airTry(inst),
785	786	.try_ptr => try self.airTryPtr(inst),
	787	.try_ptr_cold => try self.airTryPtr(inst),
786	788
787	789	.dbg_stmt => try self.airDbgStmt(inst),
788	790	.dbg_inline_block => try self.airDbgInlineBlock(inst),
...	...	@@ -5040,25 +5042,17 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
5040	5042	}
5041	5043
5042	5044	fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5043		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5044		const condition_ty = self.typeOf(pl_op.operand);
5045		const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
	5045	const switch_br = self.air.unwrapSwitch(inst);
	5046	const condition_ty = self.typeOf(switch_br.operand);
5046	5047	const liveness = try self.liveness.getSwitchBr(
5047	5048	self.gpa,
5048	5049	inst,
5049		switch_br.data.cases_len + 1,
	5050	switch_br.cases_len + 1,
5050	5051	);
5051	5052	defer self.gpa.free(liveness.deaths);
5052	5053
5053		var extra_index: usize = switch_br.end;
5054		var case_i: u32 = 0;
5055		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
5056		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
5057		const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
5058		assert(items.len > 0);
5059		const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
5060		extra_index = case.end + items.len + case_body.len;
5061
	5054	var it = switch_br.iterateCases();
	5055	while (it.next()) \|case\| {
5062	5056	// For every item, we compare it to condition and branch into
5063	5057	// the prong if they are equal. After we compared to all
5064	5058	// items, we branch into the next prong (or if no other prongs
...	...	@@ -5074,11 +5068,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5074	5068	// prong: ...
5075	5069	// ...
5076	5070	// out: ...
5077		const branch_into_prong_relocs = try self.gpa.alloc(u32, items.len);
	5071	const branch_into_prong_relocs = try self.gpa.alloc(u32, case.items.len);
5078	5072	defer self.gpa.free(branch_into_prong_relocs);
5079	5073
5080		for (items, 0..) \|item, idx\| {
5081		const cmp_result = try self.cmp(.{ .inst = pl_op.operand }, .{ .inst = item }, condition_ty, .neq);
	5074	for (case.items, 0..) \|item, idx\| {
	5075	const cmp_result = try self.cmp(.{ .inst = switch_br.operand }, .{ .inst = item }, condition_ty, .neq);
5082	5076	branch_into_prong_relocs[idx] = try self.condBr(cmp_result);
5083	5077	}
5084	5078
...	...	@@ -5104,11 +5098,11 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5104	5098	_ = self.branch_stack.pop();
5105	5099	}
5106	5100
5107		try self.ensureProcessDeathCapacity(liveness.deaths[case_i].len);
5108		for (liveness.deaths[case_i]) \|operand\| {
	5101	try self.ensureProcessDeathCapacity(liveness.deaths[case.idx].len);
	5102	for (liveness.deaths[case.idx]) \|operand\| {
5109	5103	self.processDeath(operand);
5110	5104	}
5111		try self.genBody(case_body);
	5105	try self.genBody(case.body);
5112	5106
5113	5107	// Revert to the previous register and stack allocation state.
5114	5108	var saved_case_branch = self.branch_stack.pop();
...	...	@@ -5126,8 +5120,8 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5126	5120	try self.performReloc(branch_away_from_prong_reloc);
5127	5121	}
5128	5122
5129		if (switch_br.data.else_body_len > 0) {
5130		const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	5123	if (switch_br.else_body_len > 0) {
	5124	const else_body = it.elseBody();
5131	5125
5132	5126	// Capture the state of register and stack allocation state so that we can revert to it.
5133	5127	const parent_next_stack_offset = self.next_stack_offset;
...	...	@@ -5166,7 +5160,7 @@ fn airSwitch(self: *Self, inst: Air.Inst.Index) !void {
5166	5160	// in airCondBr.
5167	5161	}
5168	5162
5169		return self.finishAir(inst, .unreach, .{ pl_op.operand, .none, .none });
	5163	return self.finishAir(inst, .unreach, .{ switch_br.operand, .none, .none });
5170	5164	}
5171	5165
5172	5166	fn performReloc(self: *Self, inst: Mir.Inst.Index) !void {

src/arch/riscv64/CodeGen.zig+16-23

...	...	@@ -1640,7 +1640,9 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1640	1640	.addrspace_cast => return func.fail("TODO: addrspace_cast", .{}),
1641	1641
1642	1642	.@"try" => try func.airTry(inst),
	1643	.try_cold => try func.airTry(inst),
1643	1644	.try_ptr => return func.fail("TODO: try_ptr", .{}),
	1645	.try_ptr_cold => return func.fail("TODO: try_ptr_cold", .{}),
1644	1646
1645	1647	.dbg_var_ptr,
1646	1648	.dbg_var_val,
...	...	@@ -5659,38 +5661,30 @@ fn lowerBlock(func: *Func, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
5659	5661	}
5660	5662
5661	5663	fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5662		const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5663		const condition_ty = func.typeOf(pl_op.operand);
5664		const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload);
5665		var extra_index: usize = switch_br.end;
5666		var case_i: u32 = 0;
5667		const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1);
	5664	const switch_br = func.air.unwrapSwitch(inst);
	5665
	5666	const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1);
5668	5667	defer func.gpa.free(liveness.deaths);
5669	5668
5670		const condition = try func.resolveInst(pl_op.operand);
	5669	const condition = try func.resolveInst(switch_br.operand);
	5670	const condition_ty = func.typeOf(switch_br.operand);
5671	5671
5672	5672	// If the condition dies here in this switch instruction, process
5673	5673	// that death now instead of later as this has an effect on
5674	5674	// whether it needs to be spilled in the branches
5675	5675	if (func.liveness.operandDies(inst, 0)) {
5676		if (pl_op.operand.toIndex()) \|op_inst\| try func.processDeath(op_inst);
	5676	if (switch_br.operand.toIndex()) \|op_inst\| try func.processDeath(op_inst);
5677	5677	}
5678	5678
5679	5679	func.scope_generation += 1;
5680	5680	const state = try func.saveState();
5681	5681
5682		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
5683		const case = func.air.extraData(Air.SwitchBr.Case, extra_index);
5684		const items: []const Air.Inst.Ref =
5685		@ptrCast(func.air.extra[case.end..][0..case.data.items_len]);
5686		const case_body: []const Air.Inst.Index =
5687		@ptrCast(func.air.extra[case.end + items.len ..][0..case.data.body_len]);
5688		extra_index = case.end + items.len + case_body.len;
5689
5690		var relocs = try func.gpa.alloc(Mir.Inst.Index, items.len);
	5682	var it = switch_br.iterateCases();
	5683	while (it.next()) \|case\| {
	5684	var relocs = try func.gpa.alloc(Mir.Inst.Index, case.items.len);
5691	5685	defer func.gpa.free(relocs);
5692	5686
5693		for (items, relocs, 0..) \|item, *reloc, i\| {
	5687	for (case.items, relocs, 0..) \|item, *reloc, i\| {
5694	5688	const item_mcv = try func.resolveInst(item);
5695	5689
5696	5690	const cond_lock = switch (condition) {
...	...	@@ -5724,10 +5718,10 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5724	5718	reloc.* = try func.condBr(condition_ty, .{ .register = cmp_reg });
5725	5719	}
5726	5720
5727		for (liveness.deaths[case_i]) \|operand\| try func.processDeath(operand);
	5721	for (liveness.deaths[case.idx]) \|operand\| try func.processDeath(operand);
5728	5722
5729	5723	for (relocs[0 .. relocs.len - 1]) \|reloc\| func.performReloc(reloc);
5730		try func.genBody(case_body);
	5724	try func.genBody(case.body);
5731	5725	try func.restoreState(state, &.{}, .{
5732	5726	.emit_instructions = false,
5733	5727	.update_tracking = true,
...	...	@@ -5738,9 +5732,8 @@ fn airSwitchBr(func: *Func, inst: Air.Inst.Index) !void {
5738	5732	func.performReloc(relocs[relocs.len - 1]);
5739	5733	}
5740	5734
5741		if (switch_br.data.else_body_len > 0) {
5742		const else_body: []const Air.Inst.Index =
5743		@ptrCast(func.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	5735	if (switch_br.else_body_len > 0) {
	5736	const else_body = it.elseBody();
5744	5737
5745	5738	const else_deaths = liveness.deaths.len - 1;
5746	5739	for (liveness.deaths[else_deaths]) \|operand\| try func.processDeath(operand);

src/arch/sparc64/CodeGen.zig+2

...	...	@@ -637,7 +637,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
637	637	.addrspace_cast => @panic("TODO try self.airAddrSpaceCast(int)"),
638	638
639	639	.@"try" => try self.airTry(inst),
	640	.try_cold => try self.airTry(inst),
640	641	.try_ptr => @panic("TODO try self.airTryPtr(inst)"),
	642	.try_ptr_cold => @panic("TODO try self.airTryPtrCold(inst)"),
641	643
642	644	.dbg_stmt => try self.airDbgStmt(inst),
643	645	.dbg_inline_block => try self.airDbgInlineBlock(inst),

src/arch/wasm/CodeGen.zig+16-20

...	...	@@ -1913,7 +1913,9 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1913	1913	.get_union_tag => func.airGetUnionTag(inst),
1914	1914
1915	1915	.@"try" => func.airTry(inst),
	1916	.try_cold => func.airTry(inst),
1916	1917	.try_ptr => func.airTryPtr(inst),
	1918	.try_ptr_cold => func.airTryPtr(inst),
1917	1919
1918	1920	.dbg_stmt => func.airDbgStmt(inst),
1919	1921	.dbg_inline_block => func.airDbgInlineBlock(inst),
...	...	@@ -4041,37 +4043,31 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4041	4043	const zcu = pt.zcu;
4042	4044	// result type is always 'noreturn'
4043	4045	const blocktype = wasm.block_empty;
4044		const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4045		const target = try func.resolveInst(pl_op.operand);
4046		const target_ty = func.typeOf(pl_op.operand);
4047		const switch_br = func.air.extraData(Air.SwitchBr, pl_op.payload);
4048		const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.data.cases_len + 1);
	4046	const switch_br = func.air.unwrapSwitch(inst);
	4047	const target = try func.resolveInst(switch_br.operand);
	4048	const target_ty = func.typeOf(switch_br.operand);
	4049	const liveness = try func.liveness.getSwitchBr(func.gpa, inst, switch_br.cases_len + 1);
4049	4050	defer func.gpa.free(liveness.deaths);
4050	4051
4051		var extra_index: usize = switch_br.end;
4052		var case_i: u32 = 0;
4053
4054	4052	// a list that maps each value with its value and body based on the order inside the list.
4055	4053	const CaseValue = struct { integer: i32, value: Value };
4056	4054	var case_list = try std.ArrayList(struct {
4057	4055	values: []const CaseValue,
4058	4056	body: []const Air.Inst.Index,
4059		}).initCapacity(func.gpa, switch_br.data.cases_len);
	4057	}).initCapacity(func.gpa, switch_br.cases_len);
4060	4058	defer for (case_list.items) \|case\| {
4061	4059	func.gpa.free(case.values);
4062	4060	} else case_list.deinit();
4063	4061
4064	4062	var lowest_maybe: ?i32 = null;
4065	4063	var highest_maybe: ?i32 = null;
4066		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
4067		const case = func.air.extraData(Air.SwitchBr.Case, extra_index);
4068		const items: []const Air.Inst.Ref = @ptrCast(func.air.extra[case.end..][0..case.data.items_len]);
4069		const case_body: []const Air.Inst.Index = @ptrCast(func.air.extra[case.end + items.len ..][0..case.data.body_len]);
4070		extra_index = case.end + items.len + case_body.len;
4071		const values = try func.gpa.alloc(CaseValue, items.len);
	4064
	4065	var it = switch_br.iterateCases();
	4066	while (it.next()) \|case\| {
	4067	const values = try func.gpa.alloc(CaseValue, case.items.len);
4072	4068	errdefer func.gpa.free(values);
4073	4069
4074		for (items, 0..) \|ref, i\| {
	4070	for (case.items, 0..) \|ref, i\| {
4075	4071	const item_val = (try func.air.value(ref, pt)).?;
4076	4072	const int_val = func.valueAsI32(item_val);
4077	4073	if (lowest_maybe == null or int_val < lowest_maybe.?) {
...	...	@@ -4083,7 +4079,7 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4083	4079	values[i] = .{ .integer = int_val, .value = item_val };
4084	4080	}
4085	4081
4086		case_list.appendAssumeCapacity(.{ .values = values, .body = case_body });
	4082	case_list.appendAssumeCapacity(.{ .values = values, .body = case.body });
4087	4083	try func.startBlock(.block, blocktype);
4088	4084	}
4089	4085
...	...	@@ -4097,7 +4093,7 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4097	4093	// TODO: Benchmark this to find a proper value, LLVM seems to draw the line at '40~45'.
4098	4094	const is_sparse = highest - lowest > 50 or target_ty.bitSize(zcu) > 32;
4099	4095
4100		const else_body: []const Air.Inst.Index = @ptrCast(func.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	4096	const else_body = it.elseBody();
4101	4097	const has_else_body = else_body.len != 0;
4102	4098	if (has_else_body) {
4103	4099	try func.startBlock(.block, blocktype);
...	...	@@ -4140,11 +4136,11 @@ fn airSwitchBr(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4140	4136	// for errors that are not present in any branch. This is fine as this default
4141	4137	// case will never be hit for those cases but we do save runtime cost and size
4142	4138	// by using a jump table for this instead of if-else chains.
4143		break :blk if (has_else_body or target_ty.zigTypeTag(zcu) == .ErrorSet) case_i else unreachable;
	4139	break :blk if (has_else_body or target_ty.zigTypeTag(zcu) == .ErrorSet) switch_br.cases_len else unreachable;
4144	4140	};
4145	4141	func.mir_extra.appendAssumeCapacity(idx);
4146	4142	} else if (has_else_body) {
4147		func.mir_extra.appendAssumeCapacity(case_i); // default branch
	4143	func.mir_extra.appendAssumeCapacity(switch_br.cases_len); // default branch
4148	4144	}
4149	4145	try func.endBlock();
4150	4146	}

src/arch/x86_64/CodeGen.zig+15-23

...	...	@@ -2262,7 +2262,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
2262	2262	.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
2263	2263
2264	2264	.@"try" => try self.airTry(inst),
	2265	.try_cold => try self.airTry(inst), // TODO
2265	2266	.try_ptr => try self.airTryPtr(inst),
	2267	.try_ptr_cold => try self.airTryPtr(inst), // TODO
2266	2268
2267	2269	.dbg_stmt => try self.airDbgStmt(inst),
2268	2270	.dbg_inline_block => try self.airDbgInlineBlock(inst),
...	...	@@ -13631,38 +13633,29 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) !
13631	13633	}
13632	13634
13633	13635	fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13634		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
13635		const condition = try self.resolveInst(pl_op.operand);
13636		const condition_ty = self.typeOf(pl_op.operand);
13637		const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
13638		var extra_index: usize = switch_br.end;
13639		var case_i: u32 = 0;
13640		const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.data.cases_len + 1);
	13636	const switch_br = self.air.unwrapSwitch(inst);
	13637	const condition = try self.resolveInst(switch_br.operand);
	13638	const condition_ty = self.typeOf(switch_br.operand);
	13639	const liveness = try self.liveness.getSwitchBr(self.gpa, inst, switch_br.cases_len + 1);
13641	13640	defer self.gpa.free(liveness.deaths);
13642	13641
13643	13642	// If the condition dies here in this switch instruction, process
13644	13643	// that death now instead of later as this has an effect on
13645	13644	// whether it needs to be spilled in the branches
13646	13645	if (self.liveness.operandDies(inst, 0)) {
13647		if (pl_op.operand.toIndex()) \|op_inst\| try self.processDeath(op_inst);
	13646	if (switch_br.operand.toIndex()) \|op_inst\| try self.processDeath(op_inst);
13648	13647	}
13649	13648
13650	13649	self.scope_generation += 1;
13651	13650	const state = try self.saveState();
13652	13651
13653		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
13654		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
13655		const items: []const Air.Inst.Ref =
13656		@ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
13657		const case_body: []const Air.Inst.Index =
13658		@ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
13659		extra_index = case.end + items.len + case_body.len;
13660
13661		var relocs = try self.gpa.alloc(Mir.Inst.Index, items.len);
	13652	var it = switch_br.iterateCases();
	13653	while (it.next()) \|case\| {
	13654	var relocs = try self.gpa.alloc(Mir.Inst.Index, case.items.len);
13662	13655	defer self.gpa.free(relocs);
13663	13656
13664	13657	try self.spillEflagsIfOccupied();
13665		for (items, relocs, 0..) \|item, *reloc, i\| {
	13658	for (case.items, relocs, 0..) \|item, *reloc, i\| {
13666	13659	const item_mcv = try self.resolveInst(item);
13667	13660	const cc: Condition = switch (condition) {
13668	13661	.eflags => \|cc\| switch (item_mcv.immediate) {
...	...	@@ -13678,10 +13671,10 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13678	13671	reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) cc else cc.negate(), undefined);
13679	13672	}
13680	13673
13681		for (liveness.deaths[case_i]) \|operand\| try self.processDeath(operand);
	13674	for (liveness.deaths[case.idx]) \|operand\| try self.processDeath(operand);
13682	13675
13683	13676	for (relocs[0 .. relocs.len - 1]) \|reloc\| self.performReloc(reloc);
13684		try self.genBody(case_body);
	13677	try self.genBody(case.body);
13685	13678	try self.restoreState(state, &.{}, .{
13686	13679	.emit_instructions = false,
13687	13680	.update_tracking = true,
...	...	@@ -13692,9 +13685,8 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
13692	13685	self.performReloc(relocs[relocs.len - 1]);
13693	13686	}
13694	13687
13695		if (switch_br.data.else_body_len > 0) {
13696		const else_body: []const Air.Inst.Index =
13697		@ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	13688	if (switch_br.else_body_len > 0) {
	13689	const else_body = it.elseBody();
13698	13690
13699	13691	const else_deaths = liveness.deaths.len - 1;
13700	13692	for (liveness.deaths[else_deaths]) \|operand\| try self.processDeath(operand);

src/codegen/c.zig+19-24

...	...	@@ -1786,7 +1786,7 @@ pub const DeclGen = struct {
1786	1786	else => unreachable,
1787	1787	}
1788	1788	}
1789		if (fn_val.getFunction(zcu)) \|func\| if (func.analysisUnordered(ip).is_cold)
	1789	if (fn_val.getFunction(zcu)) \|func\| if (func.analysisUnordered(ip).branch_hint == .cold)
1790	1790	try w.writeAll("zig_cold ");
1791	1791	if (fn_info.return_type == .noreturn_type) try w.writeAll("zig_noreturn ");
1792	1792
...	...	@@ -3290,8 +3290,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
3290	3290	.prefetch => try airPrefetch(f, inst),
3291	3291	.addrspace_cast => return f.fail("TODO: C backend: implement addrspace_cast", .{}),
3292	3292
3293		.@"try" => try airTry(f, inst),
3294		.try_ptr => try airTryPtr(f, inst),
	3293	.@"try" => try airTry(f, inst),
	3294	.try_cold => try airTry(f, inst),
	3295	.try_ptr => try airTryPtr(f, inst),
	3296	.try_ptr_cold => try airTryPtr(f, inst),
3295	3297
3296	3298	.dbg_stmt => try airDbgStmt(f, inst),
3297	3299	.dbg_inline_block => try airDbgInlineBlock(f, inst),
...	...	@@ -4988,11 +4990,10 @@ fn airCondBr(f: *Function, inst: Air.Inst.Index) !CValue {
4988	4990	fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
4989	4991	const pt = f.object.dg.pt;
4990	4992	const zcu = pt.zcu;
4991		const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4992		const condition = try f.resolveInst(pl_op.operand);
4993		try reap(f, inst, &.{pl_op.operand});
4994		const condition_ty = f.typeOf(pl_op.operand);
4995		const switch_br = f.air.extraData(Air.SwitchBr, pl_op.payload);
	4993	const switch_br = f.air.unwrapSwitch(inst);
	4994	const condition = try f.resolveInst(switch_br.operand);
	4995	try reap(f, inst, &.{switch_br.operand});
	4996	const condition_ty = f.typeOf(switch_br.operand);
4996	4997	const writer = f.object.writer();
4997	4998
4998	4999	try writer.writeAll("switch (");
...	...	@@ -5013,22 +5014,16 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
5013	5014	f.object.indent_writer.pushIndent();
5014	5015
5015	5016	const gpa = f.object.dg.gpa;
5016		const liveness = try f.liveness.getSwitchBr(gpa, inst, switch_br.data.cases_len + 1);
	5017	const liveness = try f.liveness.getSwitchBr(gpa, inst, switch_br.cases_len + 1);
5017	5018	defer gpa.free(liveness.deaths);
5018	5019
5019	5020	// On the final iteration we do not need to fix any state. This is because, like in the `else`
5020	5021	// branch of a `cond_br`, our parent has to do it for this entire body anyway.
5021		const last_case_i = switch_br.data.cases_len - @intFromBool(switch_br.data.else_body_len == 0);
5022
5023		var extra_index: usize = switch_br.end;
5024		for (0..switch_br.data.cases_len) \|case_i\| {
5025		const case = f.air.extraData(Air.SwitchBr.Case, extra_index);
5026		const items = @as([]const Air.Inst.Ref, @ptrCast(f.air.extra[case.end..][0..case.data.items_len]));
5027		const case_body: []const Air.Inst.Index =
5028		@ptrCast(f.air.extra[case.end + items.len ..][0..case.data.body_len]);
5029		extra_index = case.end + case.data.items_len + case_body.len;
	5022	const last_case_i = switch_br.cases_len - @intFromBool(switch_br.else_body_len == 0);
5030	5023
5031		for (items) \|item\| {
	5024	var it = switch_br.iterateCases();
	5025	while (it.next()) \|case\| {
	5026	for (case.items) \|item\| {
5032	5027	try f.object.indent_writer.insertNewline();
5033	5028	try writer.writeAll("case ");
5034	5029	const item_value = try f.air.value(item, pt);
...	...	@@ -5046,19 +5041,19 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index) !CValue {
5046	5041	}
5047	5042	try writer.writeByte(' ');
5048	5043
5049		if (case_i != last_case_i) {
5050		try genBodyResolveState(f, inst, liveness.deaths[case_i], case_body, false);
	5044	if (case.idx != last_case_i) {
	5045	try genBodyResolveState(f, inst, liveness.deaths[case.idx], case.body, false);
5051	5046	} else {
5052		for (liveness.deaths[case_i]) \|death\| {
	5047	for (liveness.deaths[case.idx]) \|death\| {
5053	5048	try die(f, inst, death.toRef());
5054	5049	}
5055		try genBody(f, case_body);
	5050	try genBody(f, case.body);
5056	5051	}
5057	5052
5058	5053	// The case body must be noreturn so we don't need to insert a break.
5059	5054	}
5060	5055
5061		const else_body: []const Air.Inst.Index = @ptrCast(f.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	5056	const else_body = it.elseBody();
5062	5057	try f.object.indent_writer.insertNewline();
5063	5058	if (else_body.len > 0) {
5064	5059	// Note that this must be the last case (i.e. the `last_case_i` case was not hit above)

src/codegen/llvm.zig+158-73

...	...	@@ -898,9 +898,9 @@ pub const Object = struct {
898	898	const i32_2 = try builder.intConst(.i32, 2);
899	899	const i32_3 = try builder.intConst(.i32, 3);
900	900	const debug_info_version = try builder.debugModuleFlag(
901		try builder.debugConstant(i32_2),
	901	try builder.metadataConstant(i32_2),
902	902	try builder.metadataString("Debug Info Version"),
903		try builder.debugConstant(i32_3),
	903	try builder.metadataConstant(i32_3),
904	904	);
905	905
906	906	switch (comp.config.debug_format) {
...	...	@@ -908,9 +908,9 @@ pub const Object = struct {
908	908	.dwarf => \|f\| {
909	909	const i32_4 = try builder.intConst(.i32, 4);
910	910	const dwarf_version = try builder.debugModuleFlag(
911		try builder.debugConstant(i32_2),
	911	try builder.metadataConstant(i32_2),
912	912	try builder.metadataString("Dwarf Version"),
913		try builder.debugConstant(i32_4),
	913	try builder.metadataConstant(i32_4),
914	914	);
915	915	switch (f) {
916	916	.@"32" => {
...	...	@@ -921,9 +921,9 @@ pub const Object = struct {
921	921	},
922	922	.@"64" => {
923	923	const dwarf64 = try builder.debugModuleFlag(
924		try builder.debugConstant(i32_2),
	924	try builder.metadataConstant(i32_2),
925	925	try builder.metadataString("DWARF64"),
926		try builder.debugConstant(.@"1"),
	926	try builder.metadataConstant(.@"1"),
927	927	);
928	928	try builder.debugNamed(try builder.metadataString("llvm.module.flags"), &.{
929	929	debug_info_version,
...	...	@@ -935,9 +935,9 @@ pub const Object = struct {
935	935	},
936	936	.code_view => {
937	937	const code_view = try builder.debugModuleFlag(
938		try builder.debugConstant(i32_2),
	938	try builder.metadataConstant(i32_2),
939	939	try builder.metadataString("CodeView"),
940		try builder.debugConstant(.@"1"),
	940	try builder.metadataConstant(.@"1"),
941	941	);
942	942	try builder.debugNamed(try builder.metadataString("llvm.module.flags"), &.{
943	943	debug_info_version,
...	...	@@ -1122,12 +1122,12 @@ pub const Object = struct {
1122	1122
1123	1123	self.builder.debugForwardReferenceSetType(
1124	1124	self.debug_enums_fwd_ref,
1125		try self.builder.debugTuple(self.debug_enums.items),
	1125	try self.builder.metadataTuple(self.debug_enums.items),
1126	1126	);
1127	1127
1128	1128	self.builder.debugForwardReferenceSetType(
1129	1129	self.debug_globals_fwd_ref,
1130		try self.builder.debugTuple(self.debug_globals.items),
	1130	try self.builder.metadataTuple(self.debug_globals.items),
1131	1131	);
1132	1132	}
1133	1133	}
...	...	@@ -1369,7 +1369,7 @@ pub const Object = struct {
1369	1369	_ = try attributes.removeFnAttr(.alignstack);
1370	1370	}
1371	1371
1372		if (func_analysis.is_cold) {
	1372	if (func_analysis.branch_hint == .cold) {
1373	1373	try attributes.addFnAttr(.cold, &o.builder);
1374	1374	} else {
1375	1375	_ = try attributes.removeFnAttr(.cold);
...	...	@@ -1978,7 +1978,7 @@ pub const Object = struct {
1978	1978	try o.lowerDebugType(int_ty),
1979	1979	ty.abiSize(zcu) * 8,
1980	1980	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
1981		try o.builder.debugTuple(enumerators),
	1981	try o.builder.metadataTuple(enumerators),
1982	1982	);
1983	1983
1984	1984	try o.debug_type_map.put(gpa, ty, debug_enum_type);
...	...	@@ -2087,7 +2087,7 @@ pub const Object = struct {
2087	2087	.none, // Underlying type
2088	2088	ty.abiSize(zcu) * 8,
2089	2089	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2090		try o.builder.debugTuple(&.{
	2090	try o.builder.metadataTuple(&.{
2091	2091	debug_ptr_type,
2092	2092	debug_len_type,
2093	2093	}),
...	...	@@ -2167,10 +2167,10 @@ pub const Object = struct {
2167	2167	try o.lowerDebugType(ty.childType(zcu)),
2168	2168	ty.abiSize(zcu) * 8,
2169	2169	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2170		try o.builder.debugTuple(&.{
	2170	try o.builder.metadataTuple(&.{
2171	2171	try o.builder.debugSubrange(
2172		try o.builder.debugConstant(try o.builder.intConst(.i64, 0)),
2173		try o.builder.debugConstant(try o.builder.intConst(.i64, ty.arrayLen(zcu))),
	2172	try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)),
	2173	try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.arrayLen(zcu))),
2174	2174	),
2175	2175	}),
2176	2176	);
...	...	@@ -2210,10 +2210,10 @@ pub const Object = struct {
2210	2210	debug_elem_type,
2211	2211	ty.abiSize(zcu) * 8,
2212	2212	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2213		try o.builder.debugTuple(&.{
	2213	try o.builder.metadataTuple(&.{
2214	2214	try o.builder.debugSubrange(
2215		try o.builder.debugConstant(try o.builder.intConst(.i64, 0)),
2216		try o.builder.debugConstant(try o.builder.intConst(.i64, ty.vectorLen(zcu))),
	2215	try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)),
	2216	try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.vectorLen(zcu))),
2217	2217	),
2218	2218	}),
2219	2219	);
...	...	@@ -2288,7 +2288,7 @@ pub const Object = struct {
2288	2288	.none, // Underlying type
2289	2289	ty.abiSize(zcu) * 8,
2290	2290	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2291		try o.builder.debugTuple(&.{
	2291	try o.builder.metadataTuple(&.{
2292	2292	debug_data_type,
2293	2293	debug_some_type,
2294	2294	}),
...	...	@@ -2367,7 +2367,7 @@ pub const Object = struct {
2367	2367	.none, // Underlying type
2368	2368	ty.abiSize(zcu) * 8,
2369	2369	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2370		try o.builder.debugTuple(&fields),
	2370	try o.builder.metadataTuple(&fields),
2371	2371	);
2372	2372
2373	2373	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_error_union_type);
...	...	@@ -2447,7 +2447,7 @@ pub const Object = struct {
2447	2447	.none, // Underlying type
2448	2448	ty.abiSize(zcu) * 8,
2449	2449	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2450		try o.builder.debugTuple(fields.items),
	2450	try o.builder.metadataTuple(fields.items),
2451	2451	);
2452	2452
2453	2453	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_struct_type);
...	...	@@ -2520,7 +2520,7 @@ pub const Object = struct {
2520	2520	.none, // Underlying type
2521	2521	ty.abiSize(zcu) * 8,
2522	2522	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2523		try o.builder.debugTuple(fields.items),
	2523	try o.builder.metadataTuple(fields.items),
2524	2524	);
2525	2525
2526	2526	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_struct_type);
...	...	@@ -2561,7 +2561,7 @@ pub const Object = struct {
2561	2561	.none, // Underlying type
2562	2562	ty.abiSize(zcu) * 8,
2563	2563	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2564		try o.builder.debugTuple(
	2564	try o.builder.metadataTuple(
2565	2565	&.{try o.lowerDebugType(Type.fromInterned(union_type.enum_tag_ty))},
2566	2566	),
2567	2567	);
...	...	@@ -2623,7 +2623,7 @@ pub const Object = struct {
2623	2623	.none, // Underlying type
2624	2624	ty.abiSize(zcu) * 8,
2625	2625	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2626		try o.builder.debugTuple(fields.items),
	2626	try o.builder.metadataTuple(fields.items),
2627	2627	);
2628	2628
2629	2629	o.builder.debugForwardReferenceSetType(debug_union_fwd_ref, debug_union_type);
...	...	@@ -2682,7 +2682,7 @@ pub const Object = struct {
2682	2682	.none, // Underlying type
2683	2683	ty.abiSize(zcu) * 8,
2684	2684	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
2685		try o.builder.debugTuple(&full_fields),
	2685	try o.builder.metadataTuple(&full_fields),
2686	2686	);
2687	2687
2688	2688	o.builder.debugForwardReferenceSetType(debug_fwd_ref, debug_tagged_union_type);
...	...	@@ -2735,7 +2735,7 @@ pub const Object = struct {
2735	2735	}
2736	2736
2737	2737	const debug_function_type = try o.builder.debugSubroutineType(
2738		try o.builder.debugTuple(debug_param_types.items),
	2738	try o.builder.metadataTuple(debug_param_types.items),
2739	2739	);
2740	2740
2741	2741	try o.debug_type_map.put(gpa, ty, debug_function_type);
...	...	@@ -4571,7 +4571,7 @@ pub const Object = struct {
4571	4571	const bad_value_block = try wip.block(1, "BadValue");
4572	4572	const tag_int_value = wip.arg(0);
4573	4573	var wip_switch =
4574		try wip.@"switch"(tag_int_value, bad_value_block, @intCast(enum_type.names.len));
	4574	try wip.@"switch"(tag_int_value, bad_value_block, @intCast(enum_type.names.len), .none);
4575	4575	defer wip_switch.finish(&wip);
4576	4576
4577	4577	for (0..enum_type.names.len) \|field_index\| {
...	...	@@ -4958,8 +4958,10 @@ pub const FuncGen = struct {
4958	4958	.ret_addr => try self.airRetAddr(inst),
4959	4959	.frame_addr => try self.airFrameAddress(inst),
4960	4960	.cond_br => try self.airCondBr(inst),
4961		.@"try" => try self.airTry(body[i..]),
4962		.try_ptr => try self.airTryPtr(inst),
	4961	.@"try" => try self.airTry(body[i..], false),
	4962	.try_cold => try self.airTry(body[i..], true),
	4963	.try_ptr => try self.airTryPtr(inst, false),
	4964	.try_ptr_cold => try self.airTryPtr(inst, true),
4963	4965	.intcast => try self.airIntCast(inst),
4964	4966	.trunc => try self.airTrunc(inst),
4965	4967	.fptrunc => try self.airFptrunc(inst),
...	...	@@ -5506,6 +5508,7 @@ pub const FuncGen = struct {
5506	5508	const panic_nav = ip.getNav(panic_func.owner_nav);
5507	5509	const fn_info = zcu.typeToFunc(Type.fromInterned(panic_nav.typeOf(ip))).?;
5508	5510	const panic_global = try o.resolveLlvmFunction(panic_func.owner_nav);
	5511	_ = try fg.wip.callIntrinsicAssumeCold();
5509	5512	_ = try fg.wip.call(
5510	5513	.normal,
5511	5514	toLlvmCallConv(fn_info.cc, target),
...	...	@@ -5794,7 +5797,7 @@ pub const FuncGen = struct {
5794	5797	const mixed_block = try self.wip.block(1, "Mixed");
5795	5798	const both_pl_block = try self.wip.block(1, "BothNonNull");
5796	5799	const end_block = try self.wip.block(3, "End");
5797		var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2);
	5800	var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none);
5798	5801	defer wip_switch.finish(&self.wip);
5799	5802	try wip_switch.addCase(
5800	5803	try o.builder.intConst(llvm_i2, 0b00),
...	...	@@ -5948,21 +5951,62 @@ pub const FuncGen = struct {
5948	5951	const then_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.then_body_len]);
5949	5952	const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end + then_body.len ..][0..extra.data.else_body_len]);
5950	5953
	5954	const Hint = enum {
	5955	none,
	5956	unpredictable,
	5957	then_likely,
	5958	else_likely,
	5959	then_cold,
	5960	else_cold,
	5961	};
	5962	const hint: Hint = switch (extra.data.branch_hints.true) {
	5963	.none => switch (extra.data.branch_hints.false) {
	5964	.none => .none,
	5965	.likely => .else_likely,
	5966	.unlikely => .then_likely,
	5967	.cold => .else_cold,
	5968	.unpredictable => .unpredictable,
	5969	},
	5970	.likely => switch (extra.data.branch_hints.false) {
	5971	.none => .then_likely,
	5972	.likely => .unpredictable,
	5973	.unlikely => .then_likely,
	5974	.cold => .else_cold,
	5975	.unpredictable => .unpredictable,
	5976	},
	5977	.unlikely => switch (extra.data.branch_hints.false) {
	5978	.none => .else_likely,
	5979	.likely => .else_likely,
	5980	.unlikely => .unpredictable,
	5981	.cold => .else_cold,
	5982	.unpredictable => .unpredictable,
	5983	},
	5984	.cold => .then_cold,
	5985	.unpredictable => .unpredictable,
	5986	};
	5987
5951	5988	const then_block = try self.wip.block(1, "Then");
5952	5989	const else_block = try self.wip.block(1, "Else");
5953		_ = try self.wip.brCond(cond, then_block, else_block);
	5990	_ = try self.wip.brCond(cond, then_block, else_block, switch (hint) {
	5991	.none, .then_cold, .else_cold => .none,
	5992	.unpredictable => .unpredictable,
	5993	.then_likely => .then_likely,
	5994	.else_likely => .else_likely,
	5995	});
5954	5996
5955	5997	self.wip.cursor = .{ .block = then_block };
	5998	if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
5956	5999	try self.genBodyDebugScope(null, then_body);
5957	6000
5958	6001	self.wip.cursor = .{ .block = else_block };
	6002	if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
5959	6003	try self.genBodyDebugScope(null, else_body);
5960	6004
5961	6005	// No need to reset the insert cursor since this instruction is noreturn.
5962	6006	return .none;
5963	6007	}
5964	6008
5965		fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
	6009	fn airTry(self: *FuncGen, body_tail: []const Air.Inst.Index, err_cold: bool) !Builder.Value {
5966	6010	const o = self.ng.object;
5967	6011	const pt = o.pt;
5968	6012	const zcu = pt.zcu;
...	...	@@ -5975,10 +6019,10 @@ pub const FuncGen = struct {
5975	6019	const payload_ty = self.typeOfIndex(inst);
5976	6020	const can_elide_load = if (isByRef(payload_ty, zcu)) self.canElideLoad(body_tail) else false;
5977	6021	const is_unused = self.liveness.isUnused(inst);
5978		return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused);
	6022	return lowerTry(self, err_union, body, err_union_ty, false, can_elide_load, is_unused, err_cold);
5979	6023	}
5980	6024
5981		fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
	6025	fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
5982	6026	const o = self.ng.object;
5983	6027	const zcu = o.pt.zcu;
5984	6028	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	...	@@ -5987,7 +6031,7 @@ pub const FuncGen = struct {
5987	6031	const body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len]);
5988	6032	const err_union_ty = self.typeOf(extra.data.ptr).childType(zcu);
5989	6033	const is_unused = self.liveness.isUnused(inst);
5990		return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused);
	6034	return lowerTry(self, err_union_ptr, body, err_union_ty, true, true, is_unused, err_cold);
5991	6035	}
5992	6036
5993	6037	fn lowerTry(
...	...	@@ -5998,6 +6042,7 @@ pub const FuncGen = struct {
5998	6042	operand_is_ptr: bool,
5999	6043	can_elide_load: bool,
6000	6044	is_unused: bool,
	6045	err_cold: bool,
6001	6046	) !Builder.Value {
6002	6047	const o = fg.ng.object;
6003	6048	const pt = o.pt;
...	...	@@ -6036,9 +6081,10 @@ pub const FuncGen = struct {
6036	6081
6037	6082	const return_block = try fg.wip.block(1, "TryRet");
6038	6083	const continue_block = try fg.wip.block(1, "TryCont");
6039		_ = try fg.wip.brCond(is_err, return_block, continue_block);
	6084	_ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely);
6040	6085
6041	6086	fg.wip.cursor = .{ .block = return_block };
	6087	if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold();
6042	6088	try fg.genBodyDebugScope(null, body);
6043	6089
6044	6090	fg.wip.cursor = .{ .block = continue_block };
...	...	@@ -6065,9 +6111,11 @@ pub const FuncGen = struct {
6065	6111
6066	6112	fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6067	6113	const o = self.ng.object;
6068		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6069		const cond = try self.resolveInst(pl_op.operand);
6070		const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
	6114
	6115	const switch_br = self.air.unwrapSwitch(inst);
	6116
	6117	const cond = try self.resolveInst(switch_br.operand);
	6118
6071	6119	const else_block = try self.wip.block(1, "Default");
6072	6120	const llvm_usize = try o.lowerType(Type.usize);
6073	6121	const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder))
...	...	@@ -6075,34 +6123,70 @@ pub const FuncGen = struct {
6075	6123	else
6076	6124	cond;
6077	6125
6078		var extra_index: usize = switch_br.end;
6079		var case_i: u32 = 0;
6080		var llvm_cases_len: u32 = 0;
6081		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
6082		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
6083		const items: []const Air.Inst.Ref =
6084		@ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
6085		const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
6086		extra_index = case.end + case.data.items_len + case_body.len;
	6126	const llvm_cases_len = llvm_cases_len: {
	6127	var len: u32 = 0;
	6128	var it = switch_br.iterateCases();
	6129	while (it.next()) \|case\| len += @intCast(case.items.len);
	6130	break :llvm_cases_len len;
	6131	};
	6132
	6133	const weights: Builder.Function.Instruction.BrCond.Weights = weights: {
	6134	// First pass. If any weights are `.unpredictable`, unpredictable.
	6135	// If all are `.none` or `.cold`, none.
	6136	var any_likely = false;
	6137	for (0..switch_br.cases_len) \|case_idx\| {
	6138	switch (switch_br.getHint(@intCast(case_idx))) {
	6139	.none, .cold => {},
	6140	.likely, .unlikely => any_likely = true,
	6141	.unpredictable => break :weights .unpredictable,
	6142	}
	6143	}
	6144	switch (switch_br.getElseHint()) {
	6145	.none, .cold => {},
	6146	.likely, .unlikely => any_likely = true,
	6147	.unpredictable => break :weights .unpredictable,
	6148	}
	6149	if (!any_likely) break :weights .none;
6087	6150
6088		llvm_cases_len += @intCast(items.len);
6089		}
	6151	var weights = try self.gpa.alloc(Builder.Metadata, llvm_cases_len + 1);
	6152	defer self.gpa.free(weights);
6090	6153
6091		var wip_switch = try self.wip.@"switch"(cond_int, else_block, llvm_cases_len);
6092		defer wip_switch.finish(&self.wip);
	6154	const else_weight: u32 = switch (switch_br.getElseHint()) {
	6155	.unpredictable => unreachable,
	6156	.none, .cold => 1000,
	6157	.likely => 2000,
	6158	.unlikely => 1,
	6159	};
	6160	weights[0] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight));
	6161
	6162	var weight_idx: usize = 1;
	6163	var it = switch_br.iterateCases();
	6164	while (it.next()) \|case\| {
	6165	const weight_val: u32 = switch (switch_br.getHint(case.idx)) {
	6166	.unpredictable => unreachable,
	6167	.none, .cold => 1000,
	6168	.likely => 2000,
	6169	.unlikely => 1,
	6170	};
	6171	const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val));
	6172	@memset(weights[weight_idx..][0..case.items.len], weight_meta);
	6173	weight_idx += case.items.len;
	6174	}
	6175
	6176	assert(weight_idx == weights.len);
6093	6177
6094		extra_index = switch_br.end;
6095		case_i = 0;
6096		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
6097		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
6098		const items: []const Air.Inst.Ref =
6099		@ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
6100		const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
6101		extra_index = case.end + case.data.items_len + case_body.len;
	6178	const branch_weights_str = try o.builder.metadataString("branch_weights");
	6179	const tuple = try o.builder.strTuple(branch_weights_str, weights);
	6180	break :weights @enumFromInt(@intFromEnum(tuple));
	6181	};
6102	6182
6103		const case_block = try self.wip.block(@intCast(items.len), "Case");
	6183	var wip_switch = try self.wip.@"switch"(cond_int, else_block, llvm_cases_len, weights);
	6184	defer wip_switch.finish(&self.wip);
6104	6185
6105		for (items) \|item\| {
	6186	var it = switch_br.iterateCases();
	6187	while (it.next()) \|case\| {
	6188	const case_block = try self.wip.block(@intCast(case.items.len), "Case");
	6189	for (case.items) \|item\| {
6106	6190	const llvm_item = (try self.resolveInst(item)).toConst().?;
6107	6191	const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder))
6108	6192	try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)
...	...	@@ -6110,13 +6194,14 @@ pub const FuncGen = struct {
6110	6194	llvm_item;
6111	6195	try wip_switch.addCase(llvm_int_item, case_block, &self.wip);
6112	6196	}
6113
6114	6197	self.wip.cursor = .{ .block = case_block };
6115		try self.genBodyDebugScope(null, case_body);
	6198	if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold();
	6199	try self.genBodyDebugScope(null, case.body);
6116	6200	}
6117	6201
	6202	const else_body = it.elseBody();
6118	6203	self.wip.cursor = .{ .block = else_block };
6119		const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	6204	if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold();
6120	6205	if (else_body.len != 0) {
6121	6206	try self.genBodyDebugScope(null, else_body);
6122	6207	} else {
...	...	@@ -7748,7 +7833,7 @@ pub const FuncGen = struct {
7748	7833
7749	7834	const fail_block = try fg.wip.block(1, "OverflowFail");
7750	7835	const ok_block = try fg.wip.block(1, "OverflowOk");
7751		_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block);
	7836	_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none);
7752	7837
7753	7838	fg.wip.cursor = .{ .block = fail_block };
7754	7839	try fg.buildSimplePanic(.integer_overflow);
...	...	@@ -9389,7 +9474,7 @@ pub const FuncGen = struct {
9389	9474	self.wip.cursor = .{ .block = loop_block };
9390	9475	const it_ptr = try self.wip.phi(.ptr, "");
9391	9476	const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");
9392		_ = try self.wip.brCond(end, body_block, end_block);
	9477	_ = try self.wip.brCond(end, body_block, end_block, .none);
9393	9478
9394	9479	self.wip.cursor = .{ .block = body_block };
9395	9480	const elem_abi_align = elem_ty.abiAlignment(zcu);
...	...	@@ -9427,7 +9512,7 @@ pub const FuncGen = struct {
9427	9512	const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero);
9428	9513	const memset_block = try self.wip.block(1, "MemsetTrapSkip");
9429	9514	const end_block = try self.wip.block(2, "MemsetTrapEnd");
9430		_ = try self.wip.brCond(cond, memset_block, end_block);
	9515	_ = try self.wip.brCond(cond, memset_block, end_block, .none);
9431	9516	self.wip.cursor = .{ .block = memset_block };
9432	9517	_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind);
9433	9518	_ = try self.wip.br(end_block);
...	...	@@ -9462,7 +9547,7 @@ pub const FuncGen = struct {
9462	9547	const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero);
9463	9548	const memcpy_block = try self.wip.block(1, "MemcpyTrapSkip");
9464	9549	const end_block = try self.wip.block(2, "MemcpyTrapEnd");
9465		_ = try self.wip.brCond(cond, memcpy_block, end_block);
	9550	_ = try self.wip.brCond(cond, memcpy_block, end_block, .none);
9466	9551	self.wip.cursor = .{ .block = memcpy_block };
9467	9552	_ = try self.wip.callMemCpy(
9468	9553	dest_ptr,
...	...	@@ -9632,7 +9717,7 @@ pub const FuncGen = struct {
9632	9717	const valid_block = try self.wip.block(@intCast(names.len), "Valid");
9633	9718	const invalid_block = try self.wip.block(1, "Invalid");
9634	9719	const end_block = try self.wip.block(2, "End");
9635		var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len));
	9720	var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none);
9636	9721	defer wip_switch.finish(&self.wip);
9637	9722
9638	9723	for (0..names.len) \|name_index\| {
...	...	@@ -9708,7 +9793,7 @@ pub const FuncGen = struct {
9708	9793	const named_block = try wip.block(@intCast(enum_type.names.len), "Named");
9709	9794	const unnamed_block = try wip.block(1, "Unnamed");
9710	9795	const tag_int_value = wip.arg(0);
9711		var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(enum_type.names.len));
	9796	var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(enum_type.names.len), .none);
9712	9797	defer wip_switch.finish(&wip);
9713	9798
9714	9799	for (0..enum_type.names.len) \|field_index\| {
...	...	@@ -9858,7 +9943,7 @@ pub const FuncGen = struct {
9858	9943	const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
9859	9944	const loop_then = try self.wip.block(1, "ReduceLoopThen");
9860	9945
9861		_ = try self.wip.brCond(cond, loop_then, loop_exit);
	9946	_ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
9862	9947
9863	9948	{
9864	9949	self.wip.cursor = .{ .block = loop_then };

src/codegen/llvm/Builder.zig+250-36

...	...	@@ -4817,12 +4817,22 @@ pub const Function = struct {
4817	4817	cond: Value,
4818	4818	then: Block.Index,
4819	4819	@"else": Block.Index,
	4820	weights: Weights,
	4821	pub const Weights = enum(u32) {
	4822	// We can do this as metadata indices 0 and 1 are reserved.
	4823	none = 0,
	4824	unpredictable = 1,
	4825	/// These values should be converted to `Metadata` to be used
	4826	/// in a `prof` annotation providing branch weights.
	4827	_,
	4828	};
4820	4829	};
4821	4830
4822	4831	pub const Switch = struct {
4823	4832	val: Value,
4824	4833	default: Block.Index,
4825	4834	cases_len: u32,
	4835	weights: BrCond.Weights,
4826	4836	//case_vals: [cases_len]Constant,
4827	4837	//case_blocks: [cases_len]Block.Index,
4828	4838	};
...	...	@@ -4969,7 +4979,8 @@ pub const Function = struct {
4969	4979	};
4970	4980	pub const Info = packed struct(u32) {
4971	4981	call_conv: CallConv,
4972		_: u22 = undefined,
	4982	has_op_bundle_cold: bool,
	4983	_: u21 = undefined,
4973	4984	};
4974	4985	};
4975	4986
...	...	@@ -5036,6 +5047,7 @@ pub const Function = struct {
5036	5047	FunctionAttributes,
5037	5048	Type,
5038	5049	Value,
	5050	Instruction.BrCond.Weights,
5039	5051	=> @enumFromInt(value),
5040	5052	MemoryAccessInfo,
5041	5053	Instruction.Alloca.Info,
...	...	@@ -5201,6 +5213,7 @@ pub const WipFunction = struct {
5201	5213	cond: Value,
5202	5214	then: Block.Index,
5203	5215	@"else": Block.Index,
	5216	weights: enum { none, unpredictable, then_likely, else_likely },
5204	5217	) Allocator.Error!Instruction.Index {
5205	5218	assert(cond.typeOfWip(self) == .i1);
5206	5219	try self.ensureUnusedExtraCapacity(1, Instruction.BrCond, 0);
...	...	@@ -5210,6 +5223,22 @@ pub const WipFunction = struct {
5210	5223	.cond = cond,
5211	5224	.then = then,
5212	5225	.@"else" = @"else",
	5226	.weights = switch (weights) {
	5227	.none => .none,
	5228	.unpredictable => .unpredictable,
	5229	.then_likely, .else_likely => w: {
	5230	const branch_weights_str = try self.builder.metadataString("branch_weights");
	5231	const unlikely_const = try self.builder.metadataConstant(try self.builder.intConst(.i32, 1));
	5232	const likely_const = try self.builder.metadataConstant(try self.builder.intConst(.i32, 2000));
	5233	const weight_vals: [2]Metadata = switch (weights) {
	5234	.none, .unpredictable => unreachable,
	5235	.then_likely => .{ likely_const, unlikely_const },
	5236	.else_likely => .{ unlikely_const, likely_const },
	5237	};
	5238	const tuple = try self.builder.strTuple(branch_weights_str, &weight_vals);
	5239	break :w @enumFromInt(@intFromEnum(tuple));
	5240	},
	5241	},
5213	5242	}),
5214	5243	});
5215	5244	then.ptr(self).branches += 1;
...	...	@@ -5248,6 +5277,7 @@ pub const WipFunction = struct {
5248	5277	val: Value,
5249	5278	default: Block.Index,
5250	5279	cases_len: u32,
	5280	weights: Instruction.BrCond.Weights,
5251	5281	) Allocator.Error!WipSwitch {
5252	5282	try self.ensureUnusedExtraCapacity(1, Instruction.Switch, cases_len * 2);
5253	5283	const instruction = try self.addInst(null, .{
...	...	@@ -5256,6 +5286,7 @@ pub const WipFunction = struct {
5256	5286	.val = val,
5257	5287	.default = default,
5258	5288	.cases_len = cases_len,
	5289	.weights = weights,
5259	5290	}),
5260	5291	});
5261	5292	_ = self.extra.addManyAsSliceAssumeCapacity(cases_len * 2);
...	...	@@ -5895,6 +5926,20 @@ pub const WipFunction = struct {
5895	5926	callee: Value,
5896	5927	args: []const Value,
5897	5928	name: []const u8,
	5929	) Allocator.Error!Value {
	5930	return self.callInner(kind, call_conv, function_attributes, ty, callee, args, name, false);
	5931	}
	5932
	5933	fn callInner(
	5934	self: *WipFunction,
	5935	kind: Instruction.Call.Kind,
	5936	call_conv: CallConv,
	5937	function_attributes: FunctionAttributes,
	5938	ty: Type,
	5939	callee: Value,
	5940	args: []const Value,
	5941	name: []const u8,
	5942	has_op_bundle_cold: bool,
5898	5943	) Allocator.Error!Value {
5899	5944	const ret_ty = ty.functionReturn(self.builder);
5900	5945	assert(ty.isFunction(self.builder));
...	...	@@ -5918,7 +5963,10 @@ pub const WipFunction = struct {
5918	5963	.tail_fast => .@"tail call fast",
5919	5964	},
5920	5965	.data = self.addExtraAssumeCapacity(Instruction.Call{
5921		.info = .{ .call_conv = call_conv },
	5966	.info = .{
	5967	.call_conv = call_conv,
	5968	.has_op_bundle_cold = has_op_bundle_cold,
	5969	},
5922	5970	.attributes = function_attributes,
5923	5971	.ty = ty,
5924	5972	.callee = callee,
...	...	@@ -5964,6 +6012,20 @@ pub const WipFunction = struct {
5964	6012	);
5965	6013	}
5966	6014
	6015	pub fn callIntrinsicAssumeCold(self: *WipFunction) Allocator.Error!Value {
	6016	const intrinsic = try self.builder.getIntrinsic(.assume, &.{});
	6017	return self.callInner(
	6018	.normal,
	6019	CallConv.default,
	6020	.none,
	6021	intrinsic.typeOf(self.builder),
	6022	intrinsic.toValue(self.builder),
	6023	&.{try self.builder.intValue(.i1, 1)},
	6024	"",
	6025	true,
	6026	);
	6027	}
	6028
5967	6029	pub fn callMemCpy(
5968	6030	self: *WipFunction,
5969	6031	dst: Value,
...	...	@@ -6040,7 +6102,7 @@ pub const WipFunction = struct {
6040	6102
6041	6103	break :blk metadata;
6042	6104	},
6043		.constant => \|constant\| try self.builder.debugConstant(constant),
	6105	.constant => \|constant\| try self.builder.metadataConstant(constant),
6044	6106	.metadata => \|metadata\| metadata,
6045	6107	};
6046	6108	}
...	...	@@ -6099,6 +6161,7 @@ pub const WipFunction = struct {
6099	6161	FunctionAttributes,
6100	6162	Type,
6101	6163	Value,
	6164	Instruction.BrCond.Weights,
6102	6165	=> @intFromEnum(value),
6103	6166	MemoryAccessInfo,
6104	6167	Instruction.Alloca.Info,
...	...	@@ -6380,6 +6443,7 @@ pub const WipFunction = struct {
6380	6443	.cond = instructions.map(extra.cond),
6381	6444	.then = extra.then,
6382	6445	.@"else" = extra.@"else",
	6446	.weights = extra.weights,
6383	6447	});
6384	6448	},
6385	6449	.call,
...	...	@@ -6522,6 +6586,7 @@ pub const WipFunction = struct {
6522	6586	.val = instructions.map(extra.data.val),
6523	6587	.default = extra.data.default,
6524	6588	.cases_len = extra.data.cases_len,
	6589	.weights = extra.data.weights,
6525	6590	});
6526	6591	wip_extra.appendSlice(case_vals);
6527	6592	wip_extra.appendSlice(case_blocks);
...	...	@@ -6744,6 +6809,7 @@ pub const WipFunction = struct {
6744	6809	FunctionAttributes,
6745	6810	Type,
6746	6811	Value,
	6812	Instruction.BrCond.Weights,
6747	6813	=> @intFromEnum(value),
6748	6814	MemoryAccessInfo,
6749	6815	Instruction.Alloca.Info,
...	...	@@ -6792,6 +6858,7 @@ pub const WipFunction = struct {
6792	6858	FunctionAttributes,
6793	6859	Type,
6794	6860	Value,
	6861	Instruction.BrCond.Weights,
6795	6862	=> @enumFromInt(value),
6796	6863	MemoryAccessInfo,
6797	6864	Instruction.Alloca.Info,
...	...	@@ -7735,6 +7802,7 @@ pub const Metadata = enum(u32) {
7735	7802	enumerator_signed_negative,
7736	7803	subrange,
7737	7804	tuple,
	7805	str_tuple,
7738	7806	module_flag,
7739	7807	expression,
7740	7808	local_var,
...	...	@@ -7780,6 +7848,7 @@ pub const Metadata = enum(u32) {
7780	7848	.enumerator_signed_negative,
7781	7849	.subrange,
7782	7850	.tuple,
	7851	.str_tuple,
7783	7852	.module_flag,
7784	7853	.local_var,
7785	7854	.parameter,
...	...	@@ -8044,6 +8113,13 @@ pub const Metadata = enum(u32) {
8044	8113	// elements: [elements_len]Metadata
8045	8114	};
8046	8115
	8116	pub const StrTuple = struct {
	8117	str: MetadataString,
	8118	elements_len: u32,
	8119
	8120	// elements: [elements_len]Metadata
	8121	};
	8122
8047	8123	pub const ModuleFlag = struct {
8048	8124	behavior: Metadata,
8049	8125	name: MetadataString,
...	...	@@ -8455,11 +8531,12 @@ pub fn init(options: Options) Allocator.Error!Builder {
8455	8531	assert(try self.intConst(.i32, 0) == .@"0");
8456	8532	assert(try self.intConst(.i32, 1) == .@"1");
8457	8533	assert(try self.noneConst(.token) == .none);
8458		if (!self.strip) assert(try self.debugNone() == .none);
	8534
	8535	assert(try self.metadataNone() == .none);
	8536	assert(try self.metadataTuple(&.{}) == .empty_tuple);
8459	8537
8460	8538	try self.metadata_string_indices.append(self.gpa, 0);
8461	8539	assert(try self.metadataString("") == .none);
8462		assert(try self.debugTuple(&.{}) == .empty_tuple);
8463	8540
8464	8541	return self;
8465	8542	}
...	...	@@ -9685,6 +9762,13 @@ pub fn printUnbuffered(
9685	9762	extra.then.toInst(&function).fmt(function_index, self),
9686	9763	extra.@"else".toInst(&function).fmt(function_index, self),
9687	9764	});
	9765	switch (extra.weights) {
	9766	.none => {},
	9767	.unpredictable => try writer.writeAll(", !unpredictable !{}"),
	9768	_ => try writer.print("{}", .{
	9769	try metadata_formatter.fmt(", !prof ", @as(Metadata, @enumFromInt(@intFromEnum(extra.weights)))),
	9770	}),
	9771	}
9688	9772	},
9689	9773	.call,
9690	9774	.@"call fast",
...	...	@@ -9729,6 +9813,9 @@ pub fn printUnbuffered(
9729	9813	});
9730	9814	}
9731	9815	try writer.writeByte(')');
	9816	if (extra.data.info.has_op_bundle_cold) {
	9817	try writer.writeAll(" [ \"cold\"() ]");
	9818	}
9732	9819	const call_function_attributes = extra.data.attributes.func(self);
9733	9820	if (call_function_attributes != .none) try writer.print(" #{d}", .{
9734	9821	(try attribute_groups.getOrPutValue(
...	...	@@ -9939,6 +10026,13 @@ pub fn printUnbuffered(
9939	10026	},
9940	10027	);
9941	10028	try writer.writeAll(" ]");
	10029	switch (extra.data.weights) {
	10030	.none => {},
	10031	.unpredictable => try writer.writeAll(", !unpredictable !{}"),
	10032	_ => try writer.print("{}", .{
	10033	try metadata_formatter.fmt(", !prof ", @as(Metadata, @enumFromInt(@intFromEnum(extra.data.weights)))),
	10034	}),
	10035	}
9942	10036	},
9943	10037	.va_arg => \|tag\| {
9944	10038	const extra = function.extraData(Function.Instruction.VaArg, instruction.data);
...	...	@@ -10287,6 +10381,17 @@ pub fn printUnbuffered(
10287	10381	});
10288	10382	try writer.writeAll("}\n");
10289	10383	},
	10384	.str_tuple => {
	10385	var extra = self.metadataExtraDataTrail(Metadata.StrTuple, metadata_item.data);
	10386	const elements = extra.trail.next(extra.data.elements_len, Metadata, self);
	10387	try writer.print("!{{{[str]%}", .{
	10388	.str = try metadata_formatter.fmt("", extra.data.str),
	10389	});
	10390	for (elements) \|element\| try writer.print("{[element]%}", .{
	10391	.element = try metadata_formatter.fmt("", element),
	10392	});
	10393	try writer.writeAll("}\n");
	10394	},
10290	10395	.module_flag => {
10291	10396	const extra = self.metadataExtraData(Metadata.ModuleFlag, metadata_item.data);
10292	10397	try writer.print("!{{{[behavior]%}{[name]%}{[constant]%}}}\n", .{
...	...	@@ -11799,9 +11904,9 @@ pub fn debugNamed(self: *Builder, name: MetadataString, operands: []const Metada
11799	11904	self.debugNamedAssumeCapacity(name, operands);
11800	11905	}
11801	11906
11802		fn debugNone(self: *Builder) Allocator.Error!Metadata {
	11907	fn metadataNone(self: *Builder) Allocator.Error!Metadata {
11803	11908	try self.ensureUnusedMetadataCapacity(1, NoExtra, 0);
11804		return self.debugNoneAssumeCapacity();
	11909	return self.metadataNoneAssumeCapacity();
11805	11910	}
11806	11911
11807	11912	pub fn debugFile(
...	...	@@ -12090,12 +12195,21 @@ pub fn debugExpression(
12090	12195	return self.debugExpressionAssumeCapacity(elements);
12091	12196	}
12092	12197
12093		pub fn debugTuple(
	12198	pub fn metadataTuple(
12094	12199	self: *Builder,
12095	12200	elements: []const Metadata,
12096	12201	) Allocator.Error!Metadata {
12097	12202	try self.ensureUnusedMetadataCapacity(1, Metadata.Tuple, elements.len);
12098		return self.debugTupleAssumeCapacity(elements);
	12203	return self.metadataTupleAssumeCapacity(elements);
	12204	}
	12205
	12206	pub fn strTuple(
	12207	self: *Builder,
	12208	str: MetadataString,
	12209	elements: []const Metadata,
	12210	) Allocator.Error!Metadata {
	12211	try self.ensureUnusedMetadataCapacity(1, Metadata.StrTuple, elements.len);
	12212	return self.strTupleAssumeCapacity(str, elements);
12099	12213	}
12100	12214
12101	12215	pub fn debugModuleFlag(
...	...	@@ -12166,9 +12280,9 @@ pub fn debugGlobalVarExpression(
12166	12280	return self.debugGlobalVarExpressionAssumeCapacity(variable, expression);
12167	12281	}
12168	12282
12169		pub fn debugConstant(self: *Builder, value: Constant) Allocator.Error!Metadata {
	12283	pub fn metadataConstant(self: *Builder, value: Constant) Allocator.Error!Metadata {
12170	12284	try self.ensureUnusedMetadataCapacity(1, NoExtra, 0);
12171		return self.debugConstantAssumeCapacity(value);
	12285	return self.metadataConstantAssumeCapacity(value);
12172	12286	}
12173	12287
12174	12288	pub fn debugForwardReferenceSetType(self: *Builder, fwd_ref: Metadata, ty: Metadata) void {
...	...	@@ -12263,8 +12377,7 @@ fn debugNamedAssumeCapacity(self: *Builder, name: MetadataString, operands: []co
12263	12377	};
12264	12378	}
12265	12379
12266		pub fn debugNoneAssumeCapacity(self: *Builder) Metadata {
12267		assert(!self.strip);
	12380	pub fn metadataNoneAssumeCapacity(self: *Builder) Metadata {
12268	12381	return self.metadataSimpleAssumeCapacity(.none, .{});
12269	12382	}
12270	12383
...	...	@@ -12740,11 +12853,10 @@ fn debugExpressionAssumeCapacity(
12740	12853	return @enumFromInt(gop.index);
12741	12854	}
12742	12855
12743		fn debugTupleAssumeCapacity(
	12856	fn metadataTupleAssumeCapacity(
12744	12857	self: *Builder,
12745	12858	elements: []const Metadata,
12746	12859	) Metadata {
12747		assert(!self.strip);
12748	12860	const Key = struct {
12749	12861	elements: []const Metadata,
12750	12862	};
...	...	@@ -12787,6 +12899,55 @@ fn debugTupleAssumeCapacity(
12787	12899	return @enumFromInt(gop.index);
12788	12900	}
12789	12901
	12902	fn strTupleAssumeCapacity(
	12903	self: *Builder,
	12904	str: MetadataString,
	12905	elements: []const Metadata,
	12906	) Metadata {
	12907	const Key = struct {
	12908	str: MetadataString,
	12909	elements: []const Metadata,
	12910	};
	12911	const Adapter = struct {
	12912	builder: *const Builder,
	12913	pub fn hash(_: @This(), key: Key) u32 {
	12914	var hasher = comptime std.hash.Wyhash.init(std.hash.uint32(@intFromEnum(Metadata.Tag.tuple)));
	12915	hasher.update(std.mem.sliceAsBytes(key.elements));
	12916	return @truncate(hasher.final());
	12917	}
	12918
	12919	pub fn eql(ctx: @This(), lhs_key: Key, _: void, rhs_index: usize) bool {
	12920	if (.str_tuple != ctx.builder.metadata_items.items(.tag)[rhs_index]) return false;
	12921	const rhs_data = ctx.builder.metadata_items.items(.data)[rhs_index];
	12922	var rhs_extra = ctx.builder.metadataExtraDataTrail(Metadata.StrTuple, rhs_data);
	12923	return rhs_extra.data.str == lhs_key.str and std.mem.eql(
	12924	Metadata,
	12925	lhs_key.elements,
	12926	rhs_extra.trail.next(rhs_extra.data.elements_len, Metadata, ctx.builder),
	12927	);
	12928	}
	12929	};
	12930
	12931	const gop = self.metadata_map.getOrPutAssumeCapacityAdapted(
	12932	Key{ .str = str, .elements = elements },
	12933	Adapter{ .builder = self },
	12934	);
	12935
	12936	if (!gop.found_existing) {
	12937	gop.key_ptr.* = {};
	12938	gop.value_ptr.* = {};
	12939	self.metadata_items.appendAssumeCapacity(.{
	12940	.tag = .str_tuple,
	12941	.data = self.addMetadataExtraAssumeCapacity(Metadata.StrTuple{
	12942	.str = str,
	12943	.elements_len = @intCast(elements.len),
	12944	}),
	12945	});
	12946	self.metadata_extra.appendSliceAssumeCapacity(@ptrCast(elements));
	12947	}
	12948	return @enumFromInt(gop.index);
	12949	}
	12950
12790	12951	fn debugModuleFlagAssumeCapacity(
12791	12952	self: *Builder,
12792	12953	behavior: Metadata,
...	...	@@ -12877,8 +13038,7 @@ fn debugGlobalVarExpressionAssumeCapacity(
12877	13038	});
12878	13039	}
12879	13040
12880		fn debugConstantAssumeCapacity(self: *Builder, constant: Constant) Metadata {
12881		assert(!self.strip);
	13041	fn metadataConstantAssumeCapacity(self: *Builder, constant: Constant) Metadata {
12882	13042	const Adapter = struct {
12883	13043	builder: *const Builder,
12884	13044	pub fn hash(_: @This(), key: Constant) u32 {
...	...	@@ -13757,15 +13917,18 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
13757	13917	}
13758	13918
13759	13919	// METADATA_KIND_BLOCK
13760		if (!self.strip) {
	13920	{
13761	13921	const MetadataKindBlock = ir.MetadataKindBlock;
13762	13922	var metadata_kind_block = try module_block.enterSubBlock(MetadataKindBlock, true);
13763	13923
13764	13924	inline for (@typeInfo(ir.FixedMetadataKind).Enum.fields) \|field\| {
13765		try metadata_kind_block.writeAbbrev(MetadataKindBlock.Kind{
13766		.id = field.value,
13767		.name = field.name,
13768		});
	13925	// don't include `dbg` in stripped functions
	13926	if (!(self.strip and std.mem.eql(u8, field.name, "dbg"))) {
	13927	try metadata_kind_block.writeAbbrev(MetadataKindBlock.Kind{
	13928	.id = field.value,
	13929	.name = field.name,
	13930	});
	13931	}
13769	13932	}
13770	13933
13771	13934	try metadata_kind_block.end();
...	...	@@ -13810,14 +13973,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
13810	13973	const metadata_adapter = MetadataAdapter.init(self, constant_adapter);
13811	13974
13812	13975	// METADATA_BLOCK
13813		if (!self.strip) {
	13976	{
13814	13977	const MetadataBlock = ir.MetadataBlock;
13815	13978	var metadata_block = try module_block.enterSubBlock(MetadataBlock, true);
13816	13979
13817	13980	const MetadataBlockWriter = @TypeOf(metadata_block);
13818	13981
13819	13982	// Emit all MetadataStrings
13820		{
	13983	if (self.metadata_string_map.count() > 1) {
13821	13984	const strings_offset, const strings_size = blk: {
13822	13985	var strings_offset: u32 = 0;
13823	13986	var strings_size: u32 = 0;
...	...	@@ -14087,6 +14250,22 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14087	14250	.elements = elements,
14088	14251	}, metadata_adapter);
14089	14252	},
	14253	.str_tuple => {
	14254	var extra = self.metadataExtraDataTrail(Metadata.StrTuple, data);
	14255
	14256	const elements = extra.trail.next(extra.data.elements_len, Metadata, self);
	14257
	14258	const all_elems = try self.gpa.alloc(Metadata, elements.len + 1);
	14259	defer self.gpa.free(all_elems);
	14260	all_elems[0] = @enumFromInt(metadata_adapter.getMetadataStringIndex(extra.data.str));
	14261	for (elements, all_elems[1..]) \|elem, *out_elem\| {
	14262	out_elem.* = @enumFromInt(metadata_adapter.getMetadataIndex(elem));
	14263	}
	14264
	14265	try metadata_block.writeAbbrev(MetadataBlock.Node{
	14266	.elements = all_elems,
	14267	});
	14268	},
14090	14269	.module_flag => {
14091	14270	const extra = self.metadataExtraData(Metadata.ModuleFlag, data);
14092	14271	try metadata_block.writeAbbrev(MetadataBlock.Node{
...	...	@@ -14188,6 +14367,18 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14188	14367	try metadata_block.end();
14189	14368	}
14190	14369
	14370	// OPERAND_BUNDLE_TAGS_BLOCK
	14371	{
	14372	const OperandBundleTags = ir.OperandBundleTags;
	14373	var operand_bundle_tags_block = try module_block.enterSubBlock(OperandBundleTags, true);
	14374
	14375	try operand_bundle_tags_block.writeAbbrev(OperandBundleTags.OperandBundleTag{
	14376	.tag = "cold",
	14377	});
	14378
	14379	try operand_bundle_tags_block.end();
	14380	}
	14381
14191	14382	// Block info
14192	14383	{
14193	14384	const BlockInfo = ir.BlockInfo;
...	...	@@ -14243,7 +14434,6 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14243	14434	.instruction => \|instruction\| instruction.valueIndex(adapter.func) + adapter.firstInstr(),
14244	14435	.constant => \|constant\| adapter.constant_adapter.getConstantIndex(constant),
14245	14436	.metadata => \|metadata\| {
14246		assert(!adapter.func.strip);
14247	14437	const real_metadata = metadata.unwrap(adapter.metadata_adapter.builder);
14248	14438	if (@intFromEnum(real_metadata) < Metadata.first_local_metadata)
14249	14439	return adapter.metadata_adapter.getMetadataIndex(real_metadata) - 1;
...	...	@@ -14335,6 +14525,10 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14335	14525	=> \|kind\| {
14336	14526	var extra = func.extraDataTrail(Function.Instruction.Call, data);
14337	14527
	14528	if (extra.data.info.has_op_bundle_cold) {
	14529	try function_block.writeAbbrev(FunctionBlock.ColdOperandBundle{});
	14530	}
	14531
14338	14532	const call_conv = extra.data.info.call_conv;
14339	14533	const args = extra.trail.next(extra.data.args_len, Value, &func);
14340	14534	try function_block.writeAbbrevAdapted(FunctionBlock.Call{
...	...	@@ -14358,6 +14552,10 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14358	14552	=> \|kind\| {
14359	14553	var extra = func.extraDataTrail(Function.Instruction.Call, data);
14360	14554
	14555	if (extra.data.info.has_op_bundle_cold) {
	14556	try function_block.writeAbbrev(FunctionBlock.ColdOperandBundle{});
	14557	}
	14558
14361	14559	const call_conv = extra.data.info.call_conv;
14362	14560	const args = extra.trail.next(extra.data.args_len, Value, &func);
14363	14561	try function_block.writeAbbrevAdapted(FunctionBlock.CallFast{
...	...	@@ -14837,14 +15035,14 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14837	15035	}
14838	15036
14839	15037	// METADATA_ATTACHMENT_BLOCK
14840		const any_nosanitize = true;
14841		if (!func.strip or any_nosanitize) {
	15038	{
14842	15039	const MetadataAttachmentBlock = ir.MetadataAttachmentBlock;
14843	15040	var metadata_attach_block = try function_block.enterSubBlock(MetadataAttachmentBlock, false);
14844	15041
14845		if (!func.strip) blk: {
	15042	dbg: {
	15043	if (func.strip) break :dbg;
14846	15044	const dbg = func.global.ptrConst(self).dbg;
14847		if (dbg == .none) break :blk;
	15045	if (dbg == .none) break :dbg;
14848	15046	try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentGlobalSingle{
14849	15047	.kind = .dbg,
14850	15048	.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(dbg) - 1),
...	...	@@ -14852,14 +15050,30 @@ pub fn toBitcode(self: *Builder, allocator: Allocator) bitcode_writer.Error![]co
14852	15050	}
14853	15051
14854	15052	var instr_index: u32 = 0;
14855		for (func.instructions.items(.tag)) \|instr_tag\| switch (instr_tag) {
14856		.arg, .block => {},
	15053	for (func.instructions.items(.tag), func.instructions.items(.data)) \|instr_tag, data\| switch (instr_tag) {
	15054	.arg, .block => {}, // not an actual instruction
14857	15055	else => {
14858		try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentInstructionSingle{
14859		.inst = instr_index,
14860		.kind = .nosanitize,
14861		.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(.empty_tuple) - 1),
14862		});
	15056	instr_index += 1;
	15057	},
	15058	.br_cond, .@"switch" => {
	15059	const weights = switch (instr_tag) {
	15060	.br_cond => func.extraData(Function.Instruction.BrCond, data).weights,
	15061	.@"switch" => func.extraData(Function.Instruction.Switch, data).weights,
	15062	else => unreachable,
	15063	};
	15064	switch (weights) {
	15065	.none => {},
	15066	.unpredictable => try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentInstructionSingle{
	15067	.inst = instr_index,
	15068	.kind = .unpredictable,
	15069	.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(.empty_tuple) - 1),
	15070	}),
	15071	_ => try metadata_attach_block.writeAbbrev(MetadataAttachmentBlock.AttachmentInstructionSingle{
	15072	.inst = instr_index,
	15073	.kind = .prof,
	15074	.metadata = @enumFromInt(metadata_adapter.getMetadataIndex(@enumFromInt(@intFromEnum(weights))) - 1),
	15075	}),
	15076	}
14863	15077	instr_index += 1;
14864	15078	},
14865	15079	};

src/codegen/llvm/ir.zig+25-3

...	...	@@ -25,7 +25,7 @@ const BlockAbbrev = AbbrevOp{ .vbr = 6 };
25	25	pub const FixedMetadataKind = enum(u8) {
26	26	dbg = 0,
27	27	//tbaa = 1,
28		//prof = 2,
	28	prof = 2,
29	29	//fpmath = 3,
30	30	//range = 4,
31	31	//@"tbaa.struct" = 5,
...	...	@@ -38,7 +38,7 @@ pub const FixedMetadataKind = enum(u8) {
38	38	//dereferenceable = 12,
39	39	//dereferenceable_or_null = 13,
40	40	//@"make.implicit" = 14,
41		//unpredictable = 15,
	41	unpredictable = 15,
42	42	//@"invariant.group" = 16,
43	43	//@"align" = 17,
44	44	//@"llvm.loop" = 18,
...	...	@@ -54,7 +54,7 @@ pub const FixedMetadataKind = enum(u8) {
54	54	//vcall_visibility = 28,
55	55	//noundef = 29,
56	56	//annotation = 30,
57		nosanitize = 31,
	57	//nosanitize = 31,
58	58	//func_sanitize = 32,
59	59	//exclude = 33,
60	60	//memprof = 34,
...	...	@@ -1220,6 +1220,20 @@ pub const MetadataBlock = struct {
1220	1220	};
1221	1221	};
1222	1222
	1223	pub const OperandBundleTags = struct {
	1224	pub const id = 21;
	1225
	1226	pub const abbrevs = [_]type{OperandBundleTag};
	1227
	1228	pub const OperandBundleTag = struct {
	1229	pub const ops = [_]AbbrevOp{
	1230	.{ .literal = 1 },
	1231	.array_char6,
	1232	};
	1233	tag: []const u8,
	1234	};
	1235	};
	1236
1223	1237	pub const FunctionMetadataBlock = struct {
1224	1238	pub const id = 15;
1225	1239
...	...	@@ -1279,6 +1293,7 @@ pub const FunctionBlock = struct {
1279	1293	Fence,
1280	1294	DebugLoc,
1281	1295	DebugLocAgain,
	1296	ColdOperandBundle,
1282	1297	};
1283	1298
1284	1299	pub const DeclareBlocks = struct {
...	...	@@ -1791,6 +1806,13 @@ pub const FunctionBlock = struct {
1791	1806	.{ .literal = 33 },
1792	1807	};
1793	1808	};
	1809
	1810	pub const ColdOperandBundle = struct {
	1811	pub const ops = [_]AbbrevOp{
	1812	.{ .literal = 55 },
	1813	.{ .literal = 0 },
	1814	};
	1815	};
1794	1816	};
1795	1817
1796	1818	pub const FunctionValueSymbolTable = struct {

src/codegen/spirv.zig+17-31

...	...	@@ -6173,11 +6173,10 @@ const NavGen = struct {
6173	6173	const pt = self.pt;
6174	6174	const zcu = pt.zcu;
6175	6175	const target = self.getTarget();
6176		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6177		const cond_ty = self.typeOf(pl_op.operand);
6178		const cond = try self.resolve(pl_op.operand);
	6176	const switch_br = self.air.unwrapSwitch(inst);
	6177	const cond_ty = self.typeOf(switch_br.operand);
	6178	const cond = try self.resolve(switch_br.operand);
6179	6179	var cond_indirect = try self.convertToIndirect(cond_ty, cond);
6180		const switch_br = self.air.extraData(Air.SwitchBr, pl_op.payload);
6181	6180
6182	6181	const cond_words: u32 = switch (cond_ty.zigTypeTag(zcu)) {
6183	6182	.Bool, .ErrorSet => 1,
...	...	@@ -6204,18 +6203,15 @@ const NavGen = struct {
6204	6203	else => return self.todo("implement switch for type {s}", .{@tagName(cond_ty.zigTypeTag(zcu))}),
6205	6204	};
6206	6205
6207		const num_cases = switch_br.data.cases_len;
	6206	const num_cases = switch_br.cases_len;
6208	6207
6209	6208	// Compute the total number of arms that we need.
6210	6209	// Zig switches are grouped by condition, so we need to loop through all of them
6211	6210	const num_conditions = blk: {
6212		var extra_index: usize = switch_br.end;
6213	6211	var num_conditions: u32 = 0;
6214		for (0..num_cases) \|_\| {
6215		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
6216		const case_body = self.air.extra[case.end + case.data.items_len ..][0..case.data.body_len];
6217		extra_index = case.end + case.data.items_len + case_body.len;
6218		num_conditions += case.data.items_len;
	6212	var it = switch_br.iterateCases();
	6213	while (it.next()) \|case\| {
	6214	num_conditions += @intCast(case.items.len);
6219	6215	}
6220	6216	break :blk num_conditions;
6221	6217	};
...	...	@@ -6244,17 +6240,12 @@ const NavGen = struct {
6244	6240
6245	6241	// Emit each of the cases
6246	6242	{
6247		var extra_index: usize = switch_br.end;
6248		for (0..num_cases) \|case_i\| {
	6243	var it = switch_br.iterateCases();
	6244	while (it.next()) \|case\| {
6249	6245	// SPIR-V needs a literal here, which' width depends on the case condition.
6250		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
6251		const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
6252		const case_body = self.air.extra[case.end + items.len ..][0..case.data.body_len];
6253		extra_index = case.end + case.data.items_len + case_body.len;
6254
6255		const label = case_labels.at(case_i);
	6246	const label = case_labels.at(case.idx);
6256	6247
6257		for (items) \|item\| {
	6248	for (case.items) \|item\| {
6258	6249	const value = (try self.air.value(item, pt)) orelse unreachable;
6259	6250	const int_val: u64 = switch (cond_ty.zigTypeTag(zcu)) {
6260	6251	.Bool, .Int => if (cond_ty.isSignedInt(zcu)) @bitCast(value.toSignedInt(zcu)) else value.toUnsignedInt(zcu),
...	...	@@ -6285,20 +6276,15 @@ const NavGen = struct {
6285	6276	}
6286	6277
6287	6278	// Now, finally, we can start emitting each of the cases.
6288		var extra_index: usize = switch_br.end;
6289		for (0..num_cases) \|case_i\| {
6290		const case = self.air.extraData(Air.SwitchBr.Case, extra_index);
6291		const items: []const Air.Inst.Ref = @ptrCast(self.air.extra[case.end..][0..case.data.items_len]);
6292		const case_body: []const Air.Inst.Index = @ptrCast(self.air.extra[case.end + items.len ..][0..case.data.body_len]);
6293		extra_index = case.end + case.data.items_len + case_body.len;
6294
6295		const label = case_labels.at(case_i);
	6279	var it = switch_br.iterateCases();
	6280	while (it.next()) \|case\| {
	6281	const label = case_labels.at(case.idx);
6296	6282
6297	6283	try self.beginSpvBlock(label);
6298	6284
6299	6285	switch (self.control_flow) {
6300	6286	.structured => {
6301		const next_block = try self.genStructuredBody(.selection, case_body);
	6287	const next_block = try self.genStructuredBody(.selection, case.body);
6302	6288	incoming_structured_blocks.appendAssumeCapacity(.{
6303	6289	.src_label = self.current_block_label,
6304	6290	.next_block = next_block,
...	...	@@ -6306,12 +6292,12 @@ const NavGen = struct {
6306	6292	try self.func.body.emitBranch(self.spv.gpa, merge_label.?);
6307	6293	},
6308	6294	.unstructured => {
6309		try self.genBody(case_body);
	6295	try self.genBody(case.body);
6310	6296	},
6311	6297	}
6312	6298	}
6313	6299
6314		const else_body: []const Air.Inst.Index = @ptrCast(self.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	6300	const else_body = it.elseBody();
6315	6301	try self.beginSpvBlock(default);
6316	6302	if (else_body.len != 0) {
6317	6303	switch (self.control_flow) {

src/print_air.zig+25-21

...	...	@@ -297,8 +297,8 @@ const Writer = struct {
297	297	.union_init => try w.writeUnionInit(s, inst),
298	298	.br => try w.writeBr(s, inst),
299	299	.cond_br => try w.writeCondBr(s, inst),
300		.@"try" => try w.writeTry(s, inst),
301		.try_ptr => try w.writeTryPtr(s, inst),
	300	.@"try", .try_cold => try w.writeTry(s, inst),
	301	.try_ptr, .try_ptr_cold => try w.writeTryPtr(s, inst),
302	302	.switch_br => try w.writeSwitchBr(s, inst),
303	303	.cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),
304	304	.fence => try w.writeFence(s, inst),
...	...	@@ -825,41 +825,40 @@ const Writer = struct {
825	825	}
826	826
827	827	fn writeSwitchBr(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
828		const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
829		const switch_br = w.air.extraData(Air.SwitchBr, pl_op.payload);
	828	const switch_br = w.air.unwrapSwitch(inst);
	829
830	830	const liveness = if (w.liveness) \|liveness\|
831		liveness.getSwitchBr(w.gpa, inst, switch_br.data.cases_len + 1) catch
	831	liveness.getSwitchBr(w.gpa, inst, switch_br.cases_len + 1) catch
832	832	@panic("out of memory")
833	833	else blk: {
834		const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.data.cases_len + 1) catch
	834	const slice = w.gpa.alloc([]const Air.Inst.Index, switch_br.cases_len + 1) catch
835	835	@panic("out of memory");
836	836	@memset(slice, &.{});
837	837	break :blk Liveness.SwitchBrTable{ .deaths = slice };
838	838	};
839	839	defer w.gpa.free(liveness.deaths);
840		var extra_index: usize = switch_br.end;
841		var case_i: u32 = 0;
842	840
843		try w.writeOperand(s, inst, 0, pl_op.operand);
	841	try w.writeOperand(s, inst, 0, switch_br.operand);
844	842	if (w.skip_body) return s.writeAll(", ...");
845	843	const old_indent = w.indent;
846	844	w.indent += 2;
847	845
848		while (case_i < switch_br.data.cases_len) : (case_i += 1) {
849		const case = w.air.extraData(Air.SwitchBr.Case, extra_index);
850		const items = @as([]const Air.Inst.Ref, @ptrCast(w.air.extra[case.end..][0..case.data.items_len]));
851		const case_body: []const Air.Inst.Index = @ptrCast(w.air.extra[case.end + items.len ..][0..case.data.body_len]);
852		extra_index = case.end + case.data.items_len + case_body.len;
853
	846	var it = switch_br.iterateCases();
	847	while (it.next()) \|case\| {
854	848	try s.writeAll(", [");
855		for (items, 0..) \|item, item_i\| {
	849	for (case.items, 0..) \|item, item_i\| {
856	850	if (item_i != 0) try s.writeAll(", ");
857	851	try w.writeInstRef(s, item, false);
858	852	}
859		try s.writeAll("] => {\n");
	853	try s.writeAll("] ");
	854	const hint = switch_br.getHint(case.idx);
	855	if (hint != .none) {
	856	try s.print(".{s} ", .{@tagName(hint)});
	857	}
	858	try s.writeAll("=> {\n");
860	859	w.indent += 2;
861	860
862		const deaths = liveness.deaths[case_i];
	861	const deaths = liveness.deaths[case.idx];
863	862	if (deaths.len != 0) {
864	863	try s.writeByteNTimes(' ', w.indent);
865	864	for (deaths, 0..) \|operand, i\| {
...	...	@@ -869,15 +868,20 @@ const Writer = struct {
869	868	try s.writeAll("\n");
870	869	}
871	870
872		try w.writeBody(s, case_body);
	871	try w.writeBody(s, case.body);
873	872	w.indent -= 2;
874	873	try s.writeByteNTimes(' ', w.indent);
875	874	try s.writeAll("}");
876	875	}
877	876
878		const else_body: []const Air.Inst.Index = @ptrCast(w.air.extra[extra_index..][0..switch_br.data.else_body_len]);
	877	const else_body = it.elseBody();
879	878	if (else_body.len != 0) {
880		try s.writeAll(", else => {\n");
	879	try s.writeAll(", else ");
	880	const hint = switch_br.getElseHint();
	881	if (hint != .none) {
	882	try s.print(".{s} ", .{@tagName(hint)});
	883	}
	884	try s.writeAll("=> {\n");
881	885	w.indent += 2;
882	886
883	887	const deaths = liveness.deaths[liveness.deaths.len - 1];

src/print_zir.zig+1-1

...	...	@@ -564,7 +564,6 @@ const Writer = struct {
564	564	.fence,
565	565	.set_float_mode,
566	566	.set_align_stack,
567		.set_cold,
568	567	.wasm_memory_size,
569	568	.int_from_error,
570	569	.error_from_int,
...	...	@@ -573,6 +572,7 @@ const Writer = struct {
573	572	.work_item_id,
574	573	.work_group_size,
575	574	.work_group_id,
	575	.branch_hint,
576	576	=> {
577	577	const inst_data = self.code.extraData(Zir.Inst.UnNode, extended.operand).data;
578	578	try self.writeInstRef(stream, inst_data.operand);

author	Matthew Lugg <mlugg@mlugg.co.uk> 2024-08-22 22:27:46+01:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2024-08-27 00:41:49+01:00
log	457c94d353b77b08786aa8794e1afc6a62d5c34a
tree	ef1463f066a1ed7526aadcb1b6b4e488845b34f1
parent	72e00805a61719174b1668d5ce3cbef8338e4690
signature	Commit is signed but in an unrecognized format.

compiler: implement `@branchHint`, replacing `@setCold`