| author |  Matthew Lugg <mlugg@mlugg.co.uk> 2023-11-07 07:13:25+00:00 |
| committer |  Andrew Kelley <andrew@ziglang.org> 2023-11-08 23:55:53-07:00 |
| log | d99bed1b10f85f2becca2a1c2587e1c2cb1968e6 |
| tree | e23b0ea094d2c4541c8dfa17d2a8681e892349ef |
| parent | a1d688b86aeba7ab72ae12679ff04a7730931af3 |
Sema: optimize runtime array_mul
There are two optimizations here, which work together to avoid a
pathological case.
The first optimization is that AstGen now records the result type of an
array multiplication expression where possible. This type is not used
according to the language specification, but instead as an optimization.
In the expression '.{x} ** 1000', if we know that the result must be an
array, then it is much more efficient to coerce the LHS to an array with
length 1 before doing the multiplication. Otherwise, we end up with a
1000-element tuple which we must coerce to an array by individually
extracting each field.
Secondly, the previous logic would repeatedly extract element/field
values from the LHS when initializing the result. This is unnecessary:
each element must only be extracted once, and the result reused.
These changes together give huge improvements to compiler performance on
a pathological case: AIR instructions go from 65551 to 15, and total AIR
bytes go from 1.86MiB to 264.57KiB. Codegen time spent on this function
(in a debug compiler build) goes from minutes to essentially zero.
Resolves: #175865 files changed, 80 insertions(+), 23 deletions(-)
src/AstGen.zig+5-1
| ... | ... | @@ -758,7 +758,11 @@ fn expr(gz: *GenZir, scope: *Scope, ri: ResultInfo, node: Ast.Node.Index) InnerE |
| 758 | 758 | .array_cat => return simpleBinOp(gz, scope, ri, node, .array_cat), |
| 759 | 759 | |
| 760 | 760 | .array_mult => { |
| 761 | const result = try gz.addPlNode(.array_mul, node, Zir.Inst.Bin{ | |
| 761 | // This syntax form does not currently use the result type in the language specification. | |
| 762 | // However, the result type can be used to emit more optimal code for large multiplications by | |
| 763 | // having Sema perform a coercion before the multiplication operation. | |
| 764 | const result = try gz.addPlNode(.array_mul, node, Zir.Inst.ArrayMul{ | |
| 765 | .res_ty = if (try ri.rl.resultType(gz, node)) |t| t else .none, | |
| 762 | 766 | .lhs = try expr(gz, scope, .{ .rl = .none }, node_datas[node].lhs), |
| 763 | 767 | .rhs = try comptimeExpr(gz, scope, .{ .rl = .{ .coerced_ty = .usize_type } }, node_datas[node].rhs), |
| 764 | 768 | }); |
src/Autodoc.zig-1
| ... | ... | @@ -1567,7 +1567,6 @@ fn walkInstruction( |
| 1567 | 1567 | .bit_and, |
| 1568 | 1568 | .xor, |
| 1569 | 1569 | .array_cat, |
| 1570 | .array_mul, | |
| 1571 | 1570 | => { |
| 1572 | 1571 | const pl_node = data[@intFromEnum(inst)].pl_node; |
| 1573 | 1572 | const extra = file.zir.extraData(Zir.Inst.Bin, pl_node.payload_index); |
src/Sema.zig+51-19
| ... | ... | @@ -13998,14 +13998,49 @@ fn zirArrayMul(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai |
| 13998 | 13998 | |
| 13999 | 13999 | const mod = sema.mod; |
| 14000 | 14000 | const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].pl_node; |
| 14001 | const extra = sema.code.extraData(Zir.Inst.Bin, inst_data.payload_index).data; | |
| 14002 | const lhs = try sema.resolveInst(extra.lhs); | |
| 14003 | const lhs_ty = sema.typeOf(lhs); | |
| 14001 | const extra = sema.code.extraData(Zir.Inst.ArrayMul, inst_data.payload_index).data; | |
| 14002 | const uncoerced_lhs = try sema.resolveInst(extra.lhs); | |
| 14003 | const uncoerced_lhs_ty = sema.typeOf(uncoerced_lhs); | |
| 14004 | 14004 | const src: LazySrcLoc = inst_data.src(); |
| 14005 | 14005 | const lhs_src: LazySrcLoc = .{ .node_offset_bin_lhs = inst_data.src_node }; |
| 14006 | 14006 | const operator_src: LazySrcLoc = .{ .node_offset_main_token = inst_data.src_node }; |
| 14007 | 14007 | const rhs_src: LazySrcLoc = .{ .node_offset_bin_rhs = inst_data.src_node }; |
| 14008 | 14008 | |
| 14009 | const lhs, const lhs_ty = coerced_lhs: { | |
| 14010 | // If we have a result type, we might be able to do this more efficiently | |
| 14011 | // by coercing the LHS first. Specifically, if we want an array or vector | |
| 14012 | // and have a tuple, coerce the tuple immediately. | |
| 14013 | no_coerce: { | |
| 14014 | if (extra.res_ty == .none) break :no_coerce; | |
| 14015 | const res_ty_inst = try sema.resolveInst(extra.res_ty); | |
| 14016 | const res_ty = try sema.analyzeAsType(block, src, res_ty_inst); | |
| 14017 | if (res_ty.isGenericPoison()) break :no_coerce; | |
| 14018 | if (!uncoerced_lhs_ty.isTuple(mod)) break :no_coerce; | |
| 14019 | const lhs_len = uncoerced_lhs_ty.structFieldCount(mod); | |
| 14020 | const lhs_dest_ty = switch (res_ty.zigTypeTag(mod)) { | |
| 14021 | else => break :no_coerce, | |
| 14022 | .Array => try mod.arrayType(.{ | |
| 14023 | .child = res_ty.childType(mod).toIntern(), | |
| 14024 | .len = lhs_len, | |
| 14025 | .sentinel = if (res_ty.sentinel(mod)) |s| s.toIntern() else .none, | |
| 14026 | }), | |
| 14027 | .Vector => try mod.vectorType(.{ | |
| 14028 | .child = res_ty.childType(mod).toIntern(), | |
| 14029 | .len = lhs_len, | |
| 14030 | }), | |
| 14031 | }; | |
| 14032 | // Attempt to coerce to this type, but don't emit an error if it fails. Instead, | |
| 14033 | // just exit out of this path and let the usual error happen later, so that error | |
| 14034 | // messages are consistent. | |
| 14035 | const coerced = sema.coerceExtra(block, lhs_dest_ty, uncoerced_lhs, lhs_src, .{ .report_err = false }) catch |err| switch (err) { | |
| 14036 | error.NotCoercible => break :no_coerce, | |
| 14037 | else => |e| return e, | |
| 14038 | }; | |
| 14039 | break :coerced_lhs .{ coerced, lhs_dest_ty }; | |
| 14040 | } | |
| 14041 | break :coerced_lhs .{ uncoerced_lhs, uncoerced_lhs_ty }; | |
| 14042 | }; | |
| 14043 | ||
| 14009 | 14044 | if (lhs_ty.isTuple(mod)) { |
| 14010 | 14045 | // In `**` rhs must be comptime-known, but lhs can be runtime-known |
| 14011 | 14046 | const factor = try sema.resolveInt(block, rhs_src, extra.rhs, Type.usize, .{ |
| ... | ... | @@ -14086,6 +14121,14 @@ fn zirArrayMul(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai |
| 14086 | 14121 | |
| 14087 | 14122 | try sema.requireRuntimeBlock(block, src, lhs_src); |
| 14088 | 14123 | |
| 14124 | // Grab all the LHS values ahead of time, rather than repeatedly emitting instructions | |
| 14125 | // to get the same elem values. | |
| 14126 | const lhs_vals = try sema.arena.alloc(Air.Inst.Ref, lhs_len); | |
| 14127 | for (lhs_vals, 0..) |*lhs_val, idx| { | |
| 14128 | const idx_ref = try mod.intRef(Type.usize, idx); | |
| 14129 | lhs_val.* = try sema.elemVal(block, lhs_src, lhs, idx_ref, src, false); | |
| 14130 | } | |
| 14131 | ||
| 14089 | 14132 | if (ptr_addrspace) |ptr_as| { |
| 14090 | 14133 | const alloc_ty = try sema.ptrType(.{ |
| 14091 | 14134 | .child = result_ty.toIntern(), |
| ... | ... | @@ -14099,14 +14142,11 @@ fn zirArrayMul(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai |
| 14099 | 14142 | |
| 14100 | 14143 | var elem_i: usize = 0; |
| 14101 | 14144 | while (elem_i < result_len) { |
| 14102 | var lhs_i: usize = 0; | |
| 14103 | while (lhs_i < lhs_len) : (lhs_i += 1) { | |
| 14145 | for (lhs_vals) |lhs_val| { | |
| 14104 | 14146 | const elem_index = try mod.intRef(Type.usize, elem_i); |
| 14105 | elem_i += 1; | |
| 14106 | const lhs_index = try mod.intRef(Type.usize, lhs_i); | |
| 14107 | 14147 | const elem_ptr = try block.addPtrElemPtr(alloc, elem_index, elem_ptr_ty); |
| 14108 | const init = try sema.elemVal(block, lhs_src, lhs, lhs_index, src, true); | |
| 14109 | try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store); | |
| 14148 | try sema.storePtr2(block, src, elem_ptr, src, lhs_val, lhs_src, .store); | |
| 14149 | elem_i += 1; | |
| 14110 | 14150 | } |
| 14111 | 14151 | } |
| 14112 | 14152 | if (lhs_info.sentinel) |sent_val| { |
| ... | ... | @@ -14120,17 +14160,9 @@ fn zirArrayMul(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai |
| 14120 | 14160 | } |
| 14121 | 14161 | |
| 14122 | 14162 | const element_refs = try sema.arena.alloc(Air.Inst.Ref, result_len); |
| 14123 | var elem_i: usize = 0; | |
| 14124 | while (elem_i < result_len) { | |
| 14125 | var lhs_i: usize = 0; | |
| 14126 | while (lhs_i < lhs_len) : (lhs_i += 1) { | |
| 14127 | const lhs_index = try mod.intRef(Type.usize, lhs_i); | |
| 14128 | const init = try sema.elemVal(block, lhs_src, lhs, lhs_index, src, true); | |
| 14129 | element_refs[elem_i] = init; | |
| 14130 | elem_i += 1; | |
| 14131 | } | |
| 14163 | for (0..try sema.usizeCast(block, rhs_src, factor)) |i| { | |
| 14164 | @memcpy(element_refs[i * lhs_len ..][0..lhs_len], lhs_vals); | |
| 14132 | 14165 | } |
| 14133 | ||
| 14134 | 14166 | return block.addAggregateInit(result_ty, element_refs); |
| 14135 | 14167 | } |
| 14136 | 14168 |
src/Zir.zig+10-1
| ... | ... | @@ -250,7 +250,7 @@ pub const Inst = struct { |
| 250 | 250 | /// Uses the `pl_node` union field. Payload is `Bin`. |
| 251 | 251 | array_cat, |
| 252 | 252 | /// Array multiplication `a ** b` |
| 253 | /// Uses the `pl_node` union field. Payload is `Bin`. | |
| 253 | /// Uses the `pl_node` union field. Payload is `ArrayMul`. | |
| 254 | 254 | array_mul, |
| 255 | 255 | /// `[N]T` syntax. No source location provided. |
| 256 | 256 | /// Uses the `pl_node` union field. Payload is `Bin`. lhs is length, rhs is element type. |
| ... | ... | @@ -3373,6 +3373,15 @@ pub const Inst = struct { |
| 3373 | 3373 | /// The expected field count. |
| 3374 | 3374 | expect_len: u32, |
| 3375 | 3375 | }; |
| 3376 | ||
| 3377 | pub const ArrayMul = struct { | |
| 3378 | /// The result type of the array multiplication operation, or `.none` if none was available. | |
| 3379 | res_ty: Ref, | |
| 3380 | /// The LHS of the array multiplication. | |
| 3381 | lhs: Ref, | |
| 3382 | /// The RHS of the array multiplication. | |
| 3383 | rhs: Ref, | |
| 3384 | }; | |
| 3376 | 3385 | }; |
| 3377 | 3386 | |
| 3378 | 3387 | pub const SpecialProng = enum { none, @"else", under }; |
src/print_zir.zig+14-1
| ... | ... | @@ -370,7 +370,6 @@ const Writer = struct { |
| 370 | 370 | .add_sat, |
| 371 | 371 | .add_unsafe, |
| 372 | 372 | .array_cat, |
| 373 | .array_mul, | |
| 374 | 373 | .mul, |
| 375 | 374 | .mulwrap, |
| 376 | 375 | .mul_sat, |
| ... | ... | @@ -431,6 +430,8 @@ const Writer = struct { |
| 431 | 430 | |
| 432 | 431 | .for_len => try self.writePlNodeMultiOp(stream, inst), |
| 433 | 432 | |
| 433 | .array_mul => try self.writeArrayMul(stream, inst), | |
| 434 | ||
| 434 | 435 | .elem_val_imm => try self.writeElemValImm(stream, inst), |
| 435 | 436 | |
| 436 | 437 | .@"export" => try self.writePlNodeExport(stream, inst), |
| ... | ... | @@ -977,6 +978,18 @@ const Writer = struct { |
| 977 | 978 | try self.writeSrc(stream, inst_data.src()); |
| 978 | 979 | } |
| 979 | 980 | |
| 981 | fn writeArrayMul(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void { | |
| 982 | const inst_data = self.code.instructions.items(.data)[@intFromEnum(inst)].pl_node; | |
| 983 | const extra = self.code.extraData(Zir.Inst.ArrayMul, inst_data.payload_index).data; | |
| 984 | try self.writeInstRef(stream, extra.res_ty); | |
| 985 | try stream.writeAll(", "); | |
| 986 | try self.writeInstRef(stream, extra.lhs); | |
| 987 | try stream.writeAll(", "); | |
| 988 | try self.writeInstRef(stream, extra.rhs); | |
| 989 | try stream.writeAll(") "); | |
| 990 | try self.writeSrc(stream, inst_data.src()); | |
| 991 | } | |
| 992 | ||
| 980 | 993 | fn writeElemValImm(self: *Writer, stream: anytype, inst: Zir.Inst.Index) !void { |
| 981 | 994 | const inst_data = self.code.instructions.items(.data)[@intFromEnum(inst)].elem_val_imm; |
| 982 | 995 | try self.writeInstRef(stream, inst_data.operand); |