authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-15 15:17:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-15 15:18:25-07:00
logd48e4245b68bf25c7f41804a5012ac157a5ee546
tree3e26f4794d2f8ace3484f5dd24a01aa78298f5e7
parent3901b6fb000b909e90347f6910ea6315ee03e220

stage2: implement `@prefetch`

14 files changed, 280 insertions(+), 45 deletions(-)

lib/std/builtin.zig+2-2
...@@ -643,12 +643,12 @@ pub const PrefetchOptions = struct {...@@ -643,12 +643,12 @@ pub const PrefetchOptions = struct {
643 /// The cache that the prefetch should be preformed on.643 /// The cache that the prefetch should be preformed on.
644 cache: Cache = .data,644 cache: Cache = .data,
645645
646 pub const Rw = enum {646 pub const Rw = enum(u1) {
647 read,647 read,
648 write,648 write,
649 };649 };
650650
651 pub const Cache = enum {651 pub const Cache = enum(u1) {
652 instruction,652 instruction,
653 data,653 data,
654 };654 };
src/Air.zig+12
...@@ -515,6 +515,11 @@ pub const Inst = struct {...@@ -515,6 +515,11 @@ pub const Inst = struct {
515 /// is a `Ref`. Length of the array is given by the vector type.515 /// is a `Ref`. Length of the array is given by the vector type.
516 vector_init,516 vector_init,
517517
518 /// Communicates an intent to load memory.
519 /// Result is always unused.
520 /// Uses the `prefetch` field.
521 prefetch,
522
518 pub fn fromCmpOp(op: std.math.CompareOperator) Tag {523 pub fn fromCmpOp(op: std.math.CompareOperator) Tag {
519 return switch (op) {524 return switch (op) {
520 .lt => .cmp_lt,525 .lt => .cmp_lt,
...@@ -586,6 +591,12 @@ pub const Inst = struct {...@@ -586,6 +591,12 @@ pub const Inst = struct {
586 ptr: Ref,591 ptr: Ref,
587 order: std.builtin.AtomicOrder,592 order: std.builtin.AtomicOrder,
588 },593 },
594 prefetch: struct {
595 ptr: Ref,
596 rw: std.builtin.PrefetchOptions.Rw,
597 locality: u2,
598 cache: std.builtin.PrefetchOptions.Cache,
599 },
589600
590 // Make sure we don't accidentally add a field to make this union601 // Make sure we don't accidentally add a field to make this union
591 // bigger than expected. Note that in Debug builds, Zig is allowed602 // bigger than expected. Note that in Debug builds, Zig is allowed
...@@ -823,6 +834,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {...@@ -823,6 +834,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
823 .memset,834 .memset,
824 .memcpy,835 .memcpy,
825 .set_union_tag,836 .set_union_tag,
837 .prefetch,
826 => return Type.initTag(.void),838 => return Type.initTag(.void),
827839
828 .ptrtoint,840 .ptrtoint,
src/Liveness.zig+5
...@@ -342,6 +342,11 @@ fn analyzeInst(...@@ -342,6 +342,11 @@ fn analyzeInst(
342 return trackOperands(a, new_set, inst, main_tomb, .{ operand, .none, .none });342 return trackOperands(a, new_set, inst, main_tomb, .{ operand, .none, .none });
343 },343 },
344344
345 .prefetch => {
346 const prefetch = inst_datas[inst].prefetch;
347 return trackOperands(a, new_set, inst, main_tomb, .{ prefetch.ptr, .none, .none });
348 },
349
345 .call => {350 .call => {
346 const inst_data = inst_datas[inst].pl_op;351 const inst_data = inst_datas[inst].pl_op;
347 const callee = inst_data.operand;352 const callee = inst_data.operand;
src/Sema.zig+82-38
...@@ -10274,49 +10274,53 @@ fn zirStructInitEmpty(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileE...@@ -10274,49 +10274,53 @@ fn zirStructInitEmpty(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileE
10274 const inst_data = sema.code.instructions.items(.data)[inst].un_node;10274 const inst_data = sema.code.instructions.items(.data)[inst].un_node;
10275 const src = inst_data.src();10275 const src = inst_data.src();
10276 const obj_ty = try sema.resolveType(block, src, inst_data.operand);10276 const obj_ty = try sema.resolveType(block, src, inst_data.operand);
10277 const gpa = sema.gpa;
1027810277
10279 switch (obj_ty.zigTypeTag()) {10278 switch (obj_ty.zigTypeTag()) {
10280 .Struct => {10279 .Struct => return structInitEmpty(sema, block, obj_ty, src, src),
10281 // This logic must be synchronized with that in `zirStructInit`.10280 .Array => return arrayInitEmpty(sema, obj_ty),
10282 const struct_ty = try sema.resolveTypeFields(block, src, obj_ty);
10283 const struct_obj = struct_ty.castTag(.@"struct").?.data;
10284
10285 // The init values to use for the struct instance.
10286 const field_inits = try gpa.alloc(Air.Inst.Ref, struct_obj.fields.count());
10287 defer gpa.free(field_inits);
10288
10289 var root_msg: ?*Module.ErrorMsg = null;
10290
10291 for (struct_obj.fields.values()) |field, i| {
10292 if (field.default_val.tag() == .unreachable_value) {
10293 const field_name = struct_obj.fields.keys()[i];
10294 const template = "missing struct field: {s}";
10295 const args = .{field_name};
10296 if (root_msg) |msg| {
10297 try sema.errNote(block, src, msg, template, args);
10298 } else {
10299 root_msg = try sema.errMsg(block, src, template, args);
10300 }
10301 } else {
10302 field_inits[i] = try sema.addConstant(field.ty, field.default_val);
10303 }
10304 }
10305 return sema.finishStructInit(block, src, field_inits, root_msg, struct_obj, struct_ty, false);
10306 },
10307 .Array => {
10308 if (obj_ty.sentinel()) |sentinel| {
10309 const val = try Value.Tag.empty_array_sentinel.create(sema.arena, sentinel);
10310 return sema.addConstant(obj_ty, val);
10311 } else {
10312 return sema.addConstant(obj_ty, Value.initTag(.empty_array));
10313 }
10314 },
10315 .Void => return sema.addConstant(obj_ty, Value.void),10281 .Void => return sema.addConstant(obj_ty, Value.void),
10316 else => unreachable,10282 else => unreachable,
10317 }10283 }
10318}10284}
1031910285
10286fn structInitEmpty(sema: *Sema, block: *Block, obj_ty: Type, dest_src: LazySrcLoc, init_src: LazySrcLoc) CompileError!Air.Inst.Ref {
10287 const gpa = sema.gpa;
10288 // This logic must be synchronized with that in `zirStructInit`.
10289 const struct_ty = try sema.resolveTypeFields(block, dest_src, obj_ty);
10290 const struct_obj = struct_ty.castTag(.@"struct").?.data;
10291
10292 // The init values to use for the struct instance.
10293 const field_inits = try gpa.alloc(Air.Inst.Ref, struct_obj.fields.count());
10294 defer gpa.free(field_inits);
10295
10296 var root_msg: ?*Module.ErrorMsg = null;
10297
10298 for (struct_obj.fields.values()) |field, i| {
10299 if (field.default_val.tag() == .unreachable_value) {
10300 const field_name = struct_obj.fields.keys()[i];
10301 const template = "missing struct field: {s}";
10302 const args = .{field_name};
10303 if (root_msg) |msg| {
10304 try sema.errNote(block, init_src, msg, template, args);
10305 } else {
10306 root_msg = try sema.errMsg(block, init_src, template, args);
10307 }
10308 } else {
10309 field_inits[i] = try sema.addConstant(field.ty, field.default_val);
10310 }
10311 }
10312 return sema.finishStructInit(block, dest_src, field_inits, root_msg, struct_obj, struct_ty, false);
10313}
10314
10315fn arrayInitEmpty(sema: *Sema, obj_ty: Type) CompileError!Air.Inst.Ref {
10316 if (obj_ty.sentinel()) |sentinel| {
10317 const val = try Value.Tag.empty_array_sentinel.create(sema.arena, sentinel);
10318 return sema.addConstant(obj_ty, val);
10319 } else {
10320 return sema.addConstant(obj_ty, Value.initTag(.empty_array));
10321 }
10322}
10323
10320fn zirUnionInitPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {10324fn zirUnionInitPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
10321 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;10325 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
10322 const src = inst_data.src();10326 const src = inst_data.src();
...@@ -12284,8 +12288,38 @@ fn zirPrefetch(...@@ -12284,8 +12288,38 @@ fn zirPrefetch(
12284 extended: Zir.Inst.Extended.InstData,12288 extended: Zir.Inst.Extended.InstData,
12285) CompileError!Air.Inst.Ref {12289) CompileError!Air.Inst.Ref {
12286 const extra = sema.code.extraData(Zir.Inst.BinNode, extended.operand).data;12290 const extra = sema.code.extraData(Zir.Inst.BinNode, extended.operand).data;
12287 const src: LazySrcLoc = .{ .node_offset = extra.node };12291 const ptr_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = extra.node };
12288 return sema.fail(block, src, "TODO: implement Sema.zirPrefetch", .{});12292 const opts_src: LazySrcLoc = .{ .node_offset_builtin_call_arg1 = extra.node };
12293 const options_ty = try sema.getBuiltinType(block, opts_src, "PrefetchOptions");
12294 const ptr = sema.resolveInst(extra.lhs);
12295 try sema.checkPtrType(block, ptr_src, sema.typeOf(ptr));
12296 const options = try sema.coerce(block, options_ty, sema.resolveInst(extra.rhs), opts_src);
12297
12298 const rw = try sema.fieldVal(block, opts_src, options, "rw", opts_src);
12299 const rw_val = try sema.resolveConstValue(block, opts_src, rw);
12300 const rw_tag = rw_val.toEnum(std.builtin.PrefetchOptions.Rw);
12301
12302 const locality = try sema.fieldVal(block, opts_src, options, "locality", opts_src);
12303 const locality_val = try sema.resolveConstValue(block, opts_src, locality);
12304 const locality_int = @intCast(u2, locality_val.toUnsignedInt());
12305
12306 const cache = try sema.fieldVal(block, opts_src, options, "cache", opts_src);
12307 const cache_val = try sema.resolveConstValue(block, opts_src, cache);
12308 const cache_tag = cache_val.toEnum(std.builtin.PrefetchOptions.Cache);
12309
12310 if (!block.is_comptime) {
12311 _ = try block.addInst(.{
12312 .tag = .prefetch,
12313 .data = .{ .prefetch = .{
12314 .ptr = ptr,
12315 .rw = rw_tag,
12316 .locality = locality_int,
12317 .cache = cache_tag,
12318 } },
12319 });
12320 }
12321
12322 return Air.Inst.Ref.void_value;
12289}12323}
1229012324
12291fn zirBuiltinExtern(12325fn zirBuiltinExtern(
...@@ -13739,6 +13773,11 @@ fn coerce(...@@ -13739,6 +13773,11 @@ fn coerce(
13739 },13773 },
13740 .Array => switch (inst_ty.zigTypeTag()) {13774 .Array => switch (inst_ty.zigTypeTag()) {
13741 .Vector => return sema.coerceVectorInMemory(block, dest_ty, dest_ty_src, inst, inst_src),13775 .Vector => return sema.coerceVectorInMemory(block, dest_ty, dest_ty_src, inst, inst_src),
13776 .Struct => {
13777 if (inst == .empty_struct) {
13778 return arrayInitEmpty(sema, dest_ty);
13779 }
13780 },
13742 else => {},13781 else => {},
13743 },13782 },
13744 .Vector => switch (inst_ty.zigTypeTag()) {13783 .Vector => switch (inst_ty.zigTypeTag()) {
...@@ -13746,6 +13785,11 @@ fn coerce(...@@ -13746,6 +13785,11 @@ fn coerce(
13746 .Vector => return sema.coerceVectors(block, dest_ty, dest_ty_src, inst, inst_src),13785 .Vector => return sema.coerceVectors(block, dest_ty, dest_ty_src, inst, inst_src),
13747 else => {},13786 else => {},
13748 },13787 },
13788 .Struct => {
13789 if (inst == .empty_struct) {
13790 return structInitEmpty(sema, block, dest_ty, dest_ty_src, inst_src);
13791 }
13792 },
13749 else => {},13793 else => {},
13750 }13794 }
1375113795
src/arch/aarch64/CodeGen.zig+6
...@@ -595,6 +595,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -595,6 +595,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
595 .error_name => try self.airErrorName(inst),595 .error_name => try self.airErrorName(inst),
596 .splat => try self.airSplat(inst),596 .splat => try self.airSplat(inst),
597 .vector_init => try self.airVectorInit(inst),597 .vector_init => try self.airVectorInit(inst),
598 .prefetch => try self.airPrefetch(inst),
598599
599 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),600 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
600 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),601 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -2597,6 +2598,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -2597,6 +2598,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
2597 return bt.finishAir(result);2598 return bt.finishAir(result);
2598}2599}
25992600
2601fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
2602 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
2603 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
2604}
2605
2600fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {2606fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
2601 // First section of indexes correspond to a set number of constant values.2607 // First section of indexes correspond to a set number of constant values.
2602 const ref_int = @enumToInt(inst);2608 const ref_int = @enumToInt(inst);
src/arch/arm/CodeGen.zig+6
...@@ -586,6 +586,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -586,6 +586,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
586 .error_name => try self.airErrorName(inst),586 .error_name => try self.airErrorName(inst),
587 .splat => try self.airSplat(inst),587 .splat => try self.airSplat(inst),
588 .vector_init => try self.airVectorInit(inst),588 .vector_init => try self.airVectorInit(inst),
589 .prefetch => try self.airPrefetch(inst),
589590
590 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),591 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
591 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),592 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -3696,6 +3697,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -3696,6 +3697,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
3696 return bt.finishAir(result);3697 return bt.finishAir(result);
3697}3698}
36983699
3700fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
3701 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
3702 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
3703}
3704
3699fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {3705fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
3700 // First section of indexes correspond to a set number of constant values.3706 // First section of indexes correspond to a set number of constant values.
3701 const ref_int = @enumToInt(inst);3707 const ref_int = @enumToInt(inst);
src/arch/riscv64/CodeGen.zig+6
...@@ -574,6 +574,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -574,6 +574,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
574 .error_name => try self.airErrorName(inst),574 .error_name => try self.airErrorName(inst),
575 .splat => try self.airSplat(inst),575 .splat => try self.airSplat(inst),
576 .vector_init => try self.airVectorInit(inst),576 .vector_init => try self.airVectorInit(inst),
577 .prefetch => try self.airPrefetch(inst),
577578
578 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),579 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
579 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),580 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -2096,6 +2097,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -2096,6 +2097,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
2096 return bt.finishAir(result);2097 return bt.finishAir(result);
2097}2098}
20982099
2100fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
2101 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
2102 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
2103}
2104
2099fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {2105fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
2100 // First section of indexes correspond to a set number of constant values.2106 // First section of indexes correspond to a set number of constant values.
2101 const ref_int = @enumToInt(inst);2107 const ref_int = @enumToInt(inst);
src/arch/wasm/CodeGen.zig+63-4
...@@ -1297,6 +1297,9 @@ fn copyLocal(self: *Self, value: WValue, ty: Type) InnerError!WValue {...@@ -1297,6 +1297,9 @@ fn copyLocal(self: *Self, value: WValue, ty: Type) InnerError!WValue {
1297fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {1297fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1298 const air_tags = self.air.instructions.items(.tag);1298 const air_tags = self.air.instructions.items(.tag);
1299 return switch (air_tags[inst]) {1299 return switch (air_tags[inst]) {
1300 .constant => unreachable,
1301 .const_ty => unreachable,
1302
1300 .add => self.airBinOp(inst, .add),1303 .add => self.airBinOp(inst, .add),
1301 .addwrap => self.airWrapBinOp(inst, .add),1304 .addwrap => self.airWrapBinOp(inst, .add),
1302 .sub => self.airBinOp(inst, .sub),1305 .sub => self.airBinOp(inst, .sub),
...@@ -1330,7 +1333,6 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1330,7 +1333,6 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1330 .bool_to_int => self.airBoolToInt(inst),1333 .bool_to_int => self.airBoolToInt(inst),
1331 .call => self.airCall(inst),1334 .call => self.airCall(inst),
1332 .cond_br => self.airCondBr(inst),1335 .cond_br => self.airCondBr(inst),
1333 .constant => unreachable,
1334 .dbg_stmt => WValue.none,1336 .dbg_stmt => WValue.none,
1335 .intcast => self.airIntcast(inst),1337 .intcast => self.airIntcast(inst),
1336 .float_to_int => self.airFloatToInt(inst),1338 .float_to_int => self.airFloatToInt(inst),
...@@ -1358,6 +1360,9 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1358,6 +1360,9 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1358 .ret => self.airRet(inst),1360 .ret => self.airRet(inst),
1359 .ret_ptr => self.airRetPtr(inst),1361 .ret_ptr => self.airRetPtr(inst),
1360 .ret_load => self.airRetLoad(inst),1362 .ret_load => self.airRetLoad(inst),
1363 .splat => self.airSplat(inst),
1364 .vector_init => self.airVectorInit(inst),
1365 .prefetch => self.airPrefetch(inst),
13611366
1362 .slice => self.airSlice(inst),1367 .slice => self.airSlice(inst),
1363 .slice_len => self.airSliceLen(inst),1368 .slice_len => self.airSliceLen(inst),
...@@ -1382,7 +1387,55 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {...@@ -1382,7 +1387,55 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1382 .unwrap_errunion_err => self.airUnwrapErrUnionError(inst),1387 .unwrap_errunion_err => self.airUnwrapErrUnionError(inst),
1383 .wrap_errunion_payload => self.airWrapErrUnionPayload(inst),1388 .wrap_errunion_payload => self.airWrapErrUnionPayload(inst),
1384 .wrap_errunion_err => self.airWrapErrUnionErr(inst),1389 .wrap_errunion_err => self.airWrapErrUnionErr(inst),
1385 else => |tag| self.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),1390
1391 .add_sat,
1392 .sub_sat,
1393 .mul_sat,
1394 .div_float,
1395 .div_floor,
1396 .div_exact,
1397 .rem,
1398 .mod,
1399 .max,
1400 .min,
1401 .assembly,
1402 .shl_exact,
1403 .shl_sat,
1404 .ret_addr,
1405 .clz,
1406 .ctz,
1407 .popcount,
1408 .is_err_ptr,
1409 .is_non_err_ptr,
1410 .fptrunc,
1411 .fpext,
1412 .unwrap_errunion_payload_ptr,
1413 .unwrap_errunion_err_ptr,
1414 .set_union_tag,
1415 .get_union_tag,
1416 .ptr_slice_len_ptr,
1417 .ptr_slice_ptr_ptr,
1418 .int_to_float,
1419 .memcpy,
1420 .cmpxchg_weak,
1421 .cmpxchg_strong,
1422 .fence,
1423 .atomic_load,
1424 .atomic_store_unordered,
1425 .atomic_store_monotonic,
1426 .atomic_store_release,
1427 .atomic_store_seq_cst,
1428 .atomic_rmw,
1429 .tag_name,
1430 .error_name,
1431
1432 // For these 4, probably best to wait until https://github.com/ziglang/zig/issues/10248
1433 // is implemented in the frontend before implementing them here in the wasm backend.
1434 .add_with_overflow,
1435 .sub_with_overflow,
1436 .mul_with_overflow,
1437 .shl_with_overflow,
1438 => |tag| self.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),
1386 };1439 };
1387}1440}
13881441
...@@ -3211,7 +3264,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {...@@ -3211,7 +3264,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3211 return result;3264 return result;
3212}3265}
32133266
3214fn airSplat(self: *Self, inst: Air.Inst.Index) !void {3267fn airSplat(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3215 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };3268 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };
32163269
3217 const ty_op = self.air.instructions.items(.data)[inst].ty_op;3270 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
...@@ -3222,7 +3275,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {...@@ -3222,7 +3275,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void {
3222 return self.fail("TODO: Implement wasm airSplat", .{});3275 return self.fail("TODO: Implement wasm airSplat", .{});
3223}3276}
32243277
3225fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {3278fn airVectorInit(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3226 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };3279 if (self.liveness.isUnused(inst)) return WValue{ .none = {} };
32273280
3228 const vector_ty = self.air.typeOfIndex(inst);3281 const vector_ty = self.air.typeOfIndex(inst);
...@@ -3234,6 +3287,12 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -3234,6 +3287,12 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
3234 return self.fail("TODO: Wasm backend: implement airVectorInit", .{});3287 return self.fail("TODO: Wasm backend: implement airVectorInit", .{});
3235}3288}
32363289
3290fn airPrefetch(self: *Self, inst: Air.Inst.Index) InnerError!WValue {
3291 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
3292 _ = prefetch;
3293 return WValue{ .none = {} };
3294}
3295
3237fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {3296fn cmpOptionals(self: *Self, lhs: WValue, rhs: WValue, operand_ty: Type, op: std.math.CompareOperator) InnerError!WValue {
3238 assert(operand_ty.hasCodeGenBits());3297 assert(operand_ty.hasCodeGenBits());
3239 assert(op == .eq or op == .neq);3298 assert(op == .eq or op == .neq);
src/arch/x86_64/CodeGen.zig+6
...@@ -638,6 +638,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -638,6 +638,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
638 .error_name => try self.airErrorName(inst),638 .error_name => try self.airErrorName(inst),
639 .splat => try self.airSplat(inst),639 .splat => try self.airSplat(inst),
640 .vector_init => try self.airVectorInit(inst),640 .vector_init => try self.airVectorInit(inst),
641 .prefetch => try self.airPrefetch(inst),
641642
642 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),643 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
643 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),644 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -3693,6 +3694,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -3693,6 +3694,11 @@ fn airVectorInit(self: *Self, inst: Air.Inst.Index) !void {
3693 return bt.finishAir(result);3694 return bt.finishAir(result);
3694}3695}
36953696
3697fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
3698 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
3699 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
3700}
3701
3696fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {3702fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
3697 // First section of indexes correspond to a set number of constant values.3703 // First section of indexes correspond to a set number of constant values.
3698 const ref_int = @enumToInt(inst);3704 const ref_int = @enumToInt(inst);
src/codegen/c.zig+13
...@@ -1278,6 +1278,7 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO...@@ -1278,6 +1278,7 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO
1278 .error_name => try airErrorName(f, inst),1278 .error_name => try airErrorName(f, inst),
1279 .splat => try airSplat(f, inst),1279 .splat => try airSplat(f, inst),
1280 .vector_init => try airVectorInit(f, inst),1280 .vector_init => try airVectorInit(f, inst),
1281 .prefetch => try airPrefetch(f, inst),
12811282
1282 .int_to_float,1283 .int_to_float,
1283 .float_to_int,1284 .float_to_int,
...@@ -3089,6 +3090,18 @@ fn airVectorInit(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3089,6 +3090,18 @@ fn airVectorInit(f: *Function, inst: Air.Inst.Index) !CValue {
3089 return f.fail("TODO: C backend: implement airVectorInit", .{});3090 return f.fail("TODO: C backend: implement airVectorInit", .{});
3090}3091}
30913092
3093fn airPrefetch(f: *Function, inst: Air.Inst.Index) !CValue {
3094 const prefetch = f.air.instructions.items(.data)[inst].prefetch;
3095 const ptr = try f.resolveInst(prefetch.ptr);
3096 const writer = f.object.writer();
3097 try writer.writeAll("zig_prefetch(");
3098 try f.writeCValue(writer, ptr);
3099 try writer.print(", {d}, {d});\n", .{
3100 @enumToInt(prefetch.rw), prefetch.locality,
3101 });
3102 return CValue.none;
3103}
3104
3092fn toMemoryOrder(order: std.builtin.AtomicOrder) [:0]const u8 {3105fn toMemoryOrder(order: std.builtin.AtomicOrder) [:0]const u8 {
3093 return switch (order) {3106 return switch (order) {
3094 .Unordered => "memory_order_relaxed",3107 .Unordered => "memory_order_relaxed",
src/codegen/llvm.zig+62
...@@ -2073,6 +2073,7 @@ pub const FuncGen = struct {...@@ -2073,6 +2073,7 @@ pub const FuncGen = struct {
2073 .error_name => try self.airErrorName(inst),2073 .error_name => try self.airErrorName(inst),
2074 .splat => try self.airSplat(inst),2074 .splat => try self.airSplat(inst),
2075 .vector_init => try self.airVectorInit(inst),2075 .vector_init => try self.airVectorInit(inst),
2076 .prefetch => try self.airPrefetch(inst),
20762077
2077 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),2078 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
2078 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),2079 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
...@@ -4384,6 +4385,67 @@ pub const FuncGen = struct {...@@ -4384,6 +4385,67 @@ pub const FuncGen = struct {
4384 return vector;4385 return vector;
4385 }4386 }
43864387
4388 fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
4389 const prefetch = self.air.instructions.items(.data)[inst].prefetch;
4390
4391 comptime assert(@enumToInt(std.builtin.PrefetchOptions.Rw.read) == 0);
4392 comptime assert(@enumToInt(std.builtin.PrefetchOptions.Rw.write) == 1);
4393
4394 // TODO these two asserts should be able to be comptime because the type is a u2
4395 assert(prefetch.locality >= 0);
4396 assert(prefetch.locality <= 3);
4397
4398 comptime assert(@enumToInt(std.builtin.PrefetchOptions.Cache.instruction) == 0);
4399 comptime assert(@enumToInt(std.builtin.PrefetchOptions.Cache.data) == 1);
4400
4401 // LLVM fails during codegen of instruction cache prefetchs for these architectures.
4402 // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported
4403 // by the target.
4404 // To work around this, don't emit llvm.prefetch in this case.
4405 // See https://bugs.llvm.org/show_bug.cgi?id=21037
4406 const target = self.dg.module.getTarget();
4407 switch (prefetch.cache) {
4408 .instruction => switch (target.cpu.arch) {
4409 .x86_64, .i386 => return null,
4410 .arm, .armeb, .thumb, .thumbeb => {
4411 switch (prefetch.rw) {
4412 .write => return null,
4413 else => {},
4414 }
4415 },
4416 else => {},
4417 },
4418 .data => {},
4419 }
4420
4421 const llvm_u8 = self.context.intType(8);
4422 const llvm_ptr_u8 = llvm_u8.pointerType(0);
4423 const llvm_u32 = self.context.intType(32);
4424
4425 const llvm_fn_name = "llvm.prefetch.p0i8";
4426 const fn_val = self.dg.object.llvm_module.getNamedFunction(llvm_fn_name) orelse blk: {
4427 // declare void @llvm.prefetch(i8*, i32, i32, i32)
4428 const llvm_void = self.context.voidType();
4429 const param_types = [_]*const llvm.Type{
4430 llvm_ptr_u8, llvm_u32, llvm_u32, llvm_u32,
4431 };
4432 const fn_type = llvm.functionType(llvm_void, &param_types, param_types.len, .False);
4433 break :blk self.dg.object.llvm_module.addFunction(llvm_fn_name, fn_type);
4434 };
4435
4436 const ptr = try self.resolveInst(prefetch.ptr);
4437 const ptr_u8 = self.builder.buildBitCast(ptr, llvm_ptr_u8, "");
4438
4439 const params = [_]*const llvm.Value{
4440 ptr_u8,
4441 llvm_u32.constInt(@enumToInt(prefetch.rw), .False),
4442 llvm_u32.constInt(prefetch.locality, .False),
4443 llvm_u32.constInt(@enumToInt(prefetch.cache), .False),
4444 };
4445 _ = self.builder.buildCall(fn_val, &params, params.len, .C, .Auto, "");
4446 return null;
4447 }
4448
4387 fn getErrorNameTable(self: *FuncGen) !*const llvm.Value {4449 fn getErrorNameTable(self: *FuncGen) !*const llvm.Value {
4388 if (self.dg.object.error_name_table) |table| {4450 if (self.dg.object.error_name_table) |table| {
4389 return table;4451 return table;
src/link/C/zig.h+6
...@@ -74,6 +74,12 @@...@@ -74,6 +74,12 @@
74#define zig_frame_address() 074#define zig_frame_address() 0
75#endif75#endif
7676
77#if defined(__GNUC__)
78#define zig_prefetch(addr, rw, locality) __builtin_prefetch(addr, rw, locality)
79#else
80#define zig_prefetch(addr, rw, locality)
81#endif
82
77#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)83#if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
78#include <stdatomic.h>84#include <stdatomic.h>
79#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)85#define zig_cmpxchg_strong(obj, expected, desired, succ, fail) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail)
src/print_air.zig+10
...@@ -226,6 +226,7 @@ const Writer = struct {...@@ -226,6 +226,7 @@ const Writer = struct {
226 .cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),226 .cmpxchg_weak, .cmpxchg_strong => try w.writeCmpxchg(s, inst),
227 .fence => try w.writeFence(s, inst),227 .fence => try w.writeFence(s, inst),
228 .atomic_load => try w.writeAtomicLoad(s, inst),228 .atomic_load => try w.writeAtomicLoad(s, inst),
229 .prefetch => try w.writePrefetch(s, inst),
229 .atomic_store_unordered => try w.writeAtomicStore(s, inst, .Unordered),230 .atomic_store_unordered => try w.writeAtomicStore(s, inst, .Unordered),
230 .atomic_store_monotonic => try w.writeAtomicStore(s, inst, .Monotonic),231 .atomic_store_monotonic => try w.writeAtomicStore(s, inst, .Monotonic),
231 .atomic_store_release => try w.writeAtomicStore(s, inst, .Release),232 .atomic_store_release => try w.writeAtomicStore(s, inst, .Release),
...@@ -350,6 +351,15 @@ const Writer = struct {...@@ -350,6 +351,15 @@ const Writer = struct {
350 try s.print(", {s}", .{@tagName(atomic_load.order)});351 try s.print(", {s}", .{@tagName(atomic_load.order)});
351 }352 }
352353
354 fn writePrefetch(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
355 const prefetch = w.air.instructions.items(.data)[inst].prefetch;
356
357 try w.writeOperand(s, inst, 0, prefetch.ptr);
358 try s.print(", {s}, {d}, {s}", .{
359 @tagName(prefetch.rw), prefetch.locality, @tagName(prefetch.cache),
360 });
361 }
362
353 fn writeAtomicStore(363 fn writeAtomicStore(
354 w: *Writer,364 w: *Writer,
355 s: anytype,365 s: anytype,
test/behavior.zig+1-1
...@@ -10,6 +10,7 @@ test {...@@ -10,6 +10,7 @@ test {
10 _ = @import("behavior/fn_in_struct_in_comptime.zig");10 _ = @import("behavior/fn_in_struct_in_comptime.zig");
11 _ = @import("behavior/hasdecl.zig");11 _ = @import("behavior/hasdecl.zig");
12 _ = @import("behavior/hasfield.zig");12 _ = @import("behavior/hasfield.zig");
13 _ = @import("behavior/prefetch.zig");
13 _ = @import("behavior/pub_enum.zig");14 _ = @import("behavior/pub_enum.zig");
14 _ = @import("behavior/type_info.zig");15 _ = @import("behavior/type_info.zig");
15 _ = @import("behavior/type.zig");16 _ = @import("behavior/type.zig");
...@@ -182,7 +183,6 @@ test {...@@ -182,7 +183,6 @@ test {
182 _ = @import("behavior/optional_stage1.zig");183 _ = @import("behavior/optional_stage1.zig");
183 _ = @import("behavior/pointers_stage1.zig");184 _ = @import("behavior/pointers_stage1.zig");
184 _ = @import("behavior/popcount_stage1.zig");185 _ = @import("behavior/popcount_stage1.zig");
185 _ = @import("behavior/prefetch.zig");
186 _ = @import("behavior/ptrcast_stage1.zig");186 _ = @import("behavior/ptrcast_stage1.zig");
187 _ = @import("behavior/reflection.zig");187 _ = @import("behavior/reflection.zig");
188 _ = @import("behavior/saturating_arithmetic_stage1.zig");188 _ = @import("behavior/saturating_arithmetic_stage1.zig");