authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-08-18 15:33:11+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-08-18 20:10:18+03:00
log2b45e23477605e15fecec3566b3dc90b71e2f7a7
tree28752bb999a941426b3f4570dd0d54f32f4dfb33
parente0b01bd4a98e2605f197a04f84e0c281ccc90f81
signature Commit is signed but in an unrecognized format.

stage2: character literals and multiline strings

4 files changed, 185 insertions(+), 2 deletions(-)

lib/std/zig.zig+101
...@@ -80,6 +80,107 @@ pub fn binNameAlloc(...@@ -80,6 +80,107 @@ pub fn binNameAlloc(
80 }80 }
81}81}
8282
83/// Only validates escape sequence characters.
84/// Slice must be valid utf8 starting and ending with "'" and exactly one codepoint in between.
85pub fn parseCharLiteral(
86 slice: []const u8,
87 bad_index: *usize, // populated if error.InvalidCharacter is returned)
88) error{InvalidCharacter}!u32 {
89 std.debug.assert(slice.len >= 3 and slice[0] == '\'' and slice[slice.len - 1] == '\'');
90
91 if (slice[1] == '\\') {
92 switch (slice[2]) {
93 'n' => return '\n',
94 'r' => return '\r',
95 '\\' => return '\\',
96 't' => return '\t',
97 '\'' => return '\'',
98 '"' => return '"',
99 'x' => {
100 if (slice.len != 6) {
101 bad_index.* = slice.len - 2;
102 return error.InvalidCharacter;
103 }
104
105 var value: u32 = 0;
106 for (slice[3..5]) |c, i| {
107 switch (slice[3]) {
108 '0'...'9' => {
109 value *= 16;
110 value += c - '0';
111 },
112 'a'...'f' => {
113 value *= 16;
114 value += c - 'a';
115 },
116 'A'...'F' => {
117 value *= 16;
118 value += c - 'a';
119 },
120 else => {
121 bad_index.* = i;
122 return error.InvalidCharacter;
123 },
124 }
125 }
126 return value;
127 },
128 'u' => {
129 if (slice.len < 6 or slice[3] != '{') {
130 bad_index.* = 2;
131 return error.InvalidCharacter;
132 }
133 var value: u32 = 0;
134 for (slice[4..]) |c, i| {
135 if (value > 0x10ffff) {
136 bad_index.* = i;
137 return error.InvalidCharacter;
138 }
139 switch (c) {
140 '0'...'9' => {
141 value *= 16;
142 value += c - '0';
143 },
144 'a'...'f' => {
145 value *= 16;
146 value += c - 'a';
147 },
148 'A'...'F' => {
149 value *= 16;
150 value += c - 'A';
151 },
152 '}' => break,
153 else => {
154 bad_index.* = i;
155 return error.InvalidCharacter;
156 },
157 }
158 }
159 return value;
160 },
161 else => {
162 bad_index.* = 2;
163 return error.InvalidCharacter;
164 }
165 }
166 }
167 return std.unicode.utf8Decode(slice[1 .. slice.len - 1]) catch unreachable;
168}
169
170test "parseCharLiteral" {
171 var bad_index: usize = undefined;
172 std.testing.expectEqual(try parseCharLiteral("'a'", &bad_index), 'a');
173 std.testing.expectEqual(try parseCharLiteral("'ä'", &bad_index), 'ä');
174 std.testing.expectEqual(try parseCharLiteral("'\\x00'", &bad_index), 0);
175 std.testing.expectEqual(try parseCharLiteral("'ぁ'", &bad_index), 0x3041);
176 std.testing.expectEqual(try parseCharLiteral("'\\u{3041}'", &bad_index), 0x3041);
177
178 std.testing.expectError(error.InvalidCharacter, parseCharLiteral("'\\x0'", &bad_index));
179 std.testing.expectError(error.InvalidCharacter, parseCharLiteral("'\\y'", &bad_index));
180 std.testing.expectError(error.InvalidCharacter, parseCharLiteral("'\\u'", &bad_index));
181 std.testing.expectError(error.InvalidCharacter, parseCharLiteral("'\\u{FFFFFF}'", &bad_index));
182}
183
83test "" {184test "" {
84 @import("std").meta.refAllDecls(@This());185 @import("std").meta.refAllDecls(@This());
85}186}
src-self-hosted/astgen.zig+51-2
...@@ -131,6 +131,8 @@ pub fn expr(mod: *Module, scope: *Scope, rl: ResultLoc, node: *ast.Node) InnerEr...@@ -131,6 +131,8 @@ pub fn expr(mod: *Module, scope: *Scope, rl: ResultLoc, node: *ast.Node) InnerEr
131 .ArrayType => return rlWrap(mod, scope, rl, try arrayType(mod, scope, node.castTag(.ArrayType).?)),131 .ArrayType => return rlWrap(mod, scope, rl, try arrayType(mod, scope, node.castTag(.ArrayType).?)),
132 .ArrayTypeSentinel => return rlWrap(mod, scope, rl, try arrayTypeSentinel(mod, scope, node.castTag(.ArrayTypeSentinel).?)),132 .ArrayTypeSentinel => return rlWrap(mod, scope, rl, try arrayTypeSentinel(mod, scope, node.castTag(.ArrayTypeSentinel).?)),
133 .EnumLiteral => return rlWrap(mod, scope, rl, try enumLiteral(mod, scope, node.castTag(.EnumLiteral).?)),133 .EnumLiteral => return rlWrap(mod, scope, rl, try enumLiteral(mod, scope, node.castTag(.EnumLiteral).?)),
134 .MultilineStringLiteral => return rlWrap(mod, scope, rl, try multilineStrLiteral(mod, scope, node.castTag(.MultilineStringLiteral).?)),
135 .CharLiteral => return rlWrap(mod, scope, rl, try charLiteral(mod, scope, node.castTag(.CharLiteral).?)),
134136
135 .Defer => return mod.failNode(scope, node, "TODO implement astgen.expr for .Defer", .{}),137 .Defer => return mod.failNode(scope, node, "TODO implement astgen.expr for .Defer", .{}),
136 .Catch => return mod.failNode(scope, node, "TODO implement astgen.expr for .Catch", .{}),138 .Catch => return mod.failNode(scope, node, "TODO implement astgen.expr for .Catch", .{}),
...@@ -159,8 +161,6 @@ pub fn expr(mod: *Module, scope: *Scope, rl: ResultLoc, node: *ast.Node) InnerEr...@@ -159,8 +161,6 @@ pub fn expr(mod: *Module, scope: *Scope, rl: ResultLoc, node: *ast.Node) InnerEr
159 .ErrorType => return mod.failNode(scope, node, "TODO implement astgen.expr for .ErrorType", .{}),161 .ErrorType => return mod.failNode(scope, node, "TODO implement astgen.expr for .ErrorType", .{}),
160 .FnProto => return mod.failNode(scope, node, "TODO implement astgen.expr for .FnProto", .{}),162 .FnProto => return mod.failNode(scope, node, "TODO implement astgen.expr for .FnProto", .{}),
161 .AnyFrameType => return mod.failNode(scope, node, "TODO implement astgen.expr for .AnyFrameType", .{}),163 .AnyFrameType => return mod.failNode(scope, node, "TODO implement astgen.expr for .AnyFrameType", .{}),
162 .MultilineStringLiteral => return mod.failNode(scope, node, "TODO implement astgen.expr for .MultilineStringLiteral", .{}),
163 .CharLiteral => return mod.failNode(scope, node, "TODO implement astgen.expr for .CharLiteral", .{}),
164 .ErrorSetDecl => return mod.failNode(scope, node, "TODO implement astgen.expr for .ErrorSetDecl", .{}),164 .ErrorSetDecl => return mod.failNode(scope, node, "TODO implement astgen.expr for .ErrorSetDecl", .{}),
165 .ContainerDecl => return mod.failNode(scope, node, "TODO implement astgen.expr for .ContainerDecl", .{}),165 .ContainerDecl => return mod.failNode(scope, node, "TODO implement astgen.expr for .ContainerDecl", .{}),
166 .Comptime => return mod.failNode(scope, node, "TODO implement astgen.expr for .Comptime", .{}),166 .Comptime => return mod.failNode(scope, node, "TODO implement astgen.expr for .Comptime", .{}),
...@@ -497,6 +497,7 @@ fn arrayType(mod: *Module, scope: *Scope, node: *ast.Node.ArrayType) !*zir.Inst...@@ -497,6 +497,7 @@ fn arrayType(mod: *Module, scope: *Scope, node: *ast.Node.ArrayType) !*zir.Inst
497 .val = Value.initTag(.usize_type),497 .val = Value.initTag(.usize_type),
498 });498 });
499499
500 // TODO check for [_]T
500 const len = try expr(mod, scope, .{ .ty = usize_type }, node.len_expr);501 const len = try expr(mod, scope, .{ .ty = usize_type }, node.len_expr);
501 const child_type = try expr(mod, scope, .{ .ty = meta_type }, node.rhs);502 const child_type = try expr(mod, scope, .{ .ty = meta_type }, node.rhs);
502503
...@@ -515,6 +516,7 @@ fn arrayTypeSentinel(mod: *Module, scope: *Scope, node: *ast.Node.ArrayTypeSenti...@@ -515,6 +516,7 @@ fn arrayTypeSentinel(mod: *Module, scope: *Scope, node: *ast.Node.ArrayTypeSenti
515 .val = Value.initTag(.usize_type),516 .val = Value.initTag(.usize_type),
516 });517 });
517518
519 // TODO check for [_]T
518 const len = try expr(mod, scope, .{ .ty = usize_type }, node.len_expr);520 const len = try expr(mod, scope, .{ .ty = usize_type }, node.len_expr);
519 const sentinel_uncasted = try expr(mod, scope, .none, node.sentinel);521 const sentinel_uncasted = try expr(mod, scope, .none, node.sentinel);
520 const elem_type = try expr(mod, scope, .{ .ty = meta_type }, node.rhs);522 const elem_type = try expr(mod, scope, .{ .ty = meta_type }, node.rhs);
...@@ -1120,6 +1122,53 @@ fn stringLiteral(mod: *Module, scope: *Scope, str_lit: *ast.Node.OneToken) Inner...@@ -1120,6 +1122,53 @@ fn stringLiteral(mod: *Module, scope: *Scope, str_lit: *ast.Node.OneToken) Inner
1120 return addZIRInst(mod, scope, src, zir.Inst.Str, .{ .bytes = bytes }, .{});1122 return addZIRInst(mod, scope, src, zir.Inst.Str, .{ .bytes = bytes }, .{});
1121}1123}
11221124
1125fn multilineStrLiteral(mod: *Module, scope: *Scope, node: *ast.Node.MultilineStringLiteral) !*zir.Inst {
1126 const tree = scope.tree();
1127 const lines = node.linesConst();
1128 const src = tree.token_locs[lines[0]].start;
1129
1130 // line lengths and new lines
1131 var len = lines.len - 1;
1132 for (lines) |line| {
1133 len += tree.tokenSlice(line).len - 2;
1134 }
1135
1136 const bytes = try scope.arena().alloc(u8, len);
1137 var i: usize = 0;
1138 for (lines) |line, line_i| {
1139 if (line_i != 0) {
1140 bytes[i] = '\n';
1141 i += 1;
1142 }
1143 const slice = tree.tokenSlice(line)[2..];
1144 mem.copy(u8, bytes[i..], slice);
1145 i += slice.len;
1146 }
1147
1148 return addZIRInst(mod, scope, src, zir.Inst.Str, .{ .bytes = bytes }, .{});
1149}
1150
1151fn charLiteral(mod: *Module, scope: *Scope, node: *ast.Node.OneToken) !*zir.Inst {
1152 const tree = scope.tree();
1153 const src = tree.token_locs[node.token].start;
1154 const slice = tree.tokenSlice(node.token);
1155
1156 var bad_index: usize = undefined;
1157 const value = std.zig.parseCharLiteral(slice, &bad_index) catch |err| switch (err) {
1158 error.InvalidCharacter => {
1159 const bad_byte = slice[bad_index];
1160 return mod.fail(scope, src + bad_index, "invalid character: '{c}'\n", .{bad_byte});
1161 },
1162 };
1163
1164 const int_payload = try scope.arena().create(Value.Payload.Int_u64);
1165 int_payload.* = .{ .int = value };
1166 return addZIRInstConst(mod, scope, src, .{
1167 .ty = Type.initTag(.comptime_int),
1168 .val = Value.initPayload(&int_payload.base),
1169 });
1170}
1171
1123fn integerLiteral(mod: *Module, scope: *Scope, int_lit: *ast.Node.OneToken) InnerError!*zir.Inst {1172fn integerLiteral(mod: *Module, scope: *Scope, int_lit: *ast.Node.OneToken) InnerError!*zir.Inst {
1124 const arena = scope.arena();1173 const arena = scope.arena();
1125 const tree = scope.tree();1174 const tree = scope.tree();
src-self-hosted/zir_sema.zig+1
...@@ -365,6 +365,7 @@ fn analyzeInstEnsureResultNonError(mod: *Module, scope: *Scope, inst: *zir.Inst....@@ -365,6 +365,7 @@ fn analyzeInstEnsureResultNonError(mod: *Module, scope: *Scope, inst: *zir.Inst.
365365
366fn analyzeInstAlloc(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) InnerError!*Inst {366fn analyzeInstAlloc(mod: *Module, scope: *Scope, inst: *zir.Inst.UnOp) InnerError!*Inst {
367 const var_type = try resolveType(mod, scope, inst.positionals.operand);367 const var_type = try resolveType(mod, scope, inst.positionals.operand);
368 // TODO this should happen only for var allocs
368 if (!var_type.isValidVarType()) {369 if (!var_type.isValidVarType()) {
369 return mod.fail(scope, inst.base.src, "variable of type '{}' must be const or comptime", .{var_type});370 return mod.fail(scope, inst.base.src, "variable of type '{}' must be const or comptime", .{var_type});
370 }371 }
test/stage2/compare_output.zig+32
...@@ -543,6 +543,38 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -543,6 +543,38 @@ pub fn addCases(ctx: *TestContext) !void {
543 ,543 ,
544 "",544 "",
545 );545 );
546
547 case.addCompareOutput(
548 \\export fn _start() noreturn {
549 \\ const ignore =
550 \\ \\ cool thx
551 \\ \\
552 \\ ;
553 \\ add('ぁ', '\x03');
554 \\
555 \\ exit();
556 \\}
557 \\
558 \\fn add(a: u32, b: u32) void {
559 \\ assert(a + b == 12356);
560 \\}
561 \\
562 \\pub fn assert(ok: bool) void {
563 \\ if (!ok) unreachable; // assertion failure
564 \\}
565 \\
566 \\fn exit() noreturn {
567 \\ asm volatile ("syscall"
568 \\ :
569 \\ : [number] "{rax}" (231),
570 \\ [arg1] "{rdi}" (0)
571 \\ : "rcx", "r11", "memory"
572 \\ );
573 \\ unreachable;
574 \\}
575 ,
576 "",
577 );
546 }578 }
547579
548 {580 {