authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-03-23 17:35:21-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-03-23 17:35:21-04:00
log89953ec83d8afe4fed0fc9e3cdded09c7522bf86
tree42f76e6da37b08e0022af060545c5e5d3f6bd476
parent55cb9ef138c7cf0a23e7f852a82884612a3ca663
signaturelock-open Commit is signed but in an unrecognized format.

character literals: allow unicode escapes

also make the documentation for character literals more clear. closes #2089 see #2097

7 files changed, 56 insertions(+), 30 deletions(-)

doc/langref.html.in+13-3
...@@ -501,7 +501,16 @@ pub fn main() void {...@@ -501,7 +501,16 @@ pub fn main() void {
501 </div>501 </div>
502 {#see_also|Optionals|undefined#}502 {#see_also|Optionals|undefined#}
503 {#header_close#}503 {#header_close#}
504 {#header_open|String Literals#}504 {#header_open|String Literals and Character Literals#}
505 <p>
506 String literals are UTF-8 encoded byte arrays.
507 </p>
508 <p>
509 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
510 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals
511 and character literals. Once https://github.com/ziglang/zig/issues/2097 is implemented,
512 character literals will be allowed to have a single UTF-8 encoded codepoint.
513 </p>
505 {#code_begin|test#}514 {#code_begin|test#}
506const assert = @import("std").debug.assert;515const assert = @import("std").debug.assert;
507const mem = @import("std").mem;516const mem = @import("std").mem;
...@@ -513,6 +522,7 @@ test "string literals" {...@@ -513,6 +522,7 @@ test "string literals" {
513 assert(normal_bytes.len == 5);522 assert(normal_bytes.len == 5);
514 assert(normal_bytes[1] == 'e');523 assert(normal_bytes[1] == 'e');
515 assert('e' == '\x65');524 assert('e' == '\x65');
525 assert('\U01f4a9' == 128169);
516 assert(mem.eql(u8, "hello", "h\x65llo"));526 assert(mem.eql(u8, "hello", "h\x65llo"));
517527
518 // A C string literal is a null terminated pointer.528 // A C string literal is a null terminated pointer.
...@@ -521,7 +531,7 @@ test "string literals" {...@@ -521,7 +531,7 @@ test "string literals" {
521 assert(null_terminated_bytes[5] == 0);531 assert(null_terminated_bytes[5] == 0);
522}532}
523 {#code_end#}533 {#code_end#}
524 {#see_also|Arrays|Zig Test#}534 {#see_also|Arrays|Zig Test|Source Encoding#}
525 {#header_open|Escape Sequences#}535 {#header_open|Escape Sequences#}
526 <div class="table-wrapper">536 <div class="table-wrapper">
527 <table>537 <table>
...@@ -8530,7 +8540,7 @@ pub fn main() void {...@@ -8530,7 +8540,7 @@ pub fn main() void {
8530 );8540 );
8531}8541}
8532 {#code_end#}8542 {#code_end#}
8533 {#see_also|String Literals#}8543 {#see_also|String Literals and Character Literals#}
8534 {#header_close#}8544 {#header_close#}
85358545
8536 {#header_open|Import from C Header File#}8546 {#header_open|Import from C Header File#}
src/all_types.hpp+1-1
...@@ -845,7 +845,7 @@ struct AstNodeStringLiteral {...@@ -845,7 +845,7 @@ struct AstNodeStringLiteral {
845};845};
846846
847struct AstNodeCharLiteral {847struct AstNodeCharLiteral {
848 uint8_t value;848 uint32_t value;
849};849};
850850
851struct AstNodeFloatLiteral {851struct AstNodeFloatLiteral {
src/tokenizer.cpp+8-9
...@@ -1103,11 +1103,15 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1103,11 +1103,15 @@ void tokenize(Buf *buf, Tokenization *out) {
11031103
1104 if (t.char_code_index >= t.char_code_end) {1104 if (t.char_code_index >= t.char_code_end) {
1105 if (t.unicode) {1105 if (t.unicode) {
1106 if (t.char_code <= 0x7f) {1106 if (t.char_code > 0x10ffff) {
1107 tokenize_error(&t, "unicode value out of range: %x", t.char_code);
1108 }
1109 if (t.cur_tok->id == TokenIdCharLiteral) {
1110 t.cur_tok->data.char_lit.c = t.char_code;
1111 t.state = TokenizeStateCharLiteralEnd;
1112 } else if (t.char_code <= 0x7f) {
1107 // 00000000 00000000 00000000 0xxxxxxx1113 // 00000000 00000000 00000000 0xxxxxxx
1108 handle_string_escape(&t, (uint8_t)t.char_code);1114 handle_string_escape(&t, (uint8_t)t.char_code);
1109 } else if (t.cur_tok->id == TokenIdCharLiteral) {
1110 tokenize_error(&t, "unicode value too large for character literal: %x", t.char_code);
1111 } else if (t.char_code <= 0x7ff) {1115 } else if (t.char_code <= 0x7ff) {
1112 // 00000000 00000000 00000xxx xx0000001116 // 00000000 00000000 00000xxx xx000000
1113 handle_string_escape(&t, (uint8_t)(0xc0 | (t.char_code >> 6)));1117 handle_string_escape(&t, (uint8_t)(0xc0 | (t.char_code >> 6)));
...@@ -1129,14 +1133,9 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1129,14 +1133,9 @@ void tokenize(Buf *buf, Tokenization *out) {
1129 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));1133 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1130 // 00000000 00000000 00000000 00xxxxxx1134 // 00000000 00000000 00000000 00xxxxxx
1131 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));1135 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1132 } else {
1133 tokenize_error(&t, "unicode value out of range: %x", t.char_code);
1134 }1136 }
1135 } else {1137 } else {
1136 if (t.cur_tok->id == TokenIdCharLiteral && t.char_code > UINT8_MAX) {1138 assert(t.char_code <= 255);
1137 tokenize_error(&t, "value too large for character literal: '%x'",
1138 t.char_code);
1139 }
1140 handle_string_escape(&t, (uint8_t)t.char_code);1139 handle_string_escape(&t, (uint8_t)t.char_code);
1141 }1140 }
1142 }1141 }
src/tokenizer.hpp+1-1
...@@ -148,7 +148,7 @@ struct TokenStrLit {...@@ -148,7 +148,7 @@ struct TokenStrLit {
148};148};
149149
150struct TokenCharLit {150struct TokenCharLit {
151 uint8_t c;151 uint32_t c;
152};152};
153153
154struct Token {154struct Token {
std/zig/parser_test.zig+7
...@@ -1,3 +1,10 @@...@@ -1,3 +1,10 @@
1test "zig fmt: character literal larger than u8" {
2 try testCanonical(
3 \\const x = '\U01f4a9';
4 \\
5 );
6}
7
1test "zig fmt: infix operator and then multiline string literal" {8test "zig fmt: infix operator and then multiline string literal" {
2 try testCanonical(9 try testCanonical(
3 \\const x = "" ++10 \\const x = "" ++
std/zig/tokenizer.zig+21-16
...@@ -236,8 +236,7 @@ pub const Tokenizer = struct {...@@ -236,8 +236,7 @@ pub const Tokenizer = struct {
236 MultilineStringLiteralLine,236 MultilineStringLiteralLine,
237 CharLiteral,237 CharLiteral,
238 CharLiteralBackslash,238 CharLiteralBackslash,
239 CharLiteralEscape1,239 CharLiteralHexEscape,
240 CharLiteralEscape2,
241 CharLiteralEnd,240 CharLiteralEnd,
242 Backslash,241 Backslash,
243 Equal,242 Equal,
...@@ -293,6 +292,8 @@ pub const Tokenizer = struct {...@@ -293,6 +292,8 @@ pub const Tokenizer = struct {
293 .start = self.index,292 .start = self.index,
294 .end = undefined,293 .end = undefined,
295 };294 };
295 var seen_escape_digits: usize = undefined;
296 var expected_escape_digits: usize = undefined;
296 while (self.index < self.buffer.len) : (self.index += 1) {297 while (self.index < self.buffer.len) : (self.index += 1) {
297 const c = self.buffer[self.index];298 const c = self.buffer[self.index];
298 switch (state) {299 switch (state) {
...@@ -658,26 +659,31 @@ pub const Tokenizer = struct {...@@ -658,26 +659,31 @@ pub const Tokenizer = struct {
658 break;659 break;
659 },660 },
660 'x' => {661 'x' => {
661 state = State.CharLiteralEscape1;662 state = State.CharLiteralHexEscape;
663 seen_escape_digits = 0;
664 expected_escape_digits = 2;
662 },665 },
663 else => {666 'u' => {
664 state = State.CharLiteralEnd;667 state = State.CharLiteralHexEscape;
668 seen_escape_digits = 0;
669 expected_escape_digits = 4;
665 },670 },
666 },671 'U' => {
667672 state = State.CharLiteralHexEscape;
668 State.CharLiteralEscape1 => switch (c) {673 seen_escape_digits = 0;
669 '0'...'9', 'a'...'z', 'A'...'F' => {674 expected_escape_digits = 6;
670 state = State.CharLiteralEscape2;
671 },675 },
672 else => {676 else => {
673 result.id = Token.Id.Invalid;677 state = State.CharLiteralEnd;
674 break;
675 },678 },
676 },679 },
677680
678 State.CharLiteralEscape2 => switch (c) {681 State.CharLiteralHexEscape => switch (c) {
679 '0'...'9', 'a'...'z', 'A'...'F' => {682 '0'...'9', 'a'...'z', 'A'...'F' => {
680 state = State.CharLiteralEnd;683 seen_escape_digits += 1;
684 if (seen_escape_digits == expected_escape_digits) {
685 state = State.CharLiteralEnd;
686 }
681 },687 },
682 else => {688 else => {
683 result.id = Token.Id.Invalid;689 result.id = Token.Id.Invalid;
...@@ -1045,8 +1051,7 @@ pub const Tokenizer = struct {...@@ -1045,8 +1051,7 @@ pub const Tokenizer = struct {
1045 State.Backslash,1051 State.Backslash,
1046 State.CharLiteral,1052 State.CharLiteral,
1047 State.CharLiteralBackslash,1053 State.CharLiteralBackslash,
1048 State.CharLiteralEscape1,1054 State.CharLiteralHexEscape,
1049 State.CharLiteralEscape2,
1050 State.CharLiteralEnd,1055 State.CharLiteralEnd,
1051 State.StringLiteralBackslash,1056 State.StringLiteralBackslash,
1052 State.LBracketStar,1057 State.LBracketStar,
test/stage1/behavior/misc.zig+5
...@@ -699,3 +699,8 @@ test "thread local variable" {...@@ -699,3 +699,8 @@ test "thread local variable" {
699 S.t += 1;699 S.t += 1;
700 expect(S.t == 1235);700 expect(S.t == 1235);
701}701}
702
703test "unicode escape in character literal" {
704 var a: u24 = '\U01f4a9';
705 expect(a == 128169);
706}