authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2017-12-26 23:17:33-07:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2017-12-26 23:17:33-07:00
log192a0391734d80d9876db26f7aae55ae7e8c4092
tree948f47dc2ba6440228bdaf874771588219824106
parent08dd1b553b37de24eaf24a37558b0f9993d4ca42

move utf8 parsing to std

source files no longer need to end with a newline

8 files changed, 250 insertions(+), 141 deletions(-)

CMakeLists.txt+1
...@@ -605,6 +605,7 @@ install(FILES "${CMAKE_SOURCE_DIR}/std/os/windows/index.zig" DESTINATION "${ZIG_...@@ -605,6 +605,7 @@ install(FILES "${CMAKE_SOURCE_DIR}/std/os/windows/index.zig" DESTINATION "${ZIG_
605install(FILES "${CMAKE_SOURCE_DIR}/std/os/windows/util.zig" DESTINATION "${ZIG_STD_DEST}/os/windows")605install(FILES "${CMAKE_SOURCE_DIR}/std/os/windows/util.zig" DESTINATION "${ZIG_STD_DEST}/os/windows")
606install(FILES "${CMAKE_SOURCE_DIR}/std/rand.zig" DESTINATION "${ZIG_STD_DEST}")606install(FILES "${CMAKE_SOURCE_DIR}/std/rand.zig" DESTINATION "${ZIG_STD_DEST}")
607install(FILES "${CMAKE_SOURCE_DIR}/std/sort.zig" DESTINATION "${ZIG_STD_DEST}")607install(FILES "${CMAKE_SOURCE_DIR}/std/sort.zig" DESTINATION "${ZIG_STD_DEST}")
608install(FILES "${CMAKE_SOURCE_DIR}/std/unicode.zig" DESTINATION "${ZIG_STD_DEST}")
608install(FILES "${CMAKE_SOURCE_DIR}/std/special/bootstrap.zig" DESTINATION "${ZIG_STD_DEST}/special")609install(FILES "${CMAKE_SOURCE_DIR}/std/special/bootstrap.zig" DESTINATION "${ZIG_STD_DEST}/special")
609install(FILES "${CMAKE_SOURCE_DIR}/std/special/bootstrap_lib.zig" DESTINATION "${ZIG_STD_DEST}/special")610install(FILES "${CMAKE_SOURCE_DIR}/std/special/bootstrap_lib.zig" DESTINATION "${ZIG_STD_DEST}/special")
610install(FILES "${CMAKE_SOURCE_DIR}/std/special/build_file_template.zig" DESTINATION "${ZIG_STD_DEST}/special")611install(FILES "${CMAKE_SOURCE_DIR}/std/special/build_file_template.zig" DESTINATION "${ZIG_STD_DEST}/special")
build.zig+1
...@@ -276,6 +276,7 @@ pub fn installStdLib(b: &Builder) {...@@ -276,6 +276,7 @@ pub fn installStdLib(b: &Builder) {
276 "os/windows/util.zig",276 "os/windows/util.zig",
277 "rand.zig",277 "rand.zig",
278 "sort.zig",278 "sort.zig",
279 "unicode.zig",
279 "special/bootstrap.zig",280 "special/bootstrap.zig",
280 "special/bootstrap_lib.zig",281 "special/bootstrap_lib.zig",
281 "special/build_file_template.zig",282 "special/build_file_template.zig",
doc/langref.html.in+1-1
...@@ -298,7 +298,7 @@ pub fn main() -&gt; %void {...@@ -298,7 +298,7 @@ pub fn main() -&gt; %void {
298 <li>Ascii control characters, except for U+000a (LF): U+0000 - U+0009, U+000b - U+0001f, U+007f. (Note that Windows line endings (CRLF) are not allowed, and hard tabs are not allowed.)</li>298 <li>Ascii control characters, except for U+000a (LF): U+0000 - U+0009, U+000b - U+0001f, U+007f. (Note that Windows line endings (CRLF) are not allowed, and hard tabs are not allowed.)</li>
299 <li>Non-Ascii Unicode line endings: U+0085 (NEL), U+2028 (LS), U+2029 (PS).</li>299 <li>Non-Ascii Unicode line endings: U+0085 (NEL), U+2028 (LS), U+2029 (PS).</li>
300 </ul>300 </ul>
301 <p>The codepoint U+000a (LF) (which is encoded as the single-byte value 0x0a) is the line terminator character. This character always terminates a line of zig source code. A non-empty zig source must end with the line terminator character.</p>301 <p>The codepoint U+000a (LF) (which is encoded as the single-byte value 0x0a) is the line terminator character. This character always terminates a line of zig source code (except possbly the last line of the file).</p>
302 <p>For some discussion on the rationale behind these design decisions, see <a href="https://github.com/zig-lang/zig/issues/663">issue #663</a></p>302 <p>For some discussion on the rationale behind these design decisions, see <a href="https://github.com/zig-lang/zig/issues/663">issue #663</a></p>
303 <h2 id="values">Values</h2>303 <h2 id="values">Values</h2>
304 <pre><code class="zig">const warn = @import("std").debug.warn;304 <pre><code class="zig">const warn = @import("std").debug.warn;
src-self-hosted/module.zig+4-1
...@@ -213,11 +213,14 @@ pub const Module = struct {...@@ -213,11 +213,14 @@ pub const Module = struct {
213 };213 };
214 %defer self.allocator.free(root_src_real_path);214 %defer self.allocator.free(root_src_real_path);
215215
216 const source_code = io.readFileAlloc(root_src_real_path, self.allocator) %% |err| {216 const source_code = io.readFileAllocExtra(root_src_real_path, self.allocator, 3) %% |err| {
217 %return printError("unable to open '{}': {}", root_src_real_path, err);217 %return printError("unable to open '{}': {}", root_src_real_path, err);
218 return err;218 return err;
219 };219 };
220 %defer self.allocator.free(source_code);220 %defer self.allocator.free(source_code);
221 source_code[source_code.len - 3] = '\n';
222 source_code[source_code.len - 2] = '\n';
223 source_code[source_code.len - 1] = '\n';
221224
222 warn("====input:====\n");225 warn("====input:====\n");
223226
src-self-hosted/tokenizer.zig+66-138
...@@ -70,7 +70,6 @@ pub const Token = struct {...@@ -70,7 +70,6 @@ pub const Token = struct {
70 Identifier,70 Identifier,
71 StringLiteral: StrLitKind,71 StringLiteral: StrLitKind,
72 Eof,72 Eof,
73 NoEolAtEof,
74 Builtin,73 Builtin,
75 Bang,74 Bang,
76 Equal,75 Equal,
...@@ -140,7 +139,6 @@ pub const Token = struct {...@@ -140,7 +139,6 @@ pub const Token = struct {
140pub const Tokenizer = struct {139pub const Tokenizer = struct {
141 buffer: []const u8,140 buffer: []const u8,
142 index: usize,141 index: usize,
143 actual_file_end: usize,
144 pending_invalid_token: ?Token,142 pending_invalid_token: ?Token,
145143
146 pub const Location = struct {144 pub const Location = struct {
...@@ -179,17 +177,15 @@ pub const Tokenizer = struct {...@@ -179,17 +177,15 @@ pub const Tokenizer = struct {
179 std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);177 std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
180 }178 }
181179
180 /// buffer must end with "\n\n\n". This is so that attempting to decode
181 /// a the 3 trailing bytes of a 4-byte utf8 sequence is never a buffer overflow.
182 pub fn init(buffer: []const u8) -> Tokenizer {182 pub fn init(buffer: []const u8) -> Tokenizer {
183 var source_len = buffer.len;183 std.debug.assert(buffer[buffer.len - 1] == '\n');
184 while (source_len > 0) : (source_len -= 1) {184 std.debug.assert(buffer[buffer.len - 2] == '\n');
185 if (buffer[source_len - 1] == '\n') break;185 std.debug.assert(buffer[buffer.len - 3] == '\n');
186 // last line is incomplete, so skip it, and give an error when we get there.
187 }
188
189 return Tokenizer {186 return Tokenizer {
190 .buffer = buffer[0..source_len],187 .buffer = buffer,
191 .index = 0,188 .index = 0,
192 .actual_file_end = buffer.len,
193 .pending_invalid_token = null,189 .pending_invalid_token = null,
194 };190 };
195 }191 }
...@@ -512,17 +508,14 @@ pub const Tokenizer = struct {...@@ -512,17 +508,14 @@ pub const Tokenizer = struct {
512 }508 }
513 }509 }
514 result.end = self.index;510 result.end = self.index;
511
515 if (result.id == Token.Id.Eof) {512 if (result.id == Token.Id.Eof) {
516 if (self.pending_invalid_token) |token| {513 if (self.pending_invalid_token) |token| {
517 self.pending_invalid_token = null;514 self.pending_invalid_token = null;
518 return token;515 return token;
519 }516 }
520 if (self.actual_file_end != self.buffer.len) {
521 // instead of an Eof, give an error token
522 result.id = Token.Id.NoEolAtEof;
523 result.end = self.actual_file_end;
524 }
525 }517 }
518
526 return result;519 return result;
527 }520 }
528521
...@@ -553,161 +546,96 @@ pub const Tokenizer = struct {...@@ -553,161 +546,96 @@ pub const Tokenizer = struct {
553 return 0;546 return 0;
554 } else {547 } else {
555 // check utf8-encoded character.548 // check utf8-encoded character.
556 // remember that the last byte in the buffer is guaranteed to be '\n',549 const length = std.unicode.utf8ByteSequenceLength(c0) %% return 1;
557 // which means we really don't need to do bounds checks here,550 // the last 3 bytes in the buffer are guaranteed to be '\n',
558 // as long as we check one byte at a time for being a continuation byte.551 // which means we don't need to do any bounds checking here.
559 var value: u32 = undefined;552 const bytes = self.buffer[self.index..self.index + length];
560 var length: u3 = undefined;553 switch (length) {
561 if (c0 & 0b11100000 == 0b11000000) {value = c0 & 0b00011111; length = 2;}554 2 => {
562 else if (c0 & 0b11110000 == 0b11100000) {value = c0 & 0b00001111; length = 3;}555 const value = std.unicode.utf8Decode2(bytes) %% return length;
563 else if (c0 & 0b11111000 == 0b11110000) {value = c0 & 0b00000111; length = 4;}556 if (value == 0x85) return length; // U+0085 (NEL)
564 else return 1; // unexpected continuation or too many leading 1's557 },
565558 3 => {
566 const c1 = self.buffer[self.index + 1];559 const value = std.unicode.utf8Decode3(bytes) %% return length;
567 if (c1 & 0b11000000 != 0b10000000) return 1; // expected continuation560 if (value == 0x2028) return length; // U+2028 (LS)
568 value <<= 6;561 if (value == 0x2029) return length; // U+2029 (PS)
569 value |= c1 & 0b00111111;562 },
570 if (length == 2) {563 4 => {
571 if (value < 0x80) return length; // overlong564 _ = std.unicode.utf8Decode4(bytes) %% return length;
572 if (value == 0x85) return length; // U+0085 (NEL)565 },
573 self.index += length - 1;566 else => unreachable,
574 return 0;
575 }
576 const c2 = self.buffer[self.index + 2];
577 if (c2 & 0b11000000 != 0b10000000) return 2; // expected continuation
578 value <<= 6;
579 value |= c2 & 0b00111111;
580 if (length == 3) {
581 if (value < 0x800) return length; // overlong
582 if (value == 0x2028) return length; // U+2028 (LS)
583 if (value == 0x2029) return length; // U+2029 (PS)
584 if (0xd800 <= value and value <= 0xdfff) return length; // surrogate halves not allowed in utf8
585 self.index += length - 1;
586 return 0;
587 }
588 const c3 = self.buffer[self.index + 3];
589 if (c3 & 0b11000000 != 0b10000000) return 3; // expected continuation
590 value <<= 6;
591 value |= c3 & 0b00111111;
592 if (length == 4) {
593 if (value < 0x10000) return length; // overlong
594 if (value > 0x10FFFF) return length; // out of bounds
595 self.index += length - 1;
596 return 0;
597 }567 }
598 unreachable;568 self.index += length - 1;
569 return 0;
599 }570 }
600 }571 }
601};572};
602573
603574
604575
605test "tokenizer - source must end with eol" {576test "tokenizer" {
606 testTokenizeWithEol("", []Token.Id {577 testTokenize("test", []Token.Id {
607 }, true);
608 testTokenizeWithEol("no newline", []Token.Id {
609 }, false);
610 testTokenizeWithEol("test\n", []Token.Id {
611 Token.Id.Keyword_test,
612 }, true);
613 testTokenizeWithEol("test\nno newline", []Token.Id {
614 Token.Id.Keyword_test,578 Token.Id.Keyword_test,
615 }, false);579 });
616}580}
617581
618test "tokenizer - invalid token characters" {582test "tokenizer - invalid token characters" {
619 testTokenize("#\n", []Token.Id{Token.Id.Invalid});583 testTokenize("#", []Token.Id{Token.Id.Invalid});
620 testTokenize("`\n", []Token.Id{Token.Id.Invalid});584 testTokenize("`", []Token.Id{Token.Id.Invalid});
621}585}
622586
623test "tokenizer - invalid literal/comment characters" {587test "tokenizer - invalid literal/comment characters" {
624 testTokenize("\"\x00\"\n", []Token.Id {588 testTokenize("\"\x00\"", []Token.Id {
625 Token.Id { .StringLiteral = Token.StrLitKind.Normal },589 Token.Id { .StringLiteral = Token.StrLitKind.Normal },
626 Token.Id.Invalid,590 Token.Id.Invalid,
627 });591 });
628 testTokenize("//\x00\n", []Token.Id {592 testTokenize("//\x00", []Token.Id {
629 Token.Id.Invalid,593 Token.Id.Invalid,
630 });594 });
631 testTokenize("//\x1f\n", []Token.Id {595 testTokenize("//\x1f", []Token.Id {
632 Token.Id.Invalid,596 Token.Id.Invalid,
633 });597 });
634 testTokenize("//\x7f\n", []Token.Id {598 testTokenize("//\x7f", []Token.Id {
635 Token.Id.Invalid,599 Token.Id.Invalid,
636 });600 });
637}601}
638602
639test "tokenizer - valid unicode" {603test "tokenizer - utf8" {
640 testTokenize("//\xc2\x80\n", []Token.Id{});604 testTokenize("//\xc2\x80", []Token.Id{});
641 testTokenize("//\xdf\xbf\n", []Token.Id{});605 testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{});
642 testTokenize("//\xe0\xa0\x80\n", []Token.Id{});
643 testTokenize("//\xe1\x80\x80\n", []Token.Id{});
644 testTokenize("//\xef\xbf\xbf\n", []Token.Id{});
645 testTokenize("//\xf0\x90\x80\x80\n", []Token.Id{});
646 testTokenize("//\xf1\x80\x80\x80\n", []Token.Id{});
647 testTokenize("//\xf3\xbf\xbf\xbf\n", []Token.Id{});
648 testTokenize("//\xf4\x8f\xbf\xbf\n", []Token.Id{});
649}
650
651test "tokenizer - invalid unicode continuation bytes" {
652 // unexpected continuation
653 testTokenize("//\x80\n", []Token.Id{Token.Id.Invalid});
654 testTokenize("//\xbf\n", []Token.Id{Token.Id.Invalid});
655 // too many leading 1's
656 testTokenize("//\xf8\n", []Token.Id{Token.Id.Invalid});
657 testTokenize("//\xff\n", []Token.Id{Token.Id.Invalid});
658 // expected continuation for 2 byte sequences
659 testTokenize("//\xc2\x00\n", []Token.Id{Token.Id.Invalid});
660 testTokenize("//\xc2\xc0\n", []Token.Id{Token.Id.Invalid});
661 // expected continuation for 3 byte sequences
662 testTokenize("//\xe0\x00\n", []Token.Id{Token.Id.Invalid});
663 testTokenize("//\xe0\xc0\n", []Token.Id{Token.Id.Invalid});
664 testTokenize("//\xe0\xa0\n", []Token.Id{Token.Id.Invalid});
665 testTokenize("//\xe0\xa0\x00\n", []Token.Id{Token.Id.Invalid});
666 testTokenize("//\xe0\xa0\xc0\n", []Token.Id{Token.Id.Invalid});
667 // expected continuation for 4 byte sequences
668 testTokenize("//\xf0\x00\n", []Token.Id{Token.Id.Invalid});
669 testTokenize("//\xf0\xc0\n", []Token.Id{Token.Id.Invalid});
670 testTokenize("//\xf0\x90\x00\n", []Token.Id{Token.Id.Invalid});
671 testTokenize("//\xf0\x90\xc0\n", []Token.Id{Token.Id.Invalid});
672 testTokenize("//\xf0\x90\x80\x00\n", []Token.Id{Token.Id.Invalid});
673 testTokenize("//\xf0\x90\x80\xc0\n", []Token.Id{Token.Id.Invalid});
674}606}
675607
676test "tokenizer - overlong utf8 codepoint" {608test "tokenizer - invalid utf8" {
677 testTokenize("//\xc0\x80\n", []Token.Id{Token.Id.Invalid});609 testTokenize("//\x80", []Token.Id{Token.Id.Invalid});
678 testTokenize("//\xc1\xbf\n", []Token.Id{Token.Id.Invalid});610 testTokenize("//\xbf", []Token.Id{Token.Id.Invalid});
679 testTokenize("//\xe0\x80\x80\n", []Token.Id{Token.Id.Invalid});611 testTokenize("//\xf8", []Token.Id{Token.Id.Invalid});
680 testTokenize("//\xe0\x9f\xbf\n", []Token.Id{Token.Id.Invalid});612 testTokenize("//\xff", []Token.Id{Token.Id.Invalid});
681 testTokenize("//\xf0\x80\x80\x80\n", []Token.Id{Token.Id.Invalid});613 testTokenize("//\xc2\xc0", []Token.Id{Token.Id.Invalid});
682 testTokenize("//\xf0\x8f\xbf\xbf\n", []Token.Id{Token.Id.Invalid});614 testTokenize("//\xe0", []Token.Id{Token.Id.Invalid});
615 testTokenize("//\xf0", []Token.Id{Token.Id.Invalid});
616 testTokenize("//\xf0\x90\x80\xc0", []Token.Id{Token.Id.Invalid});
683}617}
684618
685test "tokenizer - misc invalid utf8" {619test "tokenizer - illegal unicode codepoints" {
686 // codepoint out of bounds
687 testTokenize("//\xf4\x90\x80\x80\n", []Token.Id{Token.Id.Invalid});
688 testTokenize("//\xf7\xbf\xbf\xbf\n", []Token.Id{Token.Id.Invalid});
689 // unicode newline characters.U+0085, U+2028, U+2029620 // unicode newline characters.U+0085, U+2028, U+2029
690 testTokenize("//\xc2\x84\n", []Token.Id{});621 testTokenize("//\xc2\x84", []Token.Id{});
691 testTokenize("//\xc2\x85\n", []Token.Id{Token.Id.Invalid});622 testTokenize("//\xc2\x85", []Token.Id{Token.Id.Invalid});
692 testTokenize("//\xc2\x86\n", []Token.Id{});623 testTokenize("//\xc2\x86", []Token.Id{});
693 testTokenize("//\xe2\x80\xa7\n", []Token.Id{});624 testTokenize("//\xe2\x80\xa7", []Token.Id{});
694 testTokenize("//\xe2\x80\xa8\n", []Token.Id{Token.Id.Invalid});625 testTokenize("//\xe2\x80\xa8", []Token.Id{Token.Id.Invalid});
695 testTokenize("//\xe2\x80\xa9\n", []Token.Id{Token.Id.Invalid});626 testTokenize("//\xe2\x80\xa9", []Token.Id{Token.Id.Invalid});
696 testTokenize("//\xe2\x80\xaa\n", []Token.Id{});627 testTokenize("//\xe2\x80\xaa", []Token.Id{});
697 // surrogate halves
698 testTokenize("//\xed\x9f\x80\n", []Token.Id{});
699 testTokenize("//\xed\xa0\x80\n", []Token.Id{Token.Id.Invalid});
700 testTokenize("//\xed\xbf\xbf\n", []Token.Id{Token.Id.Invalid});
701 testTokenize("//\xee\x80\x80\n", []Token.Id{});
702 // surrogate halves are invalid, even in surrogate pairs
703 testTokenize("//\xed\xa0\xad\xed\xb2\xa9\n", []Token.Id{Token.Id.Invalid});
704}628}
705629
706fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) {630fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) {
707 testTokenizeWithEol(source, expected_tokens, true);631 // (test authors, just make this bigger if you need it)
708}632 var padded_source: [0x100]u8 = undefined;
709fn testTokenizeWithEol(source: []const u8, expected_tokens: []const Token.Id, expected_eol_at_eof: bool) {633 std.mem.copy(u8, padded_source[0..source.len], source);
710 var tokenizer = Tokenizer.init(source);634 padded_source[source.len + 0] = '\n';
635 padded_source[source.len + 1] = '\n';
636 padded_source[source.len + 2] = '\n';
637
638 var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
711 for (expected_tokens) |expected_token_id| {639 for (expected_tokens) |expected_token_id| {
712 const token = tokenizer.next();640 const token = tokenizer.next();
713 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));641 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
...@@ -718,5 +646,5 @@ fn testTokenizeWithEol(source: []const u8, expected_tokens: []const Token.Id, ex...@@ -718,5 +646,5 @@ fn testTokenizeWithEol(source: []const u8, expected_tokens: []const Token.Id, ex
718 else => {},646 else => {},
719 }647 }
720 }648 }
721 std.debug.assert(tokenizer.next().id == if (expected_eol_at_eof) Token.Id.Eof else Token.Id.NoEolAtEof);649 std.debug.assert(tokenizer.next().id == Token.Id.Eof);
722}650}
std/index.zig+2
...@@ -25,6 +25,7 @@ pub const net = @import("net.zig");...@@ -25,6 +25,7 @@ pub const net = @import("net.zig");
25pub const os = @import("os/index.zig");25pub const os = @import("os/index.zig");
26pub const rand = @import("rand.zig");26pub const rand = @import("rand.zig");
27pub const sort = @import("sort.zig");27pub const sort = @import("sort.zig");
28pub const unicode = @import("unicode.zig");
2829
29test "std" {30test "std" {
30 // run tests from these31 // run tests from these
...@@ -53,4 +54,5 @@ test "std" {...@@ -53,4 +54,5 @@ test "std" {
53 _ = @import("os/index.zig");54 _ = @import("os/index.zig");
54 _ = @import("rand.zig");55 _ = @import("rand.zig");
55 _ = @import("sort.zig");56 _ = @import("sort.zig");
57 _ = @import("unicode.zig");
56}58}
std/io.zig+6-1
...@@ -500,11 +500,16 @@ pub fn writeFile(path: []const u8, data: []const u8, allocator: ?&mem.Allocator)...@@ -500,11 +500,16 @@ pub fn writeFile(path: []const u8, data: []const u8, allocator: ?&mem.Allocator)
500500
501/// On success, caller owns returned buffer.501/// On success, caller owns returned buffer.
502pub fn readFileAlloc(path: []const u8, allocator: &mem.Allocator) -> %[]u8 {502pub fn readFileAlloc(path: []const u8, allocator: &mem.Allocator) -> %[]u8 {
503 return readFileAllocExtra(path, allocator, 0);
504}
505/// On success, caller owns returned buffer.
506/// Allocates extra_len extra bytes at the end of the file buffer, which are uninitialized.
507pub fn readFileAllocExtra(path: []const u8, allocator: &mem.Allocator, extra_len: usize) -> %[]u8 {
503 var file = %return File.openRead(path, allocator);508 var file = %return File.openRead(path, allocator);
504 defer file.close();509 defer file.close();
505510
506 const size = %return file.getEndPos();511 const size = %return file.getEndPos();
507 const buf = %return allocator.alloc(u8, size);512 const buf = %return allocator.alloc(u8, size + extra_len);
508 %defer allocator.free(buf);513 %defer allocator.free(buf);
509514
510 var adapter = FileInStream.init(&file);515 var adapter = FileInStream.init(&file);
std/unicode.zig created+169
...@@ -0,0 +1,169 @@
1const std = @import("./index.zig");
2
3error Utf8InvalidStartByte;
4
5/// Given the first byte of a UTF-8 codepoint,
6/// returns a number 1-4 indicating the total length of the codepoint in bytes.
7/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.
8pub fn utf8ByteSequenceLength(first_byte: u8) -> %u3 {
9 if (first_byte < 0b10000000) return u3(1);
10 if (first_byte & 0b11100000 == 0b11000000) return u3(2);
11 if (first_byte & 0b11110000 == 0b11100000) return u3(3);
12 if (first_byte & 0b11111000 == 0b11110000) return u3(4);
13 return error.Utf8InvalidStartByte;
14}
15
16error Utf8OverlongEncoding;
17error Utf8ExpectedContinuation;
18error Utf8EncodesSurrogateHalf;
19error Utf8CodepointTooLarge;
20
21/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
22/// bytes.len must be equal to %%utf8ByteSequenceLength(bytes[0]).
23/// If you already know the length at comptime, you can call one of
24/// utf8Decode2,utf8Decode3,utf8Decode4 directly instead of this function.
25pub fn utf8Decode(bytes: []const u8) -> %u32 {
26 return switch (bytes.len) {
27 1 => u32(bytes[0]),
28 2 => utf8Decode2(bytes),
29 3 => utf8Decode3(bytes),
30 4 => utf8Decode4(bytes),
31 else => unreachable,
32 };
33}
34pub fn utf8Decode2(bytes: []const u8) -> %u32 {
35 std.debug.assert(bytes.len == 2);
36 std.debug.assert(bytes[0] & 0b11100000 == 0b11000000);
37 var value: u32 = bytes[0] & 0b00011111;
38
39 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
40 value <<= 6;
41 value |= bytes[1] & 0b00111111;
42
43 if (value < 0x80) return error.Utf8OverlongEncoding;
44
45 return value;
46}
47pub fn utf8Decode3(bytes: []const u8) -> %u32 {
48 std.debug.assert(bytes.len == 3);
49 std.debug.assert(bytes[0] & 0b11110000 == 0b11100000);
50 var value: u32 = bytes[0] & 0b00001111;
51
52 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
53 value <<= 6;
54 value |= bytes[1] & 0b00111111;
55
56 if (bytes[2] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
57 value <<= 6;
58 value |= bytes[2] & 0b00111111;
59
60 if (value < 0x800) return error.Utf8OverlongEncoding;
61 if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
62
63 return value;
64}
65pub fn utf8Decode4(bytes: []const u8) -> %u32 {
66 std.debug.assert(bytes.len == 4);
67 std.debug.assert(bytes[0] & 0b11111000 == 0b11110000);
68 var value: u32 = bytes[0] & 0b00000111;
69
70 if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
71 value <<= 6;
72 value |= bytes[1] & 0b00111111;
73
74 if (bytes[2] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
75 value <<= 6;
76 value |= bytes[2] & 0b00111111;
77
78 if (bytes[3] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
79 value <<= 6;
80 value |= bytes[3] & 0b00111111;
81
82 if (value < 0x10000) return error.Utf8OverlongEncoding;
83 if (value > 0x10FFFF) return error.Utf8CodepointTooLarge;
84
85 return value;
86}
87
88error UnexpectedEof;
89test "valid utf8" {
90 testValid("\x00", 0x0);
91 testValid("\x20", 0x20);
92 testValid("\x7f", 0x7f);
93 testValid("\xc2\x80", 0x80);
94 testValid("\xdf\xbf", 0x7ff);
95 testValid("\xe0\xa0\x80", 0x800);
96 testValid("\xe1\x80\x80", 0x1000);
97 testValid("\xef\xbf\xbf", 0xffff);
98 testValid("\xf0\x90\x80\x80", 0x10000);
99 testValid("\xf1\x80\x80\x80", 0x40000);
100 testValid("\xf3\xbf\xbf\xbf", 0xfffff);
101 testValid("\xf4\x8f\xbf\xbf", 0x10ffff);
102}
103
104test "invalid utf8 continuation bytes" {
105 // unexpected continuation
106 testError("\x80", error.Utf8InvalidStartByte);
107 testError("\xbf", error.Utf8InvalidStartByte);
108 // too many leading 1's
109 testError("\xf8", error.Utf8InvalidStartByte);
110 testError("\xff", error.Utf8InvalidStartByte);
111 // expected continuation for 2 byte sequences
112 testError("\xc2", error.UnexpectedEof);
113 testError("\xc2\x00", error.Utf8ExpectedContinuation);
114 testError("\xc2\xc0", error.Utf8ExpectedContinuation);
115 // expected continuation for 3 byte sequences
116 testError("\xe0", error.UnexpectedEof);
117 testError("\xe0\x00", error.UnexpectedEof);
118 testError("\xe0\xc0", error.UnexpectedEof);
119 testError("\xe0\xa0", error.UnexpectedEof);
120 testError("\xe0\xa0\x00", error.Utf8ExpectedContinuation);
121 testError("\xe0\xa0\xc0", error.Utf8ExpectedContinuation);
122 // expected continuation for 4 byte sequences
123 testError("\xf0", error.UnexpectedEof);
124 testError("\xf0\x00", error.UnexpectedEof);
125 testError("\xf0\xc0", error.UnexpectedEof);
126 testError("\xf0\x90\x00", error.UnexpectedEof);
127 testError("\xf0\x90\xc0", error.UnexpectedEof);
128 testError("\xf0\x90\x80\x00", error.Utf8ExpectedContinuation);
129 testError("\xf0\x90\x80\xc0", error.Utf8ExpectedContinuation);
130}
131
132test "overlong utf8 codepoint" {
133 testError("\xc0\x80", error.Utf8OverlongEncoding);
134 testError("\xc1\xbf", error.Utf8OverlongEncoding);
135 testError("\xe0\x80\x80", error.Utf8OverlongEncoding);
136 testError("\xe0\x9f\xbf", error.Utf8OverlongEncoding);
137 testError("\xf0\x80\x80\x80", error.Utf8OverlongEncoding);
138 testError("\xf0\x8f\xbf\xbf", error.Utf8OverlongEncoding);
139}
140
141test "misc invalid utf8" {
142 // codepoint out of bounds
143 testError("\xf4\x90\x80\x80", error.Utf8CodepointTooLarge);
144 testError("\xf7\xbf\xbf\xbf", error.Utf8CodepointTooLarge);
145 // surrogate halves
146 testValid("\xed\x9f\xbf", 0xd7ff);
147 testError("\xed\xa0\x80", error.Utf8EncodesSurrogateHalf);
148 testError("\xed\xbf\xbf", error.Utf8EncodesSurrogateHalf);
149 testValid("\xee\x80\x80", 0xe000);
150}
151
152fn testError(bytes: []const u8, expected_err: error) {
153 if (testDecode(bytes)) |_| {
154 unreachable;
155 } else |err| {
156 std.debug.assert(err == expected_err);
157 }
158}
159
160fn testValid(bytes: []const u8, expected_codepoint: u32) {
161 std.debug.assert(%%testDecode(bytes) == expected_codepoint);
162}
163
164fn testDecode(bytes: []const u8) -> %u32 {
165 const length = %return utf8ByteSequenceLength(bytes[0]);
166 if (bytes.len < length) return error.UnexpectedEof;
167 std.debug.assert(bytes.len == length);
168 return utf8Decode(bytes);
169}