authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2018-04-29 17:28:11-04:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2018-04-29 17:28:11-04:00
log2387292f204c59259fc64d7c960d201e808af5a9
tree71a74da3726583f8ca982e3c041124133ff54c97
parent8c567d84f1f14e06286897fe1b2408a1d2fd7d76

move some checks around in utf8Encode logic to be more zig idiomatic

1 files changed, 37 insertions(+), 42 deletions(-)

std/unicode.zig+37-42
...@@ -1,12 +1,11 @@...@@ -1,12 +1,11 @@
1const std = @import("./index.zig");1const std = @import("./index.zig");
2const debug = std.debug;2const debug = std.debug;
33
4// Given a Utf8-Codepoint returns how many (1-4)4/// Returns how many bytes the UTF-8 representation would require
5// bytes there are if represented as an array of bytes.5/// for the given codepoint.
6pub fn utf8CodepointSequenceLength(c: u32) !u3 {6pub fn utf8CodepointSequenceLength(c: u32) !u3 {
7 if (c < 0x80) return u3(1);7 if (c < 0x80) return u3(1);
8 if (c < 0x800) return u3(2);8 if (c < 0x800) return u3(2);
9 if (c -% 0xd800 < 0x800) return error.InvalidCodepoint;
10 if (c < 0x10000) return u3(3);9 if (c < 0x10000) return u3(3);
11 if (c < 0x110000) return u3(4);10 if (c < 0x110000) return u3(4);
12 return error.CodepointTooLarge;11 return error.CodepointTooLarge;
...@@ -23,45 +22,39 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {...@@ -23,45 +22,39 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
23 return error.Utf8InvalidStartByte;22 return error.Utf8InvalidStartByte;
24}23}
2524
26/// Encodes a code point back into utf825/// Encodes the given codepoint into a UTF-8 byte sequence.
27/// c: the code point26/// c: the codepoint.
28/// out: the out buffer to write to27/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).
29/// Notes: out has to have a len big enough for the bytes28/// Errors: if c cannot be encoded in UTF-8.
30/// however this limit is dependent on the code point29/// Returns: the number of bytes written to out.
31/// but giving it a minimum of 4 will ensure it will work
32/// for all code points.
33/// Errors: Will return an error if the code point is invalid.
34pub fn utf8Encode(c: u32, out: []u8) !u3 {30pub fn utf8Encode(c: u32, out: []u8) !u3 {
35 if (utf8CodepointSequenceLength(c)) |length| {31 const length = try utf8CodepointSequenceLength(c);
36 debug.assert(out.len >= length);32 debug.assert(out.len >= length);
37 switch (length) {33 switch (length) {
38 // The pattern for each is the same34 // The pattern for each is the same
39 // - Increasing the initial shift by 6 each time35 // - Increasing the initial shift by 6 each time
40 // - Each time after the first shorten the shifted36 // - Each time after the first shorten the shifted
41 // value to a max of 0b111111 (63)37 // value to a max of 0b111111 (63)
42 1 => out[0] = u8(c), // Can just do 0 + codepoint for initial range38 1 => out[0] = u8(c), // Can just do 0 + codepoint for initial range
43 2 => {39 2 => {
44 out[0] = u8(0b11000000 | (c >> 6));40 out[0] = u8(0b11000000 | (c >> 6));
45 out[1] = u8(0b10000000 | (c & 0b111111));41 out[1] = u8(0b10000000 | (c & 0b111111));
46 },42 },
47 3 => {43 3 => {
48 out[0] = u8(0b11100000 | (c >> 12));44 if (0xd800 <= c and c <= 0xdfff) return error.Utf8CannotEncodeSurrogateHalf;
49 out[1] = u8(0b10000000 | ((c >> 6) & 0b111111));45 out[0] = u8(0b11100000 | (c >> 12));
50 out[2] = u8(0b10000000 | (c & 0b111111));46 out[1] = u8(0b10000000 | ((c >> 6) & 0b111111));
51 },47 out[2] = u8(0b10000000 | (c & 0b111111));
52 4 => {48 },
53 out[0] = u8(0b11110000 | (c >> 18));49 4 => {
54 out[1] = u8(0b10000000 | ((c >> 12) & 0b111111));50 out[0] = u8(0b11110000 | (c >> 18));
55 out[2] = u8(0b10000000 | ((c >> 6) & 0b111111));51 out[1] = u8(0b10000000 | ((c >> 12) & 0b111111));
56 out[3] = u8(0b10000000 | (c & 0b111111));52 out[2] = u8(0b10000000 | ((c >> 6) & 0b111111));
57 },53 out[3] = u8(0b10000000 | (c & 0b111111));
58 else => unreachable,54 },
59 }55 else => unreachable,
60
61 return length;
62 } else |err| {
63 return err;
64 }56 }
57 return length;
65}58}
6659
67/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.60/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
...@@ -249,8 +242,10 @@ test "utf8 encode" {...@@ -249,8 +242,10 @@ test "utf8 encode" {
249242
250test "utf8 encode error" {243test "utf8 encode error" {
251 var array: [4]u8 = undefined;244 var array: [4]u8 = undefined;
252 testErrorEncode(0xFFFFFF, array[0..], error.CodepointTooLarge);245 testErrorEncode(0xd800, array[0..], error.Utf8CannotEncodeSurrogateHalf);
253 testErrorEncode(0xd900, array[0..], error.InvalidCodepoint);246 testErrorEncode(0xdfff, array[0..], error.Utf8CannotEncodeSurrogateHalf);
247 testErrorEncode(0x110000, array[0..], error.CodepointTooLarge);
248 testErrorEncode(0xffffffff, array[0..], error.CodepointTooLarge);
254}249}
255250
256fn testErrorEncode(codePoint: u32, array: []u8, expectedErr: error) void {251fn testErrorEncode(codePoint: u32, array: []u8, expectedErr: error) void {