authorgravatar for Braedonww@gmail.comBraedon <Braedonww@gmail.com> 2018-04-25 14:59:03+10:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2018-04-25 14:59:03+10:00
logf6cbe9a9cca3718501272cea177ab1ad48852ffe
tree5f7c6d50aabe45abe8a97819852f3dbff522b455
parent27cbb44993389ae042a03266743379c0f15a523e
signature Signed by PGP key 4AEE18F83AFDEB23

Utf8 Encode

1 files changed, 91 insertions(+), 0 deletions(-)

std/unicode.zig+91
...@@ -1,6 +1,17 @@...@@ -1,6 +1,17 @@
1const std = @import("./index.zig");1const std = @import("./index.zig");
2const debug = std.debug;2const debug = std.debug;
33
4// Given a Utf8-Codepoint returns how many (1-4)
5// bytes there are if represented as an array of bytes.
6pub fn utf8CodepointSequenceLength(c: u32) !u3 {
7 if (c < 0x80) return u3(1);
8 if (c < 0x800) return u3(2);
9 if (c -% 0xd800 < 0x800) return error.InvalidCodepoint;
10 if (c < 0x10000) return u3(3);
11 if (c < 0x110000) return u3(4);
12 return error.CodepointTooLarge;
13}
14
4/// Given the first byte of a UTF-8 codepoint,15/// Given the first byte of a UTF-8 codepoint,
5/// returns a number 1-4 indicating the total length of the codepoint in bytes.16/// returns a number 1-4 indicating the total length of the codepoint in bytes.
6/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.17/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.
...@@ -12,6 +23,47 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {...@@ -12,6 +23,47 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
12 return error.Utf8InvalidStartByte;23 return error.Utf8InvalidStartByte;
13}24}
1425
26/// Encodes a code point back into utf8
27/// c: the code point
28/// out: the out buffer to write to
29/// Notes: out has to have a len big enough for the bytes
30/// however this limit is dependent on the code point
31/// but giving it a minimum of 4 will ensure it will work
32/// for all code points.
33/// Errors: Will return an error if the code point is invalid.
34pub fn utf8Encode(c: u32, out: []u8) !u3 {
35 if (utf8CodepointSequenceLength(c)) |length| {
36 debug.assert(out.len >= length);
37 switch (length) {
38 1 => out[0] = u8(c), // Can just do 0 + codepoint for initial range
39 2 => {
40 // 64 to convert the codepoint into its segments
41 out[0] = u8(0b11000000 + c / 64);
42 out[1] = u8(0b10000000 + c % 64);
43 },
44 3 => {
45 // Again using 64 as a conversion into their segments
46 // But using C / 4096 (64 * 64) as the first, (C/64) % 64 as the second, and just C % 64 as the last
47 out[0] = u8(0b11100000 + c / 4096);
48 out[1] = u8(0b10000000 + (c / 64) % 64);
49 out[2] = u8(0b10000000 + c % 64);
50 },
51 4 => {
52 // Same as previously but now its C / 64^3 (262144), (C / 4096) % 64, (C / 64) % 64 and C % 64
53 out[0] = u8(0b11110000 + c / 262144);
54 out[1] = u8(0b10000000 + (c / 4096) % 64);
55 out[2] = u8(0b10000000 + (c / 64) % 64);
56 out[3] = u8(0b10000000 + c % 64);
57 },
58 else => unreachable,
59 }
60
61 return length;
62 } else |err| {
63 return err;
64 }
65}
66
15/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.67/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
16/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.68/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.
17/// If you already know the length at comptime, you can call one of69/// If you already know the length at comptime, you can call one of
...@@ -25,6 +77,7 @@ pub fn utf8Decode(bytes: []const u8) !u32 {...@@ -25,6 +77,7 @@ pub fn utf8Decode(bytes: []const u8) !u32 {
25 else => unreachable,77 else => unreachable,
26 };78 };
27}79}
80
28pub fn utf8Decode2(bytes: []const u8) !u32 {81pub fn utf8Decode2(bytes: []const u8) !u32 {
29 debug.assert(bytes.len == 2);82 debug.assert(bytes.len == 2);
30 debug.assert(bytes[0] & 0b11100000 == 0b11000000);83 debug.assert(bytes[0] & 0b11100000 == 0b11000000);
...@@ -38,6 +91,7 @@ pub fn utf8Decode2(bytes: []const u8) !u32 {...@@ -38,6 +91,7 @@ pub fn utf8Decode2(bytes: []const u8) !u32 {
3891
39 return value;92 return value;
40}93}
94
41pub fn utf8Decode3(bytes: []const u8) !u32 {95pub fn utf8Decode3(bytes: []const u8) !u32 {
42 debug.assert(bytes.len == 3);96 debug.assert(bytes.len == 3);
43 debug.assert(bytes[0] & 0b11110000 == 0b11100000);97 debug.assert(bytes[0] & 0b11110000 == 0b11100000);
...@@ -56,6 +110,7 @@ pub fn utf8Decode3(bytes: []const u8) !u32 {...@@ -56,6 +110,7 @@ pub fn utf8Decode3(bytes: []const u8) !u32 {
56110
57 return value;111 return value;
58}112}
113
59pub fn utf8Decode4(bytes: []const u8) !u32 {114pub fn utf8Decode4(bytes: []const u8) !u32 {
60 debug.assert(bytes.len == 4);115 debug.assert(bytes.len == 4);
61 debug.assert(bytes[0] & 0b11111000 == 0b11110000);116 debug.assert(bytes[0] & 0b11111000 == 0b11110000);
...@@ -170,6 +225,42 @@ const Utf8Iterator = struct {...@@ -170,6 +225,42 @@ const Utf8Iterator = struct {
170 }225 }
171};226};
172227
228test "utf8 encode" {
229 // A few taken from wikipedia a few taken elsewhere
230 var array: [4]u8 = undefined;
231 debug.assert((try utf8Encode(try utf8Decode("€"), array[0..])) == 3);
232 debug.assert(array[0] == 0b11100010);
233 debug.assert(array[1] == 0b10000010);
234 debug.assert(array[2] == 0b10101100);
235
236 debug.assert((try utf8Encode(try utf8Decode("$"), array[0..])) == 1);
237 debug.assert(array[0] == 0b00100100);
238
239 debug.assert((try utf8Encode(try utf8Decode("¢"), array[0..])) == 2);
240 debug.assert(array[0] == 0b11000010);
241 debug.assert(array[1] == 0b10100010);
242
243 debug.assert((try utf8Encode(try utf8Decode("𐍈"), array[0..])) == 4);
244 debug.assert(array[0] == 0b11110000);
245 debug.assert(array[1] == 0b10010000);
246 debug.assert(array[2] == 0b10001101);
247 debug.assert(array[3] == 0b10001000);
248}
249
250test "utf8 encode error" {
251 var array: [4]u8 = undefined;
252 testErrorEncode(0xFFFFFF, array[0..], error.CodepointTooLarge);
253 testErrorEncode(0xd900, array[0..], error.InvalidCodepoint);
254}
255
256fn testErrorEncode(codePoint: u32, array: []u8, expectedErr: error) void {
257 if (utf8Encode(codePoint, array)) |_| {
258 unreachable;
259 } else |err| {
260 assert(err == expectedErr);
261 }
262}
263
173test "utf8 iterator on ascii" {264test "utf8 iterator on ascii" {
174 const s = Utf8View.initComptime("abc");265 const s = Utf8View.initComptime("abc");
175266