authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2018-04-29 16:57:29-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2018-04-29 16:57:29-04:00
log8c567d84f1f14e06286897fe1b2408a1d2fd7d76
tree6cfd3acaa871f88ceb27e502d92398425c9239b7
parentad4ee47d9fec15945d445f637987d487405e7b22
parent07af6559d8a883dcb21ff99cddb4a836d2bb66bd
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #954 from BraedonWooding/patch-2

Utf8 Encoding from Codepoint to Bytes

1 files changed, 91 insertions(+), 0 deletions(-)

std/unicode.zig+91
......@@ -1,6 +1,17 @@
11const std = @import("./index.zig");
22const debug = std.debug;
33
4// Given a Utf8-Codepoint returns how many (1-4)
5// bytes there are if represented as an array of bytes.
6pub fn utf8CodepointSequenceLength(c: u32) !u3 {
7 if (c < 0x80) return u3(1);
8 if (c < 0x800) return u3(2);
9 if (c -% 0xd800 < 0x800) return error.InvalidCodepoint;
10 if (c < 0x10000) return u3(3);
11 if (c < 0x110000) return u3(4);
12 return error.CodepointTooLarge;
13}
14
415/// Given the first byte of a UTF-8 codepoint,
516/// returns a number 1-4 indicating the total length of the codepoint in bytes.
617/// If this byte does not match the form of a UTF-8 start byte, returns Utf8InvalidStartByte.
......@@ -12,6 +23,47 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
1223 return error.Utf8InvalidStartByte;
1324}
1425
26/// Encodes a code point back into utf8
27/// c: the code point
28/// out: the out buffer to write to
29/// Notes: out has to have a len big enough for the bytes
30/// however this limit is dependent on the code point
31/// but giving it a minimum of 4 will ensure it will work
32/// for all code points.
33/// Errors: Will return an error if the code point is invalid.
34pub fn utf8Encode(c: u32, out: []u8) !u3 {
35 if (utf8CodepointSequenceLength(c)) |length| {
36 debug.assert(out.len >= length);
37 switch (length) {
38 // The pattern for each is the same
39 // - Increasing the initial shift by 6 each time
40 // - Each time after the first shorten the shifted
41 // value to a max of 0b111111 (63)
42 1 => out[0] = u8(c), // Can just do 0 + codepoint for initial range
43 2 => {
44 out[0] = u8(0b11000000 | (c >> 6));
45 out[1] = u8(0b10000000 | (c & 0b111111));
46 },
47 3 => {
48 out[0] = u8(0b11100000 | (c >> 12));
49 out[1] = u8(0b10000000 | ((c >> 6) & 0b111111));
50 out[2] = u8(0b10000000 | (c & 0b111111));
51 },
52 4 => {
53 out[0] = u8(0b11110000 | (c >> 18));
54 out[1] = u8(0b10000000 | ((c >> 12) & 0b111111));
55 out[2] = u8(0b10000000 | ((c >> 6) & 0b111111));
56 out[3] = u8(0b10000000 | (c & 0b111111));
57 },
58 else => unreachable,
59 }
60
61 return length;
62 } else |err| {
63 return err;
64 }
65}
66
1567/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
1668/// bytes.len must be equal to utf8ByteSequenceLength(bytes[0]) catch unreachable.
1769/// If you already know the length at comptime, you can call one of
......@@ -25,6 +77,7 @@ pub fn utf8Decode(bytes: []const u8) !u32 {
2577 else => unreachable,
2678 };
2779}
80
2881pub fn utf8Decode2(bytes: []const u8) !u32 {
2982 debug.assert(bytes.len == 2);
3083 debug.assert(bytes[0] & 0b11100000 == 0b11000000);
......@@ -38,6 +91,7 @@ pub fn utf8Decode2(bytes: []const u8) !u32 {
3891
3992 return value;
4093}
94
4195pub fn utf8Decode3(bytes: []const u8) !u32 {
4296 debug.assert(bytes.len == 3);
4397 debug.assert(bytes[0] & 0b11110000 == 0b11100000);
......@@ -56,6 +110,7 @@ pub fn utf8Decode3(bytes: []const u8) !u32 {
56110
57111 return value;
58112}
113
59114pub fn utf8Decode4(bytes: []const u8) !u32 {
60115 debug.assert(bytes.len == 4);
61116 debug.assert(bytes[0] & 0b11111000 == 0b11110000);
......@@ -170,6 +225,42 @@ const Utf8Iterator = struct {
170225 }
171226};
172227
228test "utf8 encode" {
229 // A few taken from wikipedia a few taken elsewhere
230 var array: [4]u8 = undefined;
231 debug.assert((try utf8Encode(try utf8Decode("€"), array[0..])) == 3);
232 debug.assert(array[0] == 0b11100010);
233 debug.assert(array[1] == 0b10000010);
234 debug.assert(array[2] == 0b10101100);
235
236 debug.assert((try utf8Encode(try utf8Decode("$"), array[0..])) == 1);
237 debug.assert(array[0] == 0b00100100);
238
239 debug.assert((try utf8Encode(try utf8Decode("¢"), array[0..])) == 2);
240 debug.assert(array[0] == 0b11000010);
241 debug.assert(array[1] == 0b10100010);
242
243 debug.assert((try utf8Encode(try utf8Decode("𐍈"), array[0..])) == 4);
244 debug.assert(array[0] == 0b11110000);
245 debug.assert(array[1] == 0b10010000);
246 debug.assert(array[2] == 0b10001101);
247 debug.assert(array[3] == 0b10001000);
248}
249
250test "utf8 encode error" {
251 var array: [4]u8 = undefined;
252 testErrorEncode(0xFFFFFF, array[0..], error.CodepointTooLarge);
253 testErrorEncode(0xd900, array[0..], error.InvalidCodepoint);
254}
255
256fn testErrorEncode(codePoint: u32, array: []u8, expectedErr: error) void {
257 if (utf8Encode(codePoint, array)) |_| {
258 unreachable;
259 } else |err| {
260 debug.assert(err == expectedErr);
261 }
262}
263
173264test "utf8 iterator on ascii" {
174265 const s = Utf8View.initComptime("abc");
175266