authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-02-03 14:23:51-08:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-02-24 14:04:58-08:00
log4ee1309a8d261360c5b80a9533535231b60780f5
tree5d278233be6e57e07631ce02d5d93cd95be636fe
parentb344ff01d380d85256929b6be2428d3c022a8580

std.unicode: Refactor and add WTF-16/WTF-8 functions

Renamed functions for consistent `Le` capitalization and conventions: - utf16leToUtf8Alloc -> utf16LeToUtf8Alloc - utf16leToUtf8AllocZ -> utf16LeToUtf8AllocZ - utf16leToUtf8 -> utf16LeToUtf8 - utf8ToUtf16LeWithNull -> utf8ToUtf16LeAllocZ - fmtUtf16le -> fmtUtf16Le New UTF related functions: - utf16LeToUtf8ArrayList - utf8ToUtf16LeArrayList - utf8ToUtf16LeAlloc - isSurrogateCodepoint (the ArrayList functions are mostly to allow the Alloc and AllocZ to share an implementation) New WTF related functions/structs: - wtf8Encode - wtf8Decode - wtf8ValidateSlice - Wtf8View - Wtf8Iterator - wtf16LeToWtf8ArrayList - wtf16LeToWtf8Alloc - wtf16LeToWtf8AllocZ - wtf16LeToWtf8 - wtf8ToWtf16LeArrayList - wtf8ToWtf16LeAlloc - wtf8ToWtf16LeAllocZ - wtf8ToWtf16Le - wtf8ToUtf8Lossy - wtf8ToUtf8LossyAlloc - wtf8ToUtf8LossyAllocZ - Wtf16LeIterator

1 files changed, 760 insertions(+), 98 deletions(-)

lib/std/unicode.zig+760-98
...@@ -39,7 +39,16 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {...@@ -39,7 +39,16 @@ pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
39/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).39/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).
40/// Errors: if c cannot be encoded in UTF-8.40/// Errors: if c cannot be encoded in UTF-8.
41/// Returns: the number of bytes written to out.41/// Returns: the number of bytes written to out.
42pub fn utf8Encode(c: u21, out: []u8) !u3 {42pub fn utf8Encode(c: u21, out: []u8) error{ Utf8CannotEncodeSurrogateHalf, CodepointTooLarge }!u3 {
43 return utf8EncodeImpl(c, out, .cannot_encode_surrogate_half);
44}
45
46const Surrogates = enum {
47 cannot_encode_surrogate_half,
48 can_encode_surrogate_half,
49};
50
51fn utf8EncodeImpl(c: u21, out: []u8, comptime surrogates: Surrogates) !u3 {
43 const length = try utf8CodepointSequenceLength(c);52 const length = try utf8CodepointSequenceLength(c);
44 assert(out.len >= length);53 assert(out.len >= length);
45 switch (length) {54 switch (length) {
...@@ -53,7 +62,9 @@ pub fn utf8Encode(c: u21, out: []u8) !u3 {...@@ -53,7 +62,9 @@ pub fn utf8Encode(c: u21, out: []u8) !u3 {
53 out[1] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));62 out[1] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));
54 },63 },
55 3 => {64 3 => {
56 if (0xd800 <= c and c <= 0xdfff) return error.Utf8CannotEncodeSurrogateHalf;65 if (surrogates == .cannot_encode_surrogate_half and isSurrogateCodepoint(c)) {
66 return error.Utf8CannotEncodeSurrogateHalf;
67 }
57 out[0] = @as(u8, @intCast(0b11100000 | (c >> 12)));68 out[0] = @as(u8, @intCast(0b11100000 | (c >> 12)));
58 out[1] = @as(u8, @intCast(0b10000000 | ((c >> 6) & 0b111111)));69 out[1] = @as(u8, @intCast(0b10000000 | ((c >> 6) & 0b111111)));
59 out[2] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));70 out[2] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));
...@@ -116,12 +127,22 @@ pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {...@@ -116,12 +127,22 @@ pub fn utf8Decode2(bytes: []const u8) Utf8Decode2Error!u21 {
116 return value;127 return value;
117}128}
118129
119const Utf8Decode3Error = error{130const Utf8Decode3Error = Utf8Decode3AllowSurrogateHalfError || error{
120 Utf8ExpectedContinuation,
121 Utf8OverlongEncoding,
122 Utf8EncodesSurrogateHalf,131 Utf8EncodesSurrogateHalf,
123};132};
124pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {133pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {
134 const value = try utf8Decode3AllowSurrogateHalf(bytes);
135
136 if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
137
138 return value;
139}
140
141const Utf8Decode3AllowSurrogateHalfError = error{
142 Utf8ExpectedContinuation,
143 Utf8OverlongEncoding,
144};
145pub fn utf8Decode3AllowSurrogateHalf(bytes: []const u8) Utf8Decode3AllowSurrogateHalfError!u21 {
125 assert(bytes.len == 3);146 assert(bytes.len == 3);
126 assert(bytes[0] & 0b11110000 == 0b11100000);147 assert(bytes[0] & 0b11110000 == 0b11100000);
127 var value: u21 = bytes[0] & 0b00001111;148 var value: u21 = bytes[0] & 0b00001111;
...@@ -135,7 +156,6 @@ pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {...@@ -135,7 +156,6 @@ pub fn utf8Decode3(bytes: []const u8) Utf8Decode3Error!u21 {
135 value |= bytes[2] & 0b00111111;156 value |= bytes[2] & 0b00111111;
136157
137 if (value < 0x800) return error.Utf8OverlongEncoding;158 if (value < 0x800) return error.Utf8OverlongEncoding;
138 if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
139159
140 return value;160 return value;
141}161}
...@@ -213,6 +233,10 @@ pub fn utf8CountCodepoints(s: []const u8) !usize {...@@ -213,6 +233,10 @@ pub fn utf8CountCodepoints(s: []const u8) !usize {
213233
214/// Returns true if the input consists entirely of UTF-8 codepoints234/// Returns true if the input consists entirely of UTF-8 codepoints
215pub fn utf8ValidateSlice(input: []const u8) bool {235pub fn utf8ValidateSlice(input: []const u8) bool {
236 return utf8ValidateSliceImpl(input, .cannot_encode_surrogate_half);
237}
238
239fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {
216 var remaining = input;240 var remaining = input;
217241
218 const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;242 const chunk_len = std.simd.suggestVectorLength(u8) orelse 1;
...@@ -240,9 +264,15 @@ pub fn utf8ValidateSlice(input: []const u8) bool {...@@ -240,9 +264,15 @@ pub fn utf8ValidateSlice(input: []const u8) bool {
240 const xx = 0xF1; // invalid: size 1264 const xx = 0xF1; // invalid: size 1
241 const as = 0xF0; // ASCII: size 1265 const as = 0xF0; // ASCII: size 1
242 const s1 = 0x02; // accept 0, size 2266 const s1 = 0x02; // accept 0, size 2
243 const s2 = 0x13; // accept 1, size 3267 const s2 = switch (surrogates) {
268 .cannot_encode_surrogate_half => 0x13, // accept 1, size 3
269 .can_encode_surrogate_half => 0x03, // accept 0, size 3
270 };
244 const s3 = 0x03; // accept 0, size 3271 const s3 = 0x03; // accept 0, size 3
245 const s4 = 0x23; // accept 2, size 3272 const s4 = switch (surrogates) {
273 .cannot_encode_surrogate_half => 0x23, // accept 2, size 3
274 .can_encode_surrogate_half => 0x03, // accept 0, size 3
275 };
246 const s5 = 0x34; // accept 3, size 4276 const s5 = 0x34; // accept 3, size 4
247 const s6 = 0x04; // accept 0, size 4277 const s6 = 0x04; // accept 0, size 4
248 const s7 = 0x44; // accept 4, size 4278 const s7 = 0x44; // accept 4, size 4
...@@ -770,11 +800,9 @@ fn testDecode(bytes: []const u8) !u21 {...@@ -770,11 +800,9 @@ fn testDecode(bytes: []const u8) !u21 {
770 return utf8Decode(bytes);800 return utf8Decode(bytes);
771}801}
772802
773/// Caller must free returned memory.803fn utf16LeToUtf8ArrayListImpl(array_list: *std.ArrayList(u8), utf16le: []const u16, comptime surrogates: Surrogates) !void {
774pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8 {
775 // optimistically guess that it will all be ascii.804 // optimistically guess that it will all be ascii.
776 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);805 try array_list.ensureTotalCapacityPrecise(utf16le.len);
777 errdefer result.deinit();
778806
779 var remaining = utf16le;807 var remaining = utf16le;
780 if (builtin.zig_backend != .stage2_x86_64) {808 if (builtin.zig_backend != .stage2_x86_64) {
...@@ -796,68 +824,69 @@ pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8...@@ -796,68 +824,69 @@ pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8
796 // We allocated enough space to encode every UTF-16 code unit824 // We allocated enough space to encode every UTF-16 code unit
797 // as ASCII, so if the entire string is ASCII then we are825 // as ASCII, so if the entire string is ASCII then we are
798 // guaranteed to have enough space allocated826 // guaranteed to have enough space allocated
799 result.appendSliceAssumeCapacity(&ascii_bytes);827 array_list.appendSliceAssumeCapacity(&ascii_bytes);
800 remaining = remaining[chunk_len..];828 remaining = remaining[chunk_len..];
801 }829 }
802 }830 }
803831
804 var out_index: usize = result.items.len;832 var out_index: usize = array_list.items.len;
805 var it = Utf16LeIterator.init(remaining);833 switch (surrogates) {
806 while (try it.nextCodepoint()) |codepoint| {834 .cannot_encode_surrogate_half => {
807 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;835 var it = Utf16LeIterator.init(remaining);
808 try result.resize(result.items.len + utf8_len);836 while (try it.nextCodepoint()) |codepoint| {
809 assert((utf8Encode(codepoint, result.items[out_index..]) catch unreachable) == utf8_len);837 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
810 out_index += utf8_len;838 try array_list.resize(array_list.items.len + utf8_len);
839 assert((utf8Encode(codepoint, array_list.items[out_index..]) catch unreachable) == utf8_len);
840 out_index += utf8_len;
841 }
842 },
843 .can_encode_surrogate_half => {
844 var it = Wtf16LeIterator.init(remaining);
845 while (it.nextCodepoint()) |codepoint| {
846 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
847 try array_list.resize(array_list.items.len + utf8_len);
848 assert((wtf8Encode(codepoint, array_list.items[out_index..]) catch unreachable) == utf8_len);
849 out_index += utf8_len;
850 }
851 },
811 }852 }
853}
854
855pub fn utf16LeToUtf8ArrayList(array_list: *std.ArrayList(u8), utf16le: []const u16) !void {
856 return utf16LeToUtf8ArrayListImpl(array_list, utf16le, .cannot_encode_surrogate_half);
857}
858
859/// Deprecated; renamed to utf16LeToUtf8Alloc
860pub const utf16leToUtf8Alloc = utf16LeToUtf8Alloc;
861
862/// Caller must free returned memory.
863pub fn utf16LeToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8 {
864 // optimistically guess that it will all be ascii.
865 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);
866 errdefer result.deinit();
867
868 try utf16LeToUtf8ArrayList(&result, utf16le);
812869
813 return result.toOwnedSlice();870 return result.toOwnedSlice();
814}871}
815872
873/// Deprecated; renamed to utf16LeToUtf8AllocZ
874pub const utf16leToUtf8AllocZ = utf16LeToUtf8AllocZ;
875
816/// Caller must free returned memory.876/// Caller must free returned memory.
817pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0]u8 {877pub fn utf16LeToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0]u8 {
818 // optimistically guess that it will all be ascii (and allocate space for the null terminator)878 // optimistically guess that it will all be ascii (and allocate space for the null terminator)
819 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);879 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);
820 errdefer result.deinit();880 errdefer result.deinit();
821881
822 var remaining = utf16le;882 try utf16LeToUtf8ArrayList(&result, utf16le);
823 if (builtin.zig_backend != .stage2_x86_64) {
824 const chunk_len = std.simd.suggestVectorLength(u16) orelse 1;
825 const Chunk = @Vector(chunk_len, u16);
826883
827 // Fast path. Check for and encode ASCII characters at the start of the input.
828 while (remaining.len >= chunk_len) {
829 const chunk: Chunk = remaining[0..chunk_len].*;
830 const mask: Chunk = @splat(std.mem.nativeToLittle(u16, 0x7F));
831 if (@reduce(.Or, chunk | mask != mask)) {
832 // found a non ASCII code unit
833 break;
834 }
835 const chunk_byte_len = chunk_len * 2;
836 const chunk_bytes: @Vector(chunk_byte_len, u8) = (std.mem.sliceAsBytes(remaining)[0..chunk_byte_len]).*;
837 const deinterlaced_bytes = std.simd.deinterlace(2, chunk_bytes);
838 const ascii_bytes: [chunk_len]u8 = deinterlaced_bytes[0];
839 // We allocated enough space to encode every UTF-16 code unit
840 // as ASCII, so if the entire string is ASCII then we are
841 // guaranteed to have enough space allocated
842 result.appendSliceAssumeCapacity(&ascii_bytes);
843 remaining = remaining[chunk_len..];
844 }
845 }
846
847 var out_index = result.items.len;
848 var it = Utf16LeIterator.init(remaining);
849 while (try it.nextCodepoint()) |codepoint| {
850 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
851 try result.resize(result.items.len + utf8_len);
852 assert((utf8Encode(codepoint, result.items[out_index..]) catch unreachable) == utf8_len);
853 out_index += utf8_len;
854 }
855 return result.toOwnedSliceSentinel(0);884 return result.toOwnedSliceSentinel(0);
856}885}
857886
858/// Asserts that the output buffer is big enough.887/// Asserts that the output buffer is big enough.
859/// Returns end byte index into utf8.888/// Returns end byte index into utf8.
860pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize {889fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surrogates) !usize {
861 var end_index: usize = 0;890 var end_index: usize = 0;
862891
863 var remaining = utf16le;892 var remaining = utf16le;
...@@ -883,30 +912,56 @@ pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize {...@@ -883,30 +912,56 @@ pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize {
883 }912 }
884 }913 }
885914
886 var it = Utf16LeIterator.init(remaining);915 switch (surrogates) {
887 while (try it.nextCodepoint()) |codepoint| {916 .cannot_encode_surrogate_half => {
888 end_index += try utf8Encode(codepoint, utf8[end_index..]);917 var it = Utf16LeIterator.init(remaining);
918 while (try it.nextCodepoint()) |codepoint| {
919 end_index += utf8Encode(codepoint, utf8[end_index..]) catch |err| switch (err) {
920 // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
921 // which is within the valid codepoint range.
922 error.CodepointTooLarge => unreachable,
923 else => |e| return e,
924 };
925 }
926 },
927 .can_encode_surrogate_half => {
928 var it = Wtf16LeIterator.init(remaining);
929 while (it.nextCodepoint()) |codepoint| {
930 end_index += wtf8Encode(codepoint, utf8[end_index..]) catch |err| switch (err) {
931 // The maximum possible codepoint encoded by UTF-16 is U+10FFFF,
932 // which is within the valid codepoint range.
933 error.CodepointTooLarge => unreachable,
934 };
935 }
936 },
889 }937 }
890 return end_index;938 return end_index;
891}939}
892940
893test "utf16leToUtf8" {941/// Deprecated; renamed to utf16LeToUtf8
942pub const utf16leToUtf8 = utf16LeToUtf8;
943
944pub fn utf16LeToUtf8(utf8: []u8, utf16le: []const u16) !usize {
945 return utf16LeToUtf8Impl(utf8, utf16le, .cannot_encode_surrogate_half);
946}
947
948test utf16LeToUtf8 {
894 var utf16le: [2]u16 = undefined;949 var utf16le: [2]u16 = undefined;
895 const utf16le_as_bytes = mem.sliceAsBytes(utf16le[0..]);950 const utf16le_as_bytes = mem.sliceAsBytes(utf16le[0..]);
896951
897 {952 {
898 mem.writeInt(u16, utf16le_as_bytes[0..2], 'A', .little);953 mem.writeInt(u16, utf16le_as_bytes[0..2], 'A', .little);
899 mem.writeInt(u16, utf16le_as_bytes[2..4], 'a', .little);954 mem.writeInt(u16, utf16le_as_bytes[2..4], 'a', .little);
900 const utf8 = try utf16leToUtf8Alloc(std.testing.allocator, &utf16le);955 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
901 defer std.testing.allocator.free(utf8);956 defer testing.allocator.free(utf8);
902 try testing.expect(mem.eql(u8, utf8, "Aa"));957 try testing.expect(mem.eql(u8, utf8, "Aa"));
903 }958 }
904959
905 {960 {
906 mem.writeInt(u16, utf16le_as_bytes[0..2], 0x80, .little);961 mem.writeInt(u16, utf16le_as_bytes[0..2], 0x80, .little);
907 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xffff, .little);962 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xffff, .little);
908 const utf8 = try utf16leToUtf8Alloc(std.testing.allocator, &utf16le);963 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
909 defer std.testing.allocator.free(utf8);964 defer testing.allocator.free(utf8);
910 try testing.expect(mem.eql(u8, utf8, "\xc2\x80" ++ "\xef\xbf\xbf"));965 try testing.expect(mem.eql(u8, utf8, "\xc2\x80" ++ "\xef\xbf\xbf"));
911 }966 }
912967
...@@ -914,8 +969,8 @@ test "utf16leToUtf8" {...@@ -914,8 +969,8 @@ test "utf16leToUtf8" {
914 // the values just outside the surrogate half range969 // the values just outside the surrogate half range
915 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd7ff, .little);970 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd7ff, .little);
916 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xe000, .little);971 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xe000, .little);
917 const utf8 = try utf16leToUtf8Alloc(std.testing.allocator, &utf16le);972 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
918 defer std.testing.allocator.free(utf8);973 defer testing.allocator.free(utf8);
919 try testing.expect(mem.eql(u8, utf8, "\xed\x9f\xbf" ++ "\xee\x80\x80"));974 try testing.expect(mem.eql(u8, utf8, "\xed\x9f\xbf" ++ "\xee\x80\x80"));
920 }975 }
921976
...@@ -923,8 +978,8 @@ test "utf16leToUtf8" {...@@ -923,8 +978,8 @@ test "utf16leToUtf8" {
923 // smallest surrogate pair978 // smallest surrogate pair
924 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd800, .little);979 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd800, .little);
925 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);980 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);
926 const utf8 = try utf16leToUtf8Alloc(std.testing.allocator, &utf16le);981 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
927 defer std.testing.allocator.free(utf8);982 defer testing.allocator.free(utf8);
928 try testing.expect(mem.eql(u8, utf8, "\xf0\x90\x80\x80"));983 try testing.expect(mem.eql(u8, utf8, "\xf0\x90\x80\x80"));
929 }984 }
930985
...@@ -932,31 +987,30 @@ test "utf16leToUtf8" {...@@ -932,31 +987,30 @@ test "utf16leToUtf8" {
932 // largest surrogate pair987 // largest surrogate pair
933 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);988 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);
934 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdfff, .little);989 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdfff, .little);
935 const utf8 = try utf16leToUtf8Alloc(std.testing.allocator, &utf16le);990 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
936 defer std.testing.allocator.free(utf8);991 defer testing.allocator.free(utf8);
937 try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xbf\xbf"));992 try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xbf\xbf"));
938 }993 }
939994
940 {995 {
941 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);996 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);
942 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);997 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);
943 const utf8 = try utf16leToUtf8Alloc(std.testing.allocator, &utf16le);998 const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
944 defer std.testing.allocator.free(utf8);999 defer testing.allocator.free(utf8);
945 try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xb0\x80"));1000 try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xb0\x80"));
946 }1001 }
9471002
948 {1003 {
949 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdcdc, .little);1004 mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdcdc, .little);
950 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdcdc, .little);1005 mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdcdc, .little);
951 const result = utf16leToUtf8Alloc(std.testing.allocator, &utf16le);1006 const result = utf16LeToUtf8Alloc(testing.allocator, &utf16le);
952 try std.testing.expectError(error.UnexpectedSecondSurrogateHalf, result);1007 try testing.expectError(error.UnexpectedSecondSurrogateHalf, result);
953 }1008 }
954}1009}
9551010
956pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u16 {1011fn utf8ToUtf16LeArrayListImpl(array_list: *std.ArrayList(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
957 // optimistically guess that it will not require surrogate pairs1012 // optimistically guess that it will not require surrogate pairs
958 var result = try std.ArrayList(u16).initCapacity(allocator, utf8.len + 1);1013 try array_list.ensureTotalCapacityPrecise(utf8.len);
959 errdefer result.deinit();
9601014
961 var remaining = utf8;1015 var remaining = utf8;
962 // Need support for std.simd.interlace1016 // Need support for std.simd.interlace
...@@ -974,26 +1028,54 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1...@@ -974,26 +1028,54 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1
974 }1028 }
975 const zeroes: Chunk = @splat(0);1029 const zeroes: Chunk = @splat(0);
976 const utf16_chunk: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });1030 const utf16_chunk: [chunk_len * 2]u8 align(@alignOf(u16)) = std.simd.interlace(.{ chunk, zeroes });
977 result.appendSliceAssumeCapacity(std.mem.bytesAsSlice(u16, &utf16_chunk));1031 array_list.appendSliceAssumeCapacity(std.mem.bytesAsSlice(u16, &utf16_chunk));
978 remaining = remaining[chunk_len..];1032 remaining = remaining[chunk_len..];
979 }1033 }
980 }1034 }
9811035
982 const view = try Utf8View.init(remaining);1036 const view = switch (surrogates) {
1037 .cannot_encode_surrogate_half => try Utf8View.init(remaining),
1038 .can_encode_surrogate_half => try Wtf8View.init(remaining),
1039 };
983 var it = view.iterator();1040 var it = view.iterator();
984 while (it.nextCodepoint()) |codepoint| {1041 while (it.nextCodepoint()) |codepoint| {
985 if (codepoint < 0x10000) {1042 if (codepoint < 0x10000) {
986 const short = @as(u16, @intCast(codepoint));1043 const short = @as(u16, @intCast(codepoint));
987 try result.append(mem.nativeToLittle(u16, short));1044 try array_list.append(mem.nativeToLittle(u16, short));
988 } else {1045 } else {
989 const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;1046 const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
990 const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;1047 const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
991 var out: [2]u16 = undefined;1048 var out: [2]u16 = undefined;
992 out[0] = mem.nativeToLittle(u16, high);1049 out[0] = mem.nativeToLittle(u16, high);
993 out[1] = mem.nativeToLittle(u16, low);1050 out[1] = mem.nativeToLittle(u16, low);
994 try result.appendSlice(out[0..]);1051 try array_list.appendSlice(out[0..]);
995 }1052 }
996 }1053 }
1054}
1055
1056pub fn utf8ToUtf16LeArrayList(array_list: *std.ArrayList(u16), utf8: []const u8) !void {
1057 return utf8ToUtf16LeArrayListImpl(array_list, utf8, .cannot_encode_surrogate_half);
1058}
1059
1060pub fn utf8ToUtf16LeAlloc(allocator: mem.Allocator, utf8: []const u8) ![]u16 {
1061 // optimistically guess that it will not require surrogate pairs
1062 var result = try std.ArrayList(u16).initCapacity(allocator, utf8.len);
1063 errdefer result.deinit();
1064
1065 try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
1066
1067 return result.toOwnedSlice();
1068}
1069
1070/// Deprecated; renamed to utf8ToUtf16LeAllocZ
1071pub const utf8ToUtf16LeWithNull = utf8ToUtf16LeAllocZ;
1072
1073pub fn utf8ToUtf16LeAllocZ(allocator: mem.Allocator, utf8: []const u8) ![:0]u16 {
1074 // optimistically guess that it will not require surrogate pairs
1075 var result = try std.ArrayList(u16).initCapacity(allocator, utf8.len + 1);
1076 errdefer result.deinit();
1077
1078 try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
9971079
998 return result.toOwnedSliceSentinel(0);1080 return result.toOwnedSliceSentinel(0);
999}1081}
...@@ -1001,6 +1083,10 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1...@@ -1001,6 +1083,10 @@ pub fn utf8ToUtf16LeWithNull(allocator: mem.Allocator, utf8: []const u8) ![:0]u1
1001/// Returns index of next character. If exact fit, returned index equals output slice length.1083/// Returns index of next character. If exact fit, returned index equals output slice length.
1002/// Assumes there is enough space for the output.1084/// Assumes there is enough space for the output.
1003pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {1085pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
1086 return utf8ToUtf16LeImpl(utf16le, utf8, .cannot_encode_surrogate_half);
1087}
1088
1089pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates: Surrogates) !usize {
1004 var dest_i: usize = 0;1090 var dest_i: usize = 0;
10051091
1006 var remaining = utf8;1092 var remaining = utf8;
...@@ -1029,7 +1115,10 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {...@@ -1029,7 +1115,10 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
1029 while (src_i < remaining.len) {1115 while (src_i < remaining.len) {
1030 const n = utf8ByteSequenceLength(remaining[src_i]) catch return error.InvalidUtf8;1116 const n = utf8ByteSequenceLength(remaining[src_i]) catch return error.InvalidUtf8;
1031 const next_src_i = src_i + n;1117 const next_src_i = src_i + n;
1032 const codepoint = utf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidUtf8;1118 const codepoint = switch (surrogates) {
1119 .cannot_encode_surrogate_half => utf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidUtf8,
1120 .can_encode_surrogate_half => wtf8Decode(remaining[src_i..next_src_i]) catch return error.InvalidUtf8,
1121 };
1033 if (codepoint < 0x10000) {1122 if (codepoint < 0x10000) {
1034 const short = @as(u16, @intCast(codepoint));1123 const short = @as(u16, @intCast(codepoint));
1035 utf16le[dest_i] = mem.nativeToLittle(u16, short);1124 utf16le[dest_i] = mem.nativeToLittle(u16, short);
...@@ -1064,21 +1153,59 @@ test "utf8ToUtf16Le" {...@@ -1064,21 +1153,59 @@ test "utf8ToUtf16Le" {
1064 }1153 }
1065}1154}
10661155
1067test "utf8ToUtf16LeWithNull" {1156test utf8ToUtf16LeArrayList {
1157 {
1158 var list = std.ArrayList(u16).init(testing.allocator);
1159 defer list.deinit();
1160 try utf8ToUtf16LeArrayList(&list, "𐐷");
1161 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(list.items));
1162 }
1163 {
1164 var list = std.ArrayList(u16).init(testing.allocator);
1165 defer list.deinit();
1166 try utf8ToUtf16LeArrayList(&list, "\u{10FFFF}");
1167 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(list.items));
1168 }
1169 {
1170 var list = std.ArrayList(u16).init(testing.allocator);
1171 defer list.deinit();
1172 const result = utf8ToUtf16LeArrayList(&list, "\xf4\x90\x80\x80");
1173 try testing.expectError(error.InvalidUtf8, result);
1174 }
1175}
1176
1177test utf8ToUtf16LeAlloc {
1178 {
1179 const utf16 = try utf8ToUtf16LeAlloc(testing.allocator, "𐐷");
1180 defer testing.allocator.free(utf16);
1181 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));
1182 }
1183 {
1184 const utf16 = try utf8ToUtf16LeAlloc(testing.allocator, "\u{10FFFF}");
1185 defer testing.allocator.free(utf16);
1186 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));
1187 }
1188 {
1189 const result = utf8ToUtf16LeAlloc(testing.allocator, "\xf4\x90\x80\x80");
1190 try testing.expectError(error.InvalidUtf8, result);
1191 }
1192}
1193
1194test utf8ToUtf16LeAllocZ {
1068 {1195 {
1069 const utf16 = try utf8ToUtf16LeWithNull(testing.allocator, "𐐷");1196 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "𐐷");
1070 defer testing.allocator.free(utf16);1197 defer testing.allocator.free(utf16);
1071 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));1198 try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));
1072 try testing.expect(utf16[2] == 0);1199 try testing.expect(utf16[2] == 0);
1073 }1200 }
1074 {1201 {
1075 const utf16 = try utf8ToUtf16LeWithNull(testing.allocator, "\u{10FFFF}");1202 const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "\u{10FFFF}");
1076 defer testing.allocator.free(utf16);1203 defer testing.allocator.free(utf16);
1077 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));1204 try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));
1078 try testing.expect(utf16[2] == 0);1205 try testing.expect(utf16[2] == 0);
1079 }1206 }
1080 {1207 {
1081 const result = utf8ToUtf16LeWithNull(testing.allocator, "\xf4\x90\x80\x80");1208 const result = utf8ToUtf16LeAllocZ(testing.allocator, "\xf4\x90\x80\x80");
1082 try testing.expectError(error.InvalidUtf8, result);1209 try testing.expectError(error.InvalidUtf8, result);
1083 }1210 }
1084}1211}
...@@ -1127,8 +1254,9 @@ test "calculate utf16 string length of given utf8 string in u16" {...@@ -1127,8 +1254,9 @@ test "calculate utf16 string length of given utf8 string in u16" {
1127 try comptime testCalcUtf16LeLen();1254 try comptime testCalcUtf16LeLen();
1128}1255}
11291256
1130/// Print the given `utf16le` string1257/// Print the given `utf16le` string, encoded as UTF-8 bytes.
1131fn formatUtf16le(1258/// Unpaired surrogates are replaced by the replacement character (U+FFFD).
1259fn formatUtf16Le(
1132 utf16le: []const u16,1260 utf16le: []const u16,
1133 comptime fmt: []const u8,1261 comptime fmt: []const u8,
1134 options: std.fmt.FormatOptions,1262 options: std.fmt.FormatOptions,
...@@ -1150,22 +1278,25 @@ fn formatUtf16le(...@@ -1150,22 +1278,25 @@ fn formatUtf16le(
1150 try writer.writeAll(buf[0..u8len]);1278 try writer.writeAll(buf[0..u8len]);
1151}1279}
11521280
1281/// Deprecated; renamed to fmtUtf16Le
1282pub const fmtUtf16le = fmtUtf16Le;
1283
1153/// Return a Formatter for a Utf16le string1284/// Return a Formatter for a Utf16le string
1154pub fn fmtUtf16le(utf16le: []const u16) std.fmt.Formatter(formatUtf16le) {1285pub fn fmtUtf16Le(utf16le: []const u16) std.fmt.Formatter(formatUtf16Le) {
1155 return .{ .data = utf16le };1286 return .{ .data = utf16le };
1156}1287}
11571288
1158test "fmtUtf16le" {1289test "fmtUtf16Le" {
1159 const expectFmt = std.testing.expectFmt;1290 const expectFmt = testing.expectFmt;
1160 try expectFmt("", "{}", .{fmtUtf16le(utf8ToUtf16LeStringLiteral(""))});1291 try expectFmt("", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral(""))});
1161 try expectFmt("foo", "{}", .{fmtUtf16le(utf8ToUtf16LeStringLiteral("foo"))});1292 try expectFmt("foo", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("foo"))});
1162 try expectFmt("𐐷", "{}", .{fmtUtf16le(utf8ToUtf16LeStringLiteral("𐐷"))});1293 try expectFmt("𐐷", "{}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("𐐷"))});
1163 try expectFmt("퟿", "{}", .{fmtUtf16le(&[_]u16{std.mem.readInt(u16, "\xff\xd7", native_endian)})});1294 try expectFmt("퟿", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xd7", native_endian)})});
1164 try expectFmt("�", "{}", .{fmtUtf16le(&[_]u16{std.mem.readInt(u16, "\x00\xd8", native_endian)})});1295 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xd8", native_endian)})});
1165 try expectFmt("�", "{}", .{fmtUtf16le(&[_]u16{std.mem.readInt(u16, "\xff\xdb", native_endian)})});1296 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xdb", native_endian)})});
1166 try expectFmt("�", "{}", .{fmtUtf16le(&[_]u16{std.mem.readInt(u16, "\x00\xdc", native_endian)})});1297 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xdc", native_endian)})});
1167 try expectFmt("�", "{}", .{fmtUtf16le(&[_]u16{std.mem.readInt(u16, "\xff\xdf", native_endian)})});1298 try expectFmt("�", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\xff\xdf", native_endian)})});
1168 try expectFmt("", "{}", .{fmtUtf16le(&[_]u16{std.mem.readInt(u16, "\x00\xe0", native_endian)})});1299 try expectFmt("", "{}", .{fmtUtf16Le(&[_]u16{std.mem.readInt(u16, "\x00\xe0", native_endian)})});
1169}1300}
11701301
1171test "utf8ToUtf16LeStringLiteral" {1302test "utf8ToUtf16LeStringLiteral" {
...@@ -1248,3 +1379,534 @@ test "utf8 valid codepoint" {...@@ -1248,3 +1379,534 @@ test "utf8 valid codepoint" {
1248 try testUtf8ValidCodepoint();1379 try testUtf8ValidCodepoint();
1249 try comptime testUtf8ValidCodepoint();1380 try comptime testUtf8ValidCodepoint();
1250}1381}
1382
1383/// Returns true if the codepoint is a surrogate (U+DC00 to U+DFFF)
1384pub fn isSurrogateCodepoint(c: u21) bool {
1385 return switch (c) {
1386 0xD800...0xDFFF => true,
1387 else => false,
1388 };
1389}
1390
1391/// Encodes the given codepoint into a WTF-8 byte sequence.
1392/// c: the codepoint.
1393/// out: the out buffer to write to. Must have a len >= utf8CodepointSequenceLength(c).
1394/// Errors: if c cannot be encoded in WTF-8.
1395/// Returns: the number of bytes written to out.
1396pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3 {
1397 return utf8EncodeImpl(c, out, .can_encode_surrogate_half);
1398}
1399
1400const Wtf8DecodeError = Utf8Decode2Error || Utf8Decode3AllowSurrogateHalfError || Utf8Decode4Error;
1401
1402pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21 {
1403 return switch (bytes.len) {
1404 1 => @as(u21, bytes[0]),
1405 2 => utf8Decode2(bytes),
1406 3 => utf8Decode3AllowSurrogateHalf(bytes),
1407 4 => utf8Decode4(bytes),
1408 else => unreachable,
1409 };
1410}
1411
1412/// Returns true if the input consists entirely of WTF-8 codepoints
1413/// (all the same restrictions as UTF-8, but allows surrogate codepoints
1414/// U+D800 to U+DFFF).
1415/// Does not check for well-formed WTF-8, meaning that this function
1416/// does not check that all surrogate halves are unpaired.
1417pub fn wtf8ValidateSlice(input: []const u8) bool {
1418 return utf8ValidateSliceImpl(input, .can_encode_surrogate_half);
1419}
1420
1421test "validate WTF-8 slice" {
1422 try testValidateWtf8Slice();
1423 try comptime testValidateWtf8Slice();
1424
1425 // We skip a variable (based on recommended vector size) chunks of
1426 // ASCII characters. Let's make sure we're chunking correctly.
1427 const str = [_]u8{'a'} ** 550 ++ "\xc0";
1428 for (0..str.len - 3) |i| {
1429 try testing.expect(!wtf8ValidateSlice(str[i..]));
1430 }
1431}
1432fn testValidateWtf8Slice() !void {
1433 // These are valid/invalid under both UTF-8 and WTF-8 rules.
1434 try testing.expect(wtf8ValidateSlice("abc"));
1435 try testing.expect(wtf8ValidateSlice("abc\xdf\xbf"));
1436 try testing.expect(wtf8ValidateSlice(""));
1437 try testing.expect(wtf8ValidateSlice("a"));
1438 try testing.expect(wtf8ValidateSlice("abc"));
1439 try testing.expect(wtf8ValidateSlice("Ж"));
1440 try testing.expect(wtf8ValidateSlice("ЖЖ"));
1441 try testing.expect(wtf8ValidateSlice("брэд-ЛГТМ"));
1442 try testing.expect(wtf8ValidateSlice("☺☻☹"));
1443 try testing.expect(wtf8ValidateSlice("a\u{fffdb}"));
1444 try testing.expect(wtf8ValidateSlice("\xf4\x8f\xbf\xbf"));
1445 try testing.expect(wtf8ValidateSlice("abc\xdf\xbf"));
1446
1447 try testing.expect(!wtf8ValidateSlice("abc\xc0"));
1448 try testing.expect(!wtf8ValidateSlice("abc\xc0abc"));
1449 try testing.expect(!wtf8ValidateSlice("aa\xe2"));
1450 try testing.expect(!wtf8ValidateSlice("\x42\xfa"));
1451 try testing.expect(!wtf8ValidateSlice("\x42\xfa\x43"));
1452 try testing.expect(!wtf8ValidateSlice("abc\xc0"));
1453 try testing.expect(!wtf8ValidateSlice("abc\xc0abc"));
1454 try testing.expect(!wtf8ValidateSlice("\xf4\x90\x80\x80"));
1455 try testing.expect(!wtf8ValidateSlice("\xf7\xbf\xbf\xbf"));
1456 try testing.expect(!wtf8ValidateSlice("\xfb\xbf\xbf\xbf\xbf"));
1457 try testing.expect(!wtf8ValidateSlice("\xc0\x80"));
1458
1459 // But surrogate codepoints are only valid in WTF-8.
1460 try testing.expect(wtf8ValidateSlice("\xed\xa0\x80"));
1461 try testing.expect(wtf8ValidateSlice("\xed\xbf\xbf"));
1462}
1463
1464/// Wtf8View iterates the code points of a WTF-8 encoded string,
1465/// including surrogate halves.
1466///
1467/// ```
1468/// var wtf8 = (try std.unicode.Wtf8View.init("hi there")).iterator();
1469/// while (wtf8.nextCodepointSlice()) |codepoint| {
1470/// // note: codepoint could be a surrogate half which is invalid
1471/// // UTF-8, avoid printing or otherwise sending/emitting this directly
1472/// }
1473/// ```
1474pub const Wtf8View = struct {
1475 bytes: []const u8,
1476
1477 pub fn init(s: []const u8) !Wtf8View {
1478 if (!wtf8ValidateSlice(s)) {
1479 return error.InvalidUtf8;
1480 }
1481
1482 return initUnchecked(s);
1483 }
1484
1485 pub fn initUnchecked(s: []const u8) Wtf8View {
1486 return Wtf8View{ .bytes = s };
1487 }
1488
1489 pub inline fn initComptime(comptime s: []const u8) Wtf8View {
1490 return comptime if (init(s)) |r| r else |err| switch (err) {
1491 error.InvalidUtf8 => {
1492 @compileError("invalid utf8 detected in wtf8 string");
1493 },
1494 };
1495 }
1496
1497 pub fn iterator(s: Wtf8View) Wtf8Iterator {
1498 return Wtf8Iterator{
1499 .bytes = s.bytes,
1500 .i = 0,
1501 };
1502 }
1503};
1504
1505/// Asserts that `bytes` is valid WTF-8
1506pub const Wtf8Iterator = struct {
1507 bytes: []const u8,
1508 i: usize,
1509
1510 pub fn nextCodepointSlice(it: *Wtf8Iterator) ?[]const u8 {
1511 if (it.i >= it.bytes.len) {
1512 return null;
1513 }
1514
1515 const cp_len = utf8ByteSequenceLength(it.bytes[it.i]) catch unreachable;
1516 it.i += cp_len;
1517 return it.bytes[it.i - cp_len .. it.i];
1518 }
1519
1520 pub fn nextCodepoint(it: *Wtf8Iterator) ?u21 {
1521 const slice = it.nextCodepointSlice() orelse return null;
1522 return wtf8Decode(slice) catch unreachable;
1523 }
1524
1525 /// Look ahead at the next n codepoints without advancing the iterator.
1526 /// If fewer than n codepoints are available, then return the remainder of the string.
1527 pub fn peek(it: *Wtf8Iterator, n: usize) []const u8 {
1528 const original_i = it.i;
1529 defer it.i = original_i;
1530
1531 var end_ix = original_i;
1532 var found: usize = 0;
1533 while (found < n) : (found += 1) {
1534 const next_codepoint = it.nextCodepointSlice() orelse return it.bytes[original_i..];
1535 end_ix += next_codepoint.len;
1536 }
1537
1538 return it.bytes[original_i..end_ix];
1539 }
1540};
1541
1542pub fn wtf16LeToWtf8ArrayList(array_list: *std.ArrayList(u8), utf16le: []const u16) !void {
1543 return utf16LeToUtf8ArrayListImpl(array_list, utf16le, .can_encode_surrogate_half);
1544}
1545
1546/// Caller must free returned memory.
1547pub fn wtf16LeToWtf8Alloc(allocator: mem.Allocator, wtf16le: []const u16) ![]u8 {
1548 // optimistically guess that it will all be ascii.
1549 var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len);
1550 errdefer result.deinit();
1551
1552 try wtf16LeToWtf8ArrayList(&result, wtf16le);
1553
1554 return result.toOwnedSlice();
1555}
1556
1557/// Caller must free returned memory.
1558pub fn wtf16LeToWtf8AllocZ(allocator: mem.Allocator, wtf16le: []const u16) ![:0]u8 {
1559 // optimistically guess that it will all be ascii (and allocate space for the null terminator)
1560 var result = try std.ArrayList(u8).initCapacity(allocator, wtf16le.len + 1);
1561 errdefer result.deinit();
1562
1563 try wtf16LeToWtf8ArrayList(&result, wtf16le);
1564
1565 return result.toOwnedSliceSentinel(0);
1566}
1567
1568pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {
1569 return utf16LeToUtf8Impl(wtf8, wtf16le, .can_encode_surrogate_half) catch |err| switch (err) {};
1570}
1571
1572pub fn wtf8ToWtf16LeArrayList(array_list: *std.ArrayList(u16), wtf8: []const u8) !void {
1573 return utf8ToUtf16LeArrayListImpl(array_list, wtf8, .can_encode_surrogate_half);
1574}
1575
1576pub fn wtf8ToWtf16LeAlloc(allocator: mem.Allocator, wtf8: []const u8) ![]u16 {
1577 // optimistically guess that it will not require surrogate pairs
1578 var result = try std.ArrayList(u16).initCapacity(allocator, wtf8.len);
1579 errdefer result.deinit();
1580
1581 try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
1582
1583 return result.toOwnedSlice();
1584}
1585
1586pub fn wtf8ToWtf16LeAllocZ(allocator: mem.Allocator, wtf8: []const u8) ![:0]u16 {
1587 // optimistically guess that it will not require surrogate pairs
1588 var result = try std.ArrayList(u16).initCapacity(allocator, wtf8.len + 1);
1589 errdefer result.deinit();
1590
1591 try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
1592
1593 return result.toOwnedSliceSentinel(0);
1594}
1595
1596/// Returns index of next character. If exact fit, returned index equals output slice length.
1597/// Assumes there is enough space for the output.
1598pub fn wtf8ToWtf16Le(wtf16le: []u16, wtf8: []const u8) !usize {
1599 return utf8ToUtf16LeImpl(wtf16le, wtf8, .can_encode_surrogate_half);
1600}
1601
1602/// Surrogate codepoints (U+D800 to U+DFFF) are replaced by the Unicode replacement
1603/// character (U+FFFD).
1604/// All surrogate codepoints and the replacement character are encoded as three
1605/// bytes, meaning the input and output slices will always be the same length.
1606/// In-place conversion is supported when `utf8` and `wtf8` refer to the same slice.
1607/// Note: If `wtf8` is entirely composed of well-formed UTF-8, then no conversion is necessary.
1608/// `utf8ValidateSlice` can be used to check if lossy conversion is worthwhile.
1609pub fn wtf8ToUtf8Lossy(utf8: []u8, wtf8: []const u8) !void {
1610 assert(utf8.len >= wtf8.len);
1611
1612 const in_place = utf8.ptr == wtf8.ptr;
1613 const replacement_char_bytes = comptime blk: {
1614 var buf: [3]u8 = undefined;
1615 assert((utf8Encode(replacement_character, &buf) catch unreachable) == 3);
1616 break :blk buf;
1617 };
1618
1619 var dest_i: usize = 0;
1620 const view = try Wtf8View.init(wtf8);
1621 var it = view.iterator();
1622 while (it.nextCodepointSlice()) |codepoint_slice| {
1623 // All surrogate codepoints are encoded as 3 bytes
1624 if (codepoint_slice.len == 3) {
1625 const codepoint = wtf8Decode(codepoint_slice) catch unreachable;
1626 if (isSurrogateCodepoint(codepoint)) {
1627 @memcpy(utf8[dest_i..][0..replacement_char_bytes.len], &replacement_char_bytes);
1628 dest_i += replacement_char_bytes.len;
1629 continue;
1630 }
1631 }
1632 if (!in_place) {
1633 @memcpy(utf8[dest_i..][0..codepoint_slice.len], codepoint_slice);
1634 }
1635 dest_i += codepoint_slice.len;
1636 }
1637}
1638
1639pub fn wtf8ToUtf8LossyAlloc(allocator: mem.Allocator, wtf8: []const u8) ![]u8 {
1640 const utf8 = try allocator.alloc(u8, wtf8.len);
1641 errdefer allocator.free(utf8);
1642
1643 try wtf8ToUtf8Lossy(utf8, wtf8);
1644
1645 return utf8;
1646}
1647
1648pub fn wtf8ToUtf8LossyAllocZ(allocator: mem.Allocator, wtf8: []const u8) ![:0]u8 {
1649 const utf8 = try allocator.allocSentinel(u8, wtf8.len, 0);
1650 errdefer allocator.free(utf8);
1651
1652 try wtf8ToUtf8Lossy(utf8, wtf8);
1653
1654 return utf8;
1655}
1656
1657test wtf8ToUtf8Lossy {
1658 var buf: [32]u8 = undefined;
1659
1660 const invalid_utf8 = "\xff";
1661 try testing.expectError(error.InvalidWtf8, wtf8ToUtf8Lossy(&buf, invalid_utf8));
1662
1663 const ascii = "abcd";
1664 try wtf8ToUtf8Lossy(&buf, ascii);
1665 try testing.expectEqualStrings("abcd", buf[0..ascii.len]);
1666
1667 const high_surrogate_half = "ab\xed\xa0\xbdcd";
1668 try wtf8ToUtf8Lossy(&buf, high_surrogate_half);
1669 try testing.expectEqualStrings("ab\u{FFFD}cd", buf[0..high_surrogate_half.len]);
1670
1671 const low_surrogate_half = "ab\xed\xb2\xa9cd";
1672 try wtf8ToUtf8Lossy(&buf, low_surrogate_half);
1673 try testing.expectEqualStrings("ab\u{FFFD}cd", buf[0..low_surrogate_half.len]);
1674
1675 // If the WTF-8 is not well-formed, each surrogate half is converted into a separate
1676 // replacement character instead of being interpreted as a surrogate pair.
1677 const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
1678 try wtf8ToUtf8Lossy(&buf, encoded_surrogate_pair);
1679 try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", buf[0..encoded_surrogate_pair.len]);
1680
1681 // in place
1682 @memcpy(buf[0..low_surrogate_half.len], low_surrogate_half);
1683 const slice = buf[0..low_surrogate_half.len];
1684 try wtf8ToUtf8Lossy(slice, slice);
1685 try testing.expectEqualStrings("ab\u{FFFD}cd", slice);
1686}
1687
1688test wtf8ToUtf8LossyAlloc {
1689 const invalid_utf8 = "\xff";
1690 try testing.expectError(error.InvalidWtf8, wtf8ToUtf8LossyAlloc(testing.allocator, invalid_utf8));
1691
1692 {
1693 const ascii = "abcd";
1694 const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, ascii);
1695 defer testing.allocator.free(utf8);
1696 try testing.expectEqualStrings("abcd", utf8);
1697 }
1698
1699 {
1700 const surrogate_half = "ab\xed\xa0\xbdcd";
1701 const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, surrogate_half);
1702 defer testing.allocator.free(utf8);
1703 try testing.expectEqualStrings("ab\u{FFFD}cd", utf8);
1704 }
1705
1706 {
1707 // If the WTF-8 is not well-formed, each surrogate half is converted into a separate
1708 // replacement character instead of being interpreted as a surrogate pair.
1709 const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
1710 const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, encoded_surrogate_pair);
1711 defer testing.allocator.free(utf8);
1712 try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", utf8);
1713 }
1714}
1715
1716test wtf8ToUtf8LossyAllocZ {
1717 const invalid_utf8 = "\xff";
1718 try testing.expectError(error.InvalidWtf8, wtf8ToUtf8LossyAllocZ(testing.allocator, invalid_utf8));
1719
1720 {
1721 const ascii = "abcd";
1722 const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, ascii);
1723 defer testing.allocator.free(utf8);
1724 try testing.expectEqualStrings("abcd", utf8);
1725 }
1726
1727 {
1728 const surrogate_half = "ab\xed\xa0\xbdcd";
1729 const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, surrogate_half);
1730 defer testing.allocator.free(utf8);
1731 try testing.expectEqualStrings("ab\u{FFFD}cd", utf8);
1732 }
1733
1734 {
1735 // If the WTF-8 is not well-formed, each surrogate half is converted into a separate
1736 // replacement character instead of being interpreted as a surrogate pair.
1737 const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
1738 const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, encoded_surrogate_pair);
1739 defer testing.allocator.free(utf8);
1740 try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", utf8);
1741 }
1742}
1743
1744pub const Wtf16LeIterator = struct {
1745 bytes: []const u8,
1746 i: usize,
1747
1748 pub fn init(s: []const u16) Wtf16LeIterator {
1749 return Wtf16LeIterator{
1750 .bytes = std.mem.sliceAsBytes(s),
1751 .i = 0,
1752 };
1753 }
1754
1755 /// If the next codepoint is encoded by a surrogate pair, returns the
1756 /// codepoint that the surrogate pair represents.
1757 /// If the next codepoint is an unpaired surrogate, returns the codepoint
1758 /// of the unpaired surrogate.
1759 pub fn nextCodepoint(it: *Wtf16LeIterator) ?u21 {
1760 assert(it.i <= it.bytes.len);
1761 if (it.i == it.bytes.len) return null;
1762 var code_units: [2]u16 = undefined;
1763 code_units[0] = std.mem.readInt(u16, it.bytes[it.i..][0..2], .little);
1764 it.i += 2;
1765 surrogate_pair: {
1766 if (utf16IsHighSurrogate(code_units[0])) {
1767 if (it.i >= it.bytes.len) break :surrogate_pair;
1768 code_units[1] = std.mem.readInt(u16, it.bytes[it.i..][0..2], .little);
1769 const codepoint = utf16DecodeSurrogatePair(&code_units) catch break :surrogate_pair;
1770 it.i += 2;
1771 return codepoint;
1772 }
1773 }
1774 return code_units[0];
1775 }
1776};
1777
1778test "non-well-formed WTF-8 does not roundtrip" {
1779 // This encodes the surrogate pair U+D83D U+DCA9.
1780 // The well-formed version of this would be U+1F4A9 which is \xF0\x9F\x92\xA9.
1781 const non_well_formed_wtf8 = "\xed\xa0\xbd\xed\xb2\xa9";
1782
1783 var wtf16_buf: [2]u16 = undefined;
1784 const wtf16_len = try wtf8ToWtf16Le(&wtf16_buf, non_well_formed_wtf8);
1785 const wtf16 = wtf16_buf[0..wtf16_len];
1786
1787 try testing.expectEqualSlices(u16, &[_]u16{
1788 mem.nativeToLittle(u16, 0xD83D), // high surrogate
1789 mem.nativeToLittle(u16, 0xDCA9), // low surrogate
1790 }, wtf16);
1791
1792 var wtf8_buf: [4]u8 = undefined;
1793 const wtf8_len = wtf16LeToWtf8(&wtf8_buf, wtf16);
1794 const wtf8 = wtf8_buf[0..wtf8_len];
1795
1796 // Converting to WTF-16 and back results in well-formed WTF-8,
1797 // but it does not match the input WTF-8
1798 try testing.expectEqualSlices(u8, "\xf0\x9f\x92\xa9", wtf8);
1799}
1800
1801fn testRoundtripWtf8(wtf8: []const u8) !void {
1802 // Buffer
1803 {
1804 var wtf16_buf: [32]u16 = undefined;
1805 const wtf16_len = try wtf8ToWtf16Le(&wtf16_buf, wtf8);
1806 const wtf16 = wtf16_buf[0..wtf16_len];
1807
1808 var roundtripped_buf: [32]u8 = undefined;
1809 const roundtripped_len = wtf16LeToWtf8(&roundtripped_buf, wtf16);
1810 const roundtripped = roundtripped_buf[0..roundtripped_len];
1811
1812 try testing.expectEqualSlices(u8, wtf8, roundtripped);
1813 }
1814 // Alloc
1815 {
1816 const wtf16 = try wtf8ToWtf16LeAlloc(testing.allocator, wtf8);
1817 defer testing.allocator.free(wtf16);
1818
1819 const roundtripped = try wtf16LeToWtf8Alloc(testing.allocator, wtf16);
1820 defer testing.allocator.free(roundtripped);
1821
1822 try testing.expectEqualSlices(u8, wtf8, roundtripped);
1823 }
1824 // AllocZ
1825 {
1826 const wtf16 = try wtf8ToWtf16LeAllocZ(testing.allocator, wtf8);
1827 defer testing.allocator.free(wtf16);
1828
1829 const roundtripped = try wtf16LeToWtf8AllocZ(testing.allocator, wtf16);
1830 defer testing.allocator.free(roundtripped);
1831
1832 try testing.expectEqualSlices(u8, wtf8, roundtripped);
1833 }
1834}
1835
1836test "well-formed WTF-8 roundtrips" {
1837 try testRoundtripWtf8("\xed\x9f\xbf"); // not a surrogate half
1838 try testRoundtripWtf8("\xed\xa0\xbd"); // high surrogate
1839 try testRoundtripWtf8("\xed\xb2\xa9"); // low surrogate
1840 try testRoundtripWtf8("\xed\xa0\xbd \xed\xb2\xa9"); // <high surrogate><space><low surrogate>
1841 try testRoundtripWtf8("\xed\xa0\x80\xed\xaf\xbf"); // <high surrogate><high surrogate>
1842 try testRoundtripWtf8("\xed\xa0\x80\xee\x80\x80"); // <high surrogate><not surrogate>
1843 try testRoundtripWtf8("\xed\x9f\xbf\xed\xb0\x80"); // <not surrogate><low surrogate>
1844 try testRoundtripWtf8("a\xed\xb0\x80"); // <not surrogate><low surrogate>
1845 try testRoundtripWtf8("\xf0\x9f\x92\xa9"); // U+1F4A9, encoded as a surrogate pair in WTF-16
1846}
1847
1848fn testRoundtripWtf16(wtf16le: []const u16) !void {
1849 // Buffer
1850 {
1851 var wtf8_buf: [32]u8 = undefined;
1852 const wtf8_len = wtf16LeToWtf8(&wtf8_buf, wtf16le);
1853 const wtf8 = wtf8_buf[0..wtf8_len];
1854
1855 var roundtripped_buf: [32]u16 = undefined;
1856 const roundtripped_len = try wtf8ToWtf16Le(&roundtripped_buf, wtf8);
1857 const roundtripped = roundtripped_buf[0..roundtripped_len];
1858
1859 try testing.expectEqualSlices(u16, wtf16le, roundtripped);
1860 }
1861 // Alloc
1862 {
1863 const wtf8 = try wtf16LeToWtf8Alloc(testing.allocator, wtf16le);
1864 defer testing.allocator.free(wtf8);
1865
1866 const roundtripped = try wtf8ToWtf16LeAlloc(testing.allocator, wtf8);
1867 defer testing.allocator.free(roundtripped);
1868
1869 try testing.expectEqualSlices(u16, wtf16le, roundtripped);
1870 }
1871 // AllocZ
1872 {
1873 const wtf8 = try wtf16LeToWtf8AllocZ(testing.allocator, wtf16le);
1874 defer testing.allocator.free(wtf8);
1875
1876 const roundtripped = try wtf8ToWtf16LeAllocZ(testing.allocator, wtf8);
1877 defer testing.allocator.free(roundtripped);
1878
1879 try testing.expectEqualSlices(u16, wtf16le, roundtripped);
1880 }
1881}
1882
1883test "well-formed WTF-16 roundtrips" {
1884 try testRoundtripWtf16(&[_]u16{
1885 std.mem.nativeToLittle(u16, 0xD83D), // high surrogate
1886 std.mem.nativeToLittle(u16, 0xDCA9), // low surrogate
1887 });
1888 try testRoundtripWtf16(&[_]u16{
1889 std.mem.nativeToLittle(u16, 0xD83D), // high surrogate
1890 std.mem.nativeToLittle(u16, ' '), // not surrogate
1891 std.mem.nativeToLittle(u16, 0xDCA9), // low surrogate
1892 });
1893 try testRoundtripWtf16(&[_]u16{
1894 std.mem.nativeToLittle(u16, 0xD800), // high surrogate
1895 std.mem.nativeToLittle(u16, 0xDBFF), // high surrogate
1896 });
1897 try testRoundtripWtf16(&[_]u16{
1898 std.mem.nativeToLittle(u16, 0xD800), // high surrogate
1899 std.mem.nativeToLittle(u16, 0xE000), // not surrogate
1900 });
1901 try testRoundtripWtf16(&[_]u16{
1902 std.mem.nativeToLittle(u16, 0xD7FF), // not surrogate
1903 std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
1904 });
1905 try testRoundtripWtf16(&[_]u16{
1906 std.mem.nativeToLittle(u16, 0x61), // not surrogate
1907 std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
1908 });
1909 try testRoundtripWtf16(&[_]u16{
1910 std.mem.nativeToLittle(u16, 0xDC00), // low surrogate
1911 });
1912}