| ... | @@ -325,6 +325,42 @@ pub const Utf16LeIterator = struct { | ... | @@ -325,6 +325,42 @@ pub const Utf16LeIterator = struct { |
| 325 | } | 325 | } |
| 326 | }; | 326 | }; |
| 327 | | 327 | |
| | 328 | /// Returns the length of a supplied UTF-16 string literal in terms of unicode |
| | 329 | /// codepoints. |
| | 330 | pub fn utf16CountCodepoints(utf16le: []const u16) !usize { |
| | 331 | var len: usize = 0; |
| | 332 | var it = Utf16LeIterator.init(utf16le); |
| | 333 | while (try it.nextCodepoint()) |_| len += 1; |
| | 334 | return len; |
| | 335 | } |
| | 336 | |
| | 337 | fn testUtf16CountCodepoints() !void { |
| | 338 | try testing.expectEqual( |
| | 339 | @as(usize, 1), |
| | 340 | try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("a")), |
| | 341 | ); |
| | 342 | try testing.expectEqual( |
| | 343 | @as(usize, 10), |
| | 344 | try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("abcdefghij")), |
| | 345 | ); |
| | 346 | try testing.expectEqual( |
| | 347 | @as(usize, 10), |
| | 348 | try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("äåéëþüúíóö")), |
| | 349 | ); |
| | 350 | try testing.expectEqual( |
| | 351 | @as(usize, 5), |
| | 352 | try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("こんにちは")), |
| | 353 | ); |
| | 354 | // testing.expectError(error.Utf8EncodesSurrogateHalf, utf8CountCodepoints("\xED\xA0\x80")); |
| | 355 | } |
| | 356 | |
| | 357 | test "utf16 count codepoints" { |
| | 358 | try testUtf16CountCodepoints(); |
| | 359 | // TODO stage1 error: out of bounds slice |
| | 360 | if (@import("builtin").zig_backend != .stage1) |
| | 361 | comptime try testUtf16CountCodepoints(); |
| | 362 | } |
| | 363 | |
| 328 | test "utf8 encode" { | 364 | test "utf8 encode" { |
| 329 | comptime try testUtf8Encode(); | 365 | comptime try testUtf8Encode(); |
| 330 | try testUtf8Encode(); | 366 | try testUtf8Encode(); |
| ... | @@ -748,9 +784,9 @@ test "utf8ToUtf16LeWithNull" { | ... | @@ -748,9 +784,9 @@ test "utf8ToUtf16LeWithNull" { |
| 748 | } | 784 | } |
| 749 | | 785 | |
| 750 | /// Converts a UTF-8 string literal into a UTF-16LE string literal. | 786 | /// Converts a UTF-8 string literal into a UTF-16LE string literal. |
| 751 | pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8):0]u16 { | 787 | pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch unreachable:0]u16 { |
| 752 | comptime { | 788 | comptime { |
| 753 | const len: usize = calcUtf16LeLen(utf8); | 789 | const len: usize = calcUtf16LeLen(utf8) catch |err| @compileError(err); |
| 754 | var utf16le: [len:0]u16 = [_:0]u16{0} ** len; | 790 | var utf16le: [len:0]u16 = [_:0]u16{0} ** len; |
| 755 | const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch |err| @compileError(err); | 791 | const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch |err| @compileError(err); |
| 756 | assert(len == utf16le_len); | 792 | assert(len == utf16le_len); |
| ... | @@ -758,13 +794,17 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le | ... | @@ -758,13 +794,17 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le |
| 758 | } | 794 | } |
| 759 | } | 795 | } |
| 760 | | 796 | |
| 761 | fn calcUtf16LeLen(utf8: []const u8) usize { | 797 | const CalcUtf16LeLenError = Utf8DecodeError || error{Utf8InvalidStartByte}; |
| | 798 | |
| | 799 | /// Returns length in UTF-16 of UTF-8 slice as length of []u16. |
| | 800 | /// Length in []u8 is 2*len16. |
| | 801 | pub fn calcUtf16LeLen(utf8: []const u8) CalcUtf16LeLenError!usize { |
| 762 | var src_i: usize = 0; | 802 | var src_i: usize = 0; |
| 763 | var dest_len: usize = 0; | 803 | var dest_len: usize = 0; |
| 764 | while (src_i < utf8.len) { | 804 | while (src_i < utf8.len) { |
| 765 | const n = utf8ByteSequenceLength(utf8[src_i]) catch unreachable; | 805 | const n = try utf8ByteSequenceLength(utf8[src_i]); |
| 766 | const next_src_i = src_i + n; | 806 | const next_src_i = src_i + n; |
| 767 | const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch unreachable; | 807 | const codepoint = try utf8Decode(utf8[src_i..next_src_i]); |
| 768 | if (codepoint < 0x10000) { | 808 | if (codepoint < 0x10000) { |
| 769 | dest_len += 1; | 809 | dest_len += 1; |
| 770 | } else { | 810 | } else { |
| ... | @@ -775,6 +815,19 @@ fn calcUtf16LeLen(utf8: []const u8) usize { | ... | @@ -775,6 +815,19 @@ fn calcUtf16LeLen(utf8: []const u8) usize { |
| 775 | return dest_len; | 815 | return dest_len; |
| 776 | } | 816 | } |
| 777 | | 817 | |
| | 818 | fn testCalcUtf16LeLen() !void { |
| | 819 | try testing.expectEqual(@as(usize, 1), try calcUtf16LeLen("a")); |
| | 820 | try testing.expectEqual(@as(usize, 10), try calcUtf16LeLen("abcdefghij")); |
| | 821 | try testing.expectEqual(@as(usize, 10), try calcUtf16LeLen("äåéëþüúíóö")); |
| | 822 | try testing.expectEqual(@as(usize, 5), try calcUtf16LeLen("こんにちは")); |
| | 823 | // testing.expectError(error.Utf8EncodesSurrogateHalf, utf8CountCodepoints("\xED\xA0\x80")); |
| | 824 | } |
| | 825 | |
| | 826 | test "calculate utf16 string length of given utf8 string in u16" { |
| | 827 | try testCalcUtf16LeLen(); |
| | 828 | comptime try testCalcUtf16LeLen(); |
| | 829 | } |
| | 830 | |
| 778 | /// Print the given `utf16le` string | 831 | /// Print the given `utf16le` string |
| 779 | fn formatUtf16le( | 832 | fn formatUtf16le( |
| 780 | utf16le: []const u16, | 833 | utf16le: []const u16, |
Jan Philipp Hafer <jan.hafer@rwth-aachen.de> 2022-05-17 13:00:07+02:00