| ... | ... | @@ -325,6 +325,42 @@ pub const Utf16LeIterator = struct { |
| 325 | 325 | } |
| 326 | 326 | }; |
| 327 | 327 | |
| 328 | /// Returns the length of a supplied UTF-16 string literal in terms of unicode |
| 329 | /// codepoints. |
| 330 | pub fn utf16CountCodepoints(utf16le: []const u16) !usize { |
| 331 | var len: usize = 0; |
| 332 | var it = Utf16LeIterator.init(utf16le); |
| 333 | while (try it.nextCodepoint()) |_| len += 1; |
| 334 | return len; |
| 335 | } |
| 336 | |
| 337 | fn testUtf16CountCodepoints() !void { |
| 338 | try testing.expectEqual( |
| 339 | @as(usize, 1), |
| 340 | try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("a")), |
| 341 | ); |
| 342 | try testing.expectEqual( |
| 343 | @as(usize, 10), |
| 344 | try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("abcdefghij")), |
| 345 | ); |
| 346 | try testing.expectEqual( |
| 347 | @as(usize, 10), |
| 348 | try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("äåéëþüúíóö")), |
| 349 | ); |
| 350 | try testing.expectEqual( |
| 351 | @as(usize, 5), |
| 352 | try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("こんにちは")), |
| 353 | ); |
| 354 | // testing.expectError(error.Utf8EncodesSurrogateHalf, utf8CountCodepoints("\xED\xA0\x80")); |
| 355 | } |
| 356 | |
| 357 | test "utf16 count codepoints" { |
| 358 | try testUtf16CountCodepoints(); |
| 359 | // TODO stage1 error: out of bounds slice |
| 360 | if (@import("builtin").zig_backend != .stage1) |
| 361 | comptime try testUtf16CountCodepoints(); |
| 362 | } |
| 363 | |
| 328 | 364 | test "utf8 encode" { |
| 329 | 365 | comptime try testUtf8Encode(); |
| 330 | 366 | try testUtf8Encode(); |
| ... | ... | @@ -748,9 +784,9 @@ test "utf8ToUtf16LeWithNull" { |
| 748 | 784 | } |
| 749 | 785 | |
| 750 | 786 | /// Converts a UTF-8 string literal into a UTF-16LE string literal. |
| 751 | | pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8):0]u16 { |
| 787 | pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch unreachable:0]u16 { |
| 752 | 788 | comptime { |
| 753 | | const len: usize = calcUtf16LeLen(utf8); |
| 789 | const len: usize = calcUtf16LeLen(utf8) catch |err| @compileError(err); |
| 754 | 790 | var utf16le: [len:0]u16 = [_:0]u16{0} ** len; |
| 755 | 791 | const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch |err| @compileError(err); |
| 756 | 792 | assert(len == utf16le_len); |
| ... | ... | @@ -758,13 +794,17 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le |
| 758 | 794 | } |
| 759 | 795 | } |
| 760 | 796 | |
| 761 | | fn calcUtf16LeLen(utf8: []const u8) usize { |
| 797 | const CalcUtf16LeLenError = Utf8DecodeError || error{Utf8InvalidStartByte}; |
| 798 | |
| 799 | /// Returns length in UTF-16 of UTF-8 slice as length of []u16. |
| 800 | /// Length in []u8 is 2*len16. |
| 801 | pub fn calcUtf16LeLen(utf8: []const u8) CalcUtf16LeLenError!usize { |
| 762 | 802 | var src_i: usize = 0; |
| 763 | 803 | var dest_len: usize = 0; |
| 764 | 804 | while (src_i < utf8.len) { |
| 765 | | const n = utf8ByteSequenceLength(utf8[src_i]) catch unreachable; |
| 805 | const n = try utf8ByteSequenceLength(utf8[src_i]); |
| 766 | 806 | const next_src_i = src_i + n; |
| 767 | | const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch unreachable; |
| 807 | const codepoint = try utf8Decode(utf8[src_i..next_src_i]); |
| 768 | 808 | if (codepoint < 0x10000) { |
| 769 | 809 | dest_len += 1; |
| 770 | 810 | } else { |
| ... | ... | @@ -775,6 +815,19 @@ fn calcUtf16LeLen(utf8: []const u8) usize { |
| 775 | 815 | return dest_len; |
| 776 | 816 | } |
| 777 | 817 | |
| 818 | fn testCalcUtf16LeLen() !void { |
| 819 | try testing.expectEqual(@as(usize, 1), try calcUtf16LeLen("a")); |
| 820 | try testing.expectEqual(@as(usize, 10), try calcUtf16LeLen("abcdefghij")); |
| 821 | try testing.expectEqual(@as(usize, 10), try calcUtf16LeLen("äåéëþüúíóö")); |
| 822 | try testing.expectEqual(@as(usize, 5), try calcUtf16LeLen("こんにちは")); |
| 823 | // testing.expectError(error.Utf8EncodesSurrogateHalf, utf8CountCodepoints("\xED\xA0\x80")); |
| 824 | } |
| 825 | |
| 826 | test "calculate utf16 string length of given utf8 string in u16" { |
| 827 | try testCalcUtf16LeLen(); |
| 828 | comptime try testCalcUtf16LeLen(); |
| 829 | } |
| 830 | |
| 778 | 831 | /// Print the given `utf16le` string |
| 779 | 832 | fn formatUtf16le( |
| 780 | 833 | utf16le: []const u16, |
Jan Philipp Hafer <jan.hafer@rwth-aachen.de> 2022-05-17 13:00:07+02:00