authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-09-14 19:25:15-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-09-14 19:25:15-04:00
log0931dda9a95e14b97a84e60aed424fd8bb5e1232
tree5e18e74a3128b36ae807b6f108623e7567a1bf4e
parentd7a0fe67b38a60a4f294d6d9034c7a342bed7094
parentcf744cf04f8cad148ae93e5c5c7d8c5f5f62c164
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11663 from matu3ba/utf16

std.unicode: add utf16 byte length and codepoints counting routines

1 files changed, 56 insertions(+), 6 deletions(-)

lib/std/unicode.zig+56-6
...@@ -164,7 +164,6 @@ pub fn utf8ValidCodepoint(value: u21) bool {...@@ -164,7 +164,6 @@ pub fn utf8ValidCodepoint(value: u21) bool {
164164
165/// Returns the length of a supplied UTF-8 string literal in terms of unicode165/// Returns the length of a supplied UTF-8 string literal in terms of unicode
166/// codepoints.166/// codepoints.
167/// Asserts that the data is valid UTF-8.
168pub fn utf8CountCodepoints(s: []const u8) !usize {167pub fn utf8CountCodepoints(s: []const u8) !usize {
169 var len: usize = 0;168 var len: usize = 0;
170169
...@@ -325,6 +324,41 @@ pub const Utf16LeIterator = struct {...@@ -325,6 +324,41 @@ pub const Utf16LeIterator = struct {
325 }324 }
326};325};
327326
327/// Returns the length of a supplied UTF-16 string literal in terms of unicode
328/// codepoints.
329pub fn utf16CountCodepoints(utf16le: []const u16) !usize {
330 var len: usize = 0;
331 var it = Utf16LeIterator.init(utf16le);
332 while (try it.nextCodepoint()) |_| len += 1;
333 return len;
334}
335
336fn testUtf16CountCodepoints() !void {
337 try testing.expectEqual(
338 @as(usize, 1),
339 try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("a")),
340 );
341 try testing.expectEqual(
342 @as(usize, 10),
343 try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("abcdefghij")),
344 );
345 try testing.expectEqual(
346 @as(usize, 10),
347 try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("äåéëþüúíóö")),
348 );
349 try testing.expectEqual(
350 @as(usize, 5),
351 try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("こんにちは")),
352 );
353}
354
355test "utf16 count codepoints" {
356 try testUtf16CountCodepoints();
357 // TODO stage1 error: out of bounds slice
358 if (@import("builtin").zig_backend != .stage1)
359 comptime try testUtf16CountCodepoints();
360}
361
328test "utf8 encode" {362test "utf8 encode" {
329 comptime try testUtf8Encode();363 comptime try testUtf8Encode();
330 try testUtf8Encode();364 try testUtf8Encode();
...@@ -748,9 +782,9 @@ test "utf8ToUtf16LeWithNull" {...@@ -748,9 +782,9 @@ test "utf8ToUtf16LeWithNull" {
748}782}
749783
750/// Converts a UTF-8 string literal into a UTF-16LE string literal.784/// Converts a UTF-8 string literal into a UTF-16LE string literal.
751pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8):0]u16 {785pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch unreachable:0]u16 {
752 comptime {786 comptime {
753 const len: usize = calcUtf16LeLen(utf8);787 const len: usize = calcUtf16LeLen(utf8) catch |err| @compileError(err);
754 var utf16le: [len:0]u16 = [_:0]u16{0} ** len;788 var utf16le: [len:0]u16 = [_:0]u16{0} ** len;
755 const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch |err| @compileError(err);789 const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch |err| @compileError(err);
756 assert(len == utf16le_len);790 assert(len == utf16le_len);
...@@ -758,13 +792,17 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le...@@ -758,13 +792,17 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le
758 }792 }
759}793}
760794
761fn calcUtf16LeLen(utf8: []const u8) usize {795const CalcUtf16LeLenError = Utf8DecodeError || error{Utf8InvalidStartByte};
796
797/// Returns length in UTF-16 of UTF-8 slice as length of []u16.
798/// Length in []u8 is 2*len16.
799pub fn calcUtf16LeLen(utf8: []const u8) CalcUtf16LeLenError!usize {
762 var src_i: usize = 0;800 var src_i: usize = 0;
763 var dest_len: usize = 0;801 var dest_len: usize = 0;
764 while (src_i < utf8.len) {802 while (src_i < utf8.len) {
765 const n = utf8ByteSequenceLength(utf8[src_i]) catch unreachable;803 const n = try utf8ByteSequenceLength(utf8[src_i]);
766 const next_src_i = src_i + n;804 const next_src_i = src_i + n;
767 const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch unreachable;805 const codepoint = try utf8Decode(utf8[src_i..next_src_i]);
768 if (codepoint < 0x10000) {806 if (codepoint < 0x10000) {
769 dest_len += 1;807 dest_len += 1;
770 } else {808 } else {
...@@ -775,6 +813,18 @@ fn calcUtf16LeLen(utf8: []const u8) usize {...@@ -775,6 +813,18 @@ fn calcUtf16LeLen(utf8: []const u8) usize {
775 return dest_len;813 return dest_len;
776}814}
777815
816fn testCalcUtf16LeLen() !void {
817 try testing.expectEqual(@as(usize, 1), try calcUtf16LeLen("a"));
818 try testing.expectEqual(@as(usize, 10), try calcUtf16LeLen("abcdefghij"));
819 try testing.expectEqual(@as(usize, 10), try calcUtf16LeLen("äåéëþüúíóö"));
820 try testing.expectEqual(@as(usize, 5), try calcUtf16LeLen("こんにちは"));
821}
822
823test "calculate utf16 string length of given utf8 string in u16" {
824 try testCalcUtf16LeLen();
825 comptime try testCalcUtf16LeLen();
826}
827
778/// Print the given `utf16le` string828/// Print the given `utf16le` string
779fn formatUtf16le(829fn formatUtf16le(
780 utf16le: []const u16,830 utf16le: []const u16,