Commit 0931dda9a95e14b97a84e60aed424fd8bb5e1232 | zig

author	Andrew Kelley <andrew@ziglang.org> 2022-09-14 19:25:15-04:00
committer	GitHub <noreply@github.com> 2022-09-14 19:25:15-04:00
log	0931dda9a95e14b97a84e60aed424fd8bb5e1232
tree	5e18e74a3128b36ae807b6f108623e7567a1bf4e
parent	d7a0fe67b38a60a4f294d6d9034c7a342bed7094
parent	cf744cf04f8cad148ae93e5c5c7d8c5f5f62c164
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11663 from matu3ba/utf16

std.unicode: add utf16 byte length and codepoints counting routines

1 files changed, 56 insertions(+), 6 deletions(-)

lib
- std
  - unicode.zig

lib/std/unicode.zig+56-6

...	...	@@ -164,7 +164,6 @@ pub fn utf8ValidCodepoint(value: u21) bool {
164	164
165	165	/// Returns the length of a supplied UTF-8 string literal in terms of unicode
166	166	/// codepoints.
167		/// Asserts that the data is valid UTF-8.
168	167	pub fn utf8CountCodepoints(s: []const u8) !usize {
169	168	var len: usize = 0;
170	169
...	...	@@ -325,6 +324,41 @@ pub const Utf16LeIterator = struct {
325	324	}
326	325	};
327	326
	327	/// Returns the length of a supplied UTF-16 string literal in terms of unicode
	328	/// codepoints.
	329	pub fn utf16CountCodepoints(utf16le: []const u16) !usize {
	330	var len: usize = 0;
	331	var it = Utf16LeIterator.init(utf16le);
	332	while (try it.nextCodepoint()) \|_\| len += 1;
	333	return len;
	334	}
	335
	336	fn testUtf16CountCodepoints() !void {
	337	try testing.expectEqual(
	338	@as(usize, 1),
	339	try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("a")),
	340	);
	341	try testing.expectEqual(
	342	@as(usize, 10),
	343	try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("abcdefghij")),
	344	);
	345	try testing.expectEqual(
	346	@as(usize, 10),
	347	try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("äåéëþüúíóö")),
	348	);
	349	try testing.expectEqual(
	350	@as(usize, 5),
	351	try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("こんにちは")),
	352	);
	353	}
	354
	355	test "utf16 count codepoints" {
	356	try testUtf16CountCodepoints();
	357	// TODO stage1 error: out of bounds slice
	358	if (@import("builtin").zig_backend != .stage1)
	359	comptime try testUtf16CountCodepoints();
	360	}
	361
328	362	test "utf8 encode" {
329	363	comptime try testUtf8Encode();
330	364	try testUtf8Encode();
...	...	@@ -748,9 +782,9 @@ test "utf8ToUtf16LeWithNull" {
748	782	}
749	783
750	784	/// Converts a UTF-8 string literal into a UTF-16LE string literal.
751		pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8):0]u16 {
	785	pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch unreachable:0]u16 {
752	786	comptime {
753		const len: usize = calcUtf16LeLen(utf8);
	787	const len: usize = calcUtf16LeLen(utf8) catch \|err\| @compileError(err);
754	788	var utf16le: [len:0]u16 = [_:0]u16{0} ** len;
755	789	const utf16le_len = utf8ToUtf16Le(&utf16le, utf8[0..]) catch \|err\| @compileError(err);
756	790	assert(len == utf16le_len);
...	...	@@ -758,13 +792,17 @@ pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16Le
758	792	}
759	793	}
760	794
761		fn calcUtf16LeLen(utf8: []const u8) usize {
	795	const CalcUtf16LeLenError = Utf8DecodeError \|\| error{Utf8InvalidStartByte};
	796
	797	/// Returns length in UTF-16 of UTF-8 slice as length of []u16.
	798	/// Length in []u8 is 2*len16.
	799	pub fn calcUtf16LeLen(utf8: []const u8) CalcUtf16LeLenError!usize {
762	800	var src_i: usize = 0;
763	801	var dest_len: usize = 0;
764	802	while (src_i < utf8.len) {
765		const n = utf8ByteSequenceLength(utf8[src_i]) catch unreachable;
	803	const n = try utf8ByteSequenceLength(utf8[src_i]);
766	804	const next_src_i = src_i + n;
767		const codepoint = utf8Decode(utf8[src_i..next_src_i]) catch unreachable;
	805	const codepoint = try utf8Decode(utf8[src_i..next_src_i]);
768	806	if (codepoint < 0x10000) {
769	807	dest_len += 1;
770	808	} else {
...	...	@@ -775,6 +813,18 @@ fn calcUtf16LeLen(utf8: []const u8) usize {
775	813	return dest_len;
776	814	}
777	815
	816	fn testCalcUtf16LeLen() !void {
	817	try testing.expectEqual(@as(usize, 1), try calcUtf16LeLen("a"));
	818	try testing.expectEqual(@as(usize, 10), try calcUtf16LeLen("abcdefghij"));
	819	try testing.expectEqual(@as(usize, 10), try calcUtf16LeLen("äåéëþüúíóö"));
	820	try testing.expectEqual(@as(usize, 5), try calcUtf16LeLen("こんにちは"));
	821	}
	822
	823	test "calculate utf16 string length of given utf8 string in u16" {
	824	try testCalcUtf16LeLen();
	825	comptime try testCalcUtf16LeLen();
	826	}
	827
778	828	/// Print the given `utf16le` string
779	829	fn formatUtf16le(
780	830	utf16le: []const u16,