authorgravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-28 13:54:42+11:00
committergravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-28 14:39:38+11:00
loga81c0ba2e730bd69e7c7a672fa795c46ce6b001b
treeae692057f5de4ba63c4ad75f02d6097aba0f279e
parent25e71216c4640a3d88c8f63912ea574ad6fa004c
signature Commit is signed but in an unrecognized format.

std: fix unicode encoding of astral plane codepoints to utf16

1 files changed, 26 insertions(+), 13 deletions(-)

lib/std/unicode.zig+26-13
...@@ -555,9 +555,8 @@ pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16...@@ -555,9 +555,8 @@ pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16
555 const short = @intCast(u16, codepoint);555 const short = @intCast(u16, codepoint);
556 try result.append(mem.nativeToLittle(u16, short));556 try result.append(mem.nativeToLittle(u16, short));
557 } else {557 } else {
558 const short = @intCast(u16, codepoint - 0x10000);558 const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
559 const high = (short >> 10) + 0xD800;559 const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
560 const low = (short & 0x3FF) + 0xDC00;
561 var out: [2]u16 = undefined;560 var out: [2]u16 = undefined;
562 out[0] = mem.nativeToLittle(u16, high);561 out[0] = mem.nativeToLittle(u16, high);
563 out[1] = mem.nativeToLittle(u16, low);562 out[1] = mem.nativeToLittle(u16, low);
...@@ -592,9 +591,8 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {...@@ -592,9 +591,8 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
592 utf16le[dest_i] = mem.nativeToLittle(u16, short);591 utf16le[dest_i] = mem.nativeToLittle(u16, short);
593 dest_i += 1;592 dest_i += 1;
594 } else {593 } else {
595 const short = @intCast(u16, codepoint - 0x10000);594 const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
596 const high = (short >> 10) + 0xD800;595 const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
597 const low = (short & 0x3FF) + 0xDC00;
598 utf16le[dest_i] = mem.nativeToLittle(u16, high);596 utf16le[dest_i] = mem.nativeToLittle(u16, high);
599 utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);597 utf16le[dest_i + 1] = mem.nativeToLittle(u16, low);
600 dest_i += 2;598 dest_i += 2;
...@@ -609,14 +607,29 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {...@@ -609,14 +607,29 @@ pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) !usize {
609607
610test "utf8ToUtf16Le" {608test "utf8ToUtf16Le" {
611 var utf16le: [2]u16 = [_]u16{0} ** 2;609 var utf16le: [2]u16 = [_]u16{0} ** 2;
612 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");610 {
613 testing.expect(@as(usize, 2) == length);611 const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
614 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", @sliceToBytes(utf16le[0..]));612 testing.expectEqual(@as(usize, 2), length);
613 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", @sliceToBytes(utf16le[0..]));
614 }
615 {
616 const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
617 testing.expectEqual(@as(usize, 2), length);
618 testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", @sliceToBytes(utf16le[0..]));
619 }
615}620}
616621
617test "utf8ToUtf16LeWithNull" {622test "utf8ToUtf16LeWithNull" {
618 var bytes: [128]u8 = undefined;623 {
619 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;624 var bytes: [128]u8 = undefined;
620 const utf16 = try utf8ToUtf16LeWithNull(allocator, "𐐷");625 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;
621 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc\x00\x00", @sliceToBytes(utf16[0..]));626 const utf16 = try utf8ToUtf16LeWithNull(allocator, "𐐷");
627 testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc\x00\x00", @sliceToBytes(utf16[0..]));
628 }
629 {
630 var bytes: [128]u8 = undefined;
631 const allocator = &std.heap.FixedBufferAllocator.init(bytes[0..]).allocator;
632 const utf16 = try utf8ToUtf16LeWithNull(allocator, "\u{10FFFF}");
633 testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf\x00\x00", @sliceToBytes(utf16[0..]));
634 }
622}635}