authorgravatar for vincenz.koop@gmail.comvinnichase <vincenz.koop@gmail.com> 2024-01-14 04:47:03+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-01-13 22:47:03-05:00
log279607cae58f7be46335793df6a4a753d0a800aa
treebf2551e56fae41101867451ae48b68bf826c083a
parentb723296e1fa65b73a43b0790bdddcbfcea7d656d
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Fix fmt UTF-8 characters as fill (#18533)

Co-authored-by: Jacob Young <jacobly0@users.noreply.github.com>

3 files changed, 88 insertions(+), 34 deletions(-)

lib/std/fmt.zig+61-34
...@@ -23,7 +23,7 @@ pub const FormatOptions = struct {...@@ -23,7 +23,7 @@ pub const FormatOptions = struct {
23 precision: ?usize = null,23 precision: ?usize = null,
24 width: ?usize = null,24 width: ?usize = null,
25 alignment: Alignment = .right,25 alignment: Alignment = .right,
26 fill: u8 = ' ',26 fill: u21 = ' ',
27};27};
2828
29/// Renders fmt string with args, calling `writer` with slices of bytes.29/// Renders fmt string with args, calling `writer` with slices of bytes.
...@@ -211,14 +211,18 @@ fn cacheString(str: anytype) []const u8 {...@@ -211,14 +211,18 @@ fn cacheString(str: anytype) []const u8 {
211211
212pub const Placeholder = struct {212pub const Placeholder = struct {
213 specifier_arg: []const u8,213 specifier_arg: []const u8,
214 fill: u8,214 fill: u21,
215 alignment: Alignment,215 alignment: Alignment,
216 arg: Specifier,216 arg: Specifier,
217 width: Specifier,217 width: Specifier,
218 precision: Specifier,218 precision: Specifier,
219219
220 pub fn parse(comptime str: anytype) Placeholder {220 pub fn parse(comptime str: anytype) Placeholder {
221 comptime var parser = Parser{ .buf = &str };221 const view = std.unicode.Utf8View.initComptime(&str);
222 comptime var parser = Parser{
223 .buf = &str,
224 .iter = view.iterator(),
225 };
222226
223 // Parse the positional argument number227 // Parse the positional argument number
224 const arg = comptime parser.specifier() catch |err|228 const arg = comptime parser.specifier() catch |err|
...@@ -230,7 +234,7 @@ pub const Placeholder = struct {...@@ -230,7 +234,7 @@ pub const Placeholder = struct {
230 // Skip the colon, if present234 // Skip the colon, if present
231 if (comptime parser.char()) |ch| {235 if (comptime parser.char()) |ch| {
232 if (ch != ':') {236 if (ch != ':') {
233 @compileError("expected : or }, found '" ++ [1]u8{ch} ++ "'");237 @compileError("expected : or }, found '" ++ unicode.utf8EncodeComptime(ch) ++ "'");
234 }238 }
235 }239 }
236240
...@@ -265,7 +269,7 @@ pub const Placeholder = struct {...@@ -265,7 +269,7 @@ pub const Placeholder = struct {
265 // Skip the dot, if present269 // Skip the dot, if present
266 if (comptime parser.char()) |ch| {270 if (comptime parser.char()) |ch| {
267 if (ch != '.') {271 if (ch != '.') {
268 @compileError("expected . or }, found '" ++ [1]u8{ch} ++ "'");272 @compileError("expected . or }, found '" ++ unicode.utf8EncodeComptime(ch) ++ "'");
269 }273 }
270 }274 }
271275
...@@ -274,7 +278,7 @@ pub const Placeholder = struct {...@@ -274,7 +278,7 @@ pub const Placeholder = struct {
274 @compileError(@errorName(err));278 @compileError(@errorName(err));
275279
276 if (comptime parser.char()) |ch| {280 if (comptime parser.char()) |ch| {
277 @compileError("extraneous trailing character '" ++ [1]u8{ch} ++ "'");281 @compileError("extraneous trailing character '" ++ unicode.utf8EncodeComptime(ch) ++ "'");
278 }282 }
279283
280 return Placeholder{284 return Placeholder{
...@@ -297,21 +301,23 @@ pub const Specifier = union(enum) {...@@ -297,21 +301,23 @@ pub const Specifier = union(enum) {
297pub const Parser = struct {301pub const Parser = struct {
298 buf: []const u8,302 buf: []const u8,
299 pos: usize = 0,303 pos: usize = 0,
304 iter: std.unicode.Utf8Iterator = undefined,
300305
301 // Returns a decimal number or null if the current character is not a306 // Returns a decimal number or null if the current character is not a
302 // digit307 // digit
303 pub fn number(self: *@This()) ?usize {308 pub fn number(self: *@This()) ?usize {
304 var r: ?usize = null;309 var r: ?usize = null;
305310
306 while (self.pos < self.buf.len) : (self.pos += 1) {311 while (self.peek(0)) |code_point| {
307 switch (self.buf[self.pos]) {312 switch (code_point) {
308 '0'...'9' => {313 '0'...'9' => {
309 if (r == null) r = 0;314 if (r == null) r = 0;
310 r.? *= 10;315 r.? *= 10;
311 r.? += self.buf[self.pos] - '0';316 r.? += code_point - '0';
312 },317 },
313 else => break,318 else => break,
314 }319 }
320 _ = self.iter.nextCodepoint();
315 }321 }
316322
317 return r;323 return r;
...@@ -319,31 +325,27 @@ pub const Parser = struct {...@@ -319,31 +325,27 @@ pub const Parser = struct {
319325
320 // Returns a substring of the input starting from the current position326 // Returns a substring of the input starting from the current position
321 // and ending where `ch` is found or until the end if not found327 // and ending where `ch` is found or until the end if not found
322 pub fn until(self: *@This(), ch: u8) []const u8 {328 pub fn until(self: *@This(), ch: u21) []const u8 {
323 const start = self.pos;329 var result: []const u8 = &[_]u8{};
324330 while (self.peek(0)) |code_point| {
325 if (start >= self.buf.len)331 if (code_point == ch)
326 return &[_]u8{};332 break;
327333 result = result ++ (self.iter.nextCodepointSlice() orelse &[_]u8{});
328 while (self.pos < self.buf.len) : (self.pos += 1) {
329 if (self.buf[self.pos] == ch) break;
330 }334 }
331 return self.buf[start..self.pos];335 return result;
332 }336 }
333337
334 // Returns one character, if available338 // Returns one character, if available
335 pub fn char(self: *@This()) ?u8 {339 pub fn char(self: *@This()) ?u21 {
336 if (self.pos < self.buf.len) {340 if (self.iter.nextCodepoint()) |code_point| {
337 const ch = self.buf[self.pos];341 return code_point;
338 self.pos += 1;
339 return ch;
340 }342 }
341 return null;343 return null;
342 }344 }
343345
344 pub fn maybe(self: *@This(), val: u8) bool {346 pub fn maybe(self: *@This(), val: u21) bool {
345 if (self.pos < self.buf.len and self.buf[self.pos] == val) {347 if (self.peek(0) == val) {
346 self.pos += 1;348 _ = self.iter.nextCodepoint();
347 return true;349 return true;
348 }350 }
349 return false;351 return false;
...@@ -367,8 +369,17 @@ pub const Parser = struct {...@@ -367,8 +369,17 @@ pub const Parser = struct {
367 }369 }
368370
369 // Returns the n-th next character or null if that's past the end371 // Returns the n-th next character or null if that's past the end
370 pub fn peek(self: *@This(), n: usize) ?u8 {372 pub fn peek(self: *@This(), n: usize) ?u21 {
371 return if (self.pos + n < self.buf.len) self.buf[self.pos + n] else null;373 const original_i = self.iter.i;
374 defer self.iter.i = original_i;
375
376 var i = 0;
377 var code_point: ?u21 = null;
378 while (i <= n) : (i += 1) {
379 code_point = self.iter.nextCodepoint();
380 if (code_point == null) return null;
381 }
382 return code_point;
372 }383 }
373};384};
374385
...@@ -965,8 +976,7 @@ pub fn formatUnicodeCodepoint(...@@ -965,8 +976,7 @@ pub fn formatUnicodeCodepoint(
965 var buf: [4]u8 = undefined;976 var buf: [4]u8 = undefined;
966 const len = unicode.utf8Encode(c, &buf) catch |err| switch (err) {977 const len = unicode.utf8Encode(c, &buf) catch |err| switch (err) {
967 error.Utf8CannotEncodeSurrogateHalf, error.CodepointTooLarge => {978 error.Utf8CannotEncodeSurrogateHalf, error.CodepointTooLarge => {
968 const len = unicode.utf8Encode(unicode.replacement_character, &buf) catch unreachable;979 return formatBuf(&unicode.utf8EncodeComptime(unicode.replacement_character), options, writer);
969 return formatBuf(buf[0..len], options, writer);
970 },980 },
971 };981 };
972 return formatBuf(buf[0..len], options, writer);982 return formatBuf(buf[0..len], options, writer);
...@@ -985,20 +995,28 @@ pub fn formatBuf(...@@ -985,20 +995,28 @@ pub fn formatBuf(
985 if (padding == 0)995 if (padding == 0)
986 return writer.writeAll(buf);996 return writer.writeAll(buf);
987997
998 var fill_buffer: [4]u8 = undefined;
999 const fill_utf8 = if (unicode.utf8Encode(options.fill, &fill_buffer)) |len|
1000 fill_buffer[0..len]
1001 else |err| switch (err) {
1002 error.Utf8CannotEncodeSurrogateHalf,
1003 error.CodepointTooLarge,
1004 => &unicode.utf8EncodeComptime(unicode.replacement_character),
1005 };
988 switch (options.alignment) {1006 switch (options.alignment) {
989 .left => {1007 .left => {
990 try writer.writeAll(buf);1008 try writer.writeAll(buf);
991 try writer.writeByteNTimes(options.fill, padding);1009 try writer.writeBytesNTimes(fill_utf8, padding);
992 },1010 },
993 .center => {1011 .center => {
994 const left_padding = padding / 2;1012 const left_padding = padding / 2;
995 const right_padding = (padding + 1) / 2;1013 const right_padding = (padding + 1) / 2;
996 try writer.writeByteNTimes(options.fill, left_padding);1014 try writer.writeBytesNTimes(fill_utf8, left_padding);
997 try writer.writeAll(buf);1015 try writer.writeAll(buf);
998 try writer.writeByteNTimes(options.fill, right_padding);1016 try writer.writeBytesNTimes(fill_utf8, right_padding);
999 },1017 },
1000 .right => {1018 .right => {
1001 try writer.writeByteNTimes(options.fill, padding);1019 try writer.writeBytesNTimes(fill_utf8, padding);
1002 try writer.writeAll(buf);1020 try writer.writeAll(buf);
1003 },1021 },
1004 }1022 }
...@@ -2793,6 +2811,15 @@ test "padding" {...@@ -2793,6 +2811,15 @@ test "padding" {
2793 try expectFmt("a====", "{c:=<5}", .{'a'});2811 try expectFmt("a====", "{c:=<5}", .{'a'});
2794}2812}
27952813
2814test "padding fill char utf" {
2815 try expectFmt("──crêpe───", "{s:─^10}", .{"crêpe"});
2816 try expectFmt("─────crêpe", "{s:─>10}", .{"crêpe"});
2817 try expectFmt("crêpe─────", "{s:─<10}", .{"crêpe"});
2818 try expectFmt("────a", "{c:─>5}", .{'a'});
2819 try expectFmt("──a──", "{c:─^5}", .{'a'});
2820 try expectFmt("a────", "{c:─<5}", .{'a'});
2821}
2822
2796test "decimal float padding" {2823test "decimal float padding" {
2797 const number: f32 = 3.1415;2824 const number: f32 = 3.1415;
2798 try expectFmt("left-pad: **3.141\n", "left-pad: {d:*>7.3}\n", .{number});2825 try expectFmt("left-pad: **3.141\n", "left-pad: {d:*>7.3}\n", .{number});
lib/std/io/writer.zig+7
...@@ -45,6 +45,13 @@ pub fn Writer(...@@ -45,6 +45,13 @@ pub fn Writer(
45 }45 }
46 }46 }
4747
48 pub fn writeBytesNTimes(self: Self, bytes: []const u8, n: usize) Error!void {
49 var i: usize = 0;
50 while (i < n) : (i += 1) {
51 try self.writeAll(bytes);
52 }
53 }
54
48 pub inline fn writeInt(self: Self, comptime T: type, value: T, endian: std.builtin.Endian) Error!void {55 pub inline fn writeInt(self: Self, comptime T: type, value: T, endian: std.builtin.Endian) Error!void {
49 var bytes: [@divExact(@typeInfo(T).Int.bits, 8)]u8 = undefined;56 var bytes: [@divExact(@typeInfo(T).Int.bits, 8)]u8 = undefined;
50 mem.writeInt(std.math.ByteAlignedInt(@TypeOf(value)), &bytes, value, endian);57 mem.writeInt(std.math.ByteAlignedInt(@TypeOf(value)), &bytes, value, endian);
lib/std/unicode.zig+20
...@@ -69,6 +69,19 @@ pub fn utf8Encode(c: u21, out: []u8) !u3 {...@@ -69,6 +69,19 @@ pub fn utf8Encode(c: u21, out: []u8) !u3 {
69 return length;69 return length;
70}70}
7171
72pub inline fn utf8EncodeComptime(comptime c: u21) [
73 utf8CodepointSequenceLength(c) catch |err|
74 @compileError(@errorName(err))
75]u8 {
76 comptime var result: [
77 utf8CodepointSequenceLength(c) catch
78 unreachable
79 ]u8 = undefined;
80 comptime assert((utf8Encode(c, &result) catch |err|
81 @compileError(@errorName(err))) == result.len);
82 return result;
83}
84
72const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error;85const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error;
7386
74/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.87/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
...@@ -525,6 +538,13 @@ fn testUtf8Encode() !void {...@@ -525,6 +538,13 @@ fn testUtf8Encode() !void {
525 try testing.expect(array[3] == 0b10001000);538 try testing.expect(array[3] == 0b10001000);
526}539}
527540
541test "utf8 encode comptime" {
542 try testing.expectEqualSlices(u8, "€", &utf8EncodeComptime('€'));
543 try testing.expectEqualSlices(u8, "$", &utf8EncodeComptime('$'));
544 try testing.expectEqualSlices(u8, "¢", &utf8EncodeComptime('¢'));
545 try testing.expectEqualSlices(u8, "𐍈", &utf8EncodeComptime('𐍈'));
546}
547
528test "utf8 encode error" {548test "utf8 encode error" {
529 try comptime testUtf8EncodeError();549 try comptime testUtf8EncodeError();
530 try testUtf8EncodeError();550 try testUtf8EncodeError();