authorgravatar for vincenz.koop@gmail.comvinnichase <vincenz.koop@gmail.com> 2024-01-14 04:47:03+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-01-13 22:47:03-05:00
log279607cae58f7be46335793df6a4a753d0a800aa
treebf2551e56fae41101867451ae48b68bf826c083a
parentb723296e1fa65b73a43b0790bdddcbfcea7d656d
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Fix fmt UTF-8 characters as fill (#18533)

Co-authored-by: Jacob Young <jacobly0@users.noreply.github.com>

3 files changed, 88 insertions(+), 34 deletions(-)

lib/std/fmt.zig+61-34
......@@ -23,7 +23,7 @@ pub const FormatOptions = struct {
2323 precision: ?usize = null,
2424 width: ?usize = null,
2525 alignment: Alignment = .right,
26 fill: u8 = ' ',
26 fill: u21 = ' ',
2727};
2828
2929/// Renders fmt string with args, calling `writer` with slices of bytes.
......@@ -211,14 +211,18 @@ fn cacheString(str: anytype) []const u8 {
211211
212212pub const Placeholder = struct {
213213 specifier_arg: []const u8,
214 fill: u8,
214 fill: u21,
215215 alignment: Alignment,
216216 arg: Specifier,
217217 width: Specifier,
218218 precision: Specifier,
219219
220220 pub fn parse(comptime str: anytype) Placeholder {
221 comptime var parser = Parser{ .buf = &str };
221 const view = std.unicode.Utf8View.initComptime(&str);
222 comptime var parser = Parser{
223 .buf = &str,
224 .iter = view.iterator(),
225 };
222226
223227 // Parse the positional argument number
224228 const arg = comptime parser.specifier() catch |err|
......@@ -230,7 +234,7 @@ pub const Placeholder = struct {
230234 // Skip the colon, if present
231235 if (comptime parser.char()) |ch| {
232236 if (ch != ':') {
233 @compileError("expected : or }, found '" ++ [1]u8{ch} ++ "'");
237 @compileError("expected : or }, found '" ++ unicode.utf8EncodeComptime(ch) ++ "'");
234238 }
235239 }
236240
......@@ -265,7 +269,7 @@ pub const Placeholder = struct {
265269 // Skip the dot, if present
266270 if (comptime parser.char()) |ch| {
267271 if (ch != '.') {
268 @compileError("expected . or }, found '" ++ [1]u8{ch} ++ "'");
272 @compileError("expected . or }, found '" ++ unicode.utf8EncodeComptime(ch) ++ "'");
269273 }
270274 }
271275
......@@ -274,7 +278,7 @@ pub const Placeholder = struct {
274278 @compileError(@errorName(err));
275279
276280 if (comptime parser.char()) |ch| {
277 @compileError("extraneous trailing character '" ++ [1]u8{ch} ++ "'");
281 @compileError("extraneous trailing character '" ++ unicode.utf8EncodeComptime(ch) ++ "'");
278282 }
279283
280284 return Placeholder{
......@@ -297,21 +301,23 @@ pub const Specifier = union(enum) {
297301pub const Parser = struct {
298302 buf: []const u8,
299303 pos: usize = 0,
304 iter: std.unicode.Utf8Iterator = undefined,
300305
301306 // Returns a decimal number or null if the current character is not a
302307 // digit
303308 pub fn number(self: *@This()) ?usize {
304309 var r: ?usize = null;
305310
306 while (self.pos < self.buf.len) : (self.pos += 1) {
307 switch (self.buf[self.pos]) {
311 while (self.peek(0)) |code_point| {
312 switch (code_point) {
308313 '0'...'9' => {
309314 if (r == null) r = 0;
310315 r.? *= 10;
311 r.? += self.buf[self.pos] - '0';
316 r.? += code_point - '0';
312317 },
313318 else => break,
314319 }
320 _ = self.iter.nextCodepoint();
315321 }
316322
317323 return r;
......@@ -319,31 +325,27 @@ pub const Parser = struct {
319325
320326 // Returns a substring of the input starting from the current position
321327 // and ending where `ch` is found or until the end if not found
322 pub fn until(self: *@This(), ch: u8) []const u8 {
323 const start = self.pos;
324
325 if (start >= self.buf.len)
326 return &[_]u8{};
327
328 while (self.pos < self.buf.len) : (self.pos += 1) {
329 if (self.buf[self.pos] == ch) break;
328 pub fn until(self: *@This(), ch: u21) []const u8 {
329 var result: []const u8 = &[_]u8{};
330 while (self.peek(0)) |code_point| {
331 if (code_point == ch)
332 break;
333 result = result ++ (self.iter.nextCodepointSlice() orelse &[_]u8{});
330334 }
331 return self.buf[start..self.pos];
335 return result;
332336 }
333337
334338 // Returns one character, if available
335 pub fn char(self: *@This()) ?u8 {
336 if (self.pos < self.buf.len) {
337 const ch = self.buf[self.pos];
338 self.pos += 1;
339 return ch;
339 pub fn char(self: *@This()) ?u21 {
340 if (self.iter.nextCodepoint()) |code_point| {
341 return code_point;
340342 }
341343 return null;
342344 }
343345
344 pub fn maybe(self: *@This(), val: u8) bool {
345 if (self.pos < self.buf.len and self.buf[self.pos] == val) {
346 self.pos += 1;
346 pub fn maybe(self: *@This(), val: u21) bool {
347 if (self.peek(0) == val) {
348 _ = self.iter.nextCodepoint();
347349 return true;
348350 }
349351 return false;
......@@ -367,8 +369,17 @@ pub const Parser = struct {
367369 }
368370
369371 // Returns the n-th next character or null if that's past the end
370 pub fn peek(self: *@This(), n: usize) ?u8 {
371 return if (self.pos + n < self.buf.len) self.buf[self.pos + n] else null;
372 pub fn peek(self: *@This(), n: usize) ?u21 {
373 const original_i = self.iter.i;
374 defer self.iter.i = original_i;
375
376 var i = 0;
377 var code_point: ?u21 = null;
378 while (i <= n) : (i += 1) {
379 code_point = self.iter.nextCodepoint();
380 if (code_point == null) return null;
381 }
382 return code_point;
372383 }
373384};
374385
......@@ -965,8 +976,7 @@ pub fn formatUnicodeCodepoint(
965976 var buf: [4]u8 = undefined;
966977 const len = unicode.utf8Encode(c, &buf) catch |err| switch (err) {
967978 error.Utf8CannotEncodeSurrogateHalf, error.CodepointTooLarge => {
968 const len = unicode.utf8Encode(unicode.replacement_character, &buf) catch unreachable;
969 return formatBuf(buf[0..len], options, writer);
979 return formatBuf(&unicode.utf8EncodeComptime(unicode.replacement_character), options, writer);
970980 },
971981 };
972982 return formatBuf(buf[0..len], options, writer);
......@@ -985,20 +995,28 @@ pub fn formatBuf(
985995 if (padding == 0)
986996 return writer.writeAll(buf);
987997
998 var fill_buffer: [4]u8 = undefined;
999 const fill_utf8 = if (unicode.utf8Encode(options.fill, &fill_buffer)) |len|
1000 fill_buffer[0..len]
1001 else |err| switch (err) {
1002 error.Utf8CannotEncodeSurrogateHalf,
1003 error.CodepointTooLarge,
1004 => &unicode.utf8EncodeComptime(unicode.replacement_character),
1005 };
9881006 switch (options.alignment) {
9891007 .left => {
9901008 try writer.writeAll(buf);
991 try writer.writeByteNTimes(options.fill, padding);
1009 try writer.writeBytesNTimes(fill_utf8, padding);
9921010 },
9931011 .center => {
9941012 const left_padding = padding / 2;
9951013 const right_padding = (padding + 1) / 2;
996 try writer.writeByteNTimes(options.fill, left_padding);
1014 try writer.writeBytesNTimes(fill_utf8, left_padding);
9971015 try writer.writeAll(buf);
998 try writer.writeByteNTimes(options.fill, right_padding);
1016 try writer.writeBytesNTimes(fill_utf8, right_padding);
9991017 },
10001018 .right => {
1001 try writer.writeByteNTimes(options.fill, padding);
1019 try writer.writeBytesNTimes(fill_utf8, padding);
10021020 try writer.writeAll(buf);
10031021 },
10041022 }
......@@ -2793,6 +2811,15 @@ test "padding" {
27932811 try expectFmt("a====", "{c:=<5}", .{'a'});
27942812}
27952813
2814test "padding fill char utf" {
2815 try expectFmt("──crêpe───", "{s:─^10}", .{"crêpe"});
2816 try expectFmt("─────crêpe", "{s:─>10}", .{"crêpe"});
2817 try expectFmt("crêpe─────", "{s:─<10}", .{"crêpe"});
2818 try expectFmt("────a", "{c:─>5}", .{'a'});
2819 try expectFmt("──a──", "{c:─^5}", .{'a'});
2820 try expectFmt("a────", "{c:─<5}", .{'a'});
2821}
2822
27962823test "decimal float padding" {
27972824 const number: f32 = 3.1415;
27982825 try expectFmt("left-pad: **3.141\n", "left-pad: {d:*>7.3}\n", .{number});
lib/std/io/writer.zig+7
......@@ -45,6 +45,13 @@ pub fn Writer(
4545 }
4646 }
4747
48 pub fn writeBytesNTimes(self: Self, bytes: []const u8, n: usize) Error!void {
49 var i: usize = 0;
50 while (i < n) : (i += 1) {
51 try self.writeAll(bytes);
52 }
53 }
54
4855 pub inline fn writeInt(self: Self, comptime T: type, value: T, endian: std.builtin.Endian) Error!void {
4956 var bytes: [@divExact(@typeInfo(T).Int.bits, 8)]u8 = undefined;
5057 mem.writeInt(std.math.ByteAlignedInt(@TypeOf(value)), &bytes, value, endian);
lib/std/unicode.zig+20
......@@ -69,6 +69,19 @@ pub fn utf8Encode(c: u21, out: []u8) !u3 {
6969 return length;
7070}
7171
72pub inline fn utf8EncodeComptime(comptime c: u21) [
73 utf8CodepointSequenceLength(c) catch |err|
74 @compileError(@errorName(err))
75]u8 {
76 comptime var result: [
77 utf8CodepointSequenceLength(c) catch
78 unreachable
79 ]u8 = undefined;
80 comptime assert((utf8Encode(c, &result) catch |err|
81 @compileError(@errorName(err))) == result.len);
82 return result;
83}
84
7285const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error;
7386
7487/// Decodes the UTF-8 codepoint encoded in the given slice of bytes.
......@@ -525,6 +538,13 @@ fn testUtf8Encode() !void {
525538 try testing.expect(array[3] == 0b10001000);
526539}
527540
541test "utf8 encode comptime" {
542 try testing.expectEqualSlices(u8, "€", &utf8EncodeComptime('€'));
543 try testing.expectEqualSlices(u8, "$", &utf8EncodeComptime('$'));
544 try testing.expectEqualSlices(u8, "¢", &utf8EncodeComptime('¢'));
545 try testing.expectEqualSlices(u8, "𐍈", &utf8EncodeComptime('𐍈'));
546}
547
528548test "utf8 encode error" {
529549 try comptime testUtf8EncodeError();
530550 try testUtf8EncodeError();