| ... | ... | @@ -134,6 +134,13 @@ pub const Header = struct { |
| 134 | 134 | } |
| 135 | 135 | return header.bytes[start..i]; |
| 136 | 136 | } |
| 137 | |
| 138 | pub fn isZeroBlock(header: Header) bool { |
| 139 | for (header.bytes) |b| { |
| 140 | if (b != 0) return false; |
| 141 | } |
| 142 | return true; |
| 143 | } |
| 137 | 144 | }; |
| 138 | 145 | |
| 139 | 146 | fn BufferedReader(comptime ReaderType: type) type { |
| ... | ... | @@ -225,7 +232,7 @@ fn Iterator(comptime ReaderType: type) type { |
| 225 | 232 | const Self = @This(); |
| 226 | 233 | |
| 227 | 234 | const File = struct { |
| 228 | | file_name: []const u8, |
| 235 | name: []const u8, |
| 229 | 236 | link_name: []const u8, |
| 230 | 237 | size: usize, |
| 231 | 238 | file_type: Header.FileType, |
| ... | ... | @@ -239,6 +246,31 @@ fn Iterator(comptime ReaderType: type) type { |
| 239 | 246 | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); |
| 240 | 247 | try self.iter.reader.skip(rounded_file_size); |
| 241 | 248 | } |
| 249 | |
| 250 | fn chksum(self: File) ![16]u8 { |
| 251 | var cs = [_]u8{0} ** 16; |
| 252 | if (self.size == 0) return cs; |
| 253 | |
| 254 | var buffer: [512]u8 = undefined; |
| 255 | var h = std.crypto.hash.Md5.init(.{}); |
| 256 | |
| 257 | var remaining_bytes: usize = self.size; |
| 258 | while (remaining_bytes > 0) { |
| 259 | const copy_size = @min(buffer.len, remaining_bytes); |
| 260 | try self.iter.reader.copy(&buffer, copy_size); |
| 261 | h.update(buffer[0..copy_size]); |
| 262 | remaining_bytes -= copy_size; |
| 263 | } |
| 264 | h.final(&cs); |
| 265 | try self.skipPadding(); |
| 266 | return cs; |
| 267 | } |
| 268 | |
| 269 | fn skipPadding(self: File) !void { |
| 270 | const rounded_file_size = std.mem.alignForward(usize, self.size, 512); |
| 271 | const pad_len: usize = rounded_file_size - self.size; |
| 272 | self.iter.reader.advance(pad_len); |
| 273 | } |
| 242 | 274 | }; |
| 243 | 275 | |
| 244 | 276 | pub fn next(self: *Self) !?File { |
| ... | ... | @@ -253,6 +285,7 @@ fn Iterator(comptime ReaderType: type) type { |
| 253 | 285 | self.reader.advance(512); |
| 254 | 286 | |
| 255 | 287 | const header: Header = .{ .bytes = chunk[0..512] }; |
| 288 | if (header.isZeroBlock()) return null; |
| 256 | 289 | const file_size = try header.fileSize(); |
| 257 | 290 | const file_type = header.fileType(); |
| 258 | 291 | const link_name = header.linkName(); |
| ... | ... | @@ -266,10 +299,10 @@ fn Iterator(comptime ReaderType: type) type { |
| 266 | 299 | switch (file_type) { |
| 267 | 300 | .directory, .normal, .symbolic_link => { |
| 268 | 301 | return File{ |
| 269 | | .file_name = file_name, |
| 270 | | .link_name = link_name, |
| 302 | .name = file_name, |
| 271 | 303 | .size = file_size, |
| 272 | 304 | .file_type = file_type, |
| 305 | .link_name = link_name, |
| 273 | 306 | .iter = self, |
| 274 | 307 | }; |
| 275 | 308 | }, |
| ... | ... | @@ -341,19 +374,19 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi |
| 341 | 374 | |
| 342 | 375 | var iter = iterator(reader, options.diagnostics); |
| 343 | 376 | |
| 344 | | while (try iter.next()) |iter_file| { |
| 345 | | switch (iter_file.file_type) { |
| 377 | while (try iter.next()) |file| { |
| 378 | switch (file.file_type) { |
| 346 | 379 | .directory => { |
| 347 | | const file_name = try stripComponents(iter_file.file_name, options.strip_components); |
| 380 | const file_name = try stripComponents(file.name, options.strip_components); |
| 348 | 381 | if (file_name.len != 0 and !options.exclude_empty_directories) { |
| 349 | 382 | try dir.makePath(file_name); |
| 350 | 383 | } |
| 351 | 384 | }, |
| 352 | 385 | .normal => { |
| 353 | | if (iter_file.size == 0 and iter_file.file_name.len == 0) return; |
| 354 | | const file_name = try stripComponents(iter_file.file_name, options.strip_components); |
| 386 | if (file.size == 0 and file.name.len == 0) return; |
| 387 | const file_name = try stripComponents(file.name, options.strip_components); |
| 355 | 388 | |
| 356 | | const file = dir.createFile(file_name, .{}) catch |err| switch (err) { |
| 389 | const fs_file = dir.createFile(file_name, .{}) catch |err| switch (err) { |
| 357 | 390 | error.FileNotFound => again: { |
| 358 | 391 | const code = code: { |
| 359 | 392 | if (std.fs.path.dirname(file_name)) |dir_name| { |
| ... | ... | @@ -373,19 +406,19 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi |
| 373 | 406 | }, |
| 374 | 407 | else => |e| return e, |
| 375 | 408 | }; |
| 376 | | defer if (file) |f| f.close(); |
| 409 | defer if (fs_file) |f| f.close(); |
| 377 | 410 | |
| 378 | | if (file) |f| { |
| 379 | | try iter_file.write(f); |
| 411 | if (fs_file) |f| { |
| 412 | try file.write(f); |
| 380 | 413 | } else { |
| 381 | | try iter_file.skip(); |
| 414 | try file.skip(); |
| 382 | 415 | } |
| 383 | 416 | }, |
| 384 | 417 | .symbolic_link => { |
| 385 | 418 | // The file system path of the symbolic link. |
| 386 | | const file_name = try stripComponents(iter_file.file_name, options.strip_components); |
| 419 | const file_name = try stripComponents(file.name, options.strip_components); |
| 387 | 420 | // The data inside the symbolic link. |
| 388 | | const link_name = iter_file.link_name; |
| 421 | const link_name = file.link_name; |
| 389 | 422 | |
| 390 | 423 | dir.symLink(link_name, file_name, .{}) catch |err| again: { |
| 391 | 424 | const code = code: { |
| ... | ... | @@ -473,3 +506,274 @@ test parsePaxAttribute { |
| 473 | 506 | |
| 474 | 507 | const std = @import("std.zig"); |
| 475 | 508 | const assert = std.debug.assert; |
| 509 | |
| 510 | const TestCase = struct { |
| 511 | const File = struct { |
| 512 | const empty_string = &[0]u8{}; |
| 513 | |
| 514 | name: []const u8, |
| 515 | size: usize = 0, |
| 516 | link_name: []const u8 = empty_string, |
| 517 | file_type: Header.FileType = .normal, |
| 518 | }; |
| 519 | |
| 520 | path: []const u8, |
| 521 | files: []const File = &[_]TestCase.File{}, |
| 522 | chksums: []const []const u8 = &[_][]const u8{}, |
| 523 | err: ?anyerror = null, |
| 524 | }; |
| 525 | |
| 526 | test "Go test cases" { |
| 527 | const test_dir = try std.fs.openDirAbsolute("/usr/local/go/src/archive/tar/testdata", .{}); |
| 528 | const cases = [_]TestCase{ |
| 529 | .{ |
| 530 | .path = "gnu.tar", |
| 531 | .files = &[_]TestCase.File{ |
| 532 | .{ |
| 533 | .name = "small.txt", |
| 534 | .size = 5, |
| 535 | .file_type = .normal, |
| 536 | }, |
| 537 | .{ |
| 538 | .name = "small2.txt", |
| 539 | .size = 11, |
| 540 | .file_type = .normal, |
| 541 | }, |
| 542 | }, |
| 543 | .chksums = &[_][]const u8{ |
| 544 | "e38b27eaccb4391bdec553a7f3ae6b2f", |
| 545 | "c65bd2e50a56a2138bf1716f2fd56fe9", |
| 546 | }, |
| 547 | }, |
| 548 | .{ |
| 549 | .path = "sparse-formats.tar", |
| 550 | .err = error.TarUnsupportedFileType, |
| 551 | }, |
| 552 | .{ |
| 553 | .path = "star.tar", |
| 554 | .files = &[_]TestCase.File{ |
| 555 | .{ |
| 556 | .name = "small.txt", |
| 557 | .size = 5, |
| 558 | .file_type = .normal, |
| 559 | }, |
| 560 | .{ |
| 561 | .name = "small2.txt", |
| 562 | .size = 11, |
| 563 | .file_type = .normal, |
| 564 | }, |
| 565 | }, |
| 566 | .chksums = &[_][]const u8{ |
| 567 | "e38b27eaccb4391bdec553a7f3ae6b2f", |
| 568 | "c65bd2e50a56a2138bf1716f2fd56fe9", |
| 569 | }, |
| 570 | }, |
| 571 | .{ |
| 572 | .path = "v7.tar", |
| 573 | .files = &[_]TestCase.File{ |
| 574 | .{ |
| 575 | .name = "small.txt", |
| 576 | .size = 5, |
| 577 | .file_type = .normal, |
| 578 | }, |
| 579 | .{ |
| 580 | .name = "small2.txt", |
| 581 | .size = 11, |
| 582 | .file_type = .normal, |
| 583 | }, |
| 584 | }, |
| 585 | .chksums = &[_][]const u8{ |
| 586 | "e38b27eaccb4391bdec553a7f3ae6b2f", |
| 587 | "c65bd2e50a56a2138bf1716f2fd56fe9", |
| 588 | }, |
| 589 | }, |
| 590 | .{ |
| 591 | .path = "pax.tar", |
| 592 | .files = &[_]TestCase.File{ |
| 593 | .{ |
| 594 | .name = "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", |
| 595 | .size = 7, |
| 596 | .file_type = .normal, |
| 597 | }, |
| 598 | .{ |
| 599 | .name = "a/b", |
| 600 | .size = 0, |
| 601 | .file_type = .symbolic_link, |
| 602 | .link_name = "1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545", |
| 603 | // TODO fix reading link name from pax header |
| 604 | // .link_name = "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", |
| 605 | }, |
| 606 | }, |
| 607 | .chksums = &[_][]const u8{ |
| 608 | "3c382e8f5b6631aa2db52643912ffd4a", |
| 609 | }, |
| 610 | }, |
| 611 | // TODO: this should fail |
| 612 | // .{ |
| 613 | // .path = "pax-bad-hdr-file.tar", |
| 614 | // .err = error.TarBadHeader, |
| 615 | // }, |
| 616 | // .{ |
| 617 | // .path = "pax-bad-mtime-file.tar", |
| 618 | // .err = error.TarBadHeader, |
| 619 | // }, |
| 620 | // |
| 621 | // TODO: giving wrong result because we are not reading pax size header |
| 622 | // .{ |
| 623 | // .path = "pax-pos-size-file.tar", |
| 624 | // .files = &[_]TestCase.File{ |
| 625 | // .{ |
| 626 | // .name = "foo", |
| 627 | // .size = 999, |
| 628 | // .file_type = .normal, |
| 629 | // }, |
| 630 | // }, |
| 631 | // .chksums = &[_][]const u8{ |
| 632 | // "0afb597b283fe61b5d4879669a350556", |
| 633 | // }, |
| 634 | // }, |
| 635 | .{ |
| 636 | // has pax records which we are not interested in |
| 637 | .path = "pax-records.tar", |
| 638 | .files = &[_]TestCase.File{ |
| 639 | .{ |
| 640 | .name = "file", |
| 641 | }, |
| 642 | }, |
| 643 | }, |
| 644 | .{ |
| 645 | // has global records which we are ignoring |
| 646 | .path = "pax-global-records.tar", |
| 647 | .files = &[_]TestCase.File{ |
| 648 | .{ |
| 649 | .name = "file1", |
| 650 | }, |
| 651 | .{ |
| 652 | .name = "file2", |
| 653 | }, |
| 654 | .{ |
| 655 | .name = "file3", |
| 656 | }, |
| 657 | .{ |
| 658 | .name = "file4", |
| 659 | }, |
| 660 | }, |
| 661 | }, |
| 662 | .{ |
| 663 | .path = "nil-uid.tar", |
| 664 | .files = &[_]TestCase.File{ |
| 665 | .{ |
| 666 | .name = "P1050238.JPG.log", |
| 667 | .size = 14, |
| 668 | .file_type = .normal, |
| 669 | }, |
| 670 | }, |
| 671 | .chksums = &[_][]const u8{ |
| 672 | "08d504674115e77a67244beac19668f5", |
| 673 | }, |
| 674 | }, |
| 675 | .{ |
| 676 | // has xattrs and pax records which we are ignoring |
| 677 | .path = "xattrs.tar", |
| 678 | .files = &[_]TestCase.File{ |
| 679 | .{ |
| 680 | .name = "small.txt", |
| 681 | .size = 5, |
| 682 | .file_type = .normal, |
| 683 | }, |
| 684 | .{ |
| 685 | .name = "small2.txt", |
| 686 | .size = 11, |
| 687 | .file_type = .normal, |
| 688 | }, |
| 689 | }, |
| 690 | .chksums = &[_][]const u8{ |
| 691 | "e38b27eaccb4391bdec553a7f3ae6b2f", |
| 692 | "c65bd2e50a56a2138bf1716f2fd56fe9", |
| 693 | }, |
| 694 | }, |
| 695 | .{ |
| 696 | .path = "gnu-multi-hdrs.tar", |
| 697 | .err = error.TarUnsupportedFileType, |
| 698 | }, |
| 699 | .{ |
| 700 | .path = "gnu-incremental.tar", |
| 701 | .err = error.TarUnsupportedFileType, |
| 702 | }, |
| 703 | // .{ |
| 704 | // .path = "pax-multi-hdrs.tar", |
| 705 | // }, |
| 706 | // .{ |
| 707 | // .path = "gnu-long-nul.tar", |
| 708 | // .files = &[_]TestCase.File{ |
| 709 | // .{ |
| 710 | // .name = "012233456789", |
| 711 | // }, |
| 712 | // }, |
| 713 | // }, |
| 714 | // .{ |
| 715 | // .path = "gnu-utf8.tar", |
| 716 | // .files = &[_]TestCase.File{ |
| 717 | // .{ |
| 718 | // .name = "012233456789", |
| 719 | // }, |
| 720 | // }, |
| 721 | // }, |
| 722 | // |
| 723 | .{ |
| 724 | .path = "gnu-not-utf8.tar", |
| 725 | .files = &[_]TestCase.File{ |
| 726 | .{ |
| 727 | .name = "hi\x80\x81\x82\x83bye", |
| 728 | }, |
| 729 | }, |
| 730 | }, |
| 731 | // TODO some files with errors: |
| 732 | // pax-nul-xattrs.tar, pax-nul-path.tar, neg-size.tar, issue10968.tar, issue11169.tar, issue12435.tar |
| 733 | .{ |
| 734 | .path = "trailing-slash.tar", |
| 735 | .files = &[_]TestCase.File{ |
| 736 | .{ |
| 737 | .name = "123456789/" ** 30, |
| 738 | .file_type = .directory, |
| 739 | }, |
| 740 | }, |
| 741 | }, |
| 742 | }; |
| 743 | |
| 744 | for (cases) |case| { |
| 745 | // if (!std.mem.eql(u8, case.path, "pax.tar")) continue; |
| 746 | |
| 747 | var fs_file = try test_dir.openFile(case.path, .{}); |
| 748 | defer fs_file.close(); |
| 749 | |
| 750 | var iter = iterator(fs_file.reader(), null); |
| 751 | var i: usize = 0; |
| 752 | while (iter.next() catch |err| { |
| 753 | if (case.err) |e| { |
| 754 | try std.testing.expectEqual(e, err); |
| 755 | continue; |
| 756 | } else { |
| 757 | return err; |
| 758 | } |
| 759 | }) |actual| { |
| 760 | const expected = case.files[i]; |
| 761 | try std.testing.expectEqualStrings(expected.name, actual.name); |
| 762 | try std.testing.expectEqual(expected.size, actual.size); |
| 763 | try std.testing.expectEqual(expected.file_type, actual.file_type); |
| 764 | try std.testing.expectEqualStrings(expected.link_name, actual.link_name); |
| 765 | |
| 766 | if (case.chksums.len > i) { |
| 767 | var actual_chksum = try actual.chksum(); |
| 768 | var hex_to_bytes_buffer: [16]u8 = undefined; |
| 769 | const expected_chksum = try std.fmt.hexToBytes(&hex_to_bytes_buffer, case.chksums[i]); |
| 770 | // std.debug.print("actual chksum: {s}\n", .{std.fmt.fmtSliceHexLower(&actual_chksum)}); |
| 771 | try std.testing.expectEqualStrings(expected_chksum, &actual_chksum); |
| 772 | } else { |
| 773 | try actual.skip(); // skip file content |
| 774 | } |
| 775 | i += 1; |
| 776 | } |
| 777 | try std.testing.expectEqual(case.files.len, i); |
| 778 | } |
| 779 | } |
Igor Anić <igor.anic@gmail.com> 2023-11-27 21:37:30+01:00
Andrew Kelley <andrew@ziglang.org> 2024-01-13 19:37:33-07:00
lib