| ... | ... | @@ -11,15 +11,13 @@ const linux = std.os.linux; |
| 11 | 11 | const windows = std.os.windows; |
| 12 | 12 | |
| 13 | 13 | io: Io, |
| 14 | | file: Io.File, |
| 15 | 14 | flags: packed struct { |
| 16 | 15 | block_size: std.mem.Alignment, |
| 17 | 16 | copy_file_range_unsupported: bool, |
| 18 | 17 | fallocate_punch_hole_unsupported: bool, |
| 19 | 18 | fallocate_insert_range_unsupported: bool, |
| 20 | 19 | }, |
| 21 | | section: if (is_windows) windows.HANDLE else void, |
| 22 | | contents: []align(std.heap.page_size_min) u8, |
| 20 | memory_map: Io.File.MemoryMap, |
| 23 | 21 | nodes: std.ArrayList(Node), |
| 24 | 22 | free_ni: Node.Index, |
| 25 | 23 | large: std.ArrayList(u64), |
| ... | ... | @@ -29,26 +27,20 @@ writers: std.SinglyLinkedList, |
| 29 | 27 | |
| 30 | 28 | pub const growth_factor = 4; |
| 31 | 29 | |
| 32 | | pub const Error = Io.File.MemoryMap.CreateError || Io.File.LengthError || error{ |
| 30 | pub const Error = error{ |
| 33 | 31 | NotFile, |
| 34 | | SystemResources, |
| 35 | | IsDir, |
| 36 | | Unseekable, |
| 37 | | NoSpaceLeft, |
| 38 | | |
| 39 | | InputOutput, |
| 40 | | FileTooBig, |
| 41 | | FileBusy, |
| 42 | | NonResizable, |
| 43 | | }; |
| 32 | } || Io.File.MemoryMap.CreateError || Io.File.MemoryMap.SetLengthError || Io.File.WritePositionalError; |
| 44 | 33 | |
| 45 | 34 | pub fn init(file: Io.File, gpa: std.mem.Allocator, io: Io) !MappedFile { |
| 46 | 35 | var mf: MappedFile = .{ |
| 47 | 36 | .io = io, |
| 48 | | .file = file, |
| 49 | 37 | .flags = undefined, |
| 50 | | .section = if (is_windows) windows.INVALID_HANDLE_VALUE else {}, |
| 51 | | .contents = &.{}, |
| 38 | .memory_map = .{ |
| 39 | .file = file, |
| 40 | .memory = &.{}, |
| 41 | .offset = 0, |
| 42 | .section = null, |
| 43 | }, |
| 52 | 44 | .nodes = .empty, |
| 53 | 45 | .free_ni = .none, |
| 54 | 46 | .large = .empty, |
| ... | ... | @@ -58,61 +50,9 @@ pub fn init(file: Io.File, gpa: std.mem.Allocator, io: Io) !MappedFile { |
| 58 | 50 | }; |
| 59 | 51 | errdefer mf.deinit(gpa); |
| 60 | 52 | const size: u64, const block_size = stat: { |
| 61 | | if (is_windows) { |
| 62 | | var sbi: windows.SYSTEM_BASIC_INFORMATION = undefined; |
| 63 | | break :stat .{ |
| 64 | | try windows.GetFileSizeEx(file.handle), |
| 65 | | switch (windows.ntdll.NtQuerySystemInformation( |
| 66 | | .SystemBasicInformation, |
| 67 | | &sbi, |
| 68 | | @sizeOf(windows.SYSTEM_BASIC_INFORMATION), |
| 69 | | null, |
| 70 | | )) { |
| 71 | | .SUCCESS => @max(sbi.PageSize, sbi.AllocationGranularity), |
| 72 | | else => std.heap.page_size_max, |
| 73 | | }, |
| 74 | | }; |
| 75 | | } |
| 76 | | if (is_linux) { |
| 77 | | const use_c = std.c.versionCheck(if (builtin.abi.isAndroid()) |
| 78 | | .{ .major = 30, .minor = 0, .patch = 0 } |
| 79 | | else |
| 80 | | .{ .major = 2, .minor = 28, .patch = 0 }); |
| 81 | | const sys = if (use_c) std.c else std.os.linux; |
| 82 | | while (true) { |
| 83 | | var statx = std.mem.zeroes(linux.Statx); |
| 84 | | const rc = sys.statx( |
| 85 | | mf.file.handle, |
| 86 | | "", |
| 87 | | std.posix.AT.EMPTY_PATH, |
| 88 | | .{ .TYPE = true, .SIZE = true, .BLOCKS = true }, |
| 89 | | &statx, |
| 90 | | ); |
| 91 | | switch (sys.errno(rc)) { |
| 92 | | .SUCCESS => { |
| 93 | | assert(statx.mask.TYPE); |
| 94 | | assert(statx.mask.SIZE); |
| 95 | | assert(statx.mask.BLOCKS); |
| 96 | | if (!std.posix.S.ISREG(statx.mode)) return error.PathAlreadyExists; |
| 97 | | break :stat .{ statx.size, @max(std.heap.pageSize(), statx.blksize) }; |
| 98 | | }, |
| 99 | | .INTR => continue, |
| 100 | | .ACCES => return error.AccessDenied, |
| 101 | | .BADF => if (std.debug.runtime_safety) unreachable else return error.Unexpected, |
| 102 | | .FAULT => if (std.debug.runtime_safety) unreachable else return error.Unexpected, |
| 103 | | .INVAL => if (std.debug.runtime_safety) unreachable else return error.Unexpected, |
| 104 | | .LOOP => return error.SymLinkLoop, |
| 105 | | .NAMETOOLONG => return error.NameTooLong, |
| 106 | | .NOENT => return error.FileNotFound, |
| 107 | | .NOTDIR => return error.FileNotFound, |
| 108 | | .NOMEM => return error.SystemResources, |
| 109 | | else => |err| return std.posix.unexpectedErrno(err), |
| 110 | | } |
| 111 | | } |
| 112 | | } |
| 113 | | const stat = try std.posix.fstat(mf.file.handle); |
| 114 | | if (!std.posix.S.ISREG(stat.mode)) return error.PathAlreadyExists; |
| 115 | | break :stat .{ @bitCast(stat.size), @max(std.heap.pageSize(), stat.blksize) }; |
| 53 | const stat = try file.stat(io); |
| 54 | if (stat.kind != .file) return error.PathAlreadyExists; |
| 55 | break :stat .{ stat.size, @max(std.heap.pageSize(), stat.block_size) }; |
| 116 | 56 | }; |
| 117 | 57 | mf.flags = .{ |
| 118 | 58 | .block_size = .fromByteUnits(std.math.ceilPowerOfTwoAssert(usize, block_size)), |
| ... | ... | @@ -348,12 +288,12 @@ pub const Node = extern struct { |
| 348 | 288 | |
| 349 | 289 | pub fn slice(ni: Node.Index, mf: *const MappedFile) []u8 { |
| 350 | 290 | const file_loc = ni.fileLocation(mf, true); |
| 351 | | return mf.contents[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; |
| 291 | return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; |
| 352 | 292 | } |
| 353 | 293 | |
| 354 | 294 | pub fn sliceConst(ni: Node.Index, mf: *const MappedFile) []const u8 { |
| 355 | 295 | const file_loc = ni.fileLocation(mf, false); |
| 356 | | return mf.contents[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; |
| 296 | return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; |
| 357 | 297 | } |
| 358 | 298 | |
| 359 | 299 | pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: std.mem.Allocator, size: u64) !void { |
| ... | ... | @@ -661,7 +601,8 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested |
| 661 | 601 | // Resize the entire file |
| 662 | 602 | if (ni == Node.Index.root) { |
| 663 | 603 | try mf.ensureCapacityForSetLocation(gpa); |
| 664 | | try mf.file.setLength(io, new_size); |
| 604 | try mf.memory_map.write(io); |
| 605 | try mf.memory_map.file.setLength(io, new_size); |
| 665 | 606 | try mf.ensureTotalCapacity(@intCast(new_size)); |
| 666 | 607 | ni.setLocationAssumeCapacity(mf, old_offset, new_size); |
| 667 | 608 | return; |
| ... | ... | @@ -685,6 +626,7 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested |
| 685 | 626 | if (is_linux and !mf.flags.fallocate_insert_range_unsupported and |
| 686 | 627 | node.flags.alignment.order(mf.flags.block_size).compare(.gte)) |
| 687 | 628 | insert_range: { |
| 629 | try mf.memory_map.write(io); |
| 688 | 630 | // Ask the filesystem driver to insert extents into the file without copying any data |
| 689 | 631 | const last_offset, const last_size = parent.last.location(mf).resolve(mf); |
| 690 | 632 | const last_end = last_offset + last_size; |
| ... | ... | @@ -696,12 +638,12 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested |
| 696 | 638 | _, const file_size = Node.Index.root.location(mf).resolve(mf); |
| 697 | 639 | while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) { |
| 698 | 640 | .lt => linux.fallocate( |
| 699 | | mf.file.handle, |
| 641 | mf.memory_map.file.handle, |
| 700 | 642 | linux.FALLOC.FL_INSERT_RANGE, |
| 701 | 643 | @intCast(range_file_offset), |
| 702 | 644 | @intCast(range_size), |
| 703 | 645 | ), |
| 704 | | .eq => linux.ftruncate(mf.file.handle, @intCast(range_file_offset + range_size)), |
| 646 | .eq => linux.ftruncate(mf.memory_map.file.handle, @intCast(range_file_offset + range_size)), |
| 705 | 647 | .gt => unreachable, |
| 706 | 648 | })) { |
| 707 | 649 | .SUCCESS => { |
| ... | ... | @@ -908,7 +850,7 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: |
| 908 | 850 | if (is_linux and !mf.flags.fallocate_punch_hole_unsupported and |
| 909 | 851 | size >= mf.flags.block_size.toByteUnits() * 2 - 1) while (true) |
| 910 | 852 | switch (linux.errno(linux.fallocate( |
| 911 | | mf.file.handle, |
| 853 | mf.memory_map.file.handle, |
| 912 | 854 | linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE, |
| 913 | 855 | @intCast(old_file_offset), |
| 914 | 856 | @intCast(size), |
| ... | ... | @@ -928,14 +870,14 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: |
| 928 | 870 | .TXTBSY => return error.FileBusy, |
| 929 | 871 | else => |e| return std.posix.unexpectedErrno(e), |
| 930 | 872 | }; |
| 931 | | @memset(mf.contents[@intCast(old_file_offset)..][0..@intCast(size)], 0); |
| 873 | @memset(mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], 0); |
| 932 | 874 | } |
| 933 | 875 | |
| 934 | 876 | fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) !void { |
| 935 | | const copy_size = try mf.copyFileRange(mf.file, old_file_offset, new_file_offset, size); |
| 877 | const copy_size = try mf.copyFileRange(mf.memory_map.file, old_file_offset, new_file_offset, size); |
| 936 | 878 | if (copy_size < size) @memcpy( |
| 937 | | mf.contents[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)], |
| 938 | | mf.contents[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)], |
| 879 | mf.memory_map.memory[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)], |
| 880 | mf.memory_map.memory[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)], |
| 939 | 881 | ); |
| 940 | 882 | } |
| 941 | 883 | |
| ... | ... | @@ -946,6 +888,8 @@ fn copyFileRange( |
| 946 | 888 | new_file_offset: u64, |
| 947 | 889 | size: u64, |
| 948 | 890 | ) !u64 { |
| 891 | const io = mf.io; |
| 892 | try mf.memory_map.write(io); |
| 949 | 893 | var remaining_size = size; |
| 950 | 894 | if (is_linux and !mf.flags.copy_file_range_unsupported) { |
| 951 | 895 | var old_file_offset_mut: i64 = @intCast(old_file_offset); |
| ... | ... | @@ -954,7 +898,7 @@ fn copyFileRange( |
| 954 | 898 | const copy_len = linux.copy_file_range( |
| 955 | 899 | old_file.handle, |
| 956 | 900 | &old_file_offset_mut, |
| 957 | | mf.file.handle, |
| 901 | mf.memory_map.file.handle, |
| 958 | 902 | &new_file_offset_mut, |
| 959 | 903 | @intCast(remaining_size), |
| 960 | 904 | 0, |
| ... | ... | @@ -990,82 +934,41 @@ fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: std.mem.Allocator) !void { |
| 990 | 934 | } |
| 991 | 935 | |
| 992 | 936 | pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) !void { |
| 993 | | if (mf.contents.len >= new_capacity) return; |
| 937 | if (mf.memory_map.memory.len >= new_capacity) return; |
| 994 | 938 | try mf.ensureTotalCapacityPrecise(new_capacity +| new_capacity / growth_factor); |
| 995 | 939 | } |
| 996 | 940 | |
| 997 | 941 | pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) !void { |
| 998 | | if (mf.contents.len >= new_capacity) return; |
| 942 | if (mf.memory_map.memory.len >= new_capacity) return; |
| 943 | const io = mf.io; |
| 999 | 944 | const aligned_capacity = mf.flags.block_size.forward(new_capacity); |
| 1000 | | if (!is_linux) mf.unmap() else if (mf.contents.len > 0) { |
| 1001 | | mf.contents = try std.posix.mremap( |
| 1002 | | mf.contents.ptr, |
| 1003 | | mf.contents.len, |
| 1004 | | aligned_capacity, |
| 1005 | | .{ .MAYMOVE = true }, |
| 1006 | | null, |
| 1007 | | ); |
| 1008 | | return; |
| 1009 | | } |
| 1010 | | if (is_windows) { |
| 1011 | | if (mf.section == windows.INVALID_HANDLE_VALUE) switch (windows.ntdll.NtCreateSection( |
| 1012 | | &mf.section, |
| 1013 | | .{ |
| 1014 | | .SPECIFIC = .{ .SECTION = .{ |
| 1015 | | .QUERY = true, |
| 1016 | | .MAP_WRITE = true, |
| 1017 | | .MAP_READ = true, |
| 1018 | | .EXTEND_SIZE = true, |
| 1019 | | } }, |
| 1020 | | .STANDARD = .{ .RIGHTS = .REQUIRED }, |
| 1021 | | }, |
| 1022 | | null, |
| 1023 | | @constCast(&@as(i64, @intCast(aligned_capacity))), |
| 1024 | | .{ .READWRITE = true }, |
| 1025 | | .{ .COMMIT = true }, |
| 1026 | | mf.file.handle, |
| 1027 | | )) { |
| 1028 | | .SUCCESS => {}, |
| 1029 | | else => return error.MemoryMappingNotSupported, |
| 1030 | | }; |
| 1031 | | var contents_ptr: ?[*]align(std.heap.page_size_min) u8 = null; |
| 1032 | | var contents_len = aligned_capacity; |
| 1033 | | switch (windows.ntdll.NtMapViewOfSection( |
| 1034 | | mf.section, |
| 1035 | | windows.GetCurrentProcess(), |
| 1036 | | @ptrCast(&contents_ptr), |
| 1037 | | null, |
| 1038 | | 0, |
| 1039 | | null, |
| 1040 | | &contents_len, |
| 1041 | | .Unmap, |
| 1042 | | .{}, |
| 1043 | | .{ .READWRITE = true }, |
| 1044 | | )) { |
| 1045 | | .SUCCESS => mf.contents = contents_ptr.?[0..contents_len], |
| 1046 | | else => return error.MemoryMappingNotSupported, |
| 945 | |
| 946 | if (mf.memory_map.memory.len > 0) { |
| 947 | if (mf.memory_map.setLength(io, aligned_capacity)) |_| { |
| 948 | return; |
| 949 | } else |err| switch (err) { |
| 950 | error.OperationUnsupported => {}, |
| 951 | else => |e| return e, |
| 1047 | 952 | } |
| 1048 | | } else mf.contents = try std.posix.mmap( |
| 1049 | | null, |
| 1050 | | aligned_capacity, |
| 1051 | | .{ .READ = true, .WRITE = true }, |
| 1052 | | .{ .TYPE = if (is_linux) .SHARED_VALIDATE else .SHARED }, |
| 1053 | | mf.file.handle, |
| 1054 | | 0, |
| 1055 | | ); |
| 953 | unmap(mf); |
| 954 | } |
| 955 | |
| 956 | const file = mf.memory_map.file; |
| 957 | mf.memory_map = try .create(io, file, .{ .len = aligned_capacity }); |
| 1056 | 958 | } |
| 1057 | 959 | |
| 1058 | 960 | pub fn unmap(mf: *MappedFile) void { |
| 1059 | | if (mf.contents.len == 0) return; |
| 1060 | | if (is_windows) |
| 1061 | | _ = windows.ntdll.NtUnmapViewOfSection(windows.GetCurrentProcess(), mf.contents.ptr) |
| 1062 | | else |
| 1063 | | std.posix.munmap(mf.contents); |
| 1064 | | mf.contents = &.{}; |
| 1065 | | if (is_windows and mf.section != windows.INVALID_HANDLE_VALUE) { |
| 1066 | | windows.CloseHandle(mf.section); |
| 1067 | | mf.section = windows.INVALID_HANDLE_VALUE; |
| 1068 | | } |
| 961 | if (mf.memory_map.memory.len == 0) return; |
| 962 | const io = mf.io; |
| 963 | const file = mf.memory_map.file; |
| 964 | mf.memory_map.destroy(io); |
| 965 | mf.memory_map.memory = &.{}; |
| 966 | mf.memory_map.file = file; |
| 967 | } |
| 968 | |
| 969 | pub fn flush(mf: *MappedFile) Io.File.WritePositionalError!void { |
| 970 | const io = mf.io; |
| 971 | try mf.memory_map.write(io); |
| 1069 | 972 | } |
| 1070 | 973 | |
| 1071 | 974 | fn verify(mf: *MappedFile) void { |