authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-08-09 16:48:44-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-08-09 16:48:44-04:00
logb219feb3f1983e9dcf0d32a7b2a3063dd6662f61
tree776347d4aba3a6a7445c25bd8b4549dc48058a95
parentc63ec9886a6742861347596478c016a14c4f4548

initial windows implementation of std.event.fs.Watch

4 files changed, 352 insertions(+), 30 deletions(-)

src-self-hosted/compilation.zig+1
......@@ -288,6 +288,7 @@ pub const Compilation = struct {
288288 InvalidDarwinVersionString,
289289 UnsupportedLinkArchitecture,
290290 UserResourceLimitReached,
291 InvalidUtf8,
291292 };
292293
293294 pub const Event = union(enum) {
std/event/fs.zig+283-28
......@@ -681,6 +681,7 @@ pub const WatchEventError = error{
681681 UserResourceLimitReached,
682682 SystemResources,
683683 AccessDenied,
684 Unexpected, // TODO remove this possibility
684685};
685686
686687pub fn Watch(comptime V: type) type {
......@@ -699,27 +700,48 @@ pub fn Watch(comptime V: type) type {
699700 value_ptr: *V,
700701 };
701702 },
702 builtin.Os.linux => struct {
703
704 builtin.Os.linux => LinuxOsData,
705 builtin.Os.windows => WindowsOsData,
706
707 else => @compileError("Unsupported OS"),
708 };
709
710 const WindowsOsData = struct {
711 table_lock: event.Lock,
712 dir_table: DirTable,
713 all_putters: std.atomic.Queue(promise),
714 ref_count: std.atomic.Int(usize),
715
716 const DirTable = std.AutoHashMap([]const u8, *Dir);
717 const FileTable = std.AutoHashMap([]const u16, V);
718
719 const Dir = struct {
703720 putter: promise,
704 inotify_fd: i32,
705 wd_table: WdTable,
721 file_table: FileTable,
706722 table_lock: event.Lock,
723 };
724 };
707725
708 const FileTable = std.AutoHashMap([]const u8, V);
709 },
710 else => @compileError("Unsupported OS"),
726 const LinuxOsData = struct {
727 putter: promise,
728 inotify_fd: i32,
729 wd_table: WdTable,
730 table_lock: event.Lock,
731
732 const WdTable = std.AutoHashMap(i32, Dir);
733 const FileTable = std.AutoHashMap([]const u8, V);
734
735 const Dir = struct {
736 dirname: []const u8,
737 file_table: FileTable,
738 };
711739 };
712740
713 const WdTable = std.AutoHashMap(i32, Dir);
714741 const FileToHandle = std.AutoHashMap([]const u8, promise);
715742
716743 const Self = this;
717744
718 const Dir = struct {
719 dirname: []const u8,
720 file_table: OsData.FileTable,
721 };
722
723745 pub const Event = struct {
724746 id: Id,
725747 data: V,
......@@ -741,6 +763,22 @@ pub fn Watch(comptime V: type) type {
741763 _ = try async<loop.allocator> linuxEventPutter(inotify_fd, channel, &result);
742764 return result;
743765 },
766
767 builtin.Os.windows => {
768 const self = try loop.allocator.createOne(Self);
769 errdefer loop.allocator.destroy(self);
770 self.* = Self{
771 .channel = channel,
772 .os_data = OsData{
773 .table_lock = event.Lock.init(loop),
774 .dir_table = OsData.DirTable.init(loop.allocator),
775 .ref_count = std.atomic.Int(usize).init(1),
776 .all_putters = std.atomic.Queue(promise).init(),
777 },
778 };
779 return self;
780 },
781
744782 builtin.Os.macosx => {
745783 const self = try loop.allocator.createOne(Self);
746784 errdefer loop.allocator.destroy(self);
......@@ -758,9 +796,11 @@ pub fn Watch(comptime V: type) type {
758796 }
759797 }
760798
799 /// All addFile calls and removeFile calls must have completed.
761800 pub fn destroy(self: *Self) void {
762801 switch (builtin.os) {
763802 builtin.Os.macosx => {
803 // TODO we need to cancel the coroutines before destroying the lock
764804 self.os_data.table_lock.deinit();
765805 var it = self.os_data.file_table.iterator();
766806 while (it.next()) |entry| {
......@@ -770,14 +810,41 @@ pub fn Watch(comptime V: type) type {
770810 self.channel.destroy();
771811 },
772812 builtin.Os.linux => cancel self.os_data.putter,
813 builtin.Os.windows => {
814 while (self.os_data.all_putters.get()) |putter_node| {
815 cancel putter_node.data;
816 }
817 self.deref();
818 },
773819 else => @compileError("Unsupported OS"),
774820 }
775821 }
776822
823 fn ref(self: *Self) void {
824 _ = self.os_data.ref_count.incr();
825 }
826
827 fn deref(self: *Self) void {
828 if (self.os_data.ref_count.decr() == 1) {
829 const allocator = self.channel.loop.allocator;
830 self.os_data.table_lock.deinit();
831 var it = self.os_data.dir_table.iterator();
832 while (it.next()) |entry| {
833 allocator.free(entry.key);
834 // TODO why does freeing this memory crash the test?
835 //allocator.destroy(entry.value);
836 }
837 self.os_data.dir_table.deinit();
838 self.channel.destroy();
839 allocator.destroy(self);
840 }
841 }
842
777843 pub async fn addFile(self: *Self, file_path: []const u8, value: V) !?V {
778844 switch (builtin.os) {
779845 builtin.Os.macosx => return await (async addFileMacosx(self, file_path, value) catch unreachable),
780846 builtin.Os.linux => return await (async addFileLinux(self, file_path, value) catch unreachable),
847 builtin.Os.windows => return await (async addFileWindows(self, file_path, value) catch unreachable),
781848 else => @compileError("Unsupported OS"),
782849 }
783850 }
......@@ -874,6 +941,8 @@ pub fn Watch(comptime V: type) type {
874941 }
875942
876943 async fn addFileLinux(self: *Self, file_path: []const u8, value: V) !?V {
944 const value_copy = value;
945
877946 const dirname = os.path.dirname(file_path) orelse ".";
878947 const dirname_with_null = try std.cstr.addNullByte(self.channel.loop.allocator, dirname);
879948 var dirname_with_null_consumed = false;
......@@ -896,7 +965,7 @@ pub fn Watch(comptime V: type) type {
896965
897966 const gop = try self.os_data.wd_table.getOrPut(wd);
898967 if (!gop.found_existing) {
899 gop.kv.value = Dir{
968 gop.kv.value = OsData.Dir{
900969 .dirname = dirname_with_null,
901970 .file_table = OsData.FileTable.init(self.channel.loop.allocator),
902971 };
......@@ -907,15 +976,201 @@ pub fn Watch(comptime V: type) type {
907976 const file_table_gop = try dir.file_table.getOrPut(basename_with_null);
908977 if (file_table_gop.found_existing) {
909978 const prev_value = file_table_gop.kv.value;
910 file_table_gop.kv.value = value;
979 file_table_gop.kv.value = value_copy;
911980 return prev_value;
912981 } else {
913 file_table_gop.kv.value = value;
982 file_table_gop.kv.value = value_copy;
914983 basename_with_null_consumed = true;
915984 return null;
916985 }
917986 }
918987
988 async fn addFileWindows(self: *Self, file_path: []const u8, value: V) !?V {
989 const value_copy = value;
990 // TODO we might need to convert dirname and basename to canonical file paths ("short"?)
991
992 const dirname = try std.mem.dupe(self.channel.loop.allocator, u8, os.path.dirname(file_path) orelse ".");
993 var dirname_consumed = false;
994 defer if (!dirname_consumed) self.channel.loop.allocator.free(dirname);
995
996 const dirname_utf16le = try std.unicode.utf8ToUtf16LeWithNull(self.channel.loop.allocator, dirname);
997 defer self.channel.loop.allocator.free(dirname_utf16le);
998
999 // TODO https://github.com/ziglang/zig/issues/265
1000 const basename = os.path.basename(file_path);
1001 const basename_utf16le_null = try std.unicode.utf8ToUtf16LeWithNull(self.channel.loop.allocator, basename);
1002 var basename_utf16le_null_consumed = false;
1003 defer if (!basename_utf16le_null_consumed) self.channel.loop.allocator.free(basename_utf16le_null);
1004 const basename_utf16le_no_null = basename_utf16le_null[0..basename_utf16le_null.len-1];
1005
1006 const dir_handle = windows.CreateFileW(
1007 dirname_utf16le.ptr,
1008 windows.FILE_LIST_DIRECTORY,
1009 windows.FILE_SHARE_READ | windows.FILE_SHARE_DELETE | windows.FILE_SHARE_WRITE,
1010 null,
1011 windows.OPEN_EXISTING,
1012 windows.FILE_FLAG_BACKUP_SEMANTICS | windows.FILE_FLAG_OVERLAPPED,
1013 null,
1014 );
1015 if (dir_handle == windows.INVALID_HANDLE_VALUE) {
1016 const err = windows.GetLastError();
1017 switch (err) {
1018 windows.ERROR.FILE_NOT_FOUND,
1019 windows.ERROR.PATH_NOT_FOUND,
1020 => return error.PathNotFound,
1021 else => return os.unexpectedErrorWindows(err),
1022 }
1023 }
1024 var dir_handle_consumed = false;
1025 defer if (!dir_handle_consumed) os.close(dir_handle);
1026
1027 const held = await (async self.os_data.table_lock.acquire() catch unreachable);
1028 defer held.release();
1029
1030 const gop = try self.os_data.dir_table.getOrPut(dirname);
1031 if (gop.found_existing) {
1032 const dir = gop.kv.value;
1033 const held_dir_lock = await (async dir.table_lock.acquire() catch unreachable);
1034 defer held_dir_lock.release();
1035
1036 const file_gop = try dir.file_table.getOrPut(basename_utf16le_no_null);
1037 if (file_gop.found_existing) {
1038 const prev_value = file_gop.kv.value;
1039 file_gop.kv.value = value_copy;
1040 return prev_value;
1041 } else {
1042 file_gop.kv.value = value_copy;
1043 basename_utf16le_null_consumed = true;
1044 return null;
1045 }
1046 } else {
1047 errdefer _ = self.os_data.dir_table.remove(dirname);
1048 const dir = try self.channel.loop.allocator.createOne(OsData.Dir);
1049 errdefer self.channel.loop.allocator.destroy(dir);
1050
1051 dir.* = OsData.Dir{
1052 .file_table = OsData.FileTable.init(self.channel.loop.allocator),
1053 .table_lock = event.Lock.init(self.channel.loop),
1054 .putter = undefined,
1055 };
1056 assert((try dir.file_table.put(basename_utf16le_no_null, value_copy)) == null);
1057 basename_utf16le_null_consumed = true;
1058
1059 dir.putter = try async self.windowsDirReader(dir_handle, dir);
1060 dir_handle_consumed = true;
1061
1062 dirname_consumed = true;
1063
1064 return null;
1065 }
1066 }
1067
1068 async fn windowsDirReader(self: *Self, dir_handle: windows.HANDLE, dir: *OsData.Dir) void {
1069 // TODO https://github.com/ziglang/zig/issues/1194
1070 suspend {
1071 resume @handle();
1072 }
1073
1074 self.ref();
1075 defer self.deref();
1076
1077 defer os.close(dir_handle);
1078
1079 var putter_node = std.atomic.Queue(promise).Node{
1080 .data = @handle(),
1081 .prev = null,
1082 .next = null,
1083 };
1084 self.os_data.all_putters.put(&putter_node);
1085 defer _ = self.os_data.all_putters.remove(&putter_node);
1086
1087 var resume_node = Loop.ResumeNode.Basic{
1088 .base = Loop.ResumeNode{
1089 .id = Loop.ResumeNode.Id.Basic,
1090 .handle = @handle(),
1091 },
1092 };
1093 const completion_key = @ptrToInt(&resume_node.base);
1094 var overlapped = windows.OVERLAPPED{
1095 .Internal = 0,
1096 .InternalHigh = 0,
1097 .Offset = 0,
1098 .OffsetHigh = 0,
1099 .hEvent = null,
1100 };
1101 var event_buf: [4096]u8 align(@alignOf(windows.FILE_NOTIFY_INFORMATION)) = undefined;
1102
1103 while (true) {
1104 _ = os.windowsCreateIoCompletionPort(
1105 dir_handle, self.channel.loop.os_data.io_port, completion_key, undefined,
1106 ) catch |err| {
1107 await (async self.channel.put(err) catch unreachable);
1108 return;
1109 };
1110 {
1111 // TODO only 1 beginOneEvent for the whole coroutine
1112 self.channel.loop.beginOneEvent();
1113 errdefer self.channel.loop.finishOneEvent();
1114 suspend {
1115 _ = windows.ReadDirectoryChangesW(
1116 dir_handle,
1117 &event_buf,
1118 @intCast(windows.DWORD, event_buf.len),
1119 windows.FALSE, // watch subtree
1120 windows.FILE_NOTIFY_CHANGE_FILE_NAME | windows.FILE_NOTIFY_CHANGE_DIR_NAME |
1121 windows.FILE_NOTIFY_CHANGE_ATTRIBUTES | windows.FILE_NOTIFY_CHANGE_SIZE |
1122 windows.FILE_NOTIFY_CHANGE_LAST_WRITE | windows.FILE_NOTIFY_CHANGE_LAST_ACCESS |
1123 windows.FILE_NOTIFY_CHANGE_CREATION | windows.FILE_NOTIFY_CHANGE_SECURITY,
1124 null, // number of bytes transferred (unused for async)
1125 &overlapped,
1126 null, // completion routine - unused because we use IOCP
1127 );
1128 }
1129 }
1130 var bytes_transferred: windows.DWORD = undefined;
1131 if (windows.GetOverlappedResult(dir_handle, &overlapped, &bytes_transferred, windows.FALSE) == 0) {
1132 const errno = windows.GetLastError();
1133 const err = switch (errno) {
1134 else => os.unexpectedErrorWindows(errno),
1135 };
1136 await (async self.channel.put(err) catch unreachable);
1137 } else {
1138 // can't use @bytesToSlice because of the special variable length name field
1139 var ptr = event_buf[0..].ptr;
1140 const end_ptr = ptr + bytes_transferred;
1141 var ev: *windows.FILE_NOTIFY_INFORMATION = undefined;
1142 while (@ptrToInt(ptr) < @ptrToInt(end_ptr)) : (ptr += ev.NextEntryOffset) {
1143 ev = @ptrCast(*windows.FILE_NOTIFY_INFORMATION, ptr);
1144 const emit = switch (ev.Action) {
1145 windows.FILE_ACTION_REMOVED => WatchEventId.Delete,
1146 windows.FILE_ACTION_MODIFIED => WatchEventId.CloseWrite,
1147 else => null,
1148 };
1149 if (emit) |id| {
1150 const basename_utf16le = ([*]u16)(&ev.FileName)[0..ev.FileNameLength/2];
1151 const user_value = blk: {
1152 const held = await (async dir.table_lock.acquire() catch unreachable);
1153 defer held.release();
1154
1155 if (dir.file_table.get(basename_utf16le)) |entry| {
1156 break :blk entry.value;
1157 } else {
1158 break :blk null;
1159 }
1160 };
1161 if (user_value) |v| {
1162 await (async self.channel.put(Event{
1163 .id = id,
1164 .data = v,
1165 }) catch unreachable);
1166 }
1167 }
1168 if (ev.NextEntryOffset == 0) break;
1169 }
1170 }
1171 }
1172 }
1173
9191174 pub async fn removeFile(self: *Self, file_path: []const u8) ?V {
9201175 @panic("TODO");
9211176 }
......@@ -933,7 +1188,7 @@ pub fn Watch(comptime V: type) type {
9331188 .os_data = OsData{
9341189 .putter = @handle(),
9351190 .inotify_fd = inotify_fd,
936 .wd_table = WdTable.init(loop.allocator),
1191 .wd_table = OsData.WdTable.init(loop.allocator),
9371192 .table_lock = event.Lock.init(loop),
9381193 },
9391194 };
......@@ -1065,15 +1320,15 @@ async fn testFsWatch(loop: *Loop) !void {
10651320 const read_contents = try await try async readFile(loop, file_path, 1024 * 1024);
10661321 assert(mem.eql(u8, read_contents, contents));
10671322
1068 //// now watch the file
1069 //var watch = try Watch(void).create(loop, 0);
1070 //defer watch.destroy();
1323 // now watch the file
1324 var watch = try Watch(void).create(loop, 0);
1325 defer watch.destroy();
10711326
1072 //assert((try await try async watch.addFile(file_path, {})) == null);
1327 assert((try await try async watch.addFile(file_path, {})) == null);
10731328
1074 //const ev = try async watch.channel.get();
1075 //var ev_consumed = false;
1076 //defer if (!ev_consumed) cancel ev;
1329 const ev = try async watch.channel.get();
1330 var ev_consumed = false;
1331 defer if (!ev_consumed) cancel ev;
10771332
10781333 // overwrite line 2
10791334 const fd = try await try async openReadWrite(loop, file_path, os.File.default_mode);
......@@ -1083,11 +1338,11 @@ async fn testFsWatch(loop: *Loop) !void {
10831338 try await try async pwritev(loop, fd, []const []const u8{"lorem ipsum"}, line2_offset);
10841339 }
10851340
1086 //ev_consumed = true;
1087 //switch ((try await ev).id) {
1088 // WatchEventId.CloseWrite => {},
1089 // WatchEventId.Delete => @panic("wrong event"),
1090 //}
1341 ev_consumed = true;
1342 switch ((try await ev).id) {
1343 WatchEventId.CloseWrite => {},
1344 WatchEventId.Delete => @panic("wrong event"),
1345 }
10911346
10921347 const contents_updated = try await try async readFile(loop, file_path, 1024 * 1024);
10931348 assert(mem.eql(u8, contents_updated,
std/os/windows/kernel32.zig+49-1
......@@ -11,7 +11,17 @@ pub extern "kernel32" stdcallcc fn CreateDirectoryA(
1111) BOOL;
1212
1313pub extern "kernel32" stdcallcc fn CreateFileA(
14 lpFileName: LPCSTR,
14 lpFileName: [*]const u8, // TODO null terminated pointer type
15 dwDesiredAccess: DWORD,
16 dwShareMode: DWORD,
17 lpSecurityAttributes: ?LPSECURITY_ATTRIBUTES,
18 dwCreationDisposition: DWORD,
19 dwFlagsAndAttributes: DWORD,
20 hTemplateFile: ?HANDLE,
21) HANDLE;
22
23pub extern "kernel32" stdcallcc fn CreateFileW(
24 lpFileName: [*]const u16, // TODO null terminated pointer type
1525 dwDesiredAccess: DWORD,
1626 dwShareMode: DWORD,
1727 lpSecurityAttributes: ?LPSECURITY_ATTRIBUTES,
......@@ -129,6 +139,17 @@ pub extern "kernel32" stdcallcc fn QueryPerformanceCounter(lpPerformanceCount: *
129139
130140pub extern "kernel32" stdcallcc fn QueryPerformanceFrequency(lpFrequency: *LARGE_INTEGER) BOOL;
131141
142pub extern "kernel32" stdcallcc fn ReadDirectoryChangesW(
143 hDirectory: HANDLE,
144 lpBuffer: [*]align(@alignOf(FILE_NOTIFY_INFORMATION)) u8,
145 nBufferLength: DWORD,
146 bWatchSubtree: BOOL,
147 dwNotifyFilter: DWORD,
148 lpBytesReturned: ?*DWORD,
149 lpOverlapped: ?*OVERLAPPED,
150 lpCompletionRoutine: LPOVERLAPPED_COMPLETION_ROUTINE,
151) BOOL;
152
132153pub extern "kernel32" stdcallcc fn ReadFile(
133154 in_hFile: HANDLE,
134155 out_lpBuffer: [*]u8,
......@@ -168,3 +189,30 @@ pub extern "kernel32" stdcallcc fn WriteFileEx(hFile: HANDLE, lpBuffer: [*]const
168189pub extern "kernel32" stdcallcc fn LoadLibraryA(lpLibFileName: LPCSTR) ?HMODULE;
169190
170191pub extern "kernel32" stdcallcc fn FreeLibrary(hModule: HMODULE) BOOL;
192
193
194pub const FILE_NOTIFY_INFORMATION = extern struct {
195 NextEntryOffset: DWORD,
196 Action: DWORD,
197 FileNameLength: DWORD,
198 FileName: [1]WCHAR,
199};
200
201pub const FILE_ACTION_ADDED = 0x00000001;
202pub const FILE_ACTION_REMOVED = 0x00000002;
203pub const FILE_ACTION_MODIFIED = 0x00000003;
204pub const FILE_ACTION_RENAMED_OLD_NAME = 0x00000004;
205pub const FILE_ACTION_RENAMED_NEW_NAME = 0x00000005;
206
207pub const LPOVERLAPPED_COMPLETION_ROUTINE = ?extern fn(DWORD, DWORD, *OVERLAPPED) void;
208
209pub const FILE_LIST_DIRECTORY = 1;
210
211pub const FILE_NOTIFY_CHANGE_CREATION = 64;
212pub const FILE_NOTIFY_CHANGE_SIZE = 8;
213pub const FILE_NOTIFY_CHANGE_SECURITY = 256;
214pub const FILE_NOTIFY_CHANGE_LAST_ACCESS = 32;
215pub const FILE_NOTIFY_CHANGE_LAST_WRITE = 16;
216pub const FILE_NOTIFY_CHANGE_DIR_NAME = 2;
217pub const FILE_NOTIFY_CHANGE_FILE_NAME = 1;
218pub const FILE_NOTIFY_CHANGE_ATTRIBUTES = 4;
std/unicode.zig+19-1
......@@ -188,6 +188,7 @@ pub const Utf8View = struct {
188188 return Utf8View{ .bytes = s };
189189 }
190190
191 /// TODO: https://github.com/ziglang/zig/issues/425
191192 pub fn initComptime(comptime s: []const u8) Utf8View {
192193 if (comptime init(s)) |r| {
193194 return r;
......@@ -199,7 +200,7 @@ pub const Utf8View = struct {
199200 }
200201 }
201202
202 pub fn iterator(s: *const Utf8View) Utf8Iterator {
203 pub fn iterator(s: Utf8View) Utf8Iterator {
203204 return Utf8Iterator{
204205 .bytes = s.bytes,
205206 .i = 0,
......@@ -530,3 +531,20 @@ test "utf16leToUtf8" {
530531 assert(mem.eql(u8, utf8, "\xf4\x8f\xb0\x80"));
531532 }
532533}
534
535/// TODO support codepoints bigger than 16 bits
536/// TODO type for null terminated pointer
537pub fn utf8ToUtf16LeWithNull(allocator: *mem.Allocator, utf8: []const u8) ![]u16 {
538 var result = std.ArrayList(u16).init(allocator);
539 // optimistically guess that it will not require surrogate pairs
540 try result.ensureCapacity(utf8.len + 1);
541
542 const view = try Utf8View.init(utf8);
543 var it = view.iterator();
544 while (it.nextCodepoint()) |codepoint| {
545 try result.append(@intCast(u16, codepoint)); // TODO surrogate pairs
546 }
547
548 try result.append(0);
549 return result.toOwnedSlice();
550}