authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-05-12 02:49:11-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-05-12 02:49:11-04:00
log5c20c7036bebe443a22a4961ee8f2cd37f65a643
tree1b54f76710d65cfee949248f919426dc10939c33
parentd383b940c2e9c9d0f2e8ef7607b38b7a74021b47
parentaef642fc0731f18514e5ffd6f743274789774f21
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #10796 from marler8997/envmap

Envmap

7 files changed, 233 insertions(+), 39 deletions(-)

doc/docgen.zig+2-2
......@@ -1708,7 +1708,7 @@ fn genHtml(
17081708 }
17091709}
17101710
1711fn exec(allocator: Allocator, env_map: *std.BufMap, args: []const []const u8) !ChildProcess.ExecResult {
1711fn exec(allocator: Allocator, env_map: *process.EnvMap, args: []const []const u8) !ChildProcess.ExecResult {
17121712 const result = try ChildProcess.exec(.{
17131713 .allocator = allocator,
17141714 .argv = args,
......@@ -1732,7 +1732,7 @@ fn exec(allocator: Allocator, env_map: *std.BufMap, args: []const []const u8) !C
17321732 return result;
17331733}
17341734
1735fn getBuiltinCode(allocator: Allocator, env_map: *std.BufMap, zig_exe: []const u8) ![]const u8 {
1735fn getBuiltinCode(allocator: Allocator, env_map: *process.EnvMap, zig_exe: []const u8) ![]const u8 {
17361736 const result = try exec(allocator, env_map, &[_][]const u8{ zig_exe, "build-obj", "--show-builtin" });
17371737 return result.stdout;
17381738}
lib/std/buf_map.zig+1-1
......@@ -82,7 +82,7 @@ pub const BufMap = struct {
8282 }
8383
8484 /// Returns the number of KV pairs stored in the map.
85 pub fn count(self: BufMap) usize {
85 pub fn count(self: BufMap) BufMapHashMap.Size {
8686 return self.hash_map.count();
8787 }
8888
lib/std/build.zig+4-4
......@@ -12,7 +12,7 @@ const StringHashMap = std.StringHashMap;
1212const Allocator = mem.Allocator;
1313const process = std.process;
1414const BufSet = std.BufSet;
15const BufMap = std.BufMap;
15const EnvMap = std.process.EnvMap;
1616const fmt_lib = std.fmt;
1717const File = std.fs.File;
1818const CrossTarget = std.zig.CrossTarget;
......@@ -48,7 +48,7 @@ pub const Builder = struct {
4848 invalid_user_input: bool,
4949 zig_exe: []const u8,
5050 default_step: *Step,
51 env_map: *BufMap,
51 env_map: *EnvMap,
5252 top_level_steps: ArrayList(*TopLevelStep),
5353 install_prefix: []const u8,
5454 dest_dir: ?[]const u8,
......@@ -167,7 +167,7 @@ pub const Builder = struct {
167167 cache_root: []const u8,
168168 global_cache_root: []const u8,
169169 ) !*Builder {
170 const env_map = try allocator.create(BufMap);
170 const env_map = try allocator.create(EnvMap);
171171 env_map.* = try process.getEnvMap(allocator);
172172
173173 const host = try NativeTargetInfo.detect(allocator, .{});
......@@ -963,7 +963,7 @@ pub const Builder = struct {
963963 warn("\n", .{});
964964 }
965965
966 pub fn spawnChildEnvMap(self: *Builder, cwd: ?[]const u8, env_map: *const BufMap, argv: []const []const u8) !void {
966 pub fn spawnChildEnvMap(self: *Builder, cwd: ?[]const u8, env_map: *const EnvMap, argv: []const []const u8) !void {
967967 if (self.verbose) {
968968 printCmd(cwd, argv);
969969 }
lib/std/build/RunStep.zig+8-19
......@@ -9,7 +9,7 @@ const fs = std.fs;
99const mem = std.mem;
1010const process = std.process;
1111const ArrayList = std.ArrayList;
12const BufMap = std.BufMap;
12const EnvMap = process.EnvMap;
1313const Allocator = mem.Allocator;
1414const ExecError = build.Builder.ExecError;
1515
......@@ -29,7 +29,7 @@ argv: ArrayList(Arg),
2929cwd: ?[]const u8,
3030
3131/// Override this field to modify the environment, or use setEnvironmentVariable
32env_map: ?*BufMap,
32env_map: ?*EnvMap,
3333
3434stdout_action: StdIoAction = .inherit,
3535stderr_action: StdIoAction = .inherit,
......@@ -91,27 +91,16 @@ pub fn addArgs(self: *RunStep, args: []const []const u8) void {
9191}
9292
9393pub fn clearEnvironment(self: *RunStep) void {
94 const new_env_map = self.builder.allocator.create(BufMap) catch unreachable;
95 new_env_map.* = BufMap.init(self.builder.allocator);
94 const new_env_map = self.builder.allocator.create(EnvMap) catch unreachable;
95 new_env_map.* = EnvMap.init(self.builder.allocator);
9696 self.env_map = new_env_map;
9797}
9898
9999pub fn addPathDir(self: *RunStep, search_path: []const u8) void {
100100 const env_map = self.getEnvMap();
101101
102 var key: []const u8 = undefined;
103 var prev_path: ?[]const u8 = undefined;
104 if (builtin.os.tag == .windows) {
105 key = "Path";
106 prev_path = env_map.get(key);
107 if (prev_path == null) {
108 key = "PATH";
109 prev_path = env_map.get(key);
110 }
111 } else {
112 key = "PATH";
113 prev_path = env_map.get(key);
114 }
102 const key = "PATH";
103 var prev_path = env_map.get(key);
115104
116105 if (prev_path) |pp| {
117106 const new_path = self.builder.fmt("{s}" ++ [1]u8{fs.path.delimiter} ++ "{s}", .{ pp, search_path });
......@@ -121,9 +110,9 @@ pub fn addPathDir(self: *RunStep, search_path: []const u8) void {
121110 }
122111}
123112
124pub fn getEnvMap(self: *RunStep) *BufMap {
113pub fn getEnvMap(self: *RunStep) *EnvMap {
125114 return self.env_map orelse {
126 const env_map = self.builder.allocator.create(BufMap) catch unreachable;
115 const env_map = self.builder.allocator.create(EnvMap) catch unreachable;
127116 env_map.* = process.getEnvMap(self.builder.allocator) catch unreachable;
128117 self.env_map = env_map;
129118 return env_map;
lib/std/child_process.zig+6-6
......@@ -12,7 +12,7 @@ const linux = os.linux;
1212const mem = std.mem;
1313const math = std.math;
1414const debug = std.debug;
15const BufMap = std.BufMap;
15const EnvMap = process.EnvMap;
1616const Os = std.builtin.Os;
1717const TailQueue = std.TailQueue;
1818const maxInt = std.math.maxInt;
......@@ -34,7 +34,7 @@ pub const ChildProcess = struct {
3434 argv: []const []const u8,
3535
3636 /// Leave as null to use the current env map using the supplied allocator.
37 env_map: ?*const BufMap,
37 env_map: ?*const EnvMap,
3838
3939 stdin_behavior: StdIo,
4040 stdout_behavior: StdIo,
......@@ -375,7 +375,7 @@ pub const ChildProcess = struct {
375375 argv: []const []const u8,
376376 cwd: ?[]const u8 = null,
377377 cwd_dir: ?fs.Dir = null,
378 env_map: ?*const BufMap = null,
378 env_map: ?*const EnvMap = null,
379379 max_output_bytes: usize = 50 * 1024,
380380 expand_arg0: Arg0Expand = .no_expand,
381381 }) !ExecResult {
......@@ -1237,7 +1237,7 @@ fn readIntFd(fd: i32) !ErrInt {
12371237}
12381238
12391239/// Caller must free result.
1240pub fn createWindowsEnvBlock(allocator: mem.Allocator, env_map: *const BufMap) ![]u16 {
1240pub fn createWindowsEnvBlock(allocator: mem.Allocator, env_map: *const EnvMap) ![]u16 {
12411241 // count bytes needed
12421242 const max_chars_needed = x: {
12431243 var max_chars_needed: usize = 4; // 4 for the final 4 null bytes
......@@ -1273,7 +1273,7 @@ pub fn createWindowsEnvBlock(allocator: mem.Allocator, env_map: *const BufMap) !
12731273 return allocator.shrink(result, i);
12741274}
12751275
1276pub fn createNullDelimitedEnvMap(arena: mem.Allocator, env_map: *const std.BufMap) ![:null]?[*:0]u8 {
1276pub fn createNullDelimitedEnvMap(arena: mem.Allocator, env_map: *const EnvMap) ![:null]?[*:0]u8 {
12771277 const envp_count = env_map.count();
12781278 const envp_buf = try arena.allocSentinel(?[*:0]u8, envp_count, null);
12791279 {
......@@ -1294,7 +1294,7 @@ pub fn createNullDelimitedEnvMap(arena: mem.Allocator, env_map: *const std.BufMa
12941294test "createNullDelimitedEnvMap" {
12951295 const testing = std.testing;
12961296 const allocator = testing.allocator;
1297 var envmap = BufMap.init(allocator);
1297 var envmap = EnvMap.init(allocator);
12981298 defer envmap.deinit();
12991299
13001300 try envmap.put("HOME", "/home/ifreund");
lib/std/os/windows/ntdll.zig+4
......@@ -229,6 +229,10 @@ pub extern "ntdll" fn RtlEqualUnicodeString(
229229 CaseInSensitive: BOOLEAN,
230230) callconv(WINAPI) BOOLEAN;
231231
232pub extern "ntdll" fn RtlUpcaseUnicodeChar(
233 SourceCharacter: u16,
234) callconv(WINAPI) u16;
235
232236pub extern "ntdll" fn NtLockFile(
233237 FileHandle: HANDLE,
234238 Event: ?HANDLE,
lib/std/process.zig+208-7
......@@ -2,7 +2,6 @@ const std = @import("std.zig");
22const builtin = @import("builtin");
33const os = std.os;
44const fs = std.fs;
5const BufMap = std.BufMap;
65const mem = std.mem;
76const math = std.math;
87const Allocator = mem.Allocator;
......@@ -53,9 +52,205 @@ test "getCwdAlloc" {
5352 testing.allocator.free(cwd);
5453}
5554
56/// Caller owns resulting `BufMap`.
57pub fn getEnvMap(allocator: Allocator) !BufMap {
58 var result = BufMap.init(allocator);
55pub const EnvMap = struct {
56 hash_map: HashMap,
57
58 const HashMap = std.HashMap(
59 []const u8,
60 []const u8,
61 EnvNameHashContext,
62 std.hash_map.default_max_load_percentage,
63 );
64
65 pub const Size = HashMap.Size;
66
67 pub const EnvNameHashContext = struct {
68 fn upcase(c: u21) u21 {
69 if (c <= std.math.maxInt(u16))
70 return std.os.windows.ntdll.RtlUpcaseUnicodeChar(@intCast(u16, c));
71 return c;
72 }
73
74 pub fn hash(self: @This(), s: []const u8) u64 {
75 _ = self;
76 if (builtin.os.tag == .windows) {
77 var h = std.hash.Wyhash.init(0);
78 var it = std.unicode.Utf8View.initUnchecked(s).iterator();
79 while (it.nextCodepoint()) |cp| {
80 const cp_upper = upcase(cp);
81 h.update(&[_]u8{
82 @intCast(u8, (cp_upper >> 16) & 0xff),
83 @intCast(u8, (cp_upper >> 8) & 0xff),
84 @intCast(u8, (cp_upper >> 0) & 0xff),
85 });
86 }
87 return h.final();
88 }
89 return std.hash_map.hashString(s);
90 }
91
92 pub fn eql(self: @This(), a: []const u8, b: []const u8) bool {
93 _ = self;
94 if (builtin.os.tag == .windows) {
95 var it_a = std.unicode.Utf8View.initUnchecked(a).iterator();
96 var it_b = std.unicode.Utf8View.initUnchecked(b).iterator();
97 while (true) {
98 const c_a = it_a.nextCodepoint() orelse break;
99 const c_b = it_b.nextCodepoint() orelse return false;
100 if (upcase(c_a) != upcase(c_b))
101 return false;
102 }
103 return if (it_b.nextCodepoint()) |_| false else true;
104 }
105 return std.hash_map.eqlString(a, b);
106 }
107 };
108
109 /// Create a EnvMap backed by a specific allocator.
110 /// That allocator will be used for both backing allocations
111 /// and string deduplication.
112 pub fn init(allocator: Allocator) EnvMap {
113 return EnvMap{ .hash_map = HashMap.init(allocator) };
114 }
115
116 /// Free the backing storage of the map, as well as all
117 /// of the stored keys and values.
118 pub fn deinit(self: *EnvMap) void {
119 var it = self.hash_map.iterator();
120 while (it.next()) |entry| {
121 self.free(entry.key_ptr.*);
122 self.free(entry.value_ptr.*);
123 }
124
125 self.hash_map.deinit();
126 }
127
128 /// Same as `put` but the key and value become owned by the EnvMap rather
129 /// than being copied.
130 /// If `putMove` fails, the ownership of key and value does not transfer.
131 /// On Windows `key` must be a valid UTF-8 string.
132 pub fn putMove(self: *EnvMap, key: []u8, value: []u8) !void {
133 const get_or_put = try self.hash_map.getOrPut(key);
134 if (get_or_put.found_existing) {
135 self.free(get_or_put.key_ptr.*);
136 self.free(get_or_put.value_ptr.*);
137 get_or_put.key_ptr.* = key;
138 }
139 get_or_put.value_ptr.* = value;
140 }
141
142 /// `key` and `value` are copied into the EnvMap.
143 /// On Windows `key` must be a valid UTF-8 string.
144 pub fn put(self: *EnvMap, key: []const u8, value: []const u8) !void {
145 const value_copy = try self.copy(value);
146 errdefer self.free(value_copy);
147 const get_or_put = try self.hash_map.getOrPut(key);
148 if (get_or_put.found_existing) {
149 self.free(get_or_put.value_ptr.*);
150 } else {
151 get_or_put.key_ptr.* = self.copy(key) catch |err| {
152 _ = self.hash_map.remove(key);
153 return err;
154 };
155 }
156 get_or_put.value_ptr.* = value_copy;
157 }
158
159 /// Find the address of the value associated with a key.
160 /// The returned pointer is invalidated if the map resizes.
161 /// On Windows `key` must be a valid UTF-8 string.
162 pub fn getPtr(self: EnvMap, key: []const u8) ?*[]const u8 {
163 return self.hash_map.getPtr(key);
164 }
165
166 /// Return the map's copy of the value associated with
167 /// a key. The returned string is invalidated if this
168 /// key is removed from the map.
169 /// On Windows `key` must be a valid UTF-8 string.
170 pub fn get(self: EnvMap, key: []const u8) ?[]const u8 {
171 return self.hash_map.get(key);
172 }
173
174 /// Removes the item from the map and frees its value.
175 /// This invalidates the value returned by get() for this key.
176 /// On Windows `key` must be a valid UTF-8 string.
177 pub fn remove(self: *EnvMap, key: []const u8) void {
178 const kv = self.hash_map.fetchRemove(key) orelse return;
179 self.free(kv.key);
180 self.free(kv.value);
181 }
182
183 /// Returns the number of KV pairs stored in the map.
184 pub fn count(self: EnvMap) HashMap.Size {
185 return self.hash_map.count();
186 }
187
188 /// Returns an iterator over entries in the map.
189 pub fn iterator(self: *const EnvMap) HashMap.Iterator {
190 return self.hash_map.iterator();
191 }
192
193 fn free(self: EnvMap, value: []const u8) void {
194 self.hash_map.allocator.free(value);
195 }
196
197 fn copy(self: EnvMap, value: []const u8) ![]u8 {
198 return self.hash_map.allocator.dupe(u8, value);
199 }
200};
201
202test "EnvMap" {
203 var env = EnvMap.init(testing.allocator);
204 defer env.deinit();
205
206 try env.put("SOMETHING_NEW", "hello");
207 try testing.expectEqualStrings("hello", env.get("SOMETHING_NEW").?);
208 try testing.expectEqual(@as(EnvMap.Size, 1), env.count());
209
210 // overwrite
211 try env.put("SOMETHING_NEW", "something");
212 try testing.expectEqualStrings("something", env.get("SOMETHING_NEW").?);
213 try testing.expectEqual(@as(EnvMap.Size, 1), env.count());
214
215 // a new longer name to test the Windows-specific conversion buffer
216 try env.put("SOMETHING_NEW_AND_LONGER", "1");
217 try testing.expectEqualStrings("1", env.get("SOMETHING_NEW_AND_LONGER").?);
218 try testing.expectEqual(@as(EnvMap.Size, 2), env.count());
219
220 // case insensitivity on Windows only
221 if (builtin.os.tag == .windows) {
222 try testing.expectEqualStrings("1", env.get("something_New_aNd_LONGER").?);
223 } else {
224 try testing.expect(null == env.get("something_New_aNd_LONGER"));
225 }
226
227 var it = env.iterator();
228 var count: EnvMap.Size = 0;
229 while (it.next()) |entry| {
230 const is_an_expected_name = std.mem.eql(u8, "SOMETHING_NEW", entry.key_ptr.*) or std.mem.eql(u8, "SOMETHING_NEW_AND_LONGER", entry.key_ptr.*);
231 try testing.expect(is_an_expected_name);
232 count += 1;
233 }
234 try testing.expectEqual(@as(EnvMap.Size, 2), count);
235
236 env.remove("SOMETHING_NEW");
237 try testing.expect(env.get("SOMETHING_NEW") == null);
238
239 try testing.expectEqual(@as(EnvMap.Size, 1), env.count());
240
241 // test Unicode case-insensitivity on Windows
242 if (builtin.os.tag == .windows) {
243 try env.put("КИРиллИЦА", "something else");
244 try testing.expectEqualStrings("something else", env.get("кириллица").?);
245 }
246}
247
248/// Returns a snapshot of the environment variables of the current process.
249/// Any modifications to the resulting EnvMap will not be not reflected in the environment, and
250/// likewise, any future modifications to the environment will not be reflected in the EnvMap.
251/// Caller owns resulting `EnvMap` and should call its `deinit` fn when done.
252pub fn getEnvMap(allocator: Allocator) !EnvMap {
253 var result = EnvMap.init(allocator);
59254 errdefer result.deinit();
60255
61256 if (builtin.os.tag == .windows) {
......@@ -65,6 +260,12 @@ pub fn getEnvMap(allocator: Allocator) !BufMap {
65260 while (ptr[i] != 0) {
66261 const key_start = i;
67262
263 // There are some special environment variables that start with =,
264 // so we need a special case to not treat = as a key/value separator
265 // if it's the first character.
266 // https://devblogs.microsoft.com/oldnewthing/20100506-00/?p=14133
267 if (ptr[key_start] == '=') i += 1;
268
68269 while (ptr[i] != 0 and ptr[i] != '=') : (i += 1) {}
69270 const key_w = ptr[key_start..i];
70271 const key = try std.unicode.utf16leToUtf8Alloc(allocator, key_w);
......@@ -140,8 +341,8 @@ pub fn getEnvMap(allocator: Allocator) !BufMap {
140341 }
141342}
142343
143test "os.getEnvMap" {
144 var env = try getEnvMap(std.testing.allocator);
344test "getEnvMap" {
345 var env = try getEnvMap(testing.allocator);
145346 defer env.deinit();
146347}
147348
......@@ -985,7 +1186,7 @@ pub fn execv(allocator: mem.Allocator, argv: []const []const u8) ExecvError {
9851186pub fn execve(
9861187 allocator: mem.Allocator,
9871188 argv: []const []const u8,
988 env_map: ?*const std.BufMap,
1189 env_map: ?*const EnvMap,
9891190) ExecvError {
9901191 if (!can_execv) @compileError("The target OS does not support execv");
9911192