Commit d2602b442e7cdea2e74584f9529917d7a53625fd | zig

author	Andrew Kelley <andrew@ziglang.org> 2019-02-06 14:32:20-05:00
committer	Andrew Kelley <andrew@ziglang.org> 2019-02-06 14:32:20-05:00
log	d2602b442e7cdea2e74584f9529917d7a53625fd
tree	656d2949eae12721fd33479ae49fc5bf0bd76e4d
parent	b1775ca168e0bcfba6753346c5226881da49c6c4
signature	Commit is signed but in an unrecognized format.

require running std lib tests coherently

this should actually improve CI times a bit too

See the description at the top of std/os/startup.zig (deleted in this
commit) for a more detailed understanding of what this commit does.

11 files changed, 65 insertions(+), 55 deletions(-)

CMakeLists.txt
src
std
- build.zig
- index.zig
- os
  - index.zig
  - startup.zig
- special
  - bootstrap.zig
test
- tests.zig

CMakeLists.txt-1

...	...	@@ -587,7 +587,6 @@ set(ZIG_STD_FILES
587	587	"os/linux/vdso.zig"
588	588	"os/linux/x86_64.zig"
589	589	"os/path.zig"
590		"os/startup.zig"
591	590	"os/time.zig"
592	591	"os/uefi.zig"
593	592	"os/windows/advapi32.zig"

src/codegen.cpp+13-5

...	...	@@ -88,7 +88,7 @@ static const char *symbols_that_llvm_depends_on[] = {
88	88	};
89	89
90	90	CodeGen codegen_create(Buf root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode,
91		Buf *zig_lib_dir)
	91	Buf zig_lib_dir, Buf override_std_dir)
92	92	{
93	93	CodeGen *g = allocate<CodeGen>(1);
94	94
...	...	@@ -96,8 +96,12 @@ CodeGen codegen_create(Buf root_src_path, const ZigTarget *target, OutType out
96	96
97	97	g->zig_lib_dir = zig_lib_dir;
98	98
99		g->zig_std_dir = buf_alloc();
100		os_path_join(zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir);
	99	if (override_std_dir == nullptr) {
	100	g->zig_std_dir = buf_alloc();
	101	os_path_join(zig_lib_dir, buf_create_from_str("std"), g->zig_std_dir);
	102	} else {
	103	g->zig_std_dir = override_std_dir;
	104	}
101	105
102	106	g->zig_c_headers_dir = buf_alloc();
103	107	os_path_join(zig_lib_dir, buf_create_from_str("include"), g->zig_c_headers_dir);
...	...	@@ -8356,8 +8360,12 @@ static void add_cache_pkg(CodeGen g, CacheHash ch, PackageTableEntry *pkg) {
8356	8360	if (!entry)
8357	8361	break;
8358	8362
8359		cache_buf(ch, entry->key);
8360		add_cache_pkg(g, ch, entry->value);
	8363	// TODO: I think we need a more sophisticated detection of
	8364	// packages we have already seen
	8365	if (entry->value != pkg) {
	8366	cache_buf(ch, entry->key);
	8367	add_cache_pkg(g, ch, entry->value);
	8368	}
8361	8369	}
8362	8370	}
8363	8371

src/codegen.hpp+1-1

...	...	@@ -15,7 +15,7 @@
15	15	#include <stdio.h>
16	16
17	17	CodeGen codegen_create(Buf root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode,
18		Buf *zig_lib_dir);
	18	Buf zig_lib_dir, Buf override_std_dir);
19	19
20	20	void codegen_set_clang_argv(CodeGen codegen, const char *args, size_t len);
21	21	void codegen_set_llvm_argv(CodeGen codegen, const char *args, size_t len);

src/link.cpp+1-1

...	...	@@ -42,7 +42,7 @@ static Buf build_a_raw(CodeGen parent_gen, const char aname, Buf full_path)
42	42	}
43	43
44	44	CodeGen *child_gen = codegen_create(full_path, child_target, child_out_type,
45		parent_gen->build_mode, parent_gen->zig_lib_dir);
	45	parent_gen->build_mode, parent_gen->zig_lib_dir, parent_gen->zig_std_dir);
46	46
47	47	child_gen->out_h_path = nullptr;
48	48	child_gen->verbose_tokenize = parent_gen->verbose_tokenize;

src/main.cpp+9-3

...	...	@@ -74,6 +74,7 @@ static int print_full_usage(const char *arg0) {
74	74	" -dirafter [dir] same as -isystem but do it last\n"
75	75	" -isystem [dir] add additional search path for other .h files\n"
76	76	" -mllvm [arg] forward an arg to LLVM's option processing\n"
	77	" --override-std-dir [arg] use an alternate Zig standard library\n"
77	78	"\n"
78	79	"Link Options:\n"
79	80	" --dynamic-linker [path] set the path to ld.so\n"
...	...	@@ -395,6 +396,7 @@ int main(int argc, char **argv) {
395	396	bool system_linker_hack = false;
396	397	TargetSubsystem subsystem = TargetSubsystemAuto;
397	398	bool is_single_threaded = false;
	399	Buf *override_std_dir = nullptr;
398	400
399	401	if (argc >= 2 && strcmp(argv[1], "build") == 0) {
400	402	Buf zig_exe_path_buf = BUF_INIT;
...	...	@@ -430,7 +432,8 @@ int main(int argc, char **argv) {
430	432	Buf *build_runner_path = buf_alloc();
431	433	os_path_join(get_zig_special_dir(), buf_create_from_str("build_runner.zig"), build_runner_path);
432	434
433		CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, get_zig_lib_dir());
	435	CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, get_zig_lib_dir(),
	436	override_std_dir);
434	437	g->enable_time_report = timing_info;
435	438	buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name);
436	439	codegen_set_out_name(g, buf_create_from_str("build"));
...	...	@@ -645,6 +648,8 @@ int main(int argc, char **argv) {
645	648	clang_argv.append(argv[i]);
646	649
647	650	llvm_argv.append(argv[i]);
	651	} else if (strcmp(arg, "--override-std-dir") == 0) {
	652	override_std_dir = buf_create_from_str(argv[i]);
648	653	} else if (strcmp(arg, "--library-path") == 0 \|\| strcmp(arg, "-L") == 0) {
649	654	lib_dirs.append(argv[i]);
650	655	} else if (strcmp(arg, "--library") == 0) {
...	...	@@ -819,7 +824,7 @@ int main(int argc, char **argv) {
819	824
820	825	switch (cmd) {
821	826	case CmdBuiltin: {
822		CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, get_zig_lib_dir());
	827	CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, get_zig_lib_dir(), override_std_dir);
823	828	g->is_single_threaded = is_single_threaded;
824	829	Buf *builtin_source = codegen_generate_builtin_source(g);
825	830	if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) {
...	...	@@ -878,7 +883,8 @@ int main(int argc, char **argv) {
878	883	if (cmd == CmdRun && buf_out_name == nullptr) {
879	884	buf_out_name = buf_create_from_str("run");
880	885	}
881		CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, get_zig_lib_dir());
	886	CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, get_zig_lib_dir(),
	887	override_std_dir);
882	888	g->subsystem = subsystem;
883	889
884	890	if (disable_pic) {

std/build.zig+10

...	...	@@ -1686,6 +1686,7 @@ pub const TestStep = struct {
1686	1686	no_rosegment: bool,
1687	1687	output_path: ?[]const u8,
1688	1688	system_linker_hack: bool,
	1689	override_std_dir: ?[]const u8,
1689	1690
1690	1691	pub fn init(builder: *Builder, root_src: []const u8) TestStep {
1691	1692	const step_name = builder.fmt("test {}", root_src);
...	...	@@ -1707,6 +1708,7 @@ pub const TestStep = struct {
1707	1708	.no_rosegment = false,
1708	1709	.output_path = null,
1709	1710	.system_linker_hack = false,
	1711	.override_std_dir = null,
1710	1712	};
1711	1713	}
1712	1714
...	...	@@ -1737,6 +1739,10 @@ pub const TestStep = struct {
1737	1739	self.build_mode = mode;
1738	1740	}
1739	1741
	1742	pub fn overrideStdDir(self: *TestStep, dir_path: []const u8) void {
	1743	self.override_std_dir = dir_path;
	1744	}
	1745
1740	1746	pub fn setOutputPath(self: *TestStep, file_path: []const u8) void {
1741	1747	self.output_path = file_path;
1742	1748
...	...	@@ -1914,6 +1920,10 @@ pub const TestStep = struct {
1914	1920	if (self.system_linker_hack) {
1915	1921	try zig_args.append("--system-linker-hack");
1916	1922	}
	1923	if (self.override_std_dir) \|dir\| {
	1924	try zig_args.append("--override-std-dir");
	1925	try zig_args.append(builder.pathFromRoot(dir));
	1926	}
1917	1927
1918	1928	try builder.spawnChild(zig_args.toSliceConst());
1919	1929	}

std/index.zig-1

...	...	@@ -45,7 +45,6 @@ pub const unicode = @import("unicode.zig");
45	45	pub const zig = @import("zig/index.zig");
46	46
47	47	pub const lazyInit = @import("lazy_init.zig").lazyInit;
48		pub const startup = @import("os/startup.zig");
49	48
50	49	test "std" {
51	50	// run tests from these

std/os/index.zig+21-10

...	...	@@ -8,8 +8,9 @@ const is_posix = switch (builtin.os) {
8	8	};
9	9	const os = @This();
10	10
11		// See the comment in startup.zig for why this does not use the `std` global above.
12		const startup = @import("std").startup;
	11	comptime {
	12	assert(@import("std") == std); // You have to run the std lib tests with --override-std-dir
	13	}
13	14
14	15	test "std.os" {
15	16	_ = @import("child_process.zig");
...	...	@@ -670,11 +671,14 @@ fn posixExecveErrnoToErr(err: usize) PosixExecveError {
670	671	}
671	672	}
672	673
	674	pub var linux_elf_aux_maybe: ?[*]std.elf.Auxv = null;
	675	pub var posix_environ_raw: [][*]u8 = undefined;
	676
673	677	/// See std.elf for the constants.
674	678	pub fn linuxGetAuxVal(index: usize) usize {
675	679	if (builtin.link_libc) {
676	680	return usize(std.c.getauxval(index));
677		} else if (startup.linux_elf_aux_maybe) \|auxv\| {
	681	} else if (linux_elf_aux_maybe) \|auxv\| {
678	682	var i: usize = 0;
679	683	while (auxv[i].a_type != std.elf.AT_NULL) : (i += 1) {
680	684	if (auxv[i].a_type == index)
...	...	@@ -734,7 +738,7 @@ pub fn getEnvMap(allocator: *Allocator) !BufMap {
734	738	try result.setMove(key, value);
735	739	}
736	740	} else {
737		for (startup.posix_environ_raw) \|ptr\| {
	741	for (posix_environ_raw) \|ptr\| {
738	742	var line_i: usize = 0;
739	743	while (ptr[line_i] != 0 and ptr[line_i] != '=') : (line_i += 1) {}
740	744	const key = ptr[0..line_i];
...	...	@@ -756,7 +760,7 @@ test "os.getEnvMap" {
756	760
757	761	/// TODO make this go through libc when we have it
758	762	pub fn getEnvPosix(key: []const u8) ?[]const u8 {
759		for (startup.posix_environ_raw) \|ptr\| {
	763	for (posix_environ_raw) \|ptr\| {
760	764	var line_i: usize = 0;
761	765	while (ptr[line_i] != 0 and ptr[line_i] != '=') : (line_i += 1) {}
762	766	const this_key = ptr[0..line_i];
...	...	@@ -1937,14 +1941,14 @@ pub const ArgIteratorPosix = struct {
1937	1941	pub fn init() ArgIteratorPosix {
1938	1942	return ArgIteratorPosix{
1939	1943	.index = 0,
1940		.count = startup.posix_argv_raw.len,
	1944	.count = raw.len,
1941	1945	};
1942	1946	}
1943	1947
1944	1948	pub fn next(self: *ArgIteratorPosix) ?[]const u8 {
1945	1949	if (self.index == self.count) return null;
1946	1950
1947		const s = startup.posix_argv_raw[self.index];
	1951	const s = raw[self.index];
1948	1952	self.index += 1;
1949	1953	return cstr.toSlice(s);
1950	1954	}
...	...	@@ -1955,6 +1959,10 @@ pub const ArgIteratorPosix = struct {
1955	1959	self.index += 1;
1956	1960	return true;
1957	1961	}
	1962
	1963	/// This is marked as public but actually it's only meant to be used
	1964	/// internally by zig's startup code.
	1965	pub var raw: [][*]u8 = undefined;
1958	1966	};
1959	1967
1960	1968	pub const ArgIteratorWindows = struct {
...	...	@@ -3000,6 +3008,9 @@ pub const SpawnThreadError = error{
3000	3008	Unexpected,
3001	3009	};
3002	3010
	3011	pub var linux_tls_phdr: ?*std.elf.Phdr = null;
	3012	pub var linux_tls_img_src: [*]const u8 = undefined; // defined if linux_tls_phdr is
	3013
3003	3014	/// caller must call wait on the returned thread
3004	3015	/// fn startFn(@typeOf(context)) T
3005	3016	/// where T is u8, noreturn, void, or !void
...	...	@@ -3109,7 +3120,7 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
3109	3120	}
3110	3121	// Finally, the Thread Local Storage, if any.
3111	3122	if (!Thread.use_pthreads) {
3112		if (startup.linux_tls_phdr) \|tls_phdr\| {
	3123	if (linux_tls_phdr) \|tls_phdr\| {
3113	3124	l = mem.alignForward(l, tls_phdr.p_align);
3114	3125	tls_start_offset = l;
3115	3126	l += tls_phdr.p_memsz;
...	...	@@ -3153,8 +3164,8 @@ pub fn spawnThread(context: var, comptime startFn: var) SpawnThreadError!*Thread
3153	3164	posix.CLONE_THREAD \| posix.CLONE_SYSVSEM \| posix.CLONE_PARENT_SETTID \| posix.CLONE_CHILD_CLEARTID \|
3154	3165	posix.CLONE_DETACHED;
3155	3166	var newtls: usize = undefined;
3156		if (startup.linux_tls_phdr) \|tls_phdr\| {
3157		@memcpy(@intToPtr([*]u8, mmap_addr + tls_start_offset), startup.linux_tls_img_src, tls_phdr.p_filesz);
	3167	if (linux_tls_phdr) \|tls_phdr\| {
	3168	@memcpy(@intToPtr([*]u8, mmap_addr + tls_start_offset), linux_tls_img_src, tls_phdr.p_filesz);
3158	3169	thread_ptr.data.tls_end_addr = mmap_addr + mmap_len;
3159	3170	newtls = @ptrToInt(&thread_ptr.data.tls_end_addr);
3160	3171	flags \|= posix.CLONE_SETTLS;

std/os/startup.zig deleted-26

...	...	@@ -1,26 +0,0 @@
1		// This file contains global variables that are initialized on startup from
2		// std/special/bootstrap.zig. There are a few things to be aware of here.
3		//
4		// First, when building an object or library, and no entry point is defined
5		// (such as pub fn main), std/special/bootstrap.zig is not included in the
6		// compilation. And so these global variables will remain set to the values
7		// you see here.
8		//
9		// Second, when using `zig test` to test the standard library, note that
10		// `zig test` is self-hosted. This means that it uses std/special/bootstrap.zig
11		// and an @import("std") from the install directory, which is distinct from
12		// the standard library files that we are directly testing with `zig test`.
13		// This means that these global variables would not get set. So the workaround
14		// here is that references to these globals from the standard library must
15		// use `@import("std").startup` rather than
16		// `@import("path/to/std/index.zig").startup` (and rather than the file path of
17		// this file directly). We also put "std" as a reference to itself in the
18		// standard library package so that this can work.
19
20		const std = @import("../index.zig");
21
22		pub var linux_tls_phdr: ?*std.elf.Phdr = null;
23		pub var linux_tls_img_src: [*]const u8 = undefined; // defined when linux_tls_phdr is non-null
24		pub var linux_elf_aux_maybe: ?[*]std.elf.Auxv = null;
25		pub var posix_environ_raw: [][*]u8 = undefined;
26		pub var posix_argv_raw: [][*]u8 = undefined;

std/special/bootstrap.zig+7-7

...	...	@@ -64,7 +64,7 @@ fn posixCallMainAndExit() noreturn {
64	64	if (builtin.os == builtin.Os.linux) {
65	65	// Scan auxiliary vector.
66	66	const auxv = @ptrCast([*]std.elf.Auxv, envp.ptr + envp_count + 1);
67		std.startup.linux_elf_aux_maybe = auxv;
	67	std.os.linux_elf_aux_maybe = auxv;
68	68	var i: usize = 0;
69	69	var at_phdr: usize = 0;
70	70	var at_phnum: usize = 0;
...	...	@@ -87,8 +87,8 @@ fn posixCallMainAndExit() noreturn {
87	87	// This is marked inline because for some reason LLVM in release mode fails to inline it,
88	88	// and we want fewer call frames in stack traces.
89	89	inline fn callMainWithArgs(argc: usize, argv: [][]u8, envp: [][*]u8) u8 {
90		std.startup.posix_argv_raw = argv[0..argc];
91		std.startup.posix_environ_raw = envp;
	90	std.os.ArgIteratorPosix.raw = argv[0..argc];
	91	std.os.posix_environ_raw = envp;
92	92	return callMain();
93	93	}
94	94
...	...	@@ -145,15 +145,15 @@ fn linuxInitializeThreadLocalStorage(at_phdr: usize, at_phnum: usize, at_phent:
145	145	// TODO look for PT_DYNAMIC when we have https://github.com/ziglang/zig/issues/1917
146	146	switch (phdr.p_type) {
147	147	std.elf.PT_PHDR => base = at_phdr - phdr.p_vaddr,
148		std.elf.PT_TLS => std.startup.linux_tls_phdr = phdr,
	148	std.elf.PT_TLS => std.os.linux_tls_phdr = phdr,
149	149	else => continue,
150	150	}
151	151	}
152		const tls_phdr = std.startup.linux_tls_phdr orelse return;
153		std.startup.linux_tls_img_src = @intToPtr([*]const u8, base + tls_phdr.p_vaddr);
	152	const tls_phdr = std.os.linux_tls_phdr orelse return;
	153	std.os.linux_tls_img_src = @intToPtr([*]const u8, base + tls_phdr.p_vaddr);
154	154	assert(main_thread_tls_bytes.len >= tls_phdr.p_memsz); // not enough preallocated Thread Local Storage
155	155	assert(main_thread_tls_align >= tls_phdr.p_align); // preallocated Thread Local Storage not aligned enough
156		@memcpy(&main_thread_tls_bytes, std.startup.linux_tls_img_src, tls_phdr.p_filesz);
	156	@memcpy(&main_thread_tls_bytes, std.os.linux_tls_img_src, tls_phdr.p_filesz);
157	157	tls_end_addr = @ptrToInt(&main_thread_tls_bytes) + tls_phdr.p_memsz;
158	158	linuxSetThreadArea(@ptrToInt(&tls_end_addr));
159	159	}

test/tests.zig+3

...	...	@@ -194,6 +194,9 @@ pub fn addPkgTests(b: *build.Builder, test_filter: ?[]const u8, root_src: []cons
194	194	if (link_libc) {
195	195	these_tests.linkSystemLibrary("c");
196	196	}
	197	if (mem.eql(u8, name, "std")) {
	198	these_tests.overrideStdDir("std");
	199	}
197	200	step.dependOn(&these_tests.step);
198	201	}
199	202	}