authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-02-25 14:03:36-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-02-25 14:03:36-05:00
log0d4db8828a9efc05b5c3622098a8337de0b62d1e
treef87768be86d630511e694ebd09173057c7b88a58
parent525c2eaf5d49f537d4ccd48ab0c5bc1f52cc3204
signature Commit is signed but in an unrecognized format.

`@cImport` works with `--cache on`

We pass -MD -MF args to clang when doing `@cImport`, which gives us a complete list of files that the C code read from. Then we add these to the cache. So even when using `@cImport` Zig's caching system remains perfect. This is a proof of concept for the mechanism that the self-hosted compiler will use to watch and rebuild files.

9 files changed, 77 insertions(+), 31 deletions(-)

src/cache_hash.cpp+34
...@@ -414,6 +414,39 @@ Error cache_add_file(CacheHash *ch, Buf *path) {...@@ -414,6 +414,39 @@ Error cache_add_file(CacheHash *ch, Buf *path) {
414 return cache_add_file_fetch(ch, resolved_path, nullptr);414 return cache_add_file_fetch(ch, resolved_path, nullptr);
415}415}
416416
417Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) {
418 Error err;
419 Buf *contents = buf_alloc();
420 if ((err = os_fetch_file_path(dep_file_path, contents, false))) {
421 if (verbose) {
422 fprintf(stderr, "unable to read .d file: %s\n", err_str(err));
423 }
424 return ErrorReadingDepFile;
425 }
426 SplitIterator it = memSplit(buf_to_slice(contents), str("\n"));
427 // skip first line
428 SplitIterator_next(&it);
429 for (;;) {
430 Optional<Slice<uint8_t>> opt_line = SplitIterator_next(&it);
431 if (!opt_line.is_some)
432 break;
433 if (opt_line.value.len == 0)
434 continue;
435 SplitIterator line_it = memSplit(opt_line.value, str(" \t"));
436 Slice<uint8_t> filename;
437 if (!SplitIterator_next(&line_it).unwrap(&filename))
438 continue;
439 Buf *filename_buf = buf_create_from_slice(filename);
440 if ((err = cache_add_file(ch, filename_buf))) {
441 if (verbose) {
442 fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err));
443 }
444 return err;
445 }
446 }
447 return ErrorNone;
448}
449
417static Error write_manifest_file(CacheHash *ch) {450static Error write_manifest_file(CacheHash *ch) {
418 Error err;451 Error err;
419 Buf contents = BUF_INIT;452 Buf contents = BUF_INIT;
...@@ -464,3 +497,4 @@ void cache_release(CacheHash *ch) {...@@ -464,3 +497,4 @@ void cache_release(CacheHash *ch) {
464497
465 os_file_close(ch->manifest_file);498 os_file_close(ch->manifest_file);
466}499}
500
src/cache_hash.hpp+2
...@@ -56,6 +56,8 @@ Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest);...@@ -56,6 +56,8 @@ Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest);
56// If you did not get a cache hit, call this function for every file56// If you did not get a cache hit, call this function for every file
57// that is depended on, and then finish with cache_final.57// that is depended on, and then finish with cache_final.
58Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path);58Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path);
59// This opens a file created by -MD -MF args to Clang
60Error ATTRIBUTE_MUST_USE cache_add_dep_file(CacheHash *ch, Buf *path, bool verbose);
5961
60// This variant of cache_add_file returns the file contents.62// This variant of cache_add_file returns the file contents.
61// Also the file path argument must be already resolved.63// Also the file path argument must be already resolved.
src/codegen.cpp+6-25
...@@ -8218,8 +8218,9 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) {...@@ -8218,8 +8218,9 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) {
8218 }8218 }
82198219
8220 if (g->verbose_cc) {8220 if (g->verbose_cc) {
8221 fprintf(stderr, "zig");
8221 for (size_t arg_i = 0; arg_i < args.length; arg_i += 1) {8222 for (size_t arg_i = 0; arg_i < args.length; arg_i += 1) {
8222 fprintf(stderr, "%s ", args.at(arg_i));8223 fprintf(stderr, " %s", args.at(arg_i));
8223 }8224 }
8224 fprintf(stderr, "\n");8225 fprintf(stderr, "\n");
8225 }8226 }
...@@ -8232,35 +8233,15 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) {...@@ -8232,35 +8233,15 @@ static void gen_c_object(CodeGen *g, Buf *self_exe_path, CFile *c_file) {
82328233
8233 g->link_objects.append(out_obj_path);8234 g->link_objects.append(out_obj_path);
82348235
8235 // add the files depended on to the cache system
8236 if (g->enable_cache) {8236 if (g->enable_cache) {
8237 Buf *contents = buf_alloc();8237 // add the files depended on to the cache system
8238 if ((err = os_fetch_file_path(out_dep_path, contents, false))) {
8239 fprintf(stderr, "unable to read .d file: %s\n", err_str(err));
8240 exit(1);
8241 }
8242 if ((err = cache_add_file(&g->cache_hash, c_source_file))) {8238 if ((err = cache_add_file(&g->cache_hash, c_source_file))) {
8243 fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(c_source_file), err_str(err));8239 fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(c_source_file), err_str(err));
8244 exit(1);8240 exit(1);
8245 }8241 }
8246 SplitIterator it = memSplit(buf_to_slice(contents), str("\n"));8242 if ((err = cache_add_dep_file(&g->cache_hash, out_dep_path, true))) {
8247 // skip first line8243 fprintf(stderr, "failed to add C source dependencies to cache: %s\n", err_str(err));
8248 SplitIterator_next(&it);8244 exit(1);
8249 for (;;) {
8250 Optional<Slice<uint8_t>> opt_line = SplitIterator_next(&it);
8251 if (!opt_line.is_some)
8252 break;
8253 if (opt_line.value.len == 0)
8254 continue;
8255 SplitIterator line_it = memSplit(opt_line.value, str(" \t"));
8256 Slice<uint8_t> filename;
8257 if (!SplitIterator_next(&line_it).unwrap(&filename))
8258 continue;
8259 Buf *filename_buf = buf_create_from_slice(filename);
8260 if ((err = cache_add_file(&g->cache_hash, filename_buf))) {
8261 fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(c_source_file), err_str(err));
8262 exit(1);
8263 }
8264 }8245 }
8265 }8246 }
8266}8247}
src/error.cpp+1
...@@ -36,6 +36,7 @@ const char *err_str(Error err) {...@@ -36,6 +36,7 @@ const char *err_str(Error err) {
36 case ErrorCacheUnavailable: return "cache unavailable";36 case ErrorCacheUnavailable: return "cache unavailable";
37 case ErrorPathTooLong: return "path too long";37 case ErrorPathTooLong: return "path too long";
38 case ErrorCCompilerCannotFindFile: return "C compiler cannot find file";38 case ErrorCCompilerCannotFindFile: return "C compiler cannot find file";
39 case ErrorReadingDepFile: return "failed to read .d file";
39 }40 }
40 return "(invalid error)";41 return "(invalid error)";
41}42}
src/error.hpp+1
...@@ -38,6 +38,7 @@ enum Error {...@@ -38,6 +38,7 @@ enum Error {
38 ErrorCacheUnavailable,38 ErrorCacheUnavailable,
39 ErrorPathTooLong,39 ErrorPathTooLong,
40 ErrorCCompilerCannotFindFile,40 ErrorCCompilerCannotFindFile,
41 ErrorReadingDepFile,
41};42};
4243
43const char *err_str(Error err);44const char *err_str(Error err);
src/ir.cpp-6
...@@ -18670,12 +18670,6 @@ static IrInstruction *ir_analyze_instruction_type_name(IrAnalyze *ira, IrInstruc...@@ -18670,12 +18670,6 @@ static IrInstruction *ir_analyze_instruction_type_name(IrAnalyze *ira, IrInstruc
18670}18670}
1867118671
18672static IrInstruction *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstructionCImport *instruction) {18672static IrInstruction *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstructionCImport *instruction) {
18673 if (ira->codegen->enable_cache) {
18674 ir_add_error(ira, &instruction->base,
18675 buf_sprintf("TODO @cImport is incompatible with --cache on. The cache system currently is unable to detect subsequent changes in .h files."));
18676 return ira->codegen->invalid_instruction;
18677 }
18678
18679 AstNode *node = instruction->base.source_node;18673 AstNode *node = instruction->base.source_node;
18680 assert(node->type == NodeTypeFnCallExpr);18674 assert(node->type == NodeTypeFnCallExpr);
18681 AstNode *block_node = node->data.fn_call_expr.params.at(0);18675 AstNode *block_node = node->data.fn_call_expr.params.at(0);
src/os.cpp+12
...@@ -1232,6 +1232,18 @@ static Error os_buf_to_tmp_file_posix(Buf *contents, Buf *suffix, Buf *out_tmp_p...@@ -1232,6 +1232,18 @@ static Error os_buf_to_tmp_file_posix(Buf *contents, Buf *suffix, Buf *out_tmp_p
1232}1232}
1233#endif1233#endif
12341234
1235Buf *os_tmp_filename(Buf *prefix, Buf *suffix) {
1236 Buf *result = buf_create_from_buf(prefix);
1237
1238 const char base64[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_";
1239 assert(array_length(base64) == 64 + 1);
1240 for (size_t i = 0; i < 12; i += 1) {
1241 buf_append_char(result, base64[rand() % 64]);
1242 }
1243 buf_append_buf(result, suffix);
1244 return result;
1245}
1246
1235#if defined(ZIG_OS_WINDOWS)1247#if defined(ZIG_OS_WINDOWS)
1236static Error os_buf_to_tmp_file_windows(Buf *contents, Buf *suffix, Buf *out_tmp_path) {1248static Error os_buf_to_tmp_file_windows(Buf *contents, Buf *suffix, Buf *out_tmp_path) {
1237 char tmp_dir[MAX_PATH + 1];1249 char tmp_dir[MAX_PATH + 1];
src/os.hpp+1
...@@ -121,6 +121,7 @@ Error ATTRIBUTE_MUST_USE os_get_cwd(Buf *out_cwd);...@@ -121,6 +121,7 @@ Error ATTRIBUTE_MUST_USE os_get_cwd(Buf *out_cwd);
121bool os_stderr_tty(void);121bool os_stderr_tty(void);
122void os_stderr_set_color(TermColor color);122void os_stderr_set_color(TermColor color);
123123
124Buf *os_tmp_filename(Buf *prefix, Buf *suffix);
124Error os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path);125Error os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path);
125Error os_delete_file(Buf *path);126Error os_delete_file(Buf *path);
126127
src/translate_c.cpp+20
...@@ -4776,6 +4776,15 @@ Error parse_h_file(ImportTableEntry *import, ZigList<ErrorMsg *> *errors, const...@@ -4776,6 +4776,15 @@ Error parse_h_file(ImportTableEntry *import, ZigList<ErrorMsg *> *errors, const
4776 clang_argv.append("-x");4776 clang_argv.append("-x");
4777 clang_argv.append("c");4777 clang_argv.append("c");
47784778
4779 Buf *out_dep_path = nullptr;
4780 if (codegen->enable_cache) {
4781 Buf *prefix = buf_sprintf("%s" OS_SEP, buf_ptr(&codegen->cache_dir));
4782 out_dep_path = os_tmp_filename(prefix, buf_create_from_str(".d"));
4783 clang_argv.append("-MD");
4784 clang_argv.append("-MF");
4785 clang_argv.append(buf_ptr(out_dep_path));
4786 }
4787
4779 if (c->codegen->zig_target->is_native) {4788 if (c->codegen->zig_target->is_native) {
4780 char *ZIG_PARSEC_CFLAGS = getenv("ZIG_NATIVE_PARSEC_CFLAGS");4789 char *ZIG_PARSEC_CFLAGS = getenv("ZIG_NATIVE_PARSEC_CFLAGS");
4781 if (ZIG_PARSEC_CFLAGS) {4790 if (ZIG_PARSEC_CFLAGS) {
...@@ -4912,6 +4921,17 @@ Error parse_h_file(ImportTableEntry *import, ZigList<ErrorMsg *> *errors, const...@@ -4912,6 +4921,17 @@ Error parse_h_file(ImportTableEntry *import, ZigList<ErrorMsg *> *errors, const
4912 return ErrorCCompileErrors;4921 return ErrorCCompileErrors;
4913 }4922 }
49144923
4924 if (codegen->enable_cache) {
4925 Error err;
4926 assert(out_dep_path != nullptr);
4927 if ((err = cache_add_dep_file(&codegen->cache_hash, out_dep_path, codegen->verbose_cimport))) {
4928 if (codegen->verbose_cimport) {
4929 fprintf(stderr, "translate-c: aborting due to failed cache operation: %s\n", err_str(err));
4930 }
4931 return err;
4932 }
4933 }
4934
4915 c->ctx = ZigClangASTUnit_getASTContext(ast_unit);4935 c->ctx = ZigClangASTUnit_getASTContext(ast_unit);
4916 c->source_manager = ZigClangASTUnit_getSourceManager(ast_unit);4936 c->source_manager = ZigClangASTUnit_getSourceManager(ast_unit);
4917 c->root = trans_create_node(c, NodeTypeContainerDecl);4937 c->root = trans_create_node(c, NodeTypeContainerDecl);