Commit 8c31eaf2a87d39fe2f9ed8f5af2a059048bfffb3 | zig

author	Andrew Kelley <andrew@ziglang.org> 2018-02-10 14:52:39-05:00
committer	Andrew Kelley <andrew@ziglang.org> 2018-02-10 14:52:39-05:00
log	8c31eaf2a87d39fe2f9ed8f5af2a059048bfffb3
tree	f4a88fab1589ee554bd74552db44360b4fbaeff7
parent	a2bd9f8912ade5149855dc6e2371aaae49093660

4 files changed, 14 insertions(+), 17 deletions(-)

src-self-hosted/main.zig+9

...	...	@@ -565,6 +565,15 @@ fn fmtMain(allocator: &mem.Allocator, file_paths: []const []const u8) !void {
565	565	var file = try io.File.openRead(allocator, file_path);
566	566	defer file.close();
567	567
	568	const source_code = io.readFileAlloc(allocator, file_path) catch \|err\| {
	569	warn("unable to open '{}': {}", file_path, err);
	570	continue;
	571	};
	572	defer allocator.free(source_code);
	573
	574	var tokenizer = std.zig.Tokenizer.init(source_code);
	575	var parser = std.zig.Parser.init(&tokenizer, allocator, file_path);
	576	defer parser.deinit();
568	577	warn("opened {} (todo tokenize and parse and render)\n", file_path);
569	578	}
570	579	}

src-self-hosted/module.zig+1-4

...	...	@@ -213,14 +213,11 @@ pub const Module = struct {
213	213	};
214	214	errdefer self.allocator.free(root_src_real_path);
215	215
216		const source_code = io.readFileAllocExtra(self.allocator, root_src_real_path, 3) catch \|err\| {
	216	const source_code = io.readFileAlloc(self.allocator, root_src_real_path) catch \|err\| {
217	217	try printError("unable to open '{}': {}", root_src_real_path, err);
218	218	return err;
219	219	};
220	220	errdefer self.allocator.free(source_code);
221		source_code[source_code.len - 3] = '\n';
222		source_code[source_code.len - 2] = '\n';
223		source_code[source_code.len - 1] = '\n';
224	221
225	222	warn("====input:====\n");
226	223

std/io.zig+1-6

...	...	@@ -524,16 +524,11 @@ pub fn writeFile(allocator: &mem.Allocator, path: []const u8, data: []const u8)
524	524
525	525	/// On success, caller owns returned buffer.
526	526	pub fn readFileAlloc(allocator: &mem.Allocator, path: []const u8) ![]u8 {
527		return readFileAllocExtra(allocator, path, 0);
528		}
529		/// On success, caller owns returned buffer.
530		/// Allocates extra_len extra bytes at the end of the file buffer, which are uninitialized.
531		pub fn readFileAllocExtra(allocator: &mem.Allocator, path: []const u8, extra_len: usize) ![]u8 {
532	527	var file = try File.openRead(allocator, path);
533	528	defer file.close();
534	529
535	530	const size = try file.getEndPos();
536		const buf = try allocator.alloc(u8, size + extra_len);
	531	const buf = try allocator.alloc(u8, size);
537	532	errdefer allocator.free(buf);
538	533
539	534	var adapter = FileInStream.init(&file);

std/zig/tokenizer.zig+3-7

...	...	@@ -175,12 +175,7 @@ pub const Tokenizer = struct {
175	175	std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
176	176	}
177	177
178		/// buffer must end with "\n\n\n". This is so that attempting to decode
179		/// a the 3 trailing bytes of a 4-byte utf8 sequence is never a buffer overflow.
180	178	pub fn init(buffer: []const u8) Tokenizer {
181		std.debug.assert(buffer[buffer.len - 1] == '\n');
182		std.debug.assert(buffer[buffer.len - 2] == '\n');
183		std.debug.assert(buffer[buffer.len - 3] == '\n');
184	179	return Tokenizer {
185	180	.buffer = buffer,
186	181	.index = 0,
...	...	@@ -556,8 +551,9 @@ pub const Tokenizer = struct {
556	551	} else {
557	552	// check utf8-encoded character.
558	553	const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
559		// the last 3 bytes in the buffer are guaranteed to be '\n',
560		// which means we don't need to do any bounds checking here.
	554	if (self.index + length >= self.buffer.len) {
	555	return u3(self.buffer.len - self.index);
	556	}
561	557	const bytes = self.buffer[self.index..self.index + length];
562	558	switch (length) {
563	559	2 => {