Commit 07898cc0dfae880b35bededa726c7e16a3c45f3d | zig

author	Andrew Kelley <andrew@ziglang.org> 2017-12-04 23:25:59-05:00
committer	Andrew Kelley <andrew@ziglang.org> 2017-12-04 23:25:59-05:00
log	07898cc0dfae880b35bededa726c7e16a3c45f3d
tree	293135e757f9599c93440dcf594df6d820cd7f4c
parent	798dbe487b678a85c7e10b78c2424c4b3508475b

tokenizing string literals

1 files changed, 50 insertions(+), 1 deletions(-)

src-self-hosted
- main.zig

src-self-hosted/main.zig+50-1

...	...	@@ -64,12 +64,15 @@ const Token = struct {
64	64	return null;
65	65	}
66	66
	67	const StrLitKind = enum {Normal, C};
67	68
68	69	const Id = union(enum) {
69	70	Invalid,
70	71	Identifier,
71	72	Keyword: Keyword,
	73	StringLiteral: StrLitKind,
72	74	Eof,
	75	Builtin,
73	76	};
74	77	};
75	78
...	...	@@ -91,6 +94,10 @@ const Tokenizer = struct {
91	94	const State = enum {
92	95	Start,
93	96	Identifier,
	97	Builtin,
	98	C,
	99	StringLiteral,
	100	StringLiteralBackslash,
94	101	};
95	102
96	103	pub fn next(self: &Tokenizer) -> Token {
...	...	@@ -107,10 +114,22 @@ const Tokenizer = struct {
107	114	' ', '\n' => {
108	115	result.start = self.index + 1;
109	116	},
110		'a'...'z', 'A'...'Z', '_' => {
	117	'c' => {
	118	state = State.C;
	119	result.id = Token.Id { .Identifier = {} };
	120	},
	121	'"' => {
	122	state = State.StringLiteral;
	123	result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
	124	},
	125	'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
111	126	state = State.Identifier;
112	127	result.id = Token.Id { .Identifier = {} };
113	128	},
	129	'@' => {
	130	state = State.Builtin;
	131	result.id = Token.Id { .Builtin = {} };
	132	},
114	133	else => {
115	134	result.id = Token.Id { .Invalid = {} };
116	135	self.index += 1;
...	...	@@ -126,6 +145,36 @@ const Tokenizer = struct {
126	145	break;
127	146	},
128	147	},
	148	State.Builtin => switch (c) {
	149	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	150	else => break,
	151	},
	152	State.C => switch (c) {
	153	'\\' => @panic("TODO"),
	154	'"' => {
	155	state = State.StringLiteral;
	156	result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
	157	},
	158	'a'...'z', 'A'...'Z', '_', '0'...'9' => {
	159	state = State.Identifier;
	160	},
	161	else => break,
	162	},
	163	State.StringLiteral => switch (c) {
	164	'\\' => {
	165	state = State.StringLiteralBackslash;
	166	},
	167	'"' => {
	168	self.index += 1;
	169	break;
	170	},
	171	'\n' => break, // Look for this error later.
	172	else => {},
	173	},
	174	State.StringLiteralBackslash => switch (c) {
	175	'\n' => break, // Look for this error later.
	176	else => {},
	177	},
129	178	}
130	179	}
131	180	result.end = self.index;