doc/docgen.zig              |  2 +-
 lib/std/zig/ast.zig         |  4 +++
 lib/std/zig/parse.zig       | 13 +++++++++
 lib/std/zig/parser_test.zig | 18 +++++++++++++
 lib/std/zig/tokenizer.zig   | 46 +++++++++++++++++++++++++++++---
 src/stage1/tokenizer.cpp    | 17 ++++++++++--
 test/compile_errors.zig     |  8 ++++++
 7 files changed, 102 insertions(+), 6 deletions(-)