| ... | @@ -42,7 +42,7 @@ pub const Node = struct { | ... | @@ -42,7 +42,7 @@ pub const Node = struct { |
| 42 | .doc => @fieldParentPtr(Node.Doc, "base", self).deinit(allocator), | 42 | .doc => @fieldParentPtr(Node.Doc, "base", self).deinit(allocator), |
| 43 | .map => @fieldParentPtr(Node.Map, "base", self).deinit(allocator), | 43 | .map => @fieldParentPtr(Node.Map, "base", self).deinit(allocator), |
| 44 | .list => @fieldParentPtr(Node.List, "base", self).deinit(allocator), | 44 | .list => @fieldParentPtr(Node.List, "base", self).deinit(allocator), |
| 45 | .value => {}, | 45 | .value => @fieldParentPtr(Node.Value, "base", self).deinit(allocator), |
| 46 | } | 46 | } |
| 47 | } | 47 | } |
| 48 | | 48 | |
| ... | @@ -82,8 +82,8 @@ pub const Node = struct { | ... | @@ -82,8 +82,8 @@ pub const Node = struct { |
| 82 | options: std.fmt.FormatOptions, | 82 | options: std.fmt.FormatOptions, |
| 83 | writer: anytype, | 83 | writer: anytype, |
| 84 | ) !void { | 84 | ) !void { |
| 85 | _ = fmt; | | |
| 86 | _ = options; | 85 | _ = options; |
| | 86 | _ = fmt; |
| 87 | if (self.directive) |id| { | 87 | if (self.directive) |id| { |
| 88 | try std.fmt.format(writer, "{{ ", .{}); | 88 | try std.fmt.format(writer, "{{ ", .{}); |
| 89 | const directive = self.base.tree.tokens[id]; | 89 | const directive = self.base.tree.tokens[id]; |
| ... | @@ -127,8 +127,8 @@ pub const Node = struct { | ... | @@ -127,8 +127,8 @@ pub const Node = struct { |
| 127 | options: std.fmt.FormatOptions, | 127 | options: std.fmt.FormatOptions, |
| 128 | writer: anytype, | 128 | writer: anytype, |
| 129 | ) !void { | 129 | ) !void { |
| 130 | _ = fmt; | | |
| 131 | _ = options; | 130 | _ = options; |
| | 131 | _ = fmt; |
| 132 | try std.fmt.format(writer, "{{ ", .{}); | 132 | try std.fmt.format(writer, "{{ ", .{}); |
| 133 | for (self.values.items) |entry| { | 133 | for (self.values.items) |entry| { |
| 134 | const key = self.base.tree.tokens[entry.key]; | 134 | const key = self.base.tree.tokens[entry.key]; |
| ... | @@ -163,8 +163,8 @@ pub const Node = struct { | ... | @@ -163,8 +163,8 @@ pub const Node = struct { |
| 163 | options: std.fmt.FormatOptions, | 163 | options: std.fmt.FormatOptions, |
| 164 | writer: anytype, | 164 | writer: anytype, |
| 165 | ) !void { | 165 | ) !void { |
| 166 | _ = fmt; | | |
| 167 | _ = options; | 166 | _ = options; |
| | 167 | _ = fmt; |
| 168 | try std.fmt.format(writer, "[ ", .{}); | 168 | try std.fmt.format(writer, "[ ", .{}); |
| 169 | for (self.values.items) |node| { | 169 | for (self.values.items) |node| { |
| 170 | try std.fmt.format(writer, "{}, ", .{node}); | 170 | try std.fmt.format(writer, "{}, ", .{node}); |
| ... | @@ -180,14 +180,19 @@ pub const Node = struct { | ... | @@ -180,14 +180,19 @@ pub const Node = struct { |
| 180 | | 180 | |
| 181 | pub const base_tag: Node.Tag = .value; | 181 | pub const base_tag: Node.Tag = .value; |
| 182 | | 182 | |
| | 183 | pub fn deinit(self: *Value, allocator: Allocator) void { |
| | 184 | _ = self; |
| | 185 | _ = allocator; |
| | 186 | } |
| | 187 | |
| 183 | pub fn format( | 188 | pub fn format( |
| 184 | self: *const Value, | 189 | self: *const Value, |
| 185 | comptime fmt: []const u8, | 190 | comptime fmt: []const u8, |
| 186 | options: std.fmt.FormatOptions, | 191 | options: std.fmt.FormatOptions, |
| 187 | writer: anytype, | 192 | writer: anytype, |
| 188 | ) !void { | 193 | ) !void { |
| 189 | _ = fmt; | | |
| 190 | _ = options; | 194 | _ = options; |
| | 195 | _ = fmt; |
| 191 | const start = self.base.tree.tokens[self.start.?]; | 196 | const start = self.base.tree.tokens[self.start.?]; |
| 192 | const end = self.base.tree.tokens[self.end.?]; | 197 | const end = self.base.tree.tokens[self.end.?]; |
| 193 | return std.fmt.format(writer, "{s}", .{ | 198 | return std.fmt.format(writer, "{s}", .{ |
| ... | @@ -197,10 +202,16 @@ pub const Node = struct { | ... | @@ -197,10 +202,16 @@ pub const Node = struct { |
| 197 | }; | 202 | }; |
| 198 | }; | 203 | }; |
| 199 | | 204 | |
| | 205 | pub const LineCol = struct { |
| | 206 | line: usize, |
| | 207 | col: usize, |
| | 208 | }; |
| | 209 | |
| 200 | pub const Tree = struct { | 210 | pub const Tree = struct { |
| 201 | allocator: Allocator, | 211 | allocator: Allocator, |
| 202 | source: []const u8, | 212 | source: []const u8, |
| 203 | tokens: []Token, | 213 | tokens: []Token, |
| | 214 | line_cols: std.AutoHashMap(TokenIndex, LineCol), |
| 204 | docs: std.ArrayListUnmanaged(*Node) = .{}, | 215 | docs: std.ArrayListUnmanaged(*Node) = .{}, |
| 205 | | 216 | |
| 206 | pub fn init(allocator: Allocator) Tree { | 217 | pub fn init(allocator: Allocator) Tree { |
| ... | @@ -208,11 +219,13 @@ pub const Tree = struct { | ... | @@ -208,11 +219,13 @@ pub const Tree = struct { |
| 208 | .allocator = allocator, | 219 | .allocator = allocator, |
| 209 | .source = undefined, | 220 | .source = undefined, |
| 210 | .tokens = undefined, | 221 | .tokens = undefined, |
| | 222 | .line_cols = std.AutoHashMap(TokenIndex, LineCol).init(allocator), |
| 211 | }; | 223 | }; |
| 212 | } | 224 | } |
| 213 | | 225 | |
| 214 | pub fn deinit(self: *Tree) void { | 226 | pub fn deinit(self: *Tree) void { |
| 215 | self.allocator.free(self.tokens); | 227 | self.allocator.free(self.tokens); |
| | 228 | self.line_cols.deinit(); |
| 216 | for (self.docs.items) |doc| { | 229 | for (self.docs.items) |doc| { |
| 217 | doc.deinit(self.allocator); | 230 | doc.deinit(self.allocator); |
| 218 | self.allocator.destroy(doc); | 231 | self.allocator.destroy(doc); |
| ... | @@ -223,12 +236,29 @@ pub const Tree = struct { | ... | @@ -223,12 +236,29 @@ pub const Tree = struct { |
| 223 | pub fn parse(self: *Tree, source: []const u8) !void { | 236 | pub fn parse(self: *Tree, source: []const u8) !void { |
| 224 | var tokenizer = Tokenizer{ .buffer = source }; | 237 | var tokenizer = Tokenizer{ .buffer = source }; |
| 225 | var tokens = std.ArrayList(Token).init(self.allocator); | 238 | var tokens = std.ArrayList(Token).init(self.allocator); |
| 226 | errdefer tokens.deinit(); | 239 | defer tokens.deinit(); |
| | 240 | |
| | 241 | var line: usize = 0; |
| | 242 | var prev_line_last_col: usize = 0; |
| 227 | | 243 | |
| 228 | while (true) { | 244 | while (true) { |
| 229 | const token = tokenizer.next(); | 245 | const token = tokenizer.next(); |
| | 246 | const tok_id = tokens.items.len; |
| 230 | try tokens.append(token); | 247 | try tokens.append(token); |
| 231 | if (token.id == .Eof) break; | 248 | |
| | 249 | try self.line_cols.putNoClobber(tok_id, .{ |
| | 250 | .line = line, |
| | 251 | .col = token.start - prev_line_last_col, |
| | 252 | }); |
| | 253 | |
| | 254 | switch (token.id) { |
| | 255 | .Eof => break, |
| | 256 | .NewLine => { |
| | 257 | line += 1; |
| | 258 | prev_line_last_col = token.end; |
| | 259 | }, |
| | 260 | else => {}, |
| | 261 | } |
| 232 | } | 262 | } |
| 233 | | 263 | |
| 234 | self.source = source; | 264 | self.source = source; |
| ... | @@ -239,15 +269,12 @@ pub const Tree = struct { | ... | @@ -239,15 +269,12 @@ pub const Tree = struct { |
| 239 | .allocator = self.allocator, | 269 | .allocator = self.allocator, |
| 240 | .tree = self, | 270 | .tree = self, |
| 241 | .token_it = &it, | 271 | .token_it = &it, |
| | 272 | .line_cols = &self.line_cols, |
| 242 | }; | 273 | }; |
| 243 | defer parser.deinit(); | | |
| 244 | | | |
| 245 | try parser.scopes.append(self.allocator, .{ | | |
| 246 | .indent = 0, | | |
| 247 | }); | | |
| 248 | | 274 | |
| 249 | while (true) { | 275 | while (true) { |
| 250 | if (parser.token_it.peek() == null) return; | 276 | if (parser.token_it.peek() == null) return; |
| | 277 | |
| 251 | const pos = parser.token_it.pos; | 278 | const pos = parser.token_it.pos; |
| 252 | const token = parser.token_it.next(); | 279 | const token = parser.token_it.next(); |
| 253 | | 280 | |
| ... | @@ -269,22 +296,12 @@ const Parser = struct { | ... | @@ -269,22 +296,12 @@ const Parser = struct { |
| 269 | allocator: Allocator, | 296 | allocator: Allocator, |
| 270 | tree: *Tree, | 297 | tree: *Tree, |
| 271 | token_it: *TokenIterator, | 298 | token_it: *TokenIterator, |
| 272 | scopes: std.ArrayListUnmanaged(Scope) = .{}, | 299 | line_cols: *const std.AutoHashMap(TokenIndex, LineCol), |
| 273 | | | |
| 274 | const Scope = struct { | | |
| 275 | indent: usize, | | |
| 276 | }; | | |
| 277 | | | |
| 278 | fn deinit(self: *Parser) void { | | |
| 279 | self.scopes.deinit(self.allocator); | | |
| 280 | } | | |
| 281 | | 300 | |
| 282 | fn doc(self: *Parser, start: TokenIndex) ParseError!*Node.Doc { | 301 | fn doc(self: *Parser, start: TokenIndex) ParseError!*Node.Doc { |
| 283 | const node = try self.allocator.create(Node.Doc); | 302 | const node = try self.allocator.create(Node.Doc); |
| 284 | errdefer self.allocator.destroy(node); | 303 | errdefer self.allocator.destroy(node); |
| 285 | node.* = .{ | 304 | node.* = .{ .start = start }; |
| 286 | .start = start, | | |
| 287 | }; | | |
| 288 | node.base.tree = self.tree; | 305 | node.base.tree = self.tree; |
| 289 | | 306 | |
| 290 | self.token_it.seekTo(start); | 307 | self.token_it.seekTo(start); |
| ... | @@ -346,19 +363,24 @@ const Parser = struct { | ... | @@ -346,19 +363,24 @@ const Parser = struct { |
| 346 | fn map(self: *Parser, start: TokenIndex) ParseError!*Node.Map { | 363 | fn map(self: *Parser, start: TokenIndex) ParseError!*Node.Map { |
| 347 | const node = try self.allocator.create(Node.Map); | 364 | const node = try self.allocator.create(Node.Map); |
| 348 | errdefer self.allocator.destroy(node); | 365 | errdefer self.allocator.destroy(node); |
| 349 | node.* = .{ | 366 | node.* = .{ .start = start }; |
| 350 | .start = start, | | |
| 351 | }; | | |
| 352 | node.base.tree = self.tree; | 367 | node.base.tree = self.tree; |
| 353 | | 368 | |
| 354 | self.token_it.seekTo(start); | 369 | self.token_it.seekTo(start); |
| 355 | | 370 | |
| 356 | log.debug("Map start: {}, {}", .{ start, self.tree.tokens[start] }); | 371 | log.debug("Map start: {}, {}", .{ start, self.tree.tokens[start] }); |
| 357 | log.debug("Current scope: {}", .{self.scopes.items[self.scopes.items.len - 1]}); | 372 | |
| | 373 | const col = self.getCol(start); |
| 358 | | 374 | |
| 359 | while (true) { | 375 | while (true) { |
| | 376 | self.eatCommentsAndSpace(); |
| | 377 | |
| 360 | // Parse key. | 378 | // Parse key. |
| 361 | const key_pos = self.token_it.pos; | 379 | const key_pos = self.token_it.pos; |
| | 380 | if (self.getCol(key_pos) != col) { |
| | 381 | break; |
| | 382 | } |
| | 383 | |
| 362 | const key = self.token_it.next(); | 384 | const key = self.token_it.next(); |
| 363 | switch (key.id) { | 385 | switch (key.id) { |
| 364 | .Literal => {}, | 386 | .Literal => {}, |
| ... | @@ -372,13 +394,13 @@ const Parser = struct { | ... | @@ -372,13 +394,13 @@ const Parser = struct { |
| 372 | | 394 | |
| 373 | // Separator | 395 | // Separator |
| 374 | _ = try self.expectToken(.MapValueInd); | 396 | _ = try self.expectToken(.MapValueInd); |
| 375 | self.eatCommentsAndSpace(); | | |
| 376 | | 397 | |
| 377 | // Parse value. | 398 | // Parse value. |
| 378 | const value: *Node = value: { | 399 | const value: *Node = value: { |
| 379 | if (self.eatToken(.NewLine)) |_| { | 400 | if (self.eatToken(.NewLine)) |_| { |
| | 401 | self.eatCommentsAndSpace(); |
| | 402 | |
| 380 | // Explicit, complex value such as list or map. | 403 | // Explicit, complex value such as list or map. |
| 381 | try self.openScope(); | | |
| 382 | const value_pos = self.token_it.pos; | 404 | const value_pos = self.token_it.pos; |
| 383 | const value = self.token_it.next(); | 405 | const value = self.token_it.next(); |
| 384 | switch (value.id) { | 406 | switch (value.id) { |
| ... | @@ -398,6 +420,8 @@ const Parser = struct { | ... | @@ -398,6 +420,8 @@ const Parser = struct { |
| 398 | }, | 420 | }, |
| 399 | } | 421 | } |
| 400 | } else { | 422 | } else { |
| | 423 | self.eatCommentsAndSpace(); |
| | 424 | |
| 401 | const value_pos = self.token_it.pos; | 425 | const value_pos = self.token_it.pos; |
| 402 | const value = self.token_it.next(); | 426 | const value = self.token_it.next(); |
| 403 | switch (value.id) { | 427 | switch (value.id) { |
| ... | @@ -424,11 +448,7 @@ const Parser = struct { | ... | @@ -424,11 +448,7 @@ const Parser = struct { |
| 424 | .value = value, | 448 | .value = value, |
| 425 | }); | 449 | }); |
| 426 | | 450 | |
| 427 | if (self.eatToken(.NewLine)) |_| { | 451 | _ = self.eatToken(.NewLine); |
| 428 | if (try self.closeScope()) { | | |
| 429 | break; | | |
| 430 | } | | |
| 431 | } | | |
| 432 | } | 452 | } |
| 433 | | 453 | |
| 434 | node.end = self.token_it.pos - 1; | 454 | node.end = self.token_it.pos - 1; |
| ... | @@ -449,14 +469,18 @@ const Parser = struct { | ... | @@ -449,14 +469,18 @@ const Parser = struct { |
| 449 | self.token_it.seekTo(start); | 469 | self.token_it.seekTo(start); |
| 450 | | 470 | |
| 451 | log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] }); | 471 | log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] }); |
| 452 | log.debug("Current scope: {}", .{self.scopes.items[self.scopes.items.len - 1]}); | 472 | |
| | 473 | const col = self.getCol(start); |
| 453 | | 474 | |
| 454 | while (true) { | 475 | while (true) { |
| | 476 | self.eatCommentsAndSpace(); |
| | 477 | |
| | 478 | if (self.getCol(self.token_it.pos) != col) { |
| | 479 | break; |
| | 480 | } |
| 455 | _ = self.eatToken(.SeqItemInd) orelse { | 481 | _ = self.eatToken(.SeqItemInd) orelse { |
| 456 | _ = try self.closeScope(); | | |
| 457 | break; | 482 | break; |
| 458 | }; | 483 | }; |
| 459 | self.eatCommentsAndSpace(); | | |
| 460 | | 484 | |
| 461 | const pos = self.token_it.pos; | 485 | const pos = self.token_it.pos; |
| 462 | const token = self.token_it.next(); | 486 | const token = self.token_it.next(); |
| ... | @@ -464,9 +488,6 @@ const Parser = struct { | ... | @@ -464,9 +488,6 @@ const Parser = struct { |
| 464 | switch (token.id) { | 488 | switch (token.id) { |
| 465 | .Literal, .SingleQuote, .DoubleQuote => { | 489 | .Literal, .SingleQuote, .DoubleQuote => { |
| 466 | if (self.eatToken(.MapValueInd)) |_| { | 490 | if (self.eatToken(.MapValueInd)) |_| { |
| 467 | if (self.eatToken(.NewLine)) |_| { | | |
| 468 | try self.openScope(); | | |
| 469 | } | | |
| 470 | // nested map | 491 | // nested map |
| 471 | const map_node = try self.map(pos); | 492 | const map_node = try self.map(pos); |
| 472 | break :value &map_node.base; | 493 | break :value &map_node.base; |
| ... | @@ -501,15 +522,12 @@ const Parser = struct { | ... | @@ -501,15 +522,12 @@ const Parser = struct { |
| 501 | fn list_bracketed(self: *Parser, start: TokenIndex) ParseError!*Node.List { | 522 | fn list_bracketed(self: *Parser, start: TokenIndex) ParseError!*Node.List { |
| 502 | const node = try self.allocator.create(Node.List); | 523 | const node = try self.allocator.create(Node.List); |
| 503 | errdefer self.allocator.destroy(node); | 524 | errdefer self.allocator.destroy(node); |
| 504 | node.* = .{ | 525 | node.* = .{ .start = start }; |
| 505 | .start = start, | | |
| 506 | }; | | |
| 507 | node.base.tree = self.tree; | 526 | node.base.tree = self.tree; |
| 508 | | 527 | |
| 509 | self.token_it.seekTo(start); | 528 | self.token_it.seekTo(start); |
| 510 | | 529 | |
| 511 | log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] }); | 530 | log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] }); |
| 512 | log.debug("Current scope: {}", .{self.scopes.items[self.scopes.items.len - 1]}); | | |
| 513 | | 531 | |
| 514 | _ = try self.expectToken(.FlowSeqStart); | 532 | _ = try self.expectToken(.FlowSeqStart); |
| 515 | | 533 | |
| ... | @@ -556,9 +574,7 @@ const Parser = struct { | ... | @@ -556,9 +574,7 @@ const Parser = struct { |
| 556 | fn leaf_value(self: *Parser, start: TokenIndex) ParseError!*Node.Value { | 574 | fn leaf_value(self: *Parser, start: TokenIndex) ParseError!*Node.Value { |
| 557 | const node = try self.allocator.create(Node.Value); | 575 | const node = try self.allocator.create(Node.Value); |
| 558 | errdefer self.allocator.destroy(node); | 576 | errdefer self.allocator.destroy(node); |
| 559 | node.* = .{ | 577 | node.* = .{ .start = start }; |
| 560 | .start = start, | | |
| 561 | }; | | |
| 562 | node.base.tree = self.tree; | 578 | node.base.tree = self.tree; |
| 563 | | 579 | |
| 564 | self.token_it.seekTo(start); | 580 | self.token_it.seekTo(start); |
| ... | @@ -625,48 +641,6 @@ const Parser = struct { | ... | @@ -625,48 +641,6 @@ const Parser = struct { |
| 625 | return node; | 641 | return node; |
| 626 | } | 642 | } |
| 627 | | 643 | |
| 628 | fn openScope(self: *Parser) !void { | | |
| 629 | const peek = self.token_it.peek() orelse return error.UnexpectedEof; | | |
| 630 | if (peek.id != .Space and peek.id != .Tab) { | | |
| 631 | // No need to open scope. | | |
| 632 | return; | | |
| 633 | } | | |
| 634 | const indent = self.token_it.next().count.?; | | |
| 635 | const prev_scope = self.scopes.items[self.scopes.items.len - 1]; | | |
| 636 | if (indent < prev_scope.indent) { | | |
| 637 | return error.MalformedYaml; | | |
| 638 | } | | |
| 639 | | | |
| 640 | log.debug("Opening scope...", .{}); | | |
| 641 | | | |
| 642 | try self.scopes.append(self.allocator, .{ | | |
| 643 | .indent = indent, | | |
| 644 | }); | | |
| 645 | } | | |
| 646 | | | |
| 647 | fn closeScope(self: *Parser) !bool { | | |
| 648 | const indent = indent: { | | |
| 649 | const peek = self.token_it.peek() orelse return error.UnexpectedEof; | | |
| 650 | switch (peek.id) { | | |
| 651 | .Space, .Tab => { | | |
| 652 | break :indent self.token_it.next().count.?; | | |
| 653 | }, | | |
| 654 | else => { | | |
| 655 | break :indent 0; | | |
| 656 | }, | | |
| 657 | } | | |
| 658 | }; | | |
| 659 | | | |
| 660 | const scope = self.scopes.items[self.scopes.items.len - 1]; | | |
| 661 | if (indent < scope.indent) { | | |
| 662 | log.debug("Closing scope...", .{}); | | |
| 663 | _ = self.scopes.pop(); | | |
| 664 | return true; | | |
| 665 | } | | |
| 666 | | | |
| 667 | return false; | | |
| 668 | } | | |
| 669 | | | |
| 670 | fn eatCommentsAndSpace(self: *Parser) void { | 644 | fn eatCommentsAndSpace(self: *Parser) void { |
| 671 | while (true) { | 645 | while (true) { |
| 672 | _ = self.token_it.peek() orelse return; | 646 | _ = self.token_it.peek() orelse return; |
| ... | @@ -701,6 +675,14 @@ const Parser = struct { | ... | @@ -701,6 +675,14 @@ const Parser = struct { |
| 701 | fn expectToken(self: *Parser, id: Token.Id) ParseError!TokenIndex { | 675 | fn expectToken(self: *Parser, id: Token.Id) ParseError!TokenIndex { |
| 702 | return self.eatToken(id) orelse error.UnexpectedToken; | 676 | return self.eatToken(id) orelse error.UnexpectedToken; |
| 703 | } | 677 | } |
| | 678 | |
| | 679 | fn getLine(self: *Parser, index: TokenIndex) usize { |
| | 680 | return self.line_cols.get(index).?.line; |
| | 681 | } |
| | 682 | |
| | 683 | fn getCol(self: *Parser, index: TokenIndex) usize { |
| | 684 | return self.line_cols.get(index).?.col; |
| | 685 | } |
| 704 | }; | 686 | }; |
| 705 | | 687 | |
| 706 | test { | 688 | test { |