authorgravatar for shachaf@gmail.comShachaf Ben-Kiki <shachaf@gmail.com> 2021-04-24 16:41:10-07:00
committergravatar for shachaf@gmail.comShachaf Ben-Kiki <shachaf@gmail.com> 2021-04-24 22:40:41-07:00
log01aa74a93fa7c18f8a2f27a4c5f76333935c3502
treede2602e3e0da264135be942426f50460c44952cc
parent37b05742ff1544bccf7c8ae9b12c6707a5a54df2

parser: Use an operator precedence table

Instead of having one function per precedence level, mirroring the BNF grammar, this uses the "precedence climbing" algorithm with a table of operator precedences (and a few special cases that don't fit into the table as it is). This is a first pass -- it's probably possible to put more of the parser into this form, e.g. to support prefix/suffix operators with precedence, if necessary, or just to simplify the code more. It may also be possible to speed this up by putting more useful information into the tokens during tokenization, to avoid the extra branch on the token in operInfo.

2 files changed, 92 insertions(+), 241 deletions(-)

lib/std/zig/parse.zig+81-241
...@@ -1309,9 +1309,8 @@ const Parser = struct {...@@ -1309,9 +1309,8 @@ const Parser = struct {
1309 return expr;1309 return expr;
1310 }1310 }
13111311
1312 /// Expr <- BoolOrExpr
1313 fn parseExpr(p: *Parser) Error!Node.Index {1312 fn parseExpr(p: *Parser) Error!Node.Index {
1314 return p.parseBoolOrExpr();1313 return p.parseExprPrecedence(0);
1315 }1314 }
13161315
1317 fn expectExpr(p: *Parser) Error!Node.Index {1316 fn expectExpr(p: *Parser) Error!Node.Index {
...@@ -1323,263 +1322,104 @@ const Parser = struct {...@@ -1323,263 +1322,104 @@ const Parser = struct {
1323 }1322 }
1324 }1323 }
13251324
1326 /// BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)*1325 const Assoc = enum {
1327 fn parseBoolOrExpr(p: *Parser) Error!Node.Index {1326 left,
1328 var res = try p.parseBoolAndExpr();1327 none,
1329 if (res == 0) return null_node;1328 };
1330
1331 while (true) {
1332 switch (p.token_tags[p.tok_i]) {
1333 .keyword_or => {
1334 const or_token = p.nextToken();
1335 const rhs = try p.parseBoolAndExpr();
1336 if (rhs == 0) {
1337 return p.fail(.invalid_token);
1338 }
1339 res = try p.addNode(.{
1340 .tag = .bool_or,
1341 .main_token = or_token,
1342 .data = .{
1343 .lhs = res,
1344 .rhs = rhs,
1345 },
1346 });
1347 },
1348 else => return res,
1349 }
1350 }
1351 }
1352
1353 /// BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)*
1354 fn parseBoolAndExpr(p: *Parser) !Node.Index {
1355 var res = try p.parseCompareExpr();
1356 if (res == 0) return null_node;
1357
1358 while (true) {
1359 switch (p.token_tags[p.tok_i]) {
1360 .keyword_and => {
1361 const and_token = p.nextToken();
1362 const rhs = try p.parseCompareExpr();
1363 if (rhs == 0) {
1364 return p.fail(.invalid_token);
1365 }
1366 res = try p.addNode(.{
1367 .tag = .bool_and,
1368 .main_token = and_token,
1369 .data = .{
1370 .lhs = res,
1371 .rhs = rhs,
1372 },
1373 });
1374 },
1375 .invalid_ampersands => {
1376 try p.warn(.invalid_and);
1377 p.tok_i += 1;
1378 return p.parseCompareExpr();
1379 },
1380 else => return res,
1381 }
1382 }
1383 }
13841329
1385 /// CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)?1330 const OperInfo = struct {
1386 /// CompareOp1331 prec: i8,
1387 /// <- EQUALEQUAL1332 tag: Node.Tag,
1388 /// / EXCLAMATIONMARKEQUAL1333 assoc: Assoc = Assoc.left,
1389 /// / LARROW1334 };
1390 /// / RARROW
1391 /// / LARROWEQUAL
1392 /// / RARROWEQUAL
1393 fn parseCompareExpr(p: *Parser) !Node.Index {
1394 const expr = try p.parseBitwiseExpr();
1395 if (expr == 0) return null_node;
13961335
1397 const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {1336 // A table of binary operator information. Higher precedence numbers are
1398 .equal_equal => .equal_equal,1337 // stickier. All operators at the same precedence level should have the same
1399 .bang_equal => .bang_equal,1338 // associativity.
1400 .angle_bracket_left => .less_than,1339 fn operInfo(tag: Token.Tag) OperInfo {
1401 .angle_bracket_right => .greater_than,1340 return switch (tag) {
1402 .angle_bracket_left_equal => .less_or_equal,1341 .keyword_or => .{ .prec = 10, .tag = .bool_or },
1403 .angle_bracket_right_equal => .greater_or_equal,1342
1404 else => return expr,1343 .keyword_and => .{ .prec = 20, .tag = .bool_and },
1344 .invalid_ampersands => .{ .prec = 20, .tag = .bool_and },
1345
1346 .equal_equal => .{ .prec = 30, .tag = .equal_equal, .assoc = Assoc.none },
1347 .bang_equal => .{ .prec = 30, .tag = .bang_equal, .assoc = Assoc.none },
1348 .angle_bracket_left => .{ .prec = 30, .tag = .less_than, .assoc = Assoc.none },
1349 .angle_bracket_right => .{ .prec = 30, .tag = .greater_than, .assoc = Assoc.none },
1350 .angle_bracket_left_equal => .{ .prec = 30, .tag = .less_or_equal, .assoc = Assoc.none },
1351 .angle_bracket_right_equal => .{ .prec = 30, .tag = .greater_or_equal, .assoc = Assoc.none },
1352
1353 .ampersand => .{ .prec = 40, .tag = .bit_and },
1354 .caret => .{ .prec = 40, .tag = .bit_xor },
1355 .pipe => .{ .prec = 40, .tag = .bit_or },
1356 .keyword_orelse => .{ .prec = 40, .tag = .@"orelse" },
1357 .keyword_catch => .{ .prec = 40, .tag = .@"catch" },
1358
1359 .angle_bracket_angle_bracket_left => .{ .prec = 50, .tag = .bit_shift_left },
1360 .angle_bracket_angle_bracket_right => .{ .prec = 50, .tag = .bit_shift_right },
1361
1362 .plus => .{ .prec = 60, .tag = .add },
1363 .minus => .{ .prec = 60, .tag = .sub },
1364 .plus_plus => .{ .prec = 60, .tag = .array_cat },
1365 .plus_percent => .{ .prec = 60, .tag = .add_wrap },
1366 .minus_percent => .{ .prec = 60, .tag = .sub_wrap },
1367
1368 .pipe_pipe => .{ .prec = 70, .tag = .merge_error_sets },
1369 .asterisk => .{ .prec = 70, .tag = .mul },
1370 .slash => .{ .prec = 70, .tag = .div },
1371 .percent => .{ .prec = 70, .tag = .mod },
1372 .asterisk_asterisk => .{ .prec = 70, .tag = .array_mult },
1373 .asterisk_percent => .{ .prec = 70, .tag = .mul_wrap },
1374
1375 else => .{ .prec = -1, .tag = ast.Node.Tag.root }, // irrelevant
1405 };1376 };
1406 return p.addNode(.{
1407 .tag = tag,
1408 .main_token = p.nextToken(),
1409 .data = .{
1410 .lhs = expr,
1411 .rhs = try p.expectBitwiseExpr(),
1412 },
1413 });
1414 }1377 }
14151378
1416 /// BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)*1379 fn parseExprPrecedence(p: *Parser, min_prec: i32) Error!Node.Index {
1417 /// BitwiseOp1380 var node = try p.parsePrefixExpr();
1418 /// <- AMPERSAND
1419 /// / CARET
1420 /// / PIPE
1421 /// / KEYWORD_orelse
1422 /// / KEYWORD_catch Payload?
1423 fn parseBitwiseExpr(p: *Parser) !Node.Index {
1424 var res = try p.parseBitShiftExpr();
1425 if (res == 0) return null_node;
1426
1427 while (true) {
1428 const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
1429 .ampersand => .bit_and,
1430 .caret => .bit_xor,
1431 .pipe => .bit_or,
1432 .keyword_orelse => .@"orelse",
1433 .keyword_catch => {
1434 const catch_token = p.nextToken();
1435 _ = try p.parsePayload();
1436 const rhs = try p.parseBitShiftExpr();
1437 if (rhs == 0) {
1438 return p.fail(.invalid_token);
1439 }
1440 res = try p.addNode(.{
1441 .tag = .@"catch",
1442 .main_token = catch_token,
1443 .data = .{
1444 .lhs = res,
1445 .rhs = rhs,
1446 },
1447 });
1448 continue;
1449 },
1450 else => return res,
1451 };
1452 res = try p.addNode(.{
1453 .tag = tag,
1454 .main_token = p.nextToken(),
1455 .data = .{
1456 .lhs = res,
1457 .rhs = try p.expectBitShiftExpr(),
1458 },
1459 });
1460 }
1461 }
1462
1463 fn expectBitwiseExpr(p: *Parser) Error!Node.Index {
1464 const node = try p.parseBitwiseExpr();
1465 if (node == 0) {1381 if (node == 0) {
1466 return p.fail(.invalid_token);1382 return null_node;
1467 } else {
1468 return node;
1469 }1383 }
1470 }
14711384
1472 /// BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)*1385 var banned_prec: i8 = -1;
1473 /// BitShiftOp
1474 /// <- LARROW2
1475 /// / RARROW2
1476 fn parseBitShiftExpr(p: *Parser) Error!Node.Index {
1477 var res = try p.parseAdditionExpr();
1478 if (res == 0) return null_node;
14791386
1480 while (true) {1387 while (true) {
1481 const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {1388 const tok_tag = p.token_tags[p.tok_i];
1482 .angle_bracket_angle_bracket_left => .bit_shift_left,1389 const info = operInfo(tok_tag);
1483 .angle_bracket_angle_bracket_right => .bit_shift_right,1390 if (info.prec < min_prec or info.prec == banned_prec) {
1484 else => return res,1391 break;
1485 };1392 }
1486 res = try p.addNode(.{1393 const oper_token = p.nextToken();
1487 .tag = tag,1394 // Special-case handling for "catch" and "&&".
1488 .main_token = p.nextToken(),1395 switch (tok_tag) {
1489 .data = .{1396 .keyword_catch => {
1490 .lhs = res,1397 _ = try p.parsePayload();
1491 .rhs = try p.expectAdditionExpr(),
1492 },1398 },
1493 });1399 .invalid_ampersands => {
1494 }1400 try p.warn(.invalid_and);
1495 }
1496
1497 fn expectBitShiftExpr(p: *Parser) Error!Node.Index {
1498 const node = try p.parseBitShiftExpr();
1499 if (node == 0) {
1500 return p.fail(.invalid_token);
1501 } else {
1502 return node;
1503 }
1504 }
1505
1506 /// AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)*
1507 /// AdditionOp
1508 /// <- PLUS
1509 /// / MINUS
1510 /// / PLUS2
1511 /// / PLUSPERCENT
1512 /// / MINUSPERCENT
1513 fn parseAdditionExpr(p: *Parser) Error!Node.Index {
1514 var res = try p.parseMultiplyExpr();
1515 if (res == 0) return null_node;
1516
1517 while (true) {
1518 const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {
1519 .plus => .add,
1520 .minus => .sub,
1521 .plus_plus => .array_cat,
1522 .plus_percent => .add_wrap,
1523 .minus_percent => .sub_wrap,
1524 else => return res,
1525 };
1526 res = try p.addNode(.{
1527 .tag = tag,
1528 .main_token = p.nextToken(),
1529 .data = .{
1530 .lhs = res,
1531 .rhs = try p.expectMultiplyExpr(),
1532 },1401 },
1533 });1402 else => {},
1534 }1403 }
1535 }1404 const rhs = try p.parseExprPrecedence(info.prec + 1);
15361405 if (rhs == 0) {
1537 fn expectAdditionExpr(p: *Parser) Error!Node.Index {1406 return p.fail(.invalid_token);
1538 const node = try p.parseAdditionExpr();1407 }
1539 if (node == 0) {
1540 return p.fail(.invalid_token);
1541 }
1542 return node;
1543 }
1544
1545 /// MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)*
1546 /// MultiplyOp
1547 /// <- PIPE2
1548 /// / ASTERISK
1549 /// / SLASH
1550 /// / PERCENT
1551 /// / ASTERISK2
1552 /// / ASTERISKPERCENT
1553 fn parseMultiplyExpr(p: *Parser) Error!Node.Index {
1554 var res = try p.parsePrefixExpr();
1555 if (res == 0) return null_node;
15561408
1557 while (true) {1409 node = try p.addNode(.{
1558 const tag: Node.Tag = switch (p.token_tags[p.tok_i]) {1410 .tag = info.tag,
1559 .pipe_pipe => .merge_error_sets,1411 .main_token = oper_token,
1560 .asterisk => .mul,
1561 .slash => .div,
1562 .percent => .mod,
1563 .asterisk_asterisk => .array_mult,
1564 .asterisk_percent => .mul_wrap,
1565 else => return res,
1566 };
1567 res = try p.addNode(.{
1568 .tag = tag,
1569 .main_token = p.nextToken(),
1570 .data = .{1412 .data = .{
1571 .lhs = res,1413 .lhs = node,
1572 .rhs = try p.expectPrefixExpr(),1414 .rhs = rhs,
1573 },1415 },
1574 });1416 });
1575 }
1576 }
15771417
1578 fn expectMultiplyExpr(p: *Parser) Error!Node.Index {1418 if (info.assoc == Assoc.none) {
1579 const node = try p.parseMultiplyExpr();1419 banned_prec = info.prec;
1580 if (node == 0) {1420 }
1581 return p.fail(.invalid_token);
1582 }1421 }
1422
1583 return node;1423 return node;
1584 }1424 }
15851425
lib/std/zig/parser_test.zig+11
...@@ -2796,6 +2796,7 @@ test "zig fmt: precedence" {...@@ -2796,6 +2796,7 @@ test "zig fmt: precedence" {
2796 \\ a or b and c;2796 \\ a or b and c;
2797 \\ (a or b) and c;2797 \\ (a or b) and c;
2798 \\ (a or b) and c;2798 \\ (a or b) and c;
2799 \\ a == b and c == d;
2799 \\}2800 \\}
2800 \\2801 \\
2801 );2802 );
...@@ -4860,6 +4861,16 @@ test "recovery: missing comma" {...@@ -4860,6 +4861,16 @@ test "recovery: missing comma" {
4860 });4861 });
4861}4862}
48624863
4864test "recovery: non-associative operators" {
4865 try testError(
4866 \\const x = a == b == c;
4867 \\const x = a == b != c;
4868 , &[_]Error{
4869 .expected_token,
4870 .expected_token,
4871 });
4872}
4873
4863test "recovery: extra qualifier" {4874test "recovery: extra qualifier" {
4864 try testError(4875 try testError(
4865 \\const a: *const const u8;4876 \\const a: *const const u8;