From 9ab49c3ee1409e33d69e7dcb461e8b569fe2b7bf Mon Sep 17 00:00:00 2001 From: Mark Tareshawty Date: Mon, 24 Aug 2026 02:13:34 -0400 Subject: [PATCH] Emit a line break before `rescue` after an omitted keyword argument value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A value-omitted keyword argument (`foo key:`, Ruby 3.1+) may take its value from the following line, as in `foo key:\n bar`. To allow that, the pair rule offers the `_no_line_break` external token as a hint, and the scanner responds by withholding the line break that would otherwise terminate the statement. The `rescue` keyword can never begin a value, so withholding the line break before it left the parser in a state where `rescue_modifier` — which requires its body and the keyword to be on the same logical line — was preferred over the `rescue` clause of the enclosing body. `begin ... foo key: ... rescue => e ... end` therefore parsed as a rescue modifier wrapping the call, with the `=> e` reported as an ERROR. The grammar declares no conflicts, so this choice is made statically at generation time and cannot be corrected in `grammar.js`; the only way to distinguish the two is to give the parser the line break. The scanner now records when it withholds a line break and, when the next token turns out to be the `rescue` keyword, produces the line break after all. The identifier lookahead used for that check is shared with the existing hash-key / `identifier!` scanning so that words merely starting with `r` keep working. Fixes tree-sitter/tree-sitter-ruby#237 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ac21705f-befb-4606-9c83-657e3e9ea0dc --- src/scanner.c | 87 +++++++++++++++++----- test/corpus/control-flow.txt | 140 +++++++++++++++++++++++++++++++++-- 2 files changed, 199 insertions(+), 28 deletions(-) diff --git a/src/scanner.c b/src/scanner.c index b3ed0549..2afb573f 100644 --- a/src/scanner.c +++ b/src/scanner.c @@ -75,6 +75,10 @@ typedef struct { typedef struct { bool has_leading_whitespace; + // Set while scanning whitespace when a newline was crossed but the parser + // asked for line breaks to be withheld, because the construct on the + // previous line may still be continued by the next one. + bool withheld_line_break; Array(Literal) literal_stack; Array(Heredoc) open_heredocs; } Scanner; @@ -141,6 +145,7 @@ static inline unsigned serialize(Scanner *scanner, char *buffer) { static inline void deserialize(Scanner *scanner, const char *buffer, unsigned length) { unsigned size = 0; scanner->has_leading_whitespace = false; + scanner->withheld_line_break = false; reset(scanner); if (length == 0) { @@ -208,6 +213,8 @@ static inline bool scan_whitespace(Scanner *scanner, TSLexer *lexer, const bool pending_heredoc < scanner->open_heredocs.size && valid_symbols[HEREDOC_BODY_START]; bool crossed_newline = false; + scanner->withheld_line_break = false; + for (;;) { if (!valid_symbols[NO_LINE_BREAK] && valid_symbols[LINE_BREAK] && lexer->is_at_included_range_start(lexer)) { lexer->mark_end(lexer); @@ -239,6 +246,9 @@ static inline bool scan_whitespace(Scanner *scanner, TSLexer *lexer, const bool skip(scanner, lexer); crossed_newline = true; } else { + if (valid_symbols[NO_LINE_BREAK] && valid_symbols[LINE_BREAK]) { + scanner->withheld_line_break = true; + } skip(scanner, lexer); } break; @@ -967,6 +977,58 @@ static inline bool scan_comment(TSLexer *lexer) { return false; } +// Scans an identifier that is only interesting to this scanner because of the +// character that follows it: `foo:` is a hash key, and `foo!` is an identifier +// whose trailing `!` must not be mistaken for the start of `!=`. The first +// character may already have been consumed by the caller. +static inline bool scan_identifier_token(TSLexer *lexer, const bool *valid_symbols, TokenType identifier_symbol) { + while (iswalnum(lexer->lookahead) || lexer->lookahead == '_') { + advance(lexer); + } + + if (valid_symbols[HASH_KEY_SYMBOL] && lexer->lookahead == ':') { + lexer->mark_end(lexer); + advance(lexer); + if (lexer->lookahead != ':') { + lexer->result_symbol = HASH_KEY_SYMBOL; + return true; + } + } else if (valid_symbols[identifier_symbol] && lexer->lookahead == '!') { + advance(lexer); + if (lexer->lookahead != '=') { + lexer->mark_end(lexer); + lexer->result_symbol = identifier_symbol; + return true; + } + } + + return false; +} + +// Line breaks are withheld while a construct may still be continued on the next +// line, as in `foo key:`, whose value is allowed to appear there. The `rescue` +// keyword can never continue such a construct, so a line break is produced +// before it. Without one, `rescue` is taken as a rescue modifier attached to the +// previous line instead of the rescue clause of the enclosing body. +static inline bool scan_line_break_before_rescue(TSLexer *lexer, const bool *valid_symbols) { + lexer->mark_end(lexer); + + for (const char *c = "rescue"; *c != '\0'; c++) { + if (lexer->lookahead != *c) { + return scan_identifier_token(lexer, valid_symbols, IDENTIFIER_SUFFIX); + } + advance(lexer); + } + + if (iswalnum(lexer->lookahead) || lexer->lookahead == '_' || lexer->lookahead == '!' || + lexer->lookahead == '?' || lexer->lookahead == ':') { + return scan_identifier_token(lexer, valid_symbols, IDENTIFIER_SUFFIX); + } + + lexer->result_symbol = LINE_BREAK; + return true; +} + static inline bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) { scanner->has_leading_whitespace = false; @@ -1012,6 +1074,10 @@ static inline bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symb return true; } + if (scanner->withheld_line_break && lexer->lookahead == 'r') { + return scan_line_break_before_rescue(lexer, valid_symbols); + } + switch (lexer->lookahead) { case '&': if (valid_symbols[BLOCK_AMPERSAND] || valid_symbols[BITWISE_AND]) { @@ -1252,26 +1318,7 @@ static inline bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symb (iswalpha(lexer->lookahead) || lexer->lookahead == '_')) || (valid_symbols[CONSTANT_SUFFIX] && iswupper(lexer->lookahead))) { TokenType validIdentifierSymbol = iswupper(lexer->lookahead) ? CONSTANT_SUFFIX : IDENTIFIER_SUFFIX; - while (iswalnum(lexer->lookahead) || lexer->lookahead == '_') { - advance(lexer); - } - - if (valid_symbols[HASH_KEY_SYMBOL] && lexer->lookahead == ':') { - lexer->mark_end(lexer); - advance(lexer); - if (lexer->lookahead != ':') { - lexer->result_symbol = HASH_KEY_SYMBOL; - return true; - } - } else if (valid_symbols[validIdentifierSymbol] && lexer->lookahead == '!') { - advance(lexer); - if (lexer->lookahead != '=') { - lexer->result_symbol = validIdentifierSymbol; - return true; - } - } - - return false; + return scan_identifier_token(lexer, valid_symbols, validIdentifierSymbol); } // Open delimiters for literals diff --git a/test/corpus/control-flow.txt b/test/corpus/control-flow.txt index b7445235..09676549 100644 --- a/test/corpus/control-flow.txt +++ b/test/corpus/control-flow.txt @@ -598,7 +598,7 @@ return (program (return)) ============================================================== -omitted keyword argument before rescue: incorrectly parsed +omitted keyword argument before rescue on the next line ============================================================== begin @@ -614,13 +614,12 @@ end (assignment (identifier) (integer)) - (rescue_modifier - (call - (identifier) - (argument_list - (pair (hash_key_symbol)))) - (ERROR) - (identifier)))) + (call + (identifier) + (argument_list + (pair (hash_key_symbol)))) + (rescue + (exception_variable (identifier))))) ========================================== omitted keyword argument before rescue @@ -642,6 +641,131 @@ begin; key = 10; foo key:; rescue => e; end (rescue (exception_variable (identifier))))) +============================================================== +omitted keyword arguments before rescue clauses +============================================================== + +begin + foo key: +rescue + bar key:, other: +rescue ArgumentError => e + baz other: +end + +def m + foo key: +rescue => e +end + +class C + foo key: +rescue => e +end + +--- + +(program + (begin + (call + (identifier) + (argument_list + (pair (hash_key_symbol)))) + (rescue + (then + (call + (identifier) + (argument_list + (pair (hash_key_symbol)) + (pair (hash_key_symbol)))))) + (rescue + (exceptions (constant)) + (exception_variable (identifier)) + (then + (call + (identifier) + (argument_list + (pair (hash_key_symbol))))))) + (method + (identifier) + (body_statement + (call + (identifier) + (argument_list + (pair (hash_key_symbol)))) + (rescue + (exception_variable (identifier))))) + (class + (constant) + (body_statement + (call + (identifier) + (argument_list + (pair (hash_key_symbol)))) + (rescue + (exception_variable (identifier)))))) + +============================================================== +omitted keyword argument value on the next line +============================================================== + +foo key: + bar + +begin + foo key: + bar +rescue => e +end + +--- + +(program + (call + (identifier) + (argument_list + (pair (hash_key_symbol) (identifier)))) + (begin + (call + (identifier) + (argument_list + (pair (hash_key_symbol) (identifier)))) + (rescue + (exception_variable (identifier))))) + +============================================================== +omitted keyword argument value starting with the letter r +============================================================== + +foo key: + rescue_handler +foo key: + reload! +foo key: + r +foo key: + Regexp + +--- + +(program + (call + (identifier) + (argument_list + (pair (hash_key_symbol) (identifier)))) + (call + (identifier) + (argument_list + (pair (hash_key_symbol) (call (identifier))))) + (call + (identifier) + (argument_list + (pair (hash_key_symbol) (identifier)))) + (call + (identifier) + (argument_list + (pair (hash_key_symbol) (constant))))) + ==== case ====