Preserve input failures while scanning string literals

Author: pablogsal

Lib/test/test_tokenize.py

@@ -2409,6 +2409,12 @@ def test_multiline_readline_chunk_with_unterminated_tail(self):
         )
         self.assertEqual(readline.call_count, 2)
 
+    def test_readline_memory_error_in_string(self):
+        readline = mock.Mock(side_effect=['"""first\n', MemoryError])
+        iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)
+        with self.assertRaises(MemoryError):
+            next(iterator)
+
     def test_readline_callback_is_not_read_ahead(self):
         readline = mock.Mock(side_effect=["x\n", "y\n", ""])
         iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)

Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst

@@ -1,3 +1,4 @@
 Correct missing-brace error messages for multiline f-strings and t-strings.
 Improve tokenization of UTF-8 input split across many readline calls.
 Preserve source order when reading files with stateful encodings.
+Preserve input errors when scanning string literals.

Parser/lexer/string.c

@@ -335,6 +335,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
             break;
         }
         if (c == EOF || (quote_size == 1 && c == '\n')) {
+            if (tok_failed(tok)) {
+                return MAKE_TOKEN(ERRORTOKEN);
+            }
             int end_lineno = tok->lineno;
             _PyTok_Loc location = tok->start_loc;
             const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col;