diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py
index 2fe1fef241e90ae..e986e0f6b74f407 100644
--- a/Lib/test/test_capi/test_tokenizer.py
+++ b/Lib/test/test_capi/test_tokenizer.py
@@ -9,9 +9,6 @@ class TokenizerTests(unittest.TestCase):
def test_source(self):
_testinternalcapi.test_tokenizer_source()
- def test_cursor(self):
- _testinternalcapi.test_tokenizer_cursor()
-
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index c1ef1a73f05c204..ae443b269fe4dde 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -1679,6 +1679,20 @@ def __repr__(self):
self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
+ self.assertEqual(f'{"""a" # inside"""=}',
+ '"""a" # inside"""=\'a" # inside\'')
+ self.assertEqual(f"{'''a' # inside'''=}",
+ "'''a' # inside'''=\"a' # inside\"")
+ self.assertEqual(f'{"""a""""#" # outside
+=}', '"""a""""#" \n=\'a#\'')
+
+ x, y = 1, 2
+ self.assertEqual(f'{x != y # outside
+=}', 'x != y \n=True')
+
+ d = {'a#b': 42}
+ self.assertEqual(f'''{f"{d["a#b"]}"=}''',
+ 'f"{d["a#b"]}"=\'42\'')
self.assertEqual(f'{ # some comment goes here
"""hello"""=}', ' \n """hello"""=\'hello\'')
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 7e02191db86be5a..0b5bdcdeda3c196 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -2427,6 +2427,31 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
(token.ENDMARKER, "", (1, 0), (1, 0), ""),
)
+ def test_fstring_offsets_survive_buffer_reallocation(self):
+ padding = " " * 9000
+ expression_line = ")=:>{2}}\n"
+ physical_lines = [
+ 'f"""\n',
+ "{(\n",
+ padding + "1\n",
+ expression_line,
+ '"""\n',
+ ]
+ source = "".join(physical_lines)
+ chunks = iter([
+ "".join(physical_lines[:2]),
+ "".join(physical_lines[2:4]),
+ physical_lines[4],
+ "",
+ ])
+
+ expected = self._get_tokens(source, extra_tokens=True)
+ tokens = list(tokenize._generate_tokens_from_c_tokenizer(
+ chunks.__next__,
+ extra_tokens=True,
+ ))
+ self.assertEqual(tokens, expected)
+
def test_extra_tokens_relaxes_lexer_errors(self):
cases = [
(
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 74653c77c55de17..c90f18a4ced5296 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -287,5 +287,22 @@ def test_triple_quoted(self):
)
self.assertEqual(fstring(t), "\n Hello,\n Python\n ")
+ t = t'{"""a" # inside"""}'
+ self.assertEqual(t.interpolations[0].expression,
+ '"""a" # inside"""')
+
+ t = t'{"""a""""#" # outside
+}'
+ self.assertEqual(t.interpolations[0].expression, '"""a""""#"')
+
+ x, y = 1, 2
+ t = t'{x != y # outside
+}'
+ self.assertEqual(t.interpolations[0].expression, 'x != y')
+
+ d = {'a#b': 42}
+ t = t'''{f"{d["a#b"]}"}'''
+ self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"')
+
if __name__ == '__main__':
unittest.main()
diff --git a/Makefile.pre.in b/Makefile.pre.in
index adcfe4c5259eb22..50f4b6770c9d7e2 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -394,12 +394,10 @@ PEGEN_OBJS= \
Parser/peg_api.o
TOKENIZER_OBJS= \
- Parser/lexer/buffer.o \
Parser/lexer/lexer.o \
Parser/lexer/number.o \
Parser/lexer/state.o \
Parser/lexer/string.o \
- Parser/tokenizer/cursor.o \
Parser/tokenizer/decoder.o \
Parser/tokenizer/reader.o \
Parser/tokenizer/source.o \
@@ -411,11 +409,9 @@ PEGEN_HEADERS= \
$(srcdir)/Parser/string_parser.h
TOKENIZER_HEADERS= \
- Parser/lexer/buffer.h \
Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
- Parser/tokenizer/cursor.h \
Parser/tokenizer/reader.h \
Parser/tokenizer/reader_internal.h \
Parser/tokenizer/source.h \
@@ -3462,7 +3458,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c
index 0b292410d3eb4ef..46ac3c40724cff1 100644
--- a/Modules/_testinternalcapi/tokenizer.c
+++ b/Modules/_testinternalcapi/tokenizer.c
@@ -1,6 +1,6 @@
#include "parts.h"
-#include "../../Parser/tokenizer/cursor.h"
+#include "../../Parser/tokenizer/source.h"
static int
check(int condition, const char *message)
@@ -23,16 +23,6 @@ check_system_error(int failed, const char *message)
return 0;
}
-static int
-same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
-{
- return left->source == right->source &&
- left->pos == right->pos &&
- left->line_start == right->line_start &&
- left->line_end == right->line_end &&
- left->lineno == right->lineno;
-}
-
static PyObject *
test_tokenizer_source(PyObject *Py_UNUSED(module),
PyObject *Py_UNUSED(args))
@@ -40,80 +30,24 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
_PyTok_SourceText source;
_PyTok_SourceInit(&source);
- _PyTok_Loc loc;
- _PyTok_Line line;
- if (check(_PyTok_SourceLocation(
- &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate empty source") < 0 ||
- check(loc.lineno == 1 && loc.byte_col == 0,
- "wrong empty source location") < 0 ||
- check(_PyTok_SourceLine(&source, 1, &line) == 0,
- "cannot find empty source line") < 0 ||
- check(line.start == 0 && line.end == 0,
- "wrong empty source line") < 0 ||
- check_system_error(
- _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
- "accepted empty source line") < 0 ||
+ if (check_system_error(
+ _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+ "accepted empty source line") < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
"accepted multiple source lines") < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
- "accepted missing implicit newline") < 0) {
- goto error;
- }
-
- if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0,
- "wrong first source offset") < 0 ||
+ "accepted missing implicit newline") < 0 ||
+ check(_PyTok_SourceAppendLine(
+ &source, "alpha\n", 6, 0) == 0,
+ "wrong first source offset") < 0 ||
check(_PyTok_SourceAppendLine(
&source, "\xce\xb2\n", 3, 1) == 6,
"wrong second source offset") < 0 ||
- check(_PyTok_SourceAppendLine(
- &source, "nul\0x\n", 6, 0) == 9,
- "wrong third source offset") < 0) {
- goto error;
- }
-
- int marker_line = 257;
- int final_line = 300;
- _PyTok_Off marker_start = -1;
- for (int lineno = 4; lineno <= final_line; lineno++) {
- const char *text = lineno == marker_line ? "marker\n" : "x\n";
- Py_ssize_t len = (Py_ssize_t)strlen(text);
- _PyTok_Off start = _PyTok_SourceAppendLine(
- &source, text, len, lineno == final_line);
- if (start < 0) {
- goto error;
- }
- if (lineno == marker_line) {
- marker_start = start;
- }
- }
-
- if (check(source.nlines == final_line, "wrong source line count") < 0 ||
- check(_PyTok_SourceLine(&source, marker_line, &line) == 0,
- "cannot find late source line") < 0 ||
- check(line.start == marker_start &&
- line.end == marker_start + 7,
- "wrong late source line") < 0 ||
- check(!line.implicit_newline && !line.contains_nul,
- "wrong late source flags") < 0 ||
- check(_PyTok_SourceLine(&source, 2, &line) == 0,
- "cannot find second source line") < 0 ||
- check(line.start == 6 && line.end == 9 &&
- line.implicit_newline && !line.contains_nul,
- "wrong second source line") < 0 ||
check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
_PyTok_SourceLineIsImplicit(&source, 2),
- "wrong early implicit newline flags") < 0 ||
- check(_PyTok_SourceLine(&source, 3, &line) == 0,
- "cannot find third source line") < 0 ||
- check(line.contains_nul, "missing null byte flag") < 0 ||
- check(_PyTok_SourceLine(&source, final_line, &line) == 0,
- "cannot find final source line") < 0 ||
- check(line.implicit_newline &&
- _PyTok_SourceLineIsImplicit(&source, final_line),
- "missing late implicit newline flag") < 0) {
+ "wrong implicit newline flags") < 0) {
goto error;
}
@@ -123,178 +57,21 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
if (check(view != NULL && view_len == 2 &&
memcmp(view, "\xce\xb2", 2) == 0,
"wrong source span view") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start,
- _PYTOK_AFFINITY_LEFT, &loc) == 0,
- "cannot locate left line boundary") < 0 ||
- check(loc.lineno == marker_line - 1 && loc.byte_col == 2,
- "wrong left boundary location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate right line boundary") < 0 ||
- check(loc.lineno == marker_line && loc.byte_col == 0,
- "wrong right boundary location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, marker_start + 1,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate late source byte") < 0 ||
- check(loc.lineno == marker_line && loc.byte_col == 1,
- "wrong late source location") < 0) {
- goto error;
- }
-
- if (check(_PyTok_SourceLocation(
- &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0,
- "cannot locate left EOF") < 0 ||
- check(loc.lineno == final_line && loc.byte_col == 2,
- "wrong left EOF location") < 0 ||
- check(_PyTok_SourceLocation(
- &source, source.len,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate right EOF") < 0 ||
- check(loc.lineno == final_line + 1 && loc.byte_col == 0,
- "wrong right EOF location") < 0 ||
- check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0,
- "cannot find virtual EOF line") < 0 ||
- check(line.start == source.len && line.end == source.len,
- "wrong virtual EOF line") < 0 ||
- check(!_PyTok_SourceLineIsImplicit(&source, 0) &&
- !_PyTok_SourceLineIsImplicit(
- &source, final_line + 1),
- "virtual or invalid line is implicit") < 0) {
- goto error;
- }
-
- view = _PyTok_SourceSpanView(
- &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len);
- if (check_system_error(view == NULL, "accepted invalid source span") < 0 ||
check_system_error(
- _PyTok_SourceLocation(
- &source, source.len + 1,
- _PYTOK_AFFINITY_RIGHT, &loc) < 0,
- "accepted invalid source offset") < 0 ||
- check_system_error(
- _PyTok_SourceLine(&source, final_line + 2, &line) < 0,
- "accepted invalid source line") < 0) {
+ _PyTok_SourceSpanView(
+ &source, _PyTok_SpanFromBounds(0, source.len + 1),
+ &view_len) == NULL,
+ "accepted invalid source span") < 0) {
goto error;
}
_PyTok_SourceClear(&source);
- _PyTok_SourceInit(&source);
if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
check_system_error(
_PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
- "appended after unterminated source line") < 0 ||
- check(_PyTok_SourceLocation(
- &source, source.len,
- _PYTOK_AFFINITY_RIGHT, &loc) == 0,
- "cannot locate unterminated EOF") < 0 ||
- check(loc.lineno == 1 && loc.byte_col == 4,
- "wrong unterminated EOF location") < 0) {
- goto error;
- }
-
- _PyTok_SourceClear(&source);
- Py_RETURN_NONE;
-
-error:
- _PyTok_SourceClear(&source);
- return NULL;
-}
-
-static PyObject *
-test_tokenizer_cursor(PyObject *Py_UNUSED(module),
- PyObject *Py_UNUSED(args))
-{
- _PyTok_SourceText source;
- _PyTok_SourceInit(&source);
- if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
- _PyTok_SourceAppendLine(&source, "cd\n", 3, 0) < 0) {
- goto error;
- }
-
- _PyTok_Cursor cursor;
- _PyTok_CursorInit(&cursor, &source);
- if (_PyTok_CursorSetOffset(&cursor, source.len) < 0 ||
- check(cursor.lineno == 3 && cursor.pos == source.len,
- "wrong cursor at virtual EOF") < 0 ||
- _PyTok_CursorSetLine(&cursor, 1) < 0) {
- goto error;
- }
-
- char large[BUFSIZ + 1];
- memset(large, 'z', sizeof(large));
- large[sizeof(large) - 1] = '\n';
- if (_PyTok_SourceAppendLine(&source, large, sizeof(large), 0) < 0) {
- goto error;
- }
-
- if (check(_PyTok_CursorPeek(&cursor, 0) == 'a',
- "wrong cursor peek after relocation") < 0 ||
- check(_PyTok_CursorPeek(&cursor, 1) == 'b',
- "wrong distant cursor peek") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == 'a',
- "wrong first cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == 'b',
- "wrong second cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == '\n',
- "wrong final cursor byte") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == EOF,
- "cursor advanced past line") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, 2) == 0,
- "cannot seek cursor offset") < 0 ||
- check(_PyTok_CursorAdvance(&cursor) == '\n',
- "wrong cursor byte after seek") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, 3) == 0,
- "cannot seek line boundary") < 0 ||
- check(cursor.lineno == 2 && cursor.line_start == 3 &&
- _PyTok_CursorAdvance(&cursor) == 'c',
- "wrong cursor at line boundary") < 0 ||
- check(_PyTok_CursorSetLine(&cursor, 3) == 0,
- "cannot advance cursor to final line") < 0 ||
- check(cursor.line_start == 6 &&
- _PyTok_CursorAdvance(&cursor) == 'z',
- "wrong cursor byte on final line") < 0) {
- goto error;
- }
-
- _PyTok_Cursor saved = cursor;
- if (check_system_error(
- _PyTok_CursorSetOffset(&cursor, source.len + 1) < 0,
- "accepted invalid cursor offset") < 0 ||
- check(same_cursor(&cursor, &saved),
- "invalid offset changed cursor") < 0 ||
- check_system_error(
- _PyTok_CursorSetLine(&cursor, source.nlines + 2) < 0,
- "accepted invalid cursor line") < 0 ||
- check(same_cursor(&cursor, &saved),
- "invalid line changed cursor") < 0 ||
- check(_PyTok_CursorSetOffset(&cursor, source.len) == 0,
- "cannot set cursor to EOF") < 0 ||
- check(cursor.lineno == 4 && cursor.pos == source.len,
- "wrong cursor at EOF") < 0) {
- goto error;
- }
-
-#if SIZEOF_VOID_P > 4
- char byte = 0;
- _PyTok_SourceText huge_source = {
- .bytes = &byte,
- .len = (_PyTok_Off)INT_MAX + 1,
- };
- _PyTok_Cursor huge_cursor = {
- .source = &huge_source,
- .pos = INT_MAX,
- .line_end = (_PyTok_Off)INT_MAX + 1,
- .lineno = 1,
- };
- if (check(_PyTok_CursorAdvance(&huge_cursor) == EOF &&
- huge_cursor.pos == INT_MAX,
- "cursor advanced past maximum column") < 0) {
+ "appended after unterminated source line") < 0) {
goto error;
}
-#endif
_PyTok_SourceClear(&source);
Py_RETURN_NONE;
@@ -306,7 +83,6 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
static PyMethodDef test_methods[] = {
{"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
- {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
{NULL},
};
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index 469fd77cc8be9dc..a6a37d7be9608f3 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -181,7 +181,6 @@
-
diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 976c99b7d24bdfd..27d47ba14e6c2a0 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -469,9 +469,6 @@
Source Files
-
- Source Files
-
Source Files
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 33647ec284061f1..055fab2d6357c34 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -422,8 +422,6 @@
-
-
@@ -592,8 +590,6 @@
-
-
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index 434dd13267fe934..ceec2cd736710aa 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -330,12 +330,6 @@
Parser
-
- Parser
-
-
- Parser
-
Parser
@@ -1358,12 +1352,6 @@
Parser
-
- Parser
-
-
- Parser
-
Parser
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
deleted file mode 100644
index cd6885a7d01040a..000000000000000
--- a/Parser/lexer/buffer.c
+++ /dev/null
@@ -1,62 +0,0 @@
-#include "Python.h"
-#include "errcode.h"
-
-#include "state.h"
-
-/* Traverse and remember all f-string buffers, in order to be able to restore
- them after reallocating tok->buf */
-void
-_PyLexer_remember_fstring_buffers(struct tok_state *tok)
-{
- int index;
- tokenizer_mode *mode;
-
- for (index = tok->tok_mode_stack_index; index >= 0; --index) {
- mode = &(tok->tok_mode_stack[index]);
- mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
- mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
- }
-}
-
-/* Traverse and restore all f-string buffers after reallocating tok->buf */
-void
-_PyLexer_restore_fstring_buffers(struct tok_state *tok)
-{
- int index;
- tokenizer_mode *mode;
-
- for (index = tok->tok_mode_stack_index; index >= 0; --index) {
- mode = &(tok->tok_mode_stack[index]);
- mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
- mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
- }
-}
-
-int
-_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
-{
- Py_ssize_t cur = tok->cur - tok->buf;
- Py_ssize_t oldsize = tok->inp - tok->buf;
- Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1);
- if (newsize > tok->end - tok->buf) {
- char *newbuf = tok->buf;
- Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf;
- Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf;
- Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf;
- _PyLexer_remember_fstring_buffers(tok);
- newbuf = (char *)PyMem_Realloc(newbuf, newsize);
- if (newbuf == NULL) {
- tok->done = E_NOMEM;
- return 0;
- }
- tok->buf = newbuf;
- tok->cur = tok->buf + cur;
- tok->inp = tok->buf + oldsize;
- tok->end = tok->buf + newsize;
- tok->start = start < 0 ? NULL : tok->buf + start;
- tok->line_start = line_start < 0 ? NULL : tok->buf + line_start;
- tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start;
- _PyLexer_restore_fstring_buffers(tok);
- }
- return 1;
-}
diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h
deleted file mode 100644
index bb218162ff48453..000000000000000
--- a/Parser/lexer/buffer.h
+++ /dev/null
@@ -1,10 +0,0 @@
-#ifndef _LEXER_BUFFER_H_
-#define _LEXER_BUFFER_H_
-
-#include "pyport.h"
-
-void _PyLexer_remember_fstring_buffers(struct tok_state *tok);
-void _PyLexer_restore_fstring_buffers(struct tok_state *tok);
-int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size);
-
-#endif
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index a96362c8961023a..ffc858e8da829ca 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -7,13 +7,26 @@
#include "../tokenizer/helpers.h"
#include "../tokenizer/reader.h"
-/* Alternate tab spacing */
+#define TABSIZE 8
#define ALTTABSIZE 1
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
-#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
- _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
+
+static int
+type_comment_token_setup(struct tok_state *tok, struct token *token, int type,
+ int col_offset, int end_col_offset,
+ const char *start, const char *end)
+{
+ _PyLexer_token_setup(tok, token, type, start, end);
+ token->start_loc = (_PyTok_Loc){tok->lineno, col_offset};
+ token->end_loc = (_PyTok_Loc){tok->lineno, end_col_offset};
+ return type;
+}
+
+#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) \
+ type_comment_token_setup(tok, token, token_type, col_offset, \
+ end_col_offset, p_start, p_end)
/* Spaces in this constant are treated as "zero or more spaces or tabs" when
tokenizing. */
@@ -32,11 +45,10 @@ _PyLexer_nextc(struct tok_state *tok)
int rc;
for (;;) {
if (tok->cur != tok->inp) {
- if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
+ if (tok->cur - tok->line_start >= INT_MAX) {
tok->done = E_COLUMNOVERFLOW;
return EOF;
}
- tok->col_offset++;
return Py_CHARMASK(*tok->cur++); /* Fast path */
}
if (tok->done != E_OK) {
@@ -76,7 +88,6 @@ _PyLexer_backup(struct tok_state *tok, int c)
if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
Py_FatalError("tok_backup: wrong character");
}
- tok->col_offset--;
}
}
@@ -90,7 +101,7 @@ verify_identifier(struct tok_state *tok)
return 1;
}
PyObject *s;
- if (tok->input_error)
+ if (tok_failed(tok))
return 0;
s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
if (s == NULL) {
@@ -167,7 +178,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
const char *p_end = NULL;
nextline:
tok->start = NULL;
- tok->starting_col_offset = -1;
+ tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
blankline = 0;
@@ -183,7 +194,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
col++, altcol++;
}
else if (c == '\t') {
- col = (col / tok->tabsize + 1) * tok->tabsize;
+ col = (col / TABSIZE + 1) * TABSIZE;
altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
}
else if (c == '\014') {/* Control-L (formfeed) */
@@ -208,15 +219,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
}
tok_backup(tok, c);
if (c == '#' || c == '\n' || c == '\r') {
+ int interactive = _PyTok_ReaderIsInteractive(tok);
/* Lines with only whitespace and/or comments
shouldn't affect the indentation and are
not passed to the parser as NEWLINE tokens,
except *totally* empty lines in interactive
mode, which signal the end of a command group. */
- if (col == 0 && c == '\n' && tok->prompt != NULL) {
+ if (col == 0 && c == '\n' && interactive) {
blankline = 0; /* Let it through */
}
- else if (tok->prompt != NULL && tok->lineno == 1) {
+ else if (interactive && tok->lineno == 1) {
/* In interactive mode, if the first line contains
only spaces and/or a comment, let it through. */
blankline = 0;
@@ -271,7 +283,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
}
tok->start = tok->cur;
- tok->starting_col_offset = tok->col_offset;
+ tok->start_loc = (_PyTok_Loc){
+ tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) : -1};
/* Return pending indents/dedents */
if (tok->pendin != 0) {
@@ -306,7 +319,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
/* Set start of current token */
tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
- tok->starting_col_offset = tok->col_offset - 1;
+ tok->start_loc = (_PyTok_Loc){
+ tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) - 1 : -1};
/* Skip comment, unless it's a type comment */
if (c == '#') {
@@ -319,13 +333,25 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
c = tok_nextc(tok);
}
+ if (INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
+ const char *comment_end = tok->cur;
+ if (c == '\n') {
+ comment_end--;
+ }
+ if (_PyLexer_record_ftstring_comment(
+ tok, tok->start, comment_end) < 0) {
+ tok->done = E_NOMEM;
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+
if (tok->tok_extra_tokens) {
p = tok->start;
}
if (tok->type_comments) {
p = tok->start;
- current_starting_col_offset = tok->starting_col_offset;
+ current_starting_col_offset = tok->start_loc.byte_col;
prefix = type_comment_prefix;
while (*prefix && p < tok->cur) {
if (*prefix == ' ') {
@@ -369,11 +395,15 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
tok_nextc(tok);
tok->atbol = 1;
}
- return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset);
+ return MAKE_TYPE_COMMENT_TOKEN(
+ TYPE_IGNORE, ignore_end_col_offset,
+ _PyLexer_ByteColumn(tok));
} else {
p_start = type_start;
p_end = tok->cur;
- return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset);
+ return MAKE_TYPE_COMMENT_TOKEN(
+ TYPE_COMMENT, current_starting_col_offset,
+ _PyLexer_ByteColumn(tok));
}
}
}
@@ -542,10 +572,18 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
int cursor_in_format_with_debug =
cursor == 1 && (current_tok->in_debug || in_format_spec);
int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
- if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
- return MAKE_TOKEN(ENDMARKER);
+ if (cursor_valid && c == '!') {
+ int c2 = tok_nextc(tok);
+ if (c2 == '=') {
+ cursor_valid = 0;
+ }
+ tok_backup(tok, c2);
+ }
+ if (cursor_valid) {
+ _PyLexer_update_ftstring_expr(tok, c);
}
- if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
+ if (cursor_valid && c != '{' &&
+ _PyLexer_set_ftstring_expr_metadata(tok, token)) {
return MAKE_TOKEN(ERRORTOKEN);
}
@@ -686,7 +724,7 @@ int
_PyTokenizer_Get(struct tok_state *tok, struct token *token)
{
int result = tok_get(tok, token);
- if (tok->input_error) {
+ if (tok_failed(tok)) {
result = ERRORTOKEN;
}
return result;
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
index 1d97ac57b745b09..a82add64263e2e5 100644
--- a/Parser/lexer/lexer.h
+++ b/Parser/lexer/lexer.h
@@ -3,8 +3,27 @@
#include "state.h"
-int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
-
int _PyTokenizer_Get(struct tok_state *, struct token *);
+/* The view points into the current input window. The next
+ _PyTokenizer_Get() call may discard it. */
+static inline const char *
+_PyToken_TextView(const struct tok_state *tok, const struct token *token,
+ Py_ssize_t *length)
+{
+ assert(length != NULL);
+ if (token->span.start < 0) {
+ assert(token->span.start == -1 && token->span.end == -1);
+ *length = 0;
+ return "";
+ }
+ assert(_PyTok_SpanIsValid(token->span));
+ assert(tok->buf != NULL);
+ assert(tok->inp >= tok->buf);
+ assert(token->span.start >= tok->buf_offset);
+ assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);
+ *length = token->span.end - token->span.start;
+ return tok->buf + (token->span.start - tok->buf_offset);
+}
+
#endif
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index c6d3b9045c72921..68273eb430e6812 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -1,6 +1,7 @@
#ifndef _PY_LEXER_INTERNAL_H_
#define _PY_LEXER_INTERNAL_H_
+#include "errcode.h"
#include "lexer.h"
#define is_potential_identifier_start(c) (\
@@ -44,9 +45,19 @@ TOK_NEXT_MODE(struct tok_state *tok)
#define tok_nextc _PyLexer_nextc
#define tok_backup _PyLexer_backup
+static inline int
+tok_failed(const struct tok_state *tok)
+{
+ return tok->done != E_OK && tok->done != E_EOF &&
+ tok->done != E_INTERACT_STOP;
+}
+
int _PyLexer_nextc(struct tok_state *);
void _PyLexer_backup(struct tok_state *, int);
-int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
+void _PyLexer_update_ftstring_expr(struct tok_state *, char);
+int _PyLexer_record_ftstring_comment(
+ struct tok_state *, const char *, const char *);
+int _PyLexer_set_ftstring_expr_metadata(struct tok_state *, struct token *);
int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 2a6408bef927a36..809df08d0d29738 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -6,9 +6,6 @@
#include "state.h"
#include "../tokenizer/reader.h"
-/* Never change this */
-#define TABSIZE 8
-
/* Create and initialize a new tok_state structure */
struct tok_state *
_PyTokenizer_tok_new(void)
@@ -22,32 +19,21 @@ _PyTokenizer_tok_new(void)
}
tok->buf = tok->cur = tok->inp = NULL;
- tok->fp_interactive = 0;
- tok->interactive_src_start = NULL;
- tok->interactive_src_end = NULL;
tok->start = NULL;
- tok->end = NULL;
tok->done = E_OK;
tok->fp = NULL;
- tok->tabsize = TABSIZE;
tok->indent = 0;
tok->indstack[0] = 0;
tok->atbol = 1;
tok->pendin = 0;
- tok->prompt = NULL;
tok->lineno = 0;
- tok->starting_col_offset = -1;
- tok->col_offset = -1;
+ tok->start_loc = (_PyTok_Loc){-1, -1};
tok->level = 0;
tok->altindstack[0] = 0;
- tok->input_error = 0;
tok->encoding = NULL;
tok->filename = NULL;
tok->module = NULL;
tok->type_comments = 0;
- tok->interactive_underflow = IUNDERFLOW_NORMAL;
- tok->str = NULL;
- tok->report_warnings = 1;
tok->tok_extra_tokens = 0;
tok->comment_newline = 0;
tok->implicit_newline = 0;
@@ -61,24 +47,6 @@ _PyTokenizer_tok_new(void)
return tok;
}
-static void
-free_fstring_expressions(struct tok_state *tok)
-{
- int index;
- tokenizer_mode *mode;
-
- for (index = tok->tok_mode_stack_index; index >= 0; --index) {
- mode = &(tok->tok_mode_stack[index]);
- if (mode->last_expr_buffer != NULL) {
- PyMem_Free(mode->last_expr_buffer);
- mode->last_expr_buffer = NULL;
- mode->last_expr_size = 0;
- mode->last_expr_end = -1;
- mode->in_format_spec = 0;
- }
- }
-}
-
/* Free a tok_state structure */
void
_PyTokenizer_Free(struct tok_state *tok)
@@ -90,7 +58,12 @@ _PyTokenizer_Free(struct tok_state *tok)
Py_XDECREF(tok->module);
_PyTok_ReaderFree(tok);
_PyTok_SourceClear(&tok->source);
- free_fstring_expressions(tok);
+ tokenizer_comments *comments = tok->ftstring_comments;
+ while (comments != NULL) {
+ tokenizer_comments *previous = comments->previous;
+ PyMem_Free(comments);
+ comments = previous;
+ }
PyMem_Free(tok);
}
@@ -101,41 +74,25 @@ _PyToken_Free(struct token *token) {
void
_PyToken_Init(struct token *token) {
+#ifdef Py_DEBUG
+ token->span = (_PyTok_Span){-1, -1};
+ token->start_loc = (_PyTok_Loc){-1, -1};
+ token->end_loc = (_PyTok_Loc){-1, -1};
+#endif
token->metadata = NULL;
}
-int
-_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
- int end_col_offset, const char *start, const char *end)
-{
- token->level = tok->level;
- token->lineno = token->end_lineno = tok->lineno;
- token->col_offset = col_offset;
- token->end_col_offset = end_col_offset;
- token->start = start;
- token->end = end;
- return type;
-}
-
int
_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
{
- assert((start == NULL && end == NULL) || (start != NULL && end != NULL));
token->level = tok->level;
- if (ISSTRINGLIT(type)) {
- token->lineno = tok->first_lineno;
+ token->span = _PyLexer_BufferSpan(tok, start, end);
+ if (start != NULL && end != NULL) {
+ token->start_loc = tok->start_loc;
+ token->end_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
}
else {
- token->lineno = tok->lineno;
- }
- token->end_lineno = tok->lineno;
- token->col_offset = token->end_col_offset = -1;
- token->start = start;
- token->end = end;
-
- if (start != NULL && end != NULL) {
- token->col_offset = tok->starting_col_offset;
- token->end_col_offset = tok->col_offset;
+ token->start_loc = token->end_loc = (_PyTok_Loc){tok->lineno, -1};
}
return type;
}
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 0824785195491ee..f9546ac4183c8cc 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -13,18 +13,11 @@
#define INSIDE_FSTRING_EXPR_AT_TOP(tok) \
(tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1)
-enum interactive_underflow_t {
- /* Normal mode of operation: return a new token when asked in interactive mode */
- IUNDERFLOW_NORMAL,
- /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
- * can be used to prevent the tokenizer to prompt the user for new tokens */
- IUNDERFLOW_STOP,
-};
-
struct token {
int level;
- int lineno, col_offset, end_lineno, end_col_offset;
- const char *start, *end;
+ _PyTok_Span span;
+ _PyTok_Loc start_loc;
+ _PyTok_Loc end_loc;
PyObject *metadata;
};
@@ -49,48 +42,42 @@ typedef struct _tokenizer_mode {
char quote;
int quote_size;
int raw;
- const char* start;
- const char* multi_line_start;
- int first_line;
-
- Py_ssize_t start_offset;
- Py_ssize_t multi_line_start_offset;
+ _PyTok_Off start;
+ _PyTok_Loc start_loc;
- Py_ssize_t last_expr_size;
- Py_ssize_t last_expr_end;
- char* last_expr_buffer;
+ _PyTok_Span expr_span;
int in_debug;
int in_format_spec;
enum string_kind_t string_kind;
} tokenizer_mode;
+typedef struct _tokenizer_comments {
+ struct _tokenizer_comments *previous;
+ Py_ssize_t count;
+ Py_ssize_t capacity;
+ int mode_index;
+ _PyTok_Span spans[];
+} tokenizer_comments;
+
/* Tokenizer state */
struct tok_state {
- /* Input state; buf <= cur <= inp <= end */
+ /* Input state; buf <= cur <= inp */
/* NB an entire line is held in the buffer */
- char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
+ char *buf; /* Owned for file/readline input; source-backed otherwise. */
char *cur; /* Next character in buffer */
char *inp; /* End of data in buffer */
- int fp_interactive; /* If the file descriptor is interactive */
- char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
- char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
- const char *end; /* End of input buffer if buf != NULL */
+ _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
const char *start; /* Start of current token if not NULL */
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
FILE *fp; /* Rest of input; NULL if tokenizing a string */
- int tabsize; /* Tab spacing */
int indent; /* Current indentation index */
int indstack[MAXINDENT]; /* Stack of indents */
int atbol; /* Nonzero if at begin of new line */
int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
- const char *prompt; /* For interactive prompting */
int lineno; /* Current line number */
- int first_lineno; /* First line of a single line or multi line string
- expression (cf. issue 16806) */
- int starting_col_offset; /* The column offset at the beginning of a token */
- int col_offset; /* Current col offset */
+ _PyTok_Loc start_loc;
int level; /* () [] {} Parentheses nesting level */
/* Used to allow free continuations inside them */
char parenstack[MAXLEVEL];
@@ -101,25 +88,16 @@ struct tok_state {
/* Stuff for checking on different tab sizes */
int altindstack[MAXINDENT]; /* Stack of alternate indents */
/* Stuff for PEP 0263 */
- int input_error;
char *encoding; /* Source encoding. */
const char* line_start; /* pointer to start of current line */
- const char* multi_line_start; /* pointer to start of first line of
- a single line or multi line string
- expression (cf. issue 16806) */
- char* str; /* Source string being tokenized (if tokenizing from a string)*/
-
_PyTok_SourceText source;
struct _PyTok_Reader *reader;
int type_comments; /* Whether to look for type comments */
- /* How to proceed when asked for a new token in interactive mode */
- enum interactive_underflow_t interactive_underflow;
- int report_warnings;
- // TODO: Factor this into its own thing
tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
int tok_mode_stack_index;
+ tokenizer_comments *ftstring_comments;
int tok_extra_tokens;
int comment_newline;
int implicit_newline;
@@ -128,8 +106,42 @@ struct tok_state {
#endif
};
-int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
- int end_col_offset, const char *start, const char *end);
+static inline _PyTok_Off
+_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
+{
+ assert(tok->buf != NULL);
+ assert(tok->inp >= tok->buf);
+ assert(position >= tok->buf && position <= tok->inp);
+ Py_ssize_t offset = position - tok->buf;
+ assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset);
+ return tok->buf_offset + offset;
+}
+
+static inline int
+_PyLexer_ByteColumn(const struct tok_state *tok)
+{
+ assert(tok->line_start != NULL);
+ assert(tok->cur >= tok->line_start);
+ Py_ssize_t column = tok->cur - tok->line_start;
+ assert(column <= INT_MAX);
+ return (int)column;
+}
+
+static inline _PyTok_Span
+_PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
+ const char *end)
+{
+ if (start == NULL) {
+ assert(end == NULL);
+ return (_PyTok_Span){-1, -1};
+ }
+ assert(end != NULL);
+ assert(start <= end);
+ return _PyTok_SpanFromBounds(
+ _PyLexer_BufferOffset(tok, start),
+ _PyLexer_BufferOffset(tok, end));
+}
+
int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
struct tok_state *_PyTokenizer_tok_new(void);
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index d67c48f7f678eda..989951abce558ba 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -7,110 +7,142 @@
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
-int
-_PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
- assert(token != NULL);
- assert(c == '}' || c == ':' || c == '!');
- tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
+static char *
+offset_pointer(const struct tok_state *tok, _PyTok_Off offset)
+{
+ if (offset >= tok->buf_offset &&
+ offset - tok->buf_offset <= tok->inp - tok->buf) {
+ return tok->buf + (offset - tok->buf_offset);
+ }
+ assert(offset >= 0 && offset <= tok->source.len);
+ return tok->source.bytes + offset;
+}
- if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
- return 0;
+static const char *
+span_view(const struct tok_state *tok, _PyTok_Span span, Py_ssize_t *length)
+{
+ assert(length != NULL);
+ assert(_PyTok_SpanIsValid(span));
+ if (span.start >= tok->buf_offset &&
+ span.end - tok->buf_offset <= tok->inp - tok->buf) {
+ *length = span.end - span.start;
+ return tok->buf + (span.start - tok->buf_offset);
}
- PyObject *res = NULL;
+ return _PyTok_SourceSpanView(&tok->source, span, length);
+}
- // Look for a # character outside of string literals
- int hash_detected = 0;
- int in_string = 0;
- char quote_char = 0;
+static void
+rewind_to_string_start(struct tok_state *tok, const char *start,
+ _PyTok_Loc location)
+{
+ tok->cur = (char *)start + 1;
+ tok->line_start = start - location.byte_col;
+ tok->lineno = location.lineno;
+}
- for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
- char ch = tok_mode->last_expr_buffer[i];
+static tokenizer_comments *
+current_comments(const struct tok_state *tok)
+{
+ tokenizer_comments *comments = tok->ftstring_comments;
+ return comments != NULL &&
+ comments->mode_index == tok->tok_mode_stack_index
+ ? comments : NULL;
+}
- // Skip escaped characters
- if (ch == '\\') {
- i++;
- continue;
+int
+_PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
+ const char *end)
+{
+ tokenizer_mode *mode = TOK_GET_MODE(tok);
+ if (mode->expr_span.end >= 0) {
+ return 0;
+ }
+ assert(mode->expr_span.start >= 0);
+ tokenizer_comments *comments = current_comments(tok);
+ if (comments == NULL || comments->count == comments->capacity) {
+ int create = comments == NULL;
+ Py_ssize_t max_capacity = (PY_SSIZE_T_MAX -
+ (Py_ssize_t)sizeof(*comments)) /
+ (Py_ssize_t)sizeof(*comments->spans);
+ if (comments != NULL && comments->capacity > max_capacity / 2) {
+ PyErr_NoMemory();
+ return -1;
}
-
- // Handle quotes
- if (ch == '"' || ch == '\'') {
- // The following if/else block works becase there is an off number
- // of quotes in STRING tokens and the lexer only ever reaches this
- // function with valid STRING tokens.
- // For example: """hello"""
- // First quote: in_string = 1
- // Second quote: in_string = 0
- // Third quote: in_string = 1
- if (!in_string) {
- in_string = 1;
- quote_char = ch;
- }
- else if (ch == quote_char) {
- in_string = 0;
- }
- continue;
+ Py_ssize_t capacity = comments == NULL ? 4 : comments->capacity * 2;
+ size_t size = sizeof(*comments) +
+ (size_t)capacity * sizeof(*comments->spans);
+ tokenizer_comments *resized = PyMem_Realloc(comments, size);
+ if (resized == NULL) {
+ PyErr_NoMemory();
+ return -1;
}
-
- // Check for # outside strings
- if (ch == '#' && !in_string) {
- hash_detected = 1;
- break;
+ comments = resized;
+ if (create) {
+ comments->previous = tok->ftstring_comments;
+ comments->count = 0;
+ comments->mode_index = tok->tok_mode_stack_index;
}
+ comments->capacity = capacity;
+ tok->ftstring_comments = comments;
}
- // If we found a # character in the expression, we need to handle comments
- if (hash_detected) {
- // Allocate buffer for processed result
- char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
- if (!result) {
+ comments->spans[comments->count++] =
+ _PyLexer_BufferSpan(tok, start, end);
+ return 0;
+}
+
+int
+_PyLexer_set_ftstring_expr_metadata(struct tok_state *tok, struct token *token)
+{
+ assert(token != NULL);
+ tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
+
+ if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
+ return 0;
+ }
+ Py_ssize_t expr_len;
+ const char *expr = span_view(tok, tok_mode->expr_span, &expr_len);
+ if (expr == NULL) {
+ return -1;
+ }
+ tokenizer_comments *comments = current_comments(tok);
+ PyObject *res;
+ if (comments != NULL && comments->count > 0) {
+ Py_ssize_t stripped_size = expr_len;
+ _PyTok_Off previous_end = tok_mode->expr_span.start;
+ for (Py_ssize_t i = 0; i < comments->count; i++) {
+ _PyTok_Span comment = comments->spans[i];
+ assert(_PyTok_SpanIsValid(comment));
+ assert(comment.start >= previous_end);
+ assert(comment.end <= tok_mode->expr_span.end);
+ stripped_size -= comment.end - comment.start;
+ previous_end = comment.end;
+ }
+ char *stripped = PyMem_Malloc((size_t)stripped_size);
+ if (stripped == NULL) {
+ PyErr_NoMemory();
return -1;
}
-
- Py_ssize_t i = 0; // Input position
- Py_ssize_t j = 0; // Output position
- in_string = 0; // Whether we're in a string
- quote_char = 0; // Current string quote char
-
- // Process each character
- while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
- char ch = tok_mode->last_expr_buffer[i];
-
- // Handle string quotes
- if (ch == '"' || ch == '\'') {
- // See comment above to understand this part
- if (!in_string) {
- in_string = 1;
- quote_char = ch;
- } else if (ch == quote_char) {
- in_string = 0;
- }
- result[j++] = ch;
- }
- // Skip comments
- else if (ch == '#' && !in_string) {
- while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
- tok_mode->last_expr_buffer[i] != '\n') {
- i++;
- }
- if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
- result[j++] = '\n';
- }
- }
- // Copy other chars
- else {
- result[j++] = ch;
- }
- i++;
+ _PyTok_Off copied_to = tok_mode->expr_span.start;
+ Py_ssize_t stripped_len = 0;
+ for (Py_ssize_t i = 0; i < comments->count; i++) {
+ _PyTok_Span comment = comments->spans[i];
+ Py_ssize_t length = comment.start - copied_to;
+ memcpy(stripped + stripped_len,
+ expr + copied_to - tok_mode->expr_span.start,
+ (size_t)length);
+ stripped_len += length;
+ copied_to = comment.end;
}
-
- result[j] = '\0'; // Null-terminate the result string
- res = PyUnicode_DecodeUTF8(result, j, NULL);
- PyMem_Free(result);
- } else {
- res = PyUnicode_DecodeUTF8(
- tok_mode->last_expr_buffer,
- tok_mode->last_expr_size - tok_mode->last_expr_end,
- NULL
- );
+ Py_ssize_t length = tok_mode->expr_span.end - copied_to;
+ memcpy(stripped + stripped_len,
+ expr + copied_to - tok_mode->expr_span.start,
+ (size_t)length);
+ stripped_len += length;
+ res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL);
+ PyMem_Free(stripped);
+ }
+ else {
+ res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
}
if (!res) {
@@ -120,59 +152,33 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
return 0;
}
-int
+void
_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
{
- assert(tok->cur != NULL);
-
- Py_ssize_t size = strlen(tok->cur);
tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
switch (cur) {
- case 0:
- if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
- return 1;
- }
- char *new_buffer = PyMem_Realloc(
- tok_mode->last_expr_buffer,
- tok_mode->last_expr_size + size
- );
- if (new_buffer == NULL) {
- PyMem_Free(tok_mode->last_expr_buffer);
- goto error;
- }
- tok_mode->last_expr_buffer = new_buffer;
- strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
- tok_mode->last_expr_size += size;
- break;
case '{':
- if (tok_mode->last_expr_buffer != NULL) {
- PyMem_Free(tok_mode->last_expr_buffer);
- }
- tok_mode->last_expr_buffer = PyMem_Malloc(size);
- if (tok_mode->last_expr_buffer == NULL) {
- goto error;
+ tok_mode->expr_span = (_PyTok_Span){
+ _PyLexer_BufferOffset(tok, tok->cur), -1};
+ tokenizer_comments *comments = current_comments(tok);
+ if (comments != NULL) {
+ comments->count = 0;
}
- tok_mode->last_expr_size = size;
- tok_mode->last_expr_end = -1;
- strncpy(tok_mode->last_expr_buffer, tok->cur, size);
break;
case '}':
case '!':
- tok_mode->last_expr_end = strlen(tok->start);
+ tok_mode->expr_span.end = _PyLexer_BufferOffset(tok, tok->start);
break;
case ':':
- if (tok_mode->last_expr_end == -1) {
- tok_mode->last_expr_end = strlen(tok->start);
+ if (tok_mode->expr_span.end < 0) {
+ tok_mode->expr_span.end =
+ _PyLexer_BufferOffset(tok, tok->start);
}
break;
default:
Py_UNREACHABLE();
}
- return 1;
-error:
- tok->done = E_NOMEM;
- return 0;
}
int
@@ -229,13 +235,6 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
int quote = c;
int quote_size = 1; /* 1 or 3 */
- /* Nodes of type STRING, especially multi line strings
- must be handled differently in order to get both
- the starting line number and the column offset right.
- (cf. issue 16806) */
- tok->first_lineno = tok->lineno;
- tok->multi_line_start = tok->line_start;
-
/* Find the quote size and start of string */
int after_quote = tok_nextc(tok);
if (after_quote == quote) {
@@ -244,7 +243,6 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
quote_size = 3;
}
else {
- // TODO: Check this
tok_backup(tok, after_after_quote);
tok_backup(tok, after_quote);
}
@@ -263,14 +261,9 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
the_current_tok->kind = TOK_FSTRING_MODE;
the_current_tok->quote = quote;
the_current_tok->quote_size = quote_size;
- the_current_tok->start = tok->start;
- the_current_tok->multi_line_start = tok->line_start;
- the_current_tok->first_line = tok->lineno;
- the_current_tok->start_offset = -1;
- the_current_tok->multi_line_start_offset = -1;
- the_current_tok->last_expr_buffer = NULL;
- the_current_tok->last_expr_size = 0;
- the_current_tok->last_expr_end = -1;
+ the_current_tok->start = _PyLexer_BufferOffset(tok, tok->start);
+ the_current_tok->start_loc = tok->start_loc;
+ the_current_tok->expr_span = (_PyTok_Span){-1, -1};
the_current_tok->in_format_spec = 0;
the_current_tok->in_debug = 0;
@@ -313,13 +306,6 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
int end_quote_size = 0;
int has_escaped_quote = 0;
- /* Nodes of type STRING, especially multi line strings
- must be handled differently in order to get both
- the starting line number and the column offset right.
- (cf. issue 16806) */
- tok->first_lineno = tok->lineno;
- tok->multi_line_start = tok->line_start;
-
/* Find the quote size and start of string */
c = tok_nextc(tok);
if (c == quote) {
@@ -345,15 +331,8 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
break;
}
if (c == EOF || (quote_size == 1 && c == '\n')) {
- assert(tok->multi_line_start != NULL);
- // shift the tok_state's location into
- // the start of string, and report the error
- // from the initial quote character
- tok->cur = (char *)tok->start;
- tok->cur++;
- tok->line_start = tok->multi_line_start;
- int start = tok->lineno;
- tok->lineno = tok->first_lineno;
+ int end_lineno = tok->lineno;
+ rewind_to_string_start(tok, tok->start, tok->start_loc);
if (INSIDE_FSTRING(tok)) {
/* When we are in an f-string, before raising the
@@ -371,7 +350,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
if (quote_size == 3) {
_PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
- " (detected at line %d)", start);
+ " (detected at line %d)", end_lineno);
if (c != '\n') {
tok->done = E_EOFS;
}
@@ -383,11 +362,11 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
tok,
"unterminated string literal (detected at line %d); "
"perhaps you escaped the end quote?",
- start
+ end_lineno
);
} else {
_PyTokenizer_syntaxerror(
- tok, "unterminated string literal (detected at line %d)", start
+ tok, "unterminated string literal (detected at line %d)", end_lineno
);
}
if (c != '\n') {
@@ -427,8 +406,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
int unicode_escape = 0;
tok->start = tok->cur;
- tok->first_lineno = tok->lineno;
- tok->starting_col_offset = tok->col_offset;
+ tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
// If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize
// before it.
@@ -460,23 +438,18 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
}
}
- if (current_tok->last_expr_buffer != NULL) {
- PyMem_Free(current_tok->last_expr_buffer);
- current_tok->last_expr_buffer = NULL;
- current_tok->last_expr_size = 0;
- current_tok->last_expr_end = -1;
- }
-
p_start = tok->start;
p_end = tok->cur;
+ tokenizer_comments *comments = current_comments(tok);
+ if (comments != NULL) {
+ tok->ftstring_comments = comments->previous;
+ PyMem_Free(comments);
+ }
tok->tok_mode_stack_index--;
return MAKE_TOKEN(FTSTRING_END(current_tok));
f_string_middle:
- // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
- // this.
- tok->multi_line_start = tok->line_start;
while (end_quote_size != current_tok->quote_size) {
int c = tok_nextc(tok);
if (tok->done == E_ERROR || tok->done == E_DECODE) {
@@ -489,7 +462,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
);
if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) {
- if (tok->input_error) {
+ if (tok_failed(tok)) {
return MAKE_TOKEN(ERRORTOKEN);
}
@@ -514,23 +487,16 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
}
- assert(tok->multi_line_start != NULL);
- // shift the tok_state's location into
- // the start of string, and report the error
- // from the initial quote character
- tok->cur = (char *)current_tok->start;
- tok->cur++;
- tok->line_start = current_tok->multi_line_start;
- int start = tok->lineno;
-
- tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
- tok->lineno = the_current_tok->first_line;
+ int end_lineno = tok->lineno;
+ rewind_to_string_start(tok,
+ offset_pointer(tok, current_tok->start),
+ current_tok->start_loc);
if (current_tok->quote_size == 3) {
_PyTokenizer_syntaxerror(tok,
"unterminated triple-quoted %c-string literal"
" (detected at line %d)",
- TOK_GET_STRING_PREFIX(tok), start);
+ TOK_GET_STRING_PREFIX(tok), end_lineno);
if (c != '\n') {
tok->done = E_EOFS;
}
@@ -539,7 +505,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
else {
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
"unterminated %c-string literal (detected at"
- " line %d)", TOK_GET_STRING_PREFIX(tok), start));
+ " line %d)", TOK_GET_STRING_PREFIX(tok), end_lineno));
}
}
@@ -551,9 +517,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
}
if (c == '{') {
- if (!_PyLexer_update_ftstring_expr(tok, c)) {
- return MAKE_TOKEN(ENDMARKER);
- }
+ _PyLexer_update_ftstring_expr(tok, c);
int peek = tok_nextc(tok);
if (peek != '{' || in_format_spec) {
tok_backup(tok, peek);
diff --git a/Parser/pegen.c b/Parser/pegen.c
index fcec810037e98d4..49d2355793f0d7d 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -8,8 +8,9 @@
#include
#include "lexer/lexer.h"
-#include "tokenizer/tokenizer.h"
#include "tokenizer/helpers.h"
+#include "tokenizer/reader.h"
+#include "tokenizer/tokenizer.h"
#include "pegen.h"
#define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two.
@@ -171,18 +172,17 @@ growable_comment_array_deallocate(growable_comment_array *arr) {
}
static int
-_get_keyword_or_name_type(Parser *p, struct token *new_token)
+_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length)
{
- Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset;
- assert(name_len > 0);
+ assert(length > 0);
- if (name_len >= p->n_keyword_lists ||
- p->keywords[name_len] == NULL ||
- p->keywords[name_len]->type == -1) {
+ if (length >= p->n_keyword_lists ||
+ p->keywords[length] == NULL ||
+ p->keywords[length]->type == -1) {
return NAME;
}
- for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
- if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) {
+ for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) {
+ if (memcmp(k->str, text, (size_t)length) == 0) {
return k->type;
}
}
@@ -193,8 +193,11 @@ static int
initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) {
assert(parser_token != NULL);
- parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type;
- parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start);
+ Py_ssize_t length;
+ const char *text = _PyToken_TextView(p->tok, new_token, &length);
+ parser_token->type = token_type == NAME
+ ? _get_keyword_or_name_type(p, text, length) : token_type;
+ parser_token->bytes = PyBytes_FromStringAndSize(text, length);
if (parser_token->bytes == NULL) {
return -1;
}
@@ -214,12 +217,14 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
}
parser_token->level = new_token->level;
- parser_token->lineno = new_token->lineno;
- parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset
- : new_token->col_offset;
- parser_token->end_lineno = new_token->end_lineno;
- parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset
- : new_token->end_col_offset;
+ parser_token->lineno = new_token->start_loc.lineno;
+ parser_token->col_offset = p->tok->lineno == p->starting_lineno
+ ? p->starting_col_offset + new_token->start_loc.byte_col
+ : new_token->start_loc.byte_col;
+ parser_token->end_lineno = new_token->end_loc.lineno;
+ parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+ ? p->starting_col_offset + new_token->end_loc.byte_col
+ : new_token->end_loc.byte_col;
p->fill += 1;
@@ -261,13 +266,14 @@ _PyPegen_fill_token(Parser *p)
// Record and skip '# type: ignore' comments
while (type == TYPE_IGNORE) {
- Py_ssize_t len = new_token.end_col_offset - new_token.col_offset;
+ Py_ssize_t len;
+ const char *text = _PyToken_TextView(p->tok, &new_token, &len);
char *tag = PyMem_Malloc((size_t)len + 1);
if (tag == NULL) {
PyErr_NoMemory();
goto error;
}
- strncpy(tag, new_token.start, (size_t)len);
+ memcpy(tag, text, (size_t)len);
tag[len] = '\0';
// Ownership of tag passes to the growable array
if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
@@ -938,9 +944,7 @@ reset_parser_state_for_error_pass(Parser *p)
}
p->mark = 0;
p->call_invalid_rules = 1;
- // Don't try to get extra tokens in interactive mode when trying to
- // raise specialized errors in the second pass.
- p->tok->interactive_underflow = IUNDERFLOW_STOP;
+ _PyTok_ReaderStopInteractive(p->tok);
}
static inline int
@@ -956,12 +960,9 @@ _PyPegen_set_syntax_error_metadata(Parser *p) {
PyErr_SetRaisedException(exc);
return;
}
- const char *source = NULL;
- if (p->tok->str != NULL) {
- source = p->tok->str;
- }
- if (!source && p->tok->fp_interactive && p->tok->interactive_src_start) {
- source = p->tok->interactive_src_start;
+ const char *source = p->tok->source.bytes;
+ if (source == NULL && p->tok->fp == NULL) {
+ source = _PyTok_SourceData(&p->tok->source);
}
PyObject* the_source = NULL;
if (source) {
@@ -1065,10 +1066,6 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
}
return NULL;
}
- if (!tok->fp || ps1 != NULL || ps2 != NULL ||
- PyUnicode_CompareWithASCIIString(filename_ob, "") == 0) {
- tok->fp_interactive = 1;
- }
// This transfers the ownership to the tokenizer
tok->filename = Py_NewRef(filename_ob);
@@ -1090,8 +1087,8 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
result = _PyPegen_run_parser(p);
_PyPegen_Parser_Free(p);
- if (tok->fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) {
- *interactive_src = PyUnicode_FromString(tok->interactive_src_start);
+ if (tok->source.bytes != NULL && result && interactive_src != NULL) {
+ *interactive_src = PyUnicode_FromString(tok->source.bytes);
if (!*interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) {
Py_XDECREF(*interactive_src);
result = NULL;
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index b13e1c079220a92..72c22cea77cddf3 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -7,6 +7,7 @@
#include "lexer/state.h"
#include "lexer/lexer.h"
#include "pegen.h"
+#include "tokenizer/reader.h"
// TOKENIZER ERRORS
@@ -122,7 +123,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
// before the one that we had for the generic error.
// We don't want to tokenize to the end for interactive input
- if (p->tok->prompt != NULL) {
+ if (_PyTok_ReaderIsInteractive(p->tok)) {
return 0;
}
@@ -225,45 +226,23 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
}
static PyObject *
-get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno)
+get_error_line_from_source(Parser *p, Py_ssize_t lineno)
{
- /* If the file descriptor is interactive, the source lines of the current
- * (multi-line) statement are stored in p->tok->interactive_src_start.
- * If not, we're parsing from a string, which means that the whole source
- * is stored in p->tok->str. */
- assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp != NULL);
-
- char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
- if (cur_line == NULL) {
- assert(p->tok->fp_interactive);
- // We can reach this point if the tokenizer buffers for interactive source have not been
- // initialized because we failed to decode the original source with the given locale.
- return Py_GetConstant(Py_CONSTANT_EMPTY_STR);
- }
+ const char *cur_line = _PyTok_SourceData(&p->tok->source);
Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno;
- const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp;
-
- if (buf_end < cur_line) {
- buf_end = cur_line + strlen(cur_line);
- }
+ const char *buf_end = cur_line + p->tok->source.len;
for (int i = 0; i < relative_lineno - 1; i++) {
- char *new_line = strchr(cur_line, '\n');
- // The assert is here for debug builds but the conditional that
- // follows is there so in release builds we do not crash at the cost
- // to report a potentially wrong line.
- assert(new_line != NULL && new_line + 1 < buf_end);
- if (new_line == NULL || new_line + 1 > buf_end) {
+ const char *new_line = memchr(cur_line, '\n', buf_end - cur_line);
+ if (new_line == NULL) {
break;
}
cur_line = new_line + 1;
}
- char *next_newline;
- if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
- next_newline = cur_line + strlen(cur_line);
- }
+ const char *next_newline = memchr(cur_line, '\n', buf_end - cur_line);
+ next_newline = next_newline != NULL ? next_newline : buf_end;
return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
}
@@ -295,8 +274,9 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
goto error;
}
- if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) {
- error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+ if (_PyTok_ReaderIsInteractive(p->tok) &&
+ p->tok->source.bytes != NULL) {
+ error_line = get_error_line_from_source(p, lineno);
}
else if (p->start_rule == Py_file_input) {
error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename,
@@ -318,7 +298,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace");
}
else if (p->tok->fp == NULL || p->tok->fp == stdin) {
- error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+ error_line = get_error_line_from_source(p, lineno);
}
else {
error_line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c
deleted file mode 100644
index 698a26a740fd249..000000000000000
--- a/Parser/tokenizer/cursor.c
+++ /dev/null
@@ -1,81 +0,0 @@
-#include "Python.h"
-
-#include "cursor.h"
-
-static void
-set_line(_PyTok_Cursor *cursor, int lineno, _PyTok_Off start,
- _PyTok_Off end)
-{
- cursor->pos = start;
- cursor->line_start = start;
- cursor->line_end = end;
- cursor->lineno = lineno;
-}
-
-int
-_PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno)
-{
- if (cursor->source == NULL) {
- PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
- return -1;
- }
- const _PyTok_SourceText *source = cursor->source;
- if (lineno > 0 && cursor->lineno == lineno - 1 &&
- lineno <= source->nlines) {
- _PyTok_Off start = cursor->line_end;
- _PyTok_Off end = source->len;
- if (lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- set_line(cursor, lineno, start, end);
- return 0;
- }
-
- _PyTok_Line line;
- if (_PyTok_SourceLine(source, lineno, &line) < 0) {
- return -1;
- }
- set_line(cursor, lineno, line.start, line.end);
- return 0;
-}
-
-int
-_PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
-{
- if (cursor->source == NULL) {
- PyErr_SetString(PyExc_SystemError, "cursor has no tokenizer source");
- return -1;
- }
- const _PyTok_SourceText *source = cursor->source;
- int stays_on_line = cursor->lineno > 0 &&
- offset >= cursor->line_start && offset < cursor->line_end;
- if (!stays_on_line && cursor->lineno > 0 &&
- offset == cursor->line_end && offset == source->len &&
- (offset == 0 || source->bytes[offset - 1] != '\n')) {
- stays_on_line = 1;
- }
- if (stays_on_line) {
- cursor->pos = offset;
- return 0;
- }
-
- _PyTok_Loc loc;
- if (_PyTok_SourceLocation(
- source, offset, _PYTOK_AFFINITY_RIGHT, &loc) < 0) {
- return -1;
- }
- _PyTok_Off start = offset - loc.byte_col;
- _PyTok_Off end = source->len;
- if (loc.lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- set_line(cursor, loc.lineno, start, end);
- cursor->pos = offset;
- return 0;
-}
diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h
deleted file mode 100644
index d0fd9cf80b77b71..000000000000000
--- a/Parser/tokenizer/cursor.h
+++ /dev/null
@@ -1,65 +0,0 @@
-#ifndef Py_TOKENIZER_CURSOR_H
-#define Py_TOKENIZER_CURSOR_H
-
-#include "source.h"
-
-typedef struct {
- /* The source must remain initialized at this address while in use. */
- const _PyTok_SourceText *source;
- _PyTok_Off pos;
- _PyTok_Off line_start;
- _PyTok_Off line_end;
- int lineno;
-} _PyTok_Cursor;
-
-/* Move to the start of a 1-based line. Both setters preserve the cursor on
- error. */
-PyAPI_FUNC(int) _PyTok_CursorSetLine(_PyTok_Cursor *, int);
-/* Move to an offset. A line boundary selects the following line. */
-PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
-
-static inline void
-_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
-{
- *cursor = (_PyTok_Cursor){
- .source = source,
- };
-}
-
-/* Read one byte from the current line, including its terminating newline.
- EOF marks the line boundary, not necessarily the end of the source. It is
- also returned if advancing would make the byte column unrepresentable. */
-static inline int
-_PyTok_CursorAdvance(_PyTok_Cursor *cursor)
-{
- assert(cursor->source != NULL);
- assert(cursor->pos >= cursor->line_start);
- assert(cursor->pos <= cursor->line_end);
- assert(cursor->line_end <= cursor->source->len);
- if (cursor->pos >= cursor->line_end) {
- return EOF;
- }
- if (cursor->pos - cursor->line_start >= INT_MAX) {
- return EOF;
- }
- return Py_CHARMASK(cursor->source->bytes[cursor->pos++]);
-}
-
-/* Return the byte at a nonnegative distance within the current line, or EOF
- if the distance reaches or crosses the line boundary. */
-static inline int
-_PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
-{
- assert(cursor->source != NULL);
- assert(cursor->pos >= cursor->line_start);
- assert(cursor->pos <= cursor->line_end);
- assert(cursor->line_end <= cursor->source->len);
- assert(distance >= 0);
- if (distance < 0 ||
- distance >= cursor->line_end - cursor->pos) {
- return EOF;
- }
- return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]);
-}
-
-#endif
diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c
index af17b8b63235f51..5564b69645b500d 100644
--- a/Parser/tokenizer/decoder.c
+++ b/Parser/tokenizer/decoder.c
@@ -399,10 +399,9 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
if (stored < 0) {
return -1;
}
- tok->str = tok->source.bytes != NULL ? tok->source.bytes : (char *)"";
if (!utf8_only &&
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
- !_PyTokenizer_ensure_utf8(tok->str, tok, 1)) {
+ !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
return -1;
}
return 0;
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index bbd64760a18a664..eded08a388ed70e 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -98,10 +98,6 @@ _PyTokenizer_indenterror(struct tok_state *tok)
int
_PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char)
{
- if (!tok->report_warnings) {
- return 0;
- }
-
PyObject *msg = PyUnicode_FromFormat(
"\"\\%c\" is an invalid escape sequence. "
"Such sequences will not work in the future. "
@@ -187,10 +183,6 @@ _PyTokenizer_raise_init_error(PyObject *filename)
int
_PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...)
{
- if (!tok->report_warnings) {
- return 0;
- }
-
PyObject *errmsg;
va_list vargs;
va_start(vargs, format);
diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h
index 5edf5a3dfd2e0d1..51de0cbb156f833 100644
--- a/Parser/tokenizer/helpers.h
+++ b/Parser/tokenizer/helpers.h
@@ -5,10 +5,6 @@
#include "../lexer/state.h"
-#define ADVANCE_LINENO() \
- tok->lineno++; \
- tok->col_offset = 0;
-
int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...);
int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...);
int _PyTokenizer_indenterror(struct tok_state *tok);
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 82b824f56374fcf..e312f47fa1bc3f7 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -5,14 +5,51 @@
#include "helpers.h"
#include "reader.h"
#include "reader_internal.h"
-#include "../lexer/buffer.h"
-#include "../lexer/lexer.h"
#include "../lexer/state.h"
#ifdef HAVE_UNISTD_H
# include
#endif
+static inline int
+reader_is_streaming(_PyTok_ReaderKind kind)
+{
+ return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE;
+}
+
+typedef struct {
+ Py_ssize_t buf;
+ Py_ssize_t cur;
+ Py_ssize_t inp;
+ Py_ssize_t start;
+ Py_ssize_t line_start;
+} BufferOffsets;
+
+static BufferOffsets
+save_buffer_offsets(const struct tok_state *tok, const char *base)
+{
+ return (BufferOffsets) {
+ .buf = tok->buf - base,
+ .cur = tok->cur - base,
+ .inp = tok->inp - base,
+ .start = tok->start == NULL ? -1 : tok->start - base,
+ .line_start = tok->line_start == NULL
+ ? -1 : tok->line_start - base,
+ };
+}
+
+static void
+restore_buffer_offsets(struct tok_state *tok, char *base,
+ const BufferOffsets *offsets)
+{
+ tok->buf = base + offsets->buf;
+ tok->cur = base + offsets->cur;
+ tok->inp = base + offsets->inp;
+ tok->start = offsets->start < 0 ? NULL : base + offsets->start;
+ tok->line_start = offsets->line_start < 0
+ ? NULL : base + offsets->line_start;
+}
+
void
_PyTok_ReaderFree(struct tok_state *tok)
{
@@ -28,10 +65,10 @@ _PyTok_ReaderFree(struct tok_state *tok)
}
PyMem_Free(reader->file_buffer);
PyMem_Free(reader->decoded);
- if (reader->kind != _PYTOK_READER_PREPARED) {
+ if (reader_is_streaming(reader->kind)) {
PyMem_Free(tok->buf);
- tok->buf = NULL;
}
+ tok->buf = NULL;
PyMem_Free(reader);
tok->reader = NULL;
}
@@ -60,6 +97,25 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed)
return 0;
}
+static int
+reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed)
+{
+ _PyTok_Reader *reader = tok->reader;
+ if (needed <= reader->input_buffer_cap) {
+ return 0;
+ }
+ assert(tok->buf != NULL);
+ assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+ assert(tok->inp - tok->buf <= reader->input_buffer_cap);
+ BufferOffsets offsets = save_buffer_offsets(tok, tok->buf);
+ if (reserve_buffer(
+ &tok->buf, &reader->input_buffer_cap, needed) < 0) {
+ return -1;
+ }
+ restore_buffer_offsets(tok, tok->buf, &offsets);
+ return 0;
+}
+
static int
append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len)
{
@@ -472,13 +528,13 @@ static _PyTok_ReadResult
next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
{
_PyTok_Reader *reader = tok->reader;
- if (tok->interactive_underflow == IUNDERFLOW_STOP) {
+ if (reader->stop_interactive) {
return _PYTOK_READ_STOPPED;
}
char *input = PyOS_Readline(
- tok->fp != NULL ? tok->fp : stdin, stdout, tok->prompt);
+ tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt);
if (reader->nextprompt != NULL) {
- tok->prompt = reader->nextprompt;
+ reader->prompt = reader->nextprompt;
}
if (input == NULL) {
return _PYTOK_READ_INTERRUPT;
@@ -512,6 +568,20 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
return _PYTOK_READ_LINE;
}
+int
+_PyTok_ReaderIsInteractive(const struct tok_state *tok)
+{
+ return tok->reader->kind == _PYTOK_READER_INTERACTIVE;
+}
+
+void
+_PyTok_ReaderStopInteractive(struct tok_state *tok)
+{
+ if (_PyTok_ReaderIsInteractive(tok)) {
+ tok->reader->stop_interactive = 1;
+ }
+}
+
static _PyTok_ReadResult
reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
{
@@ -529,19 +599,32 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
Py_UNREACHABLE();
}
+static void
+reset_streaming_buffer(struct tok_state *tok)
+{
+ assert(tok->buf != NULL);
+ assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+ Py_ssize_t consumed = tok->inp - tok->buf;
+ assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed);
+ tok->buf_offset += consumed;
+ tok->cur = tok->inp = tok->buf;
+ tok->line_start = tok->buf;
+}
+
int
_PyTok_ReaderUnderflow(struct tok_state *tok)
{
- int prepared = tok->reader->kind == _PYTOK_READER_PREPARED;
+ _PyTok_ReaderKind kind = tok->reader->kind;
+ int prepared = kind == _PYTOK_READER_PREPARED;
+ int streaming = reader_is_streaming(kind);
int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok);
- if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) {
- tok->cur = tok->inp = tok->buf;
- }
-
_PyTok_Chunk chunk;
_PyTok_ReadResult result = reader_next(tok, &chunk);
if (result != _PYTOK_READ_LINE) {
+ if (reset_buffer && streaming) {
+ reset_streaming_buffer(tok);
+ }
if (result == _PYTOK_READ_EOF) {
tok->done = E_EOF;
}
@@ -552,70 +635,81 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
tok->done = E_INTR;
}
else {
- tok->input_error = 1;
if (tok->done == E_OK) {
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
? E_NOMEM : E_ERROR;
}
}
- if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
+ if (kind == _PYTOK_READER_INTERACTIVE &&
result != _PYTOK_READ_STOPPED) {
PySys_WriteStderr("\n");
}
return 0;
}
- Py_ssize_t copy_len = chunk.len;
- if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
+ Py_ssize_t scan_len = chunk.len;
+ if (kind == _PYTOK_READER_INTERACTIVE &&
chunk.implicit_newline) {
- copy_len--;
- }
- if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
- tok->cur = tok->inp = tok->buf;
- }
- if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) {
- _PyTok_ChunkClear(&chunk);
- tok->input_error = 1;
- return 0;
+ scan_len--;
}
- if (tok->reader->kind == _PYTOK_READER_INTERACTIVE &&
- _PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len,
- chunk.implicit_newline) < 0) {
- _PyTok_ChunkClear(&chunk);
- tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
- ? E_NOMEM : E_ERROR;
- tok->input_error = 1;
- return 0;
+ if (streaming) {
+ if (reset_buffer) {
+ reset_streaming_buffer(tok);
+ }
+ Py_ssize_t used = tok->inp - tok->buf;
+ int overflow = scan_len > PY_SSIZE_T_MAX - used - 1 ||
+ tok->buf_offset > PY_SSIZE_T_MAX - used - scan_len;
+ if (overflow) {
+ PyErr_NoMemory();
+ }
+ if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) {
+ _PyTok_ChunkClear(&chunk);
+ tok->done = E_NOMEM;
+ return 0;
+ }
+ memcpy(tok->inp, chunk.data, (size_t)scan_len);
+ tok->inp += scan_len;
+ *tok->inp = '\0';
}
- if (tok->fp_interactive) {
- tok->interactive_src_start = tok->source.bytes;
- tok->interactive_src_end = tok->source.bytes + tok->source.len;
+ else if (!prepared) {
+ BufferOffsets offsets;
+ if (!reset_buffer) {
+ offsets = save_buffer_offsets(tok, tok->source.bytes);
+ }
+ _PyTok_Off source_start = _PyTok_SourceAppendLine(
+ &tok->source, chunk.data, chunk.len,
+ chunk.implicit_newline);
+ if (source_start < 0) {
+ _PyTok_ChunkClear(&chunk);
+ tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
+ ? E_NOMEM : E_ERROR;
+ return 0;
+ }
+ if (reset_buffer) {
+ tok->buf = tok->cur = tok->source.bytes + source_start;
+ tok->buf_offset = source_start;
+ tok->line_start = tok->buf;
+ tok->start = NULL;
+ }
+ else {
+ restore_buffer_offsets(tok, tok->source.bytes, &offsets);
+ }
+ tok->inp = tok->source.bytes + source_start + scan_len;
}
if (prepared) {
if (tok->start == NULL) {
tok->buf = tok->cur;
+ tok->buf_offset = chunk.data - tok->source.bytes;
}
tok->inp = chunk.data + chunk.len;
}
- else {
- memcpy(tok->inp, chunk.data, (size_t)copy_len);
- tok->inp += copy_len;
- *tok->inp = '\0';
- }
tok->implicit_newline = chunk.implicit_newline;
- if (!prepared && tok->tok_mode_stack_index &&
- !_PyLexer_update_ftstring_expr(tok, 0)) {
- _PyTok_ChunkClear(&chunk);
- tok->input_error = 1;
- return 0;
- }
- ADVANCE_LINENO();
- if (tok->reader->kind == _PYTOK_READER_FILE &&
+ tok->lineno++;
+ if (kind == _PYTOK_READER_FILE &&
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
!_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {
_PyTok_ChunkClear(&chunk);
- tok->input_error = 1;
return 0;
}
_PyTok_ChunkClear(&chunk);
@@ -639,14 +733,16 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
if (kind == _PYTOK_READER_PREPARED) {
return tok;
}
- tok->buf = PyMem_Malloc(BUFSIZ);
- if (tok->buf == NULL) {
- PyErr_NoMemory();
- _PyTokenizer_Free(tok);
- return NULL;
+ if (reader_is_streaming(kind)) {
+ if (reserve_buffer(
+ &tok->buf, &tok->reader->input_buffer_cap, BUFSIZ) < 0) {
+ _PyTokenizer_Free(tok);
+ return NULL;
+ }
+ tok->cur = tok->inp = tok->buf;
+ tok->line_start = tok->buf;
+ tok->buf[0] = '\0';
}
- tok->cur = tok->inp = tok->buf;
- tok->end = tok->buf + BUFSIZ;
return tok;
}
@@ -663,8 +759,9 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
_PyTokenizer_Free(tok);
return NULL;
}
- tok->buf = tok->cur = tok->inp = tok->str;
- tok->end = tok->buf;
+ char *source = (char *)_PyTok_SourceData(&tok->source);
+ tok->buf = tok->cur = tok->inp = source;
+ tok->line_start = source;
return tok;
}
@@ -710,7 +807,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding,
return NULL;
}
tok->fp = fp;
- tok->prompt = ps1;
+ tok->reader->prompt = ps1;
tok->reader->nextprompt = ps2;
return tok;
}
@@ -764,13 +861,10 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
_PyTokenizer_Free(tok);
return NULL;
}
- /* Reporting a warning here could recursively ask for the encoding. */
- tok->report_warnings = 0;
- while (tok->lineno < 2 && tok->done == E_OK) {
- struct token token;
- _PyToken_Init(&token);
- _PyTokenizer_Get(tok, &token);
- _PyToken_Free(&token);
+ if (initialize_file(tok) < 0) {
+ fclose(fp);
+ _PyTokenizer_Free(tok);
+ return NULL;
}
fclose(fp);
char *encoding = tok->encoding == NULL
diff --git a/Parser/tokenizer/reader.h b/Parser/tokenizer/reader.h
index c27bc2aa3fb8197..2913e52b9d563b4 100644
--- a/Parser/tokenizer/reader.h
+++ b/Parser/tokenizer/reader.h
@@ -5,5 +5,7 @@ struct tok_state;
void _PyTok_ReaderFree(struct tok_state *);
int _PyTok_ReaderUnderflow(struct tok_state *);
+int _PyTok_ReaderIsInteractive(const struct tok_state *);
+void _PyTok_ReaderStopInteractive(struct tok_state *);
#endif
diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h
index 121d0f96f6698a2..edb185a2f74223a 100644
--- a/Parser/tokenizer/reader_internal.h
+++ b/Parser/tokenizer/reader_internal.h
@@ -42,8 +42,11 @@ typedef struct _PyTok_Reader {
_PyTok_ReaderKind kind;
PyObject *readline;
PyObject *decoder;
+ const char *prompt;
const char *nextprompt;
+ Py_ssize_t input_buffer_cap;
+
char *file_buffer;
Py_ssize_t file_buffer_cap;
_PyTok_Chunk prefetched_lines[2];
@@ -59,6 +62,7 @@ typedef struct _PyTok_Reader {
int file_initialized;
int file_eof;
int decoder_finalized;
+ int stop_interactive;
} _PyTok_Reader;
struct tok_state;
diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c
index c0f7925e33f8b97..6f6601279944fdb 100644
--- a/Parser/tokenizer/source.c
+++ b/Parser/tokenizer/source.c
@@ -2,8 +2,6 @@
#include "source.h"
-#define LINE_CHECKPOINT_INTERVAL 256
-
void
_PyTok_SourceInit(_PyTok_SourceText *source)
{
@@ -14,7 +12,6 @@ void
_PyTok_SourceClear(_PyTok_SourceText *source)
{
PyMem_Free(source->bytes);
- PyMem_Free(source->line_checkpoints);
PyMem_Free(source->implicit_lines);
_PyTok_SourceInit(source);
}
@@ -58,34 +55,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
return 0;
}
-static int
-reserve_checkpoints(_PyTok_SourceText *source, int needed)
-{
- if (needed <= source->checkpoints_cap) {
- return 0;
- }
- int cap;
- if (source->checkpoints_cap == 0) {
- cap = 16;
- }
- else if (source->checkpoints_cap <= INT_MAX / 2) {
- cap = source->checkpoints_cap * 2;
- }
- else {
- PyErr_NoMemory();
- return -1;
- }
- _PyTok_Off *checkpoints = source->line_checkpoints;
- PyMem_Resize(checkpoints, _PyTok_Off, cap);
- if (checkpoints == NULL) {
- PyErr_NoMemory();
- return -1;
- }
- source->line_checkpoints = checkpoints;
- source->checkpoints_cap = cap;
- return 0;
-}
-
static int
reserve_implicit_lines(_PyTok_SourceText *source, int nlines)
{
@@ -148,11 +117,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
return -1;
}
int nlines = source->nlines + 1;
- int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0;
- int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
- if ((checkpoint &&
- reserve_checkpoints(source, checkpoint_count) < 0) ||
- (implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
+ if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) ||
reserve_bytes(source, source->len + len + 1) < 0) {
return -1;
}
@@ -161,9 +126,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
memcpy(source->bytes + start, bytes, len);
source->len += len;
source->bytes[source->len] = '\0';
- if (checkpoint) {
- source->line_checkpoints[checkpoint_count - 1] = start;
- }
if (implicit_newline) {
source->implicit_lines[(nlines - 1) / 8] |=
(unsigned char)(1U << ((nlines - 1) & 7));
@@ -181,7 +143,7 @@ _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
return NULL;
}
*len = span.end - span.start;
- return source->bytes == NULL ? "" : source->bytes + span.start;
+ return _PyTok_SourceData(source) + span.start;
}
int
@@ -194,121 +156,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno)
return (source->implicit_lines[(lineno - 1) / 8] >>
((lineno - 1) & 7)) & 1;
}
-
-static int
-source_ends_in_newline(const _PyTok_SourceText *source)
-{
- return source->len > 0 && source->bytes[source->len - 1] == '\n';
-}
-
-static int
-eof_lineno(const _PyTok_SourceText *source)
-{
- if (source->nlines == 0) {
- return 1;
- }
- return source->nlines + source_ends_in_newline(source);
-}
-
-int
-_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno,
- _PyTok_Line *line)
-{
- if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) {
- PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
- return -1;
- }
- if (lineno > source->nlines) {
- *line = (_PyTok_Line){
- .start = source->len,
- .end = source->len,
- };
- return 0;
- }
-
- int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL;
- int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
- _PyTok_Off start = source->line_checkpoints[checkpoint];
- while (current < lineno) {
- start = _PyTok_SourceFindLineEnd(source, start);
- if (start < 0) {
- return -1;
- }
- current++;
- }
- _PyTok_Off end = source->len;
- if (lineno < source->nlines) {
- end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- }
- *line = (_PyTok_Line){
- .start = start,
- .end = end,
- .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno),
- .contains_nul = memchr(
- source->bytes + start, 0, end - start) != NULL,
- };
- return 0;
-}
-
-int
-_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset,
- _PyTok_Affinity affinity, _PyTok_Loc *loc)
-{
- if (offset < 0 || offset > source->len || loc == NULL ||
- (affinity != _PYTOK_AFFINITY_LEFT &&
- affinity != _PYTOK_AFFINITY_RIGHT)) {
- PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset");
- return -1;
- }
- if (source->nlines == 0 ||
- (offset == source->len && source_ends_in_newline(source) &&
- affinity == _PYTOK_AFFINITY_RIGHT)) {
- *loc = (_PyTok_Loc){eof_lineno(source), 0};
- return 0;
- }
-
- _PyTok_Off key = offset;
- if (affinity == _PYTOK_AFFINITY_LEFT && key > 0) {
- key--;
- }
- int low = 0;
- int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1;
- while (low < high) {
- int middle = low + (high - low) / 2;
- if (source->line_checkpoints[middle] <= key) {
- low = middle + 1;
- }
- else {
- high = middle;
- }
- }
- int checkpoint = low - 1;
- if (checkpoint < 0) {
- PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index");
- return -1;
- }
- int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1;
- _PyTok_Off start = source->line_checkpoints[checkpoint];
- while (lineno < source->nlines) {
- _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start);
- if (end < 0) {
- return -1;
- }
- if (offset < end ||
- (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) {
- break;
- }
- start = end;
- lineno++;
- }
- _PyTok_Off byte_col = offset - start;
- if (byte_col > INT_MAX) {
- PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large");
- return -1;
- }
- *loc = (_PyTok_Loc){lineno, (int)byte_col};
- return 0;
-}
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index b42ecda1b31aa50..11521fd682f39fe 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -5,7 +5,8 @@
typedef Py_ssize_t _PyTok_Off;
-/* Half-open byte offsets into a _PyTok_SourceText. */
+/* Spans use half-open logical byte offsets into decoded input. Their backing
+ storage may retain only the current input window. */
typedef struct {
_PyTok_Off start;
_PyTok_Off end;
@@ -17,32 +18,23 @@ typedef struct {
int byte_col;
} _PyTok_Loc;
-typedef enum {
- _PYTOK_AFFINITY_LEFT,
- _PYTOK_AFFINITY_RIGHT,
-} _PyTok_Affinity;
-
-/* The half-open range includes the terminating newline when present. */
-typedef struct {
- _PyTok_Off start;
- _PyTok_Off end;
- unsigned implicit_newline : 1;
- unsigned contains_nul : 1;
-} _PyTok_Line;
-
typedef struct {
char *bytes;
_PyTok_Off len;
_PyTok_Off cap;
- _PyTok_Off *line_checkpoints;
unsigned char *implicit_lines;
int nlines;
- int checkpoints_cap;
Py_ssize_t implicit_cap;
} _PyTok_SourceText;
+static inline const char *
+_PyTok_SourceData(const _PyTok_SourceText *source)
+{
+ return source->bytes != NULL ? source->bytes : "";
+}
+
PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
-/* Clear invalidates all cursors, spans, and views for the source. */
+/* Clear invalidates all spans and views for the source. */
PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
/* Append one nonempty logical line and return its start offset. The input may
contain one newline, as its final byte. An unterminated line must be the
@@ -54,17 +46,9 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
/* The returned view is invalidated by SourceAppendLine and SourceClear. */
PyAPI_FUNC(const char *) _PyTok_SourceSpanView(
const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *);
-/* Look up a 1-based line. Empty and newline-terminated sources have an empty
- virtual line at EOF. */
-PyAPI_FUNC(int) _PyTok_SourceLine(
- const _PyTok_SourceText *, int, _PyTok_Line *);
/* Return false for invalid line numbers and the virtual EOF line. */
PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
const _PyTok_SourceText *, int);
-/* At a line boundary, left affinity selects the preceding line at its end;
- right affinity selects the following line at byte column zero. */
-PyAPI_FUNC(int) _PyTok_SourceLocation(
- const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);
static inline _PyTok_Span
_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end)
@@ -78,22 +62,4 @@ _PyTok_SpanIsValid(_PyTok_Span span)
return span.start >= 0 && span.end >= span.start;
}
-static inline _PyTok_Off
-_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start)
-{
- if (source->bytes == NULL || start < 0 || start >= source->len) {
- PyErr_SetString(PyExc_SystemError,
- "corrupt tokenizer source line index");
- return -1;
- }
- const char *newline = memchr(
- source->bytes + start, '\n', source->len - start);
- if (newline == NULL) {
- PyErr_SetString(PyExc_SystemError,
- "corrupt tokenizer source line index");
- return -1;
- }
- return newline - source->bytes + 1;
-}
-
#endif
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 762b7b3e4c8d71d..eb5c0b86a8fbe45 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -203,14 +203,19 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si
}
static void
-_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start,
- PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno,
+_get_col_offsets(tokenizeriterobject *it, const struct token *token,
+ const char *token_start, const char *line_start,
+ PyObject *line, int line_changed,
Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
+ const char *token_end = token_start == NULL
+ ? NULL : token_start + token->span.end - token->span.start;
+ Py_ssize_t lineno = token->start_loc.lineno;
+ Py_ssize_t end_lineno = token->end_loc.lineno;
Py_ssize_t byte_offset = -1;
- if (token.start != NULL && token.start >= line_start) {
- byte_offset = token.start - line_start;
+ if (token_start != NULL && token_start >= line_start) {
+ byte_offset = token_start - line_start;
if (line_changed) {
*col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
it->byte_col_offset_diff = byte_offset - *col_offset;
@@ -220,15 +225,13 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s
}
}
- if (token.end != NULL && token.end >= it->tok->line_start) {
- Py_ssize_t end_byte_offset = token.end - it->tok->line_start;
+ if (token_end != NULL && token_end >= it->tok->line_start) {
+ Py_ssize_t end_byte_offset = token_end - it->tok->line_start;
if (lineno == end_lineno) {
- // If the whole token is at the same line, we can just use the token.start
- // buffer for figuring out the new column offset, since using line is not
- // performant for very long lines.
+ // Avoid rescanning the prefix of a very long line.
Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
*end_col_offset = *col_offset + token_col_offset;
- it->byte_col_offset_diff += token.end - token.start - token_col_offset;
+ it->byte_col_offset_diff += token_end - token_start - token_col_offset;
}
else {
*end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset);
@@ -263,12 +266,17 @@ tokenizeriter_next(PyObject *op)
it->done = 1;
goto exit;
}
- PyObject *str = NULL;
- if (token.start == NULL || token.end == NULL) {
+ const char *token_start = NULL;
+ PyObject *str;
+ if (token.span.start < 0) {
+ assert(token.span.start == -1 && token.span.end == -1);
str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
else {
- str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);
+ Py_ssize_t token_length;
+ token_start = _PyToken_TextView(
+ it->tok, &token, &token_length);
+ str = PyUnicode_FromStringAndSize(token_start, token_length);
}
if (str == NULL) {
goto exit;
@@ -279,7 +287,8 @@ tokenizeriter_next(PyObject *op)
is_trailing_token = 1;
}
- const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
+ const char *line_start = ISSTRINGLIT(type)
+ ? token_start - token.start_loc.byte_col : it->tok->line_start;
PyObject* line = NULL;
int line_changed = 1;
if (it->tok->tok_extra_tokens && is_trailing_token) {
@@ -297,12 +306,12 @@ tokenizeriter_next(PyObject *op)
goto exit;
}
- Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
- Py_ssize_t end_lineno = it->tok->lineno;
+ Py_ssize_t lineno = token.start_loc.lineno;
+ Py_ssize_t end_lineno = token.end_loc.lineno;
Py_ssize_t col_offset = -1;
Py_ssize_t end_col_offset = -1;
- _get_col_offsets(it, token, line_start, line, line_changed,
- lineno, end_lineno, &col_offset, &end_col_offset);
+ _get_col_offsets(it, &token, token_start, line_start, line, line_changed,
+ &col_offset, &end_col_offset);
if (it->tok->tok_extra_tokens) {
if (is_trailing_token) {
@@ -317,7 +326,8 @@ tokenizeriter_next(PyObject *op)
else if (type == NEWLINE) {
Py_DECREF(str);
if (!it->tok->implicit_newline) {
- if (it->tok->start[0] == '\r') {
+ assert(token_start != NULL);
+ if (token_start[0] == '\r') {
str = PyUnicode_FromString("\r\n");
} else {
str = PyUnicode_FromString("\n");
diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py
index bfd8e43c6912e86..af8027db27234ad 100644
--- a/Tools/peg_generator/pegen/build.py
+++ b/Tools/peg_generator/pegen/build.py
@@ -128,7 +128,6 @@ def compile_c_extension(
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),