diff --git a/1027.patch b/1027.patch new file mode 100644 index 0000000..2c4bc2a --- /dev/null +++ b/1027.patch @@ -0,0 +1,89 @@ +From be26d03a477a8c7a3919e2ba00121a0f7954bdb5 Mon Sep 17 00:00:00 2001 +From: Petr Viktorin +Date: Thu, 21 Sep 2023 12:45:32 +0200 +Subject: [PATCH] Add F f-string parsing for Python 3.12 (PEP 701) + +Since Python 3.12, f-strings are tokenized and parsed like the rest +of Python's grammar, using the new tokens FSTRING_START, FSTRING_MIDDLE +and FSTRING_END. + +Make the babel message extractor concatenate these three if they're +adjacent to each other. If they're not, that means there are dynamic +substitutions, so the f-string is ignored. +--- + babel/messages/extract.py | 34 ++++++++++++++++++++++++++++++++++ + 1 file changed, 34 insertions(+) + +diff --git a/babel/messages/extract.py b/babel/messages/extract.py +index b6dce6fd..5b67d2b7 100644 +--- a/babel/messages/extract.py ++++ b/babel/messages/extract.py +@@ -33,6 +33,7 @@ + from textwrap import dedent + from tokenize import COMMENT, NAME, OP, STRING, generate_tokens + from typing import TYPE_CHECKING, Any ++import tokenize + + from babel.util import parse_encoding, parse_future_flags, pathmatch + +@@ -89,6 +90,11 @@ def tell(self) -> int: ... + + DEFAULT_MAPPING: list[tuple[str, str]] = [('**.py', 'python')] + ++# New tokens in Python 3.12, or None on older versions ++FSTRING_START = getattr(tokenize, "FSTRING_START", None) ++FSTRING_MIDDLE = getattr(tokenize, "FSTRING_MIDDLE", None) ++FSTRING_END = getattr(tokenize, "FSTRING_END", None) ++ + + def _strip_comment_tags(comments: MutableSequence[str], tags: Iterable[str]): + """Helper function for `extract` that strips comment tags from strings +@@ -497,6 +503,11 @@ def extract_python( + next_line = lambda: fileobj.readline().decode(encoding) + + tokens = generate_tokens(next_line) ++ ++ # Current prefix of a Python 3.12 (PEP 701) f-string, or None if we're not ++ # currently parsing one. ++ current_fstring_start = None ++ + for tok, value, (lineno, _), _, _ in tokens: + if call_stack == -1 and tok == NAME and value in ('def', 'class'): + in_def = True +@@ -558,6 +569,20 @@ def extract_python( + val = _parse_python_string(value, encoding, future_flags) + if val is not None: + buf.append(val) ++ ++ # Python 3.12+, see https://peps.python.org/pep-0701/#new-tokens ++ elif tok == FSTRING_START: ++ current_fstring_start = value ++ elif tok == FSTRING_MIDDLE: ++ if current_fstring_start is not None: ++ current_fstring_start += value ++ elif tok == FSTRING_END: ++ if current_fstring_start is not None: ++ fstring = current_fstring_start + value ++ val = _parse_python_string(fstring, encoding, future_flags) ++ if val is not None: ++ buf.append(val) ++ + elif tok == OP and value == ',': + if buf: + messages.append(''.join(buf)) +@@ -578,6 +603,15 @@ def extract_python( + elif tok == NAME and value in keywords: + funcname = value + ++ if (current_fstring_start is not None ++ and tok not in {FSTRING_START, FSTRING_MIDDLE} ++ ): ++ # In Python 3.12, tokens other than FSTRING_* mean the ++ # f-string is dynamic, so we don't wan't to extract it. ++ # And if it's FSTRING_END, we've already handled it above. ++ # Let's forget that we're in an f-string. ++ current_fstring_start = None ++ + + def _parse_python_string(value: str, encoding: str, future_flags: int) -> str | None: + # Unwrap quotes in a safe manner, maintaining the string's encoding diff --git a/babel.spec b/babel.spec index 48d644c..8f7207f 100644 --- a/babel.spec +++ b/babel.spec @@ -28,6 +28,9 @@ Source: %{pypi_source Babel} # 2 doctests are still deselected (as doctests are not part of this fix) Patch: https://github.com/python-babel/babel/pull/998.patch +# New code for extracting f-strings for Pytohn 3.12 (PEP 701) +Patch: https://github.com/python-babel/babel/pull/1027.patch + BuildArch: noarch BuildRequires: python3-devel