Add patch for Python 3.12 compatibility (PEP 701 f-strings)

Resolves: RHBZ#2225718
This commit is contained in:
Petr Viktorin 2023-09-21 13:45:42 +02:00
parent 1d55f9c163
commit cd7ec00bec
2 changed files with 92 additions and 0 deletions

89
1027.patch Normal file
View File

@ -0,0 +1,89 @@
From be26d03a477a8c7a3919e2ba00121a0f7954bdb5 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Thu, 21 Sep 2023 12:45:32 +0200
Subject: [PATCH] Add F f-string parsing for Python 3.12 (PEP 701)
Since Python 3.12, f-strings are tokenized and parsed like the rest
of Python's grammar, using the new tokens FSTRING_START, FSTRING_MIDDLE
and FSTRING_END.
Make the babel message extractor concatenate these three if they're
adjacent to each other. If they're not, that means there are dynamic
substitutions, so the f-string is ignored.
---
babel/messages/extract.py | 34 ++++++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/babel/messages/extract.py b/babel/messages/extract.py
index b6dce6fd..5b67d2b7 100644
--- a/babel/messages/extract.py
+++ b/babel/messages/extract.py
@@ -33,6 +33,7 @@
from textwrap import dedent
from tokenize import COMMENT, NAME, OP, STRING, generate_tokens
from typing import TYPE_CHECKING, Any
+import tokenize
from babel.util import parse_encoding, parse_future_flags, pathmatch
@@ -89,6 +90,11 @@ def tell(self) -> int: ...
DEFAULT_MAPPING: list[tuple[str, str]] = [('**.py', 'python')]
+# New tokens in Python 3.12, or None on older versions
+FSTRING_START = getattr(tokenize, "FSTRING_START", None)
+FSTRING_MIDDLE = getattr(tokenize, "FSTRING_MIDDLE", None)
+FSTRING_END = getattr(tokenize, "FSTRING_END", None)
+
def _strip_comment_tags(comments: MutableSequence[str], tags: Iterable[str]):
"""Helper function for `extract` that strips comment tags from strings
@@ -497,6 +503,11 @@ def extract_python(
next_line = lambda: fileobj.readline().decode(encoding)
tokens = generate_tokens(next_line)
+
+ # Current prefix of a Python 3.12 (PEP 701) f-string, or None if we're not
+ # currently parsing one.
+ current_fstring_start = None
+
for tok, value, (lineno, _), _, _ in tokens:
if call_stack == -1 and tok == NAME and value in ('def', 'class'):
in_def = True
@@ -558,6 +569,20 @@ def extract_python(
val = _parse_python_string(value, encoding, future_flags)
if val is not None:
buf.append(val)
+
+ # Python 3.12+, see https://peps.python.org/pep-0701/#new-tokens
+ elif tok == FSTRING_START:
+ current_fstring_start = value
+ elif tok == FSTRING_MIDDLE:
+ if current_fstring_start is not None:
+ current_fstring_start += value
+ elif tok == FSTRING_END:
+ if current_fstring_start is not None:
+ fstring = current_fstring_start + value
+ val = _parse_python_string(fstring, encoding, future_flags)
+ if val is not None:
+ buf.append(val)
+
elif tok == OP and value == ',':
if buf:
messages.append(''.join(buf))
@@ -578,6 +603,15 @@ def extract_python(
elif tok == NAME and value in keywords:
funcname = value
+ if (current_fstring_start is not None
+ and tok not in {FSTRING_START, FSTRING_MIDDLE}
+ ):
+ # In Python 3.12, tokens other than FSTRING_* mean the
+ # f-string is dynamic, so we don't wan't to extract it.
+ # And if it's FSTRING_END, we've already handled it above.
+ # Let's forget that we're in an f-string.
+ current_fstring_start = None
+
def _parse_python_string(value: str, encoding: str, future_flags: int) -> str | None:
# Unwrap quotes in a safe manner, maintaining the string's encoding

View File

@ -28,6 +28,9 @@ Source: %{pypi_source Babel}
# 2 doctests are still deselected (as doctests are not part of this fix)
Patch: https://github.com/python-babel/babel/pull/998.patch
# New code for extracting f-strings for Pytohn 3.12 (PEP 701)
Patch: https://github.com/python-babel/babel/pull/1027.patch
BuildArch: noarch
BuildRequires: python3-devel