rpms/python-cffi
6
0
Fork 0
Code Issues Pull Requests Releases Activity

Add patches for compatibility with Python 3.12

Browse Source
This commit is contained in:
Tomáš Hrnčiar 2023-06-14 09:02:20 +02:00
parent e15554b384
commit 2c992c872b
6 changed files with 550 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

315
482f0af.patch Normal file
View File

@ -0,0 +1,315 @@
From 482f0af7e188755ec50e278ee9862e85aa3bf2fc Mon Sep 17 00:00:00 2001
From: Armin Rigo <arigo@tunes.org>
Date: Thu, 8 Jun 2023 16:14:19 +0200
Subject: [PATCH] Python 3.12 compatibility, step 1
---
cffi/_imp_emulation.py | 83 ++++++++++++++++++++++++++++++++
cffi/vengine_cpy.py | 3 +-
setup.py | 2 +
testing/cffi0/test_verify.py | 3 +-
testing/cffi0/test_zdistutils.py | 7 +--
testing/cffi1/test_new_ffi_1.py | 10 ++--
testing/cffi1/test_verify1.py | 3 +-
testing/cffi1/test_zdist.py | 9 ++--
testing/support.py | 8 +--
9 files changed, 108 insertions(+), 20 deletions(-)
create mode 100644 cffi/_imp_emulation.py
diff --git a/cffi/_imp_emulation.py b/cffi/_imp_emulation.py
new file mode 100644
index 00000000..136abddd
--- /dev/null
+++ b/cffi/_imp_emulation.py
@@ -0,0 +1,83 @@
+
+try:
+ # this works on Python < 3.12
+ from imp import *
+
+except ImportError:
+ # this is a limited emulation for Python >= 3.12.
+ # Note that this is used only for tests or for the old ffi.verify().
+ # This is copied from the source code of Python 3.11.
+
+ from _imp import (acquire_lock, release_lock,
+ is_builtin, is_frozen)
+
+ from importlib._bootstrap import _load
+
+ from importlib import machinery
+ import os
+ import sys
+ import tokenize
+
+ SEARCH_ERROR = 0
+ PY_SOURCE = 1
+ PY_COMPILED = 2
+ C_EXTENSION = 3
+ PY_RESOURCE = 4
+ PKG_DIRECTORY = 5
+ C_BUILTIN = 6
+ PY_FROZEN = 7
+ PY_CODERESOURCE = 8
+ IMP_HOOK = 9
+
+ def get_suffixes():
+ extensions = [(s, 'rb', C_EXTENSION)
+ for s in machinery.EXTENSION_SUFFIXES]
+ source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
+ bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
+ return extensions + source + bytecode
+
+ def find_module(name, path=None):
+ if not isinstance(name, str):
+ raise TypeError("'name' must be a str, not {}".format(type(name)))
+ elif not isinstance(path, (type(None), list)):
+ # Backwards-compatibility
+ raise RuntimeError("'path' must be None or a list, "
+ "not {}".format(type(path)))
+
+ if path is None:
+ if is_builtin(name):
+ return None, None, ('', '', C_BUILTIN)
+ elif is_frozen(name):
+ return None, None, ('', '', PY_FROZEN)
+ else:
+ path = sys.path
+
+ for entry in path:
+ package_directory = os.path.join(entry, name)
+ for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]:
+ package_file_name = '__init__' + suffix
+ file_path = os.path.join(package_directory, package_file_name)
+ if os.path.isfile(file_path):
+ return None, package_directory, ('', '', PKG_DIRECTORY)
+ for suffix, mode, type_ in get_suffixes():
+ file_name = name + suffix
+ file_path = os.path.join(entry, file_name)
+ if os.path.isfile(file_path):
+ break
+ else:
+ continue
+ break # Break out of outer loop when breaking out of inner loop.
+ else:
+ raise ImportError(name, name=name)
+
+ encoding = None
+ if 'b' not in mode:
+ with open(file_path, 'rb') as file:
+ encoding = tokenize.detect_encoding(file.readline)[0]
+ file = open(file_path, mode, encoding=encoding)
+ return file, file_path, (suffix, mode, type_)
+
+ def load_dynamic(name, path, file=None):
+ loader = machinery.ExtensionFileLoader(name, path)
+ spec = machinery.ModuleSpec(name=name, loader=loader, origin=path)
+ return _load(spec)
diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
index 6de0df0e..49727d36 100644
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -1,9 +1,10 @@
#
# DEPRECATED: implementation for ffi.verify()
#
-import sys, imp
+import sys
from . import model
from .error import VerificationError
+from . import _imp_emulation as imp
class VCPythonEngine(object):
diff --git a/setup.py b/setup.py
index 7fd8f46d..99ab397e 100644
--- a/setup.py
+++ b/setup.py
@@ -65,6 +65,8 @@ def no_working_compiler_found():
no_compiler_found = True
def get_config():
+ if sys.version_info >= (3, 12):
+ import setuptools # makes 'distutils' available
from distutils.core import Distribution
from distutils.sysconfig import get_config_vars
get_config_vars() # workaround for a bug of distutils, e.g. on OS/X
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
index 96d752d7..31fcc6dd 100644
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -1575,7 +1575,8 @@ def test_addressof():
def test_callback_in_thread():
if sys.platform == 'win32':
pytest.skip("pthread only")
- import os, subprocess, imp
+ import os, subprocess
+ from cffi import _imp_emulation as imp
arg = os.path.join(os.path.dirname(__file__), 'callback_in_thread.py')
g = subprocess.Popen([sys.executable, arg,
os.path.dirname(imp.find_module('cffi')[1])])
diff --git a/testing/cffi0/test_zdistutils.py b/testing/cffi0/test_zdistutils.py
index 6f46fa82..08c432c7 100644
--- a/testing/cffi0/test_zdistutils.py
+++ b/testing/cffi0/test_zdistutils.py
@@ -1,9 +1,10 @@
-import sys, os, imp, math, shutil
+import sys, os, math, shutil
import pytest
from cffi import FFI, FFIError
from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes
from cffi.ffiplatform import maybe_relative_path
from testing.udir import udir
+from testing.support import load_dynamic
class DistUtilsTest(object):
@@ -80,7 +81,7 @@ class DistUtilsTest(object):
v.compile_module()
assert v.get_module_name().startswith('_cffi_')
if v.generates_python_module():
- mod = imp.load_dynamic(v.get_module_name(), v.modulefilename)
+ mod = load_dynamic(v.get_module_name(), v.modulefilename)
assert hasattr(mod, '_cffi_setup')
def test_compile_module_explicit_filename(self):
@@ -95,7 +96,7 @@ class DistUtilsTest(object):
assert filename == v.modulefilename
assert v.get_module_name() == basename
if v.generates_python_module():
- mod = imp.load_dynamic(v.get_module_name(), v.modulefilename)
+ mod = load_dynamic(v.get_module_name(), v.modulefilename)
assert hasattr(mod, '_cffi_setup')
def test_name_from_checksum_of_cdef(self):
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
index c874a362..a461176c 100644
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1,5 +1,5 @@
import pytest
-import platform, imp
+import platform
import sys, os, ctypes
import cffi
from testing.udir import udir
@@ -91,7 +91,7 @@ def setup_module():
outputfilename = recompile(ffi1, "test_new_ffi_1", CCODE,
tmpdir=str(udir))
- module = imp.load_dynamic("test_new_ffi_1", outputfilename)
+ module = load_dynamic("test_new_ffi_1", outputfilename)
ffi = module.ffi
construction_params = (ffi1, CCODE)
@@ -1619,7 +1619,7 @@ class TestNewFFI1:
ffi2 = cffi.FFI(); ffi2.cdef(CDEF2)
outputfilename = recompile(ffi2, "test_multiple_independent_structs",
CDEF2, tmpdir=str(udir))
- module = imp.load_dynamic("test_multiple_independent_structs",
+ module = load_dynamic("test_multiple_independent_structs",
outputfilename)
ffi1 = module.ffi
foo1 = ffi1.new("struct ab *", [10])
@@ -1635,7 +1635,7 @@ class TestNewFFI1:
outputfilename = recompile(ffi2,
"test_include_struct_union_enum_typedef",
CCODE, tmpdir=str(udir))
- module = imp.load_dynamic("test_include_struct_union_enum_typedef",
+ module = load_dynamic("test_include_struct_union_enum_typedef",
outputfilename)
ffi2 = module.ffi
#
@@ -1783,7 +1783,7 @@ class TestNewFFI1:
"int myfunc(int x) { return x + 1; }\n"
"int myvar = -5;\n"
"#define MYFOO 42", tmpdir=str(udir))
- imp.load_dynamic("_test_import_from_lib", outputfilename)
+ load_dynamic("_test_import_from_lib", outputfilename)
from _test_import_from_lib.lib import myfunc, myvar, MYFOO
assert MYFOO == 42
assert myfunc(43) == 44
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
index 6245281b..f1a5fa14 100644
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -1535,7 +1535,8 @@ def test_callback_in_thread():
pytest.xfail("adapt or remove")
if sys.platform == 'win32':
pytest.skip("pthread only")
- import os, subprocess, imp
+ import os, subprocess
+ from cffi import _imp_emulation as imp
arg = os.path.join(os.path.dirname(__file__), 'callback_in_thread.py')
g = subprocess.Popen([sys.executable, arg,
os.path.dirname(imp.find_module('cffi')[1])])
diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
index 58d5bc8d..b1013da1 100644
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -41,11 +41,10 @@ class TestDist(object):
tmp_home = mkdtemp()
assert tmp_home != None, "cannot create temporary homedir"
env['HOME'] = tmp_home
+ pathlist = sys.path[:]
if cwd is None:
- newpath = self.rootdir
- if 'PYTHONPATH' in env:
- newpath += os.pathsep + env['PYTHONPATH']
- env['PYTHONPATH'] = newpath
+ pathlist.insert(0, self.rootdir)
+ env['PYTHONPATH'] = os.pathsep.join(pathlist)
try:
subprocess.check_call([self.executable] + args, cwd=cwd, env=env)
finally:
@@ -291,7 +290,7 @@ class TestDist(object):
f.write("""if 1:
# https://bugs.python.org/issue23246
import sys
- if sys.platform == 'win32':
+ if sys.platform == 'win32' or sys.version_info >= (3, 12):
try:
import setuptools
except ImportError:
diff --git a/testing/support.py b/testing/support.py
index f0677194..666c351b 100644
--- a/testing/support.py
+++ b/testing/support.py
@@ -1,7 +1,8 @@
import sys, os
+from cffi._imp_emulation import load_dynamic
if sys.version_info < (3,):
- __all__ = ['u', 'arraytostring']
+ __all__ = ['u', 'arraytostring', 'load_dynamic']
class U(object):
def __add__(self, other):
@@ -16,7 +17,7 @@ if sys.version_info < (3,):
return a.tostring()
else:
- __all__ = ['u', 'unicode', 'long', 'arraytostring']
+ __all__ = ['u', 'unicode', 'long', 'arraytostring', 'load_dynamic']
u = ""
unicode = str
long = int
@@ -72,14 +73,13 @@ class FdWriteCapture(object):
return self._value
def _verify(ffi, module_name, preamble, *args, **kwds):
- import imp
from cffi.recompiler import recompile
from .udir import udir
assert module_name not in sys.modules, "module name conflict: %r" % (
module_name,)
kwds.setdefault('tmpdir', str(udir))
outputfilename = recompile(ffi, module_name, preamble, *args, **kwds)
- module = imp.load_dynamic(module_name, outputfilename)
+ module = load_dynamic(module_name, outputfilename)
#
# hack hack hack: copy all *bound methods* from module.ffi back to the
# ffi instance. Then calls like ffi.new() will invoke module.ffi.new().
--
GitLab

25
69660bd.patch Normal file
View File

@ -0,0 +1,25 @@
From 69660bdf6544338edd524b73004b7accfa464c68 Mon Sep 17 00:00:00 2001
From: Armin Rigo <arigo@tunes.org>
Date: Thu, 8 Jun 2023 19:02:08 +0200
Subject: [PATCH] gah
---
c/misc_thread_common.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/c/misc_thread_common.h b/c/misc_thread_common.h
index 9e9a818d..ead9c83c 100644
--- a/c/misc_thread_common.h
+++ b/c/misc_thread_common.h
@@ -166,7 +166,7 @@ thread_canary_free_zombies(void)
break;
PyThreadState_Clear(tstate); /* calls thread_canary_dealloc on 'ob',
but now ob->zombie_next == NULL. */
-#if PY_HEX_VERSION >= 0x030C0000
+#if PY_VERSION_HEX >= 0x030C0000
/* this might be a bug introduced in 3.12, or just me abusing the
API around there. The issue is that PyThreadState_Delete()
called on a random old tstate will clear the *current* thread
--
GitLab

45
814c55e.patch Normal file
View File

@ -0,0 +1,45 @@
From 814c55ebf46da49cdb0733756b9568486c682d03 Mon Sep 17 00:00:00 2001
From: Armin Rigo <arigo@tunes.org>
Date: Thu, 8 Jun 2023 19:18:37 +0200
Subject: [PATCH] fix for multiple embedded modules loaded in parallel from
multiple threads
---
cffi/_embedding.h | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
index 265630ea..c5eaf889 100644
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -321,10 +321,11 @@ static int _cffi_carefully_make_gil(void)
int old_value, locked_value = -42;
assert(!(PyCapsule_Type.tp_flags & Py_TPFLAGS_HAVE_VERSION_TAG));
# else
- static PyBufferProcs empty_buffer_procs;
+ static struct ebp_s { PyBufferProcs buf; int mark; } empty_buffer_procs;
+ empty_buffer_procs.mark = -42;
PyBufferProcs *volatile *lock = (PyBufferProcs *volatile *)
&PyCapsule_Type.tp_as_buffer;
- PyBufferProcs *old_value, *locked_value = &empty_buffer_procs;
+ PyBufferProcs *old_value, *locked_value = &empty_buffer_procs.buf;
# endif
while (1) { /* spin loop */
@@ -334,7 +335,13 @@ static int _cffi_carefully_make_gil(void)
break;
}
else {
+# if PY_VERSION_HEX < 0x030C0000
assert(old_value == locked_value);
+# else
+ /* The pointer should point to a possibly different
+ empty_buffer_procs from another C extension module */
+ assert(((struct ebp_s *)old_value)->mark == -42);
+# endif
/* should ideally do a spin loop instruction here, but
hard to do it portably and doesn't really matter I
think: PyEval_InitThreads() should be very fast, and
--
GitLab

103
87f514b.patch Normal file
View File

@ -0,0 +1,103 @@
From 87f514b8a4fc6b29df8febf1176ad846007a5b56 Mon Sep 17 00:00:00 2001
From: Armin Rigo <arigo@tunes.org>
Date: Thu, 8 Jun 2023 18:34:20 +0200
Subject: [PATCH] Fix for what may be a CPython 3.12 bug, or just me abusing
the API
---
c/misc_thread_common.h | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/c/misc_thread_common.h b/c/misc_thread_common.h
index 66e28354..9e9a818d 100644
--- a/c/misc_thread_common.h
+++ b/c/misc_thread_common.h
@@ -102,6 +102,7 @@ thread_canary_dealloc(ThreadCanaryObj *ob)
'local_thread_canary' accesses need to be protected with
the TLS_ZOM_LOCK.
*/
+ //fprintf(stderr, "entering dealloc(%p)\n", ob);
TLS_ZOM_LOCK();
if (ob->zombie_next != NULL) {
//fprintf(stderr, "thread_canary_dealloc(%p): ZOMBIE\n", ob);
@@ -136,6 +137,7 @@ thread_canary_make_zombie(ThreadCanaryObj *ob)
ob->zombie_prev = last;
last->zombie_next = ob;
cffi_zombie_head.zombie_prev = ob;
+ //fprintf(stderr, "thread_canary_make_zombie(%p) DONE\n", ob);
}
static void
@@ -164,6 +166,15 @@ thread_canary_free_zombies(void)
break;
PyThreadState_Clear(tstate); /* calls thread_canary_dealloc on 'ob',
but now ob->zombie_next == NULL. */
+#if PY_HEX_VERSION >= 0x030C0000
+ /* this might be a bug introduced in 3.12, or just me abusing the
+ API around there. The issue is that PyThreadState_Delete()
+ called on a random old tstate will clear the *current* thread
+ notion of what PyGILState_GetThisThreadState() should be, at
+ least if the internal 'bound_gilstate' flag is set in the old
+ tstate. Workaround: clear that flag. */
+ tstate->_status.bound_gilstate = 0;
+#endif
PyThreadState_Delete(tstate);
//fprintf(stderr, "thread_canary_free_zombie: cleared and deleted tstate=%p\n", tstate);
}
@@ -213,9 +224,11 @@ thread_canary_register(PyThreadState *tstate)
tstate->gilstate_counter++;
/* ^^^ this means 'tstate' will never be automatically freed by
PyGILState_Release() */
+ //fprintf(stderr, "CANARY: ready, tstate=%p, tls=%p, canary=%p\n", tstate, tls, canary);
return;
ignore_error:
+ //fprintf(stderr, "CANARY: IGNORED ERROR\n");
PyErr_Clear();
}
@@ -334,6 +347,7 @@ static PyGILState_STATE gil_ensure(void)
*/
PyGILState_STATE result;
PyThreadState *ts = PyGILState_GetThisThreadState();
+ //fprintf(stderr, "%p: gil_ensure(), tstate=%p, tls=%p\n", get_cffi_tls(), ts, get_cffi_tls());
if (ts != NULL) {
ts->gilstate_counter++;
@@ -341,9 +355,11 @@ static PyGILState_STATE gil_ensure(void)
/* common case: 'ts' is our non-current thread state and
we have to make it current and acquire the GIL */
PyEval_RestoreThread(ts);
+ //fprintf(stderr, "%p: gil_ensure(), tstate=%p MADE CURRENT\n", get_cffi_tls(), ts);
return PyGILState_UNLOCKED;
}
else {
+ //fprintf(stderr, "%p: gil_ensure(), tstate=%p ALREADY CURRENT\n", get_cffi_tls(), ts);
return PyGILState_LOCKED;
}
}
@@ -353,6 +369,7 @@ static PyGILState_STATE gil_ensure(void)
assert(result == PyGILState_UNLOCKED);
ts = PyGILState_GetThisThreadState();
+ //fprintf(stderr, "%p: gil_ensure(), made a new tstate=%p\n", get_cffi_tls(), ts);
assert(ts != NULL);
assert(ts == get_current_ts());
assert(ts->gilstate_counter >= 1);
@@ -361,11 +378,13 @@ static PyGILState_STATE gil_ensure(void)
thread really shuts down */
thread_canary_register(ts);
+ assert(ts == PyGILState_GetThisThreadState());
return result;
}
}
static void gil_release(PyGILState_STATE oldstate)
{
+ //fprintf(stderr, "%p: gil_release(%d), tls=%p\n", get_cffi_tls(), (int)oldstate, get_cffi_tls());
PyGILState_Release(oldstate);
}
--
GitLab

54
c3593e4.patch Normal file
View File

@ -0,0 +1,54 @@
From c3593e41b9aaf8d8bfce2c97fa6bfbd8a54e8974 Mon Sep 17 00:00:00 2001
From: Armin Rigo <arigo@tunes.org>
Date: Thu, 8 Jun 2023 16:57:32 +0200
Subject: [PATCH] step 2: partially fix embedding for 3.12
---
cffi/_embedding.h | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
index 8e8df882..265630ea 100644
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -283,6 +283,15 @@ static int _cffi_carefully_make_gil(void)
Python < 3.8 because someone might use a mixture of cffi
embedded modules, some of which were compiled before this file
changed.
+
+ In Python >= 3.12, this stopped working because that particular
+ tp_version_tag gets modified during interpreter startup. It's
+ arguably a bad idea before 3.12 too, but again we can't change
+ that because someone might use a mixture of cffi embedded
+ modules, and no-one reported a bug so far. In Python >= 3.12
+ we go instead for PyCapsuleType.tp_as_buffer, which is supposed
+ to always be NULL. We write to it temporarily a pointer to
+ a struct full of NULLs, which is semantically the same.
*/
#ifdef WITH_THREAD
@@ -307,13 +316,19 @@ static int _cffi_carefully_make_gil(void)
}
}
# else
+# if PY_VERSION_HEX < 0x030C0000
int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag;
- int old_value, locked_value;
+ int old_value, locked_value = -42;
assert(!(PyCapsule_Type.tp_flags & Py_TPFLAGS_HAVE_VERSION_TAG));
+# else
+ static PyBufferProcs empty_buffer_procs;
+ PyBufferProcs *volatile *lock = (PyBufferProcs *volatile *)
+ &PyCapsule_Type.tp_as_buffer;
+ PyBufferProcs *old_value, *locked_value = &empty_buffer_procs;
+# endif
while (1) { /* spin loop */
old_value = *lock;
- locked_value = -42;
if (old_value == 0) {
if (cffi_compare_and_swap(lock, old_value, locked_value))
break;
--
GitLab

9
python-cffi.spec
View File

@ -3,7 +3,8 @@ Name: python-cffi
Version: %{general_version}%{?prerel:~%{prerel}}
Release: 5%{?dist}
Summary: Foreign Function Interface for Python to call C code
License: MIT
# Patch 482f0af has bits copied from CPython (PSF-2.0)
License: MIT AND PSF-2.0
URL: https://cffi.readthedocs.org/
Source: https://foss.heptapod.net/pypy/cffi/-/archive/v%{version}/cffi-v%{version}.tar.bz2
@ -14,6 +15,12 @@ Patch: https://foss.heptapod.net/pypy/cffi/-/merge_requests/113.patch
Patch: https://foss.heptapod.net/pypy/cffi/-/merge_requests/115.patch
# Drop usage of the deprecated py.code package
Patch: https://foss.heptapod.net/pypy/cffi/-/merge_requests/116.patch
# Python 3.12 compatibility
Patch: https://foss.heptapod.net/pypy/cffi/-/commit/482f0af.patch
Patch: https://foss.heptapod.net/pypy/cffi/-/commit/c3593e4.patch
Patch: https://foss.heptapod.net/pypy/cffi/-/commit/87f514b.patch
Patch: https://foss.heptapod.net/pypy/cffi/-/commit/69660bd.patch
Patch: https://foss.heptapod.net/pypy/cffi/-/commit/814c55e.patch
BuildRequires: make
BuildRequires: libffi-devel