From 482f0af7e188755ec50e278ee9862e85aa3bf2fc Mon Sep 17 00:00:00 2001 From: Armin Rigo Date: Thu, 8 Jun 2023 16:14:19 +0200 Subject: [PATCH] Python 3.12 compatibility, step 1 --- cffi/_imp_emulation.py | 83 ++++++++++++++++++++++++++++++++ cffi/vengine_cpy.py | 3 +- setup.py | 2 + testing/cffi0/test_verify.py | 3 +- testing/cffi0/test_zdistutils.py | 7 +-- testing/cffi1/test_new_ffi_1.py | 10 ++-- testing/cffi1/test_verify1.py | 3 +- testing/cffi1/test_zdist.py | 9 ++-- testing/support.py | 8 +-- 9 files changed, 108 insertions(+), 20 deletions(-) create mode 100644 cffi/_imp_emulation.py diff --git a/cffi/_imp_emulation.py b/cffi/_imp_emulation.py new file mode 100644 index 00000000..136abddd --- /dev/null +++ b/cffi/_imp_emulation.py @@ -0,0 +1,83 @@ + +try: + # this works on Python < 3.12 + from imp import * + +except ImportError: + # this is a limited emulation for Python >= 3.12. + # Note that this is used only for tests or for the old ffi.verify(). + # This is copied from the source code of Python 3.11. + + from _imp import (acquire_lock, release_lock, + is_builtin, is_frozen) + + from importlib._bootstrap import _load + + from importlib import machinery + import os + import sys + import tokenize + + SEARCH_ERROR = 0 + PY_SOURCE = 1 + PY_COMPILED = 2 + C_EXTENSION = 3 + PY_RESOURCE = 4 + PKG_DIRECTORY = 5 + C_BUILTIN = 6 + PY_FROZEN = 7 + PY_CODERESOURCE = 8 + IMP_HOOK = 9 + + def get_suffixes(): + extensions = [(s, 'rb', C_EXTENSION) + for s in machinery.EXTENSION_SUFFIXES] + source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] + bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] + return extensions + source + bytecode + + def find_module(name, path=None): + if not isinstance(name, str): + raise TypeError("'name' must be a str, not {}".format(type(name))) + elif not isinstance(path, (type(None), list)): + # Backwards-compatibility + raise RuntimeError("'path' must be None or a list, " + "not {}".format(type(path))) + + if path is None: + if is_builtin(name): + return None, None, ('', '', C_BUILTIN) + elif is_frozen(name): + return None, None, ('', '', PY_FROZEN) + else: + path = sys.path + + for entry in path: + package_directory = os.path.join(entry, name) + for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: + package_file_name = '__init__' + suffix + file_path = os.path.join(package_directory, package_file_name) + if os.path.isfile(file_path): + return None, package_directory, ('', '', PKG_DIRECTORY) + for suffix, mode, type_ in get_suffixes(): + file_name = name + suffix + file_path = os.path.join(entry, file_name) + if os.path.isfile(file_path): + break + else: + continue + break # Break out of outer loop when breaking out of inner loop. + else: + raise ImportError(name, name=name) + + encoding = None + if 'b' not in mode: + with open(file_path, 'rb') as file: + encoding = tokenize.detect_encoding(file.readline)[0] + file = open(file_path, mode, encoding=encoding) + return file, file_path, (suffix, mode, type_) + + def load_dynamic(name, path, file=None): + loader = machinery.ExtensionFileLoader(name, path) + spec = machinery.ModuleSpec(name=name, loader=loader, origin=path) + return _load(spec) diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py index 6de0df0e..49727d36 100644 --- a/cffi/vengine_cpy.py +++ b/cffi/vengine_cpy.py @@ -1,9 +1,10 @@ # # DEPRECATED: implementation for ffi.verify() # -import sys, imp +import sys from . import model from .error import VerificationError +from . import _imp_emulation as imp class VCPythonEngine(object): diff --git a/setup.py b/setup.py index 7fd8f46d..99ab397e 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,8 @@ def no_working_compiler_found(): no_compiler_found = True def get_config(): + if sys.version_info >= (3, 12): + import setuptools # makes 'distutils' available from distutils.core import Distribution from distutils.sysconfig import get_config_vars get_config_vars() # workaround for a bug of distutils, e.g. on OS/X diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py index 96d752d7..31fcc6dd 100644 --- a/testing/cffi0/test_verify.py +++ b/testing/cffi0/test_verify.py @@ -1575,7 +1575,8 @@ def test_addressof(): def test_callback_in_thread(): if sys.platform == 'win32': pytest.skip("pthread only") - import os, subprocess, imp + import os, subprocess + from cffi import _imp_emulation as imp arg = os.path.join(os.path.dirname(__file__), 'callback_in_thread.py') g = subprocess.Popen([sys.executable, arg, os.path.dirname(imp.find_module('cffi')[1])]) diff --git a/testing/cffi0/test_zdistutils.py b/testing/cffi0/test_zdistutils.py index 6f46fa82..08c432c7 100644 --- a/testing/cffi0/test_zdistutils.py +++ b/testing/cffi0/test_zdistutils.py @@ -1,9 +1,10 @@ -import sys, os, imp, math, shutil +import sys, os, math, shutil import pytest from cffi import FFI, FFIError from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffixes from cffi.ffiplatform import maybe_relative_path from testing.udir import udir +from testing.support import load_dynamic class DistUtilsTest(object): @@ -80,7 +81,7 @@ class DistUtilsTest(object): v.compile_module() assert v.get_module_name().startswith('_cffi_') if v.generates_python_module(): - mod = imp.load_dynamic(v.get_module_name(), v.modulefilename) + mod = load_dynamic(v.get_module_name(), v.modulefilename) assert hasattr(mod, '_cffi_setup') def test_compile_module_explicit_filename(self): @@ -95,7 +96,7 @@ class DistUtilsTest(object): assert filename == v.modulefilename assert v.get_module_name() == basename if v.generates_python_module(): - mod = imp.load_dynamic(v.get_module_name(), v.modulefilename) + mod = load_dynamic(v.get_module_name(), v.modulefilename) assert hasattr(mod, '_cffi_setup') def test_name_from_checksum_of_cdef(self): diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py index c874a362..a461176c 100644 --- a/testing/cffi1/test_new_ffi_1.py +++ b/testing/cffi1/test_new_ffi_1.py @@ -1,5 +1,5 @@ import pytest -import platform, imp +import platform import sys, os, ctypes import cffi from testing.udir import udir @@ -91,7 +91,7 @@ def setup_module(): outputfilename = recompile(ffi1, "test_new_ffi_1", CCODE, tmpdir=str(udir)) - module = imp.load_dynamic("test_new_ffi_1", outputfilename) + module = load_dynamic("test_new_ffi_1", outputfilename) ffi = module.ffi construction_params = (ffi1, CCODE) @@ -1619,7 +1619,7 @@ class TestNewFFI1: ffi2 = cffi.FFI(); ffi2.cdef(CDEF2) outputfilename = recompile(ffi2, "test_multiple_independent_structs", CDEF2, tmpdir=str(udir)) - module = imp.load_dynamic("test_multiple_independent_structs", + module = load_dynamic("test_multiple_independent_structs", outputfilename) ffi1 = module.ffi foo1 = ffi1.new("struct ab *", [10]) @@ -1635,7 +1635,7 @@ class TestNewFFI1: outputfilename = recompile(ffi2, "test_include_struct_union_enum_typedef", CCODE, tmpdir=str(udir)) - module = imp.load_dynamic("test_include_struct_union_enum_typedef", + module = load_dynamic("test_include_struct_union_enum_typedef", outputfilename) ffi2 = module.ffi # @@ -1783,7 +1783,7 @@ class TestNewFFI1: "int myfunc(int x) { return x + 1; }\n" "int myvar = -5;\n" "#define MYFOO 42", tmpdir=str(udir)) - imp.load_dynamic("_test_import_from_lib", outputfilename) + load_dynamic("_test_import_from_lib", outputfilename) from _test_import_from_lib.lib import myfunc, myvar, MYFOO assert MYFOO == 42 assert myfunc(43) == 44 diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py index 6245281b..f1a5fa14 100644 --- a/testing/cffi1/test_verify1.py +++ b/testing/cffi1/test_verify1.py @@ -1535,7 +1535,8 @@ def test_callback_in_thread(): pytest.xfail("adapt or remove") if sys.platform == 'win32': pytest.skip("pthread only") - import os, subprocess, imp + import os, subprocess + from cffi import _imp_emulation as imp arg = os.path.join(os.path.dirname(__file__), 'callback_in_thread.py') g = subprocess.Popen([sys.executable, arg, os.path.dirname(imp.find_module('cffi')[1])]) diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py index 58d5bc8d..b1013da1 100644 --- a/testing/cffi1/test_zdist.py +++ b/testing/cffi1/test_zdist.py @@ -41,11 +41,10 @@ class TestDist(object): tmp_home = mkdtemp() assert tmp_home != None, "cannot create temporary homedir" env['HOME'] = tmp_home + pathlist = sys.path[:] if cwd is None: - newpath = self.rootdir - if 'PYTHONPATH' in env: - newpath += os.pathsep + env['PYTHONPATH'] - env['PYTHONPATH'] = newpath + pathlist.insert(0, self.rootdir) + env['PYTHONPATH'] = os.pathsep.join(pathlist) try: subprocess.check_call([self.executable] + args, cwd=cwd, env=env) finally: @@ -291,7 +290,7 @@ class TestDist(object): f.write("""if 1: # https://bugs.python.org/issue23246 import sys - if sys.platform == 'win32': + if sys.platform == 'win32' or sys.version_info >= (3, 12): try: import setuptools except ImportError: diff --git a/testing/support.py b/testing/support.py index f0677194..666c351b 100644 --- a/testing/support.py +++ b/testing/support.py @@ -1,7 +1,8 @@ import sys, os +from cffi._imp_emulation import load_dynamic if sys.version_info < (3,): - __all__ = ['u', 'arraytostring'] + __all__ = ['u', 'arraytostring', 'load_dynamic'] class U(object): def __add__(self, other): @@ -16,7 +17,7 @@ if sys.version_info < (3,): return a.tostring() else: - __all__ = ['u', 'unicode', 'long', 'arraytostring'] + __all__ = ['u', 'unicode', 'long', 'arraytostring', 'load_dynamic'] u = "" unicode = str long = int @@ -72,14 +73,13 @@ class FdWriteCapture(object): return self._value def _verify(ffi, module_name, preamble, *args, **kwds): - import imp from cffi.recompiler import recompile from .udir import udir assert module_name not in sys.modules, "module name conflict: %r" % ( module_name,) kwds.setdefault('tmpdir', str(udir)) outputfilename = recompile(ffi, module_name, preamble, *args, **kwds) - module = imp.load_dynamic(module_name, outputfilename) + module = load_dynamic(module_name, outputfilename) # # hack hack hack: copy all *bound methods* from module.ffi back to the # ffi instance. Then calls like ffi.new() will invoke module.ffi.new(). -- GitLab