From 2bf599d7ed9d4b557b3806fa29439e740cae7f95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 1 Mar 2018 17:39:10 +0100 Subject: [PATCH 1/2] Fix getting pyc mtime on Python 3.7 Fixes https://github.com/rpm-software-management/rpmlint/issues/128 --- FilesCheck.py | 28 +++++++++++++++++++++++++--- test/pyc/.gitignore | 1 + test/pyc/__future__.cpython-36.pyc | Bin 0 -> 4169 bytes test/pyc/__future__.cpython-37.pyc | Bin 0 -> 4116 bytes test/test_files.py | 19 +++++++++++++++++++ tools/Testing.py | 8 ++++++-- 6 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 test/pyc/.gitignore create mode 100644 test/pyc/__future__.cpython-36.pyc create mode 100644 test/pyc/__future__.cpython-37.pyc diff --git a/FilesCheck.py b/FilesCheck.py index 232a91888..c8666b966 100644 --- a/FilesCheck.py +++ b/FilesCheck.py @@ -379,6 +379,27 @@ def py_demarshal_long(b): return (b[0] + (b[1] << 8) + (b[2] << 16) + (b[3] << 24)) +def pyc_magic_from_chunk(chunk): + """From given chunk (beginning of the file), return Python magic number""" + return py_demarshal_long(chunk[:4]) & 0xffff + + +def pyc_mtime_from_chunk(chunk): + """From given chunk (beginning of the file), return mtime or None + + From Python 3.7, mtime is not always present. + + See https://www.python.org/dev/peps/pep-0552/#specification + """ + magic = pyc_magic_from_chunk(chunk) + second = py_demarshal_long(chunk[4:8]) + if magic >= _python_magic_values['3.7'][0]: + if second == 0: + return py_demarshal_long(chunk[8:12]) + return None # No mtime saved, TODO check hashes instead + return second + + def python_bytecode_to_script(path): """ Given a python bytecode path, give the path of the .py file @@ -731,7 +752,7 @@ def check(self, pkg): if chunk: # Verify that the magic ABI value embedded in the # .pyc header is correct - found_magic = py_demarshal_long(chunk[:4]) & 0xffff + found_magic = pyc_magic_from_chunk(chunk) exp_magic, exp_version = get_expected_pyc_magic(f) if exp_magic and found_magic not in exp_magic: found_version = 'unknown' @@ -754,13 +775,14 @@ def check(self, pkg): # Verify that the timestamp embedded in the .pyc # header matches the mtime of the .py file: - pyc_timestamp = py_demarshal_long(chunk[4:8]) + pyc_timestamp = pyc_mtime_from_chunk(chunk) # If it's a symlink, check target file mtime. srcfile = pkg.readlink(files[source_file]) if not srcfile: printWarning( pkg, 'python-bytecode-without-source', f) - elif pyc_timestamp != srcfile.mtime: + elif (pyc_timestamp is not None and + pyc_timestamp != srcfile.mtime): cts = datetime.fromtimestamp( pyc_timestamp).isoformat() sts = datetime.fromtimestamp( diff --git a/test/pyc/.gitignore b/test/pyc/.gitignore new file mode 100644 index 000000000..0cc313a69 --- /dev/null +++ b/test/pyc/.gitignore @@ -0,0 +1 @@ +!*.pyc diff --git a/test/test_files.py b/test/test_files.py index 0192c13a5..a209a83a2 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -1,8 +1,10 @@ import os +import pytest import FilesCheck from FilesCheck import python_bytecode_to_script as pbts from FilesCheck import script_interpreter as se +from FilesCheck import pyc_magic_from_chunk, pyc_mtime_from_chunk import Testing @@ -33,6 +35,23 @@ def test_python_bytecode_magic(self): out = self._rpm_test_output(os.path.join("binary", package)) assert "python-bytecode-wrong-magic-value" not in "\n".join(out) + @pytest.mark.parametrize('version, magic', ((36, 3379), (37, 3393))) + def test_pyc_magic_from_chunk(self, version, magic): + path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) + with open(path, 'rb') as f: + chunk = f.read(16) + assert pyc_magic_from_chunk(chunk) == magic + + +class TestPythonBytecodeMtime(object): + + @pytest.mark.parametrize('version, mtime', ((36, 1513659236), (37, 1519778958))) + def test_pyc_mtime_from_chunk(self, version, mtime): + path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) + with open(path, 'rb') as f: + chunk = f.read(16) + assert pyc_mtime_from_chunk(chunk) == mtime + class TestDevelFiles(Testing.OutputTest): diff --git a/tools/Testing.py b/tools/Testing.py index 34b707931..2b9012701 100644 --- a/tools/Testing.py +++ b/tools/Testing.py @@ -43,13 +43,17 @@ def getOutput(): return output +def getTestedPath(path): + return os.path.join(_testpath(), path) + + def getTestedPackage(name): - pkg_path = glob.glob(os.path.join(_testpath(), name) + "-*.rpm")[0] + pkg_path = glob.glob(getTestedPath(name) + "-*.rpm")[0] return Pkg.Pkg(pkg_path, tempfile.gettempdir()) def getTestedSpecPackage(name): - pkg_path = glob.glob(os.path.join(_testpath(), name) + ".spec")[0] + pkg_path = glob.glob(getTestedPath(name) + ".spec")[0] return Pkg.FakePkg(pkg_path) From aba676f98dc3852cf39b459bd6d86bad0450746a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 2 Mar 2018 10:46:30 +0100 Subject: [PATCH 2/2] pyc related tests: DRY --- test/test_files.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/test_files.py b/test/test_files.py index a209a83a2..c6a162c5d 100644 --- a/test/test_files.py +++ b/test/test_files.py @@ -24,6 +24,13 @@ def test_pep0488(self): assert pbts("/usr/lib/python3.5/site-packages/__pycache__/pytest.cpython-35.pyc") == "/usr/lib/python3.5/site-packages/pytest.py" +def chunk_from_pyc(version, size=16): + """Helper to get start of an example pyc file as bytes""" + path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) + with open(path, 'rb') as f: + return f.read(size) + + class TestPythonBytecodeMagic(Testing.OutputTest): @classmethod @@ -37,9 +44,7 @@ def test_python_bytecode_magic(self): @pytest.mark.parametrize('version, magic', ((36, 3379), (37, 3393))) def test_pyc_magic_from_chunk(self, version, magic): - path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) - with open(path, 'rb') as f: - chunk = f.read(16) + chunk = chunk_from_pyc(version) assert pyc_magic_from_chunk(chunk) == magic @@ -47,9 +52,7 @@ class TestPythonBytecodeMtime(object): @pytest.mark.parametrize('version, mtime', ((36, 1513659236), (37, 1519778958))) def test_pyc_mtime_from_chunk(self, version, mtime): - path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) - with open(path, 'rb') as f: - chunk = f.read(16) + chunk = chunk_from_pyc(version) assert pyc_mtime_from_chunk(chunk) == mtime