rpms/rpmlint
7
0
Fork 0
Code Issues Pull Requests Releases Activity

Correctly load .pyc mtime for Python 3.7+ bytecode cache

Browse Source 
Resolves: RHEL-20900
This commit is contained in:
Miro Hrončok 2024-01-05 17:51:22 +01:00
parent 6a689565a6
commit dd844654c3
4 changed files with 208 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
__future__.cpython-36.pyc Normal file
View File

Binary file not shown.

BIN
__future__.cpython-37.pyc Normal file
View File

Binary file not shown.

196
rpmlint-1.10-python3.7-pyc-mtime.patch Normal file
View File

@ -0,0 +1,196 @@
From 2bf599d7ed9d4b557b3806fa29439e740cae7f95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= <miro@hroncok.cz>
Date: Thu, 1 Mar 2018 17:39:10 +0100
Subject: [PATCH 1/2] Fix getting pyc mtime on Python 3.7
Fixes https://github.com/rpm-software-management/rpmlint/issues/128
---
FilesCheck.py | 28 +++++++++++++++++++++++++---
test/pyc/.gitignore | 1 +
test/pyc/__future__.cpython-36.pyc | Bin 0 -> 4169 bytes
test/pyc/__future__.cpython-37.pyc | Bin 0 -> 4116 bytes
test/test_files.py | 19 +++++++++++++++++++
tools/Testing.py | 8 ++++++--
6 files changed, 51 insertions(+), 5 deletions(-)
create mode 100644 test/pyc/.gitignore
create mode 100644 test/pyc/__future__.cpython-36.pyc
create mode 100644 test/pyc/__future__.cpython-37.pyc
diff --git a/FilesCheck.py b/FilesCheck.py
index 232a91888..c8666b966 100644
--- a/FilesCheck.py
+++ b/FilesCheck.py
@@ -379,6 +379,27 @@ def py_demarshal_long(b):
return (b[0] + (b[1] << 8) + (b[2] << 16) + (b[3] << 24))
+def pyc_magic_from_chunk(chunk):
+ """From given chunk (beginning of the file), return Python magic number"""
+ return py_demarshal_long(chunk[:4]) & 0xffff
+
+
+def pyc_mtime_from_chunk(chunk):
+ """From given chunk (beginning of the file), return mtime or None
+
+ From Python 3.7, mtime is not always present.
+
+ See https://www.python.org/dev/peps/pep-0552/#specification
+ """
+ magic = pyc_magic_from_chunk(chunk)
+ second = py_demarshal_long(chunk[4:8])
+ if magic >= _python_magic_values['3.7'][0]:
+ if second == 0:
+ return py_demarshal_long(chunk[8:12])
+ return None # No mtime saved, TODO check hashes instead
+ return second
+
+
def python_bytecode_to_script(path):
"""
Given a python bytecode path, give the path of the .py file
@@ -731,7 +752,7 @@ def check(self, pkg):
if chunk:
# Verify that the magic ABI value embedded in the
# .pyc header is correct
- found_magic = py_demarshal_long(chunk[:4]) & 0xffff
+ found_magic = pyc_magic_from_chunk(chunk)
exp_magic, exp_version = get_expected_pyc_magic(f)
if exp_magic and found_magic not in exp_magic:
found_version = 'unknown'
@@ -754,13 +775,14 @@ def check(self, pkg):
# Verify that the timestamp embedded in the .pyc
# header matches the mtime of the .py file:
- pyc_timestamp = py_demarshal_long(chunk[4:8])
+ pyc_timestamp = pyc_mtime_from_chunk(chunk)
# If it's a symlink, check target file mtime.
srcfile = pkg.readlink(files[source_file])
if not srcfile:
printWarning(
pkg, 'python-bytecode-without-source', f)
- elif pyc_timestamp != srcfile.mtime:
+ elif (pyc_timestamp is not None and
+ pyc_timestamp != srcfile.mtime):
cts = datetime.fromtimestamp(
pyc_timestamp).isoformat()
sts = datetime.fromtimestamp(
diff --git a/test/pyc/.gitignore b/test/pyc/.gitignore
new file mode 100644
index 000000000..0cc313a69
--- /dev/null
+++ b/test/pyc/.gitignore
@@ -0,0 +1 @@
+!*.pyc
diff --git a/test/test_files.py b/test/test_files.py
index 0192c13a5..a209a83a2 100644
--- a/test/test_files.py
+++ b/test/test_files.py
@@ -1,8 +1,10 @@
import os
+import pytest
import FilesCheck
from FilesCheck import python_bytecode_to_script as pbts
from FilesCheck import script_interpreter as se
+from FilesCheck import pyc_magic_from_chunk, pyc_mtime_from_chunk
import Testing
@@ -33,6 +35,23 @@ def test_python_bytecode_magic(self):
out = self._rpm_test_output(os.path.join("binary", package))
assert "python-bytecode-wrong-magic-value" not in "\n".join(out)
+ @pytest.mark.parametrize('version, magic', ((36, 3379), (37, 3393)))
+ def test_pyc_magic_from_chunk(self, version, magic):
+ path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
+ with open(path, 'rb') as f:
+ chunk = f.read(16)
+ assert pyc_magic_from_chunk(chunk) == magic
+
+
+class TestPythonBytecodeMtime(object):
+
+ @pytest.mark.parametrize('version, mtime', ((36, 1513659236), (37, 1519778958)))
+ def test_pyc_mtime_from_chunk(self, version, mtime):
+ path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
+ with open(path, 'rb') as f:
+ chunk = f.read(16)
+ assert pyc_mtime_from_chunk(chunk) == mtime
+
class TestDevelFiles(Testing.OutputTest):
diff --git a/tools/Testing.py b/tools/Testing.py
index 34b707931..2b9012701 100644
--- a/tools/Testing.py
+++ b/tools/Testing.py
@@ -43,13 +43,17 @@ def getOutput():
return output
+def getTestedPath(path):
+ return os.path.join(_testpath(), path)
+
+
def getTestedPackage(name):
- pkg_path = glob.glob(os.path.join(_testpath(), name) + "-*.rpm")[0]
+ pkg_path = glob.glob(getTestedPath(name) + "-*.rpm")[0]
return Pkg.Pkg(pkg_path, tempfile.gettempdir())
def getTestedSpecPackage(name):
- pkg_path = glob.glob(os.path.join(_testpath(), name) + ".spec")[0]
+ pkg_path = glob.glob(getTestedPath(name) + ".spec")[0]
return Pkg.FakePkg(pkg_path)
From aba676f98dc3852cf39b459bd6d86bad0450746a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= <miro@hroncok.cz>
Date: Fri, 2 Mar 2018 10:46:30 +0100
Subject: [PATCH 2/2] pyc related tests: DRY
---
test/test_files.py | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/test/test_files.py b/test/test_files.py
index a209a83a2..c6a162c5d 100644
--- a/test/test_files.py
+++ b/test/test_files.py
@@ -24,6 +24,13 @@ def test_pep0488(self):
assert pbts("/usr/lib/python3.5/site-packages/__pycache__/pytest.cpython-35.pyc") == "/usr/lib/python3.5/site-packages/pytest.py"
+def chunk_from_pyc(version, size=16):
+ """Helper to get start of an example pyc file as bytes"""
+ path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
+ with open(path, 'rb') as f:
+ return f.read(size)
+
+
class TestPythonBytecodeMagic(Testing.OutputTest):
@classmethod
@@ -37,9 +44,7 @@ def test_python_bytecode_magic(self):
@pytest.mark.parametrize('version, magic', ((36, 3379), (37, 3393)))
def test_pyc_magic_from_chunk(self, version, magic):
- path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
- with open(path, 'rb') as f:
- chunk = f.read(16)
+ chunk = chunk_from_pyc(version)
assert pyc_magic_from_chunk(chunk) == magic
@@ -47,9 +52,7 @@ class TestPythonBytecodeMtime(object):
@pytest.mark.parametrize('version, mtime', ((36, 1513659236), (37, 1519778958)))
def test_pyc_mtime_from_chunk(self, version, mtime):
- path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
- with open(path, 'rb') as f:
- chunk = f.read(16)
+ chunk = chunk_from_pyc(version)
assert pyc_mtime_from_chunk(chunk) == mtime

13
rpmlint.spec
View File

@ -18,7 +18,7 @@
Name: rpmlint
Version: 1.10
Release: 14%{?dist}
Release: 15%{?dist}
Summary: Tool for checking common errors in RPM packages
Group: Development/Tools
License: GPLv2
@ -40,6 +40,11 @@ Patch2: rpmlint-1.10-no_python2.patch
Patch3: rpmlint-1.10-fix_test.patch
Patch4: rpmlint-1.10-rpm_surrogate_escaped_utf8.patch
Patch5: rpmlint-1.10-update-crypto-example.patch
# https://github.com/rpm-software-management/rpmlint/commit/e52dcc73bab5c4310e9bb773e6aedea020e340ff
Patch6: rpmlint-1.10-python3.7-pyc-mtime.patch
# binary files from the above
Source61: __future__.cpython-36.pyc
Source62: __future__.cpython-37.pyc
BuildArch: noarch
%if %{with python3}
%if 0%{?rhel} > 7
@ -98,6 +103,8 @@ and source packages as well as spec files can be checked.
%patch3 -p1 -b .fix_test
%patch4 -p1 -b .rpm_surrogate_escaped_utf8
%patch5 -p1 -b .update_crypto_example
%patch6 -p1 -b .python3.7-pyc-mtime
cp -p %{SOURCE61} %{SOURCE62} test/pyc/
sed -i -e /MenuCheck/d Config.py
cp -p config config.example
install -pm 644 %{SOURCE3} config
@ -139,6 +146,10 @@ make check PYTHON=%{python} PYTEST=%{pytest} FLAKE8=%{flake8}
%{_mandir}/man1/rpmlint.1*
%changelog
* Fri Jan 05 2024 Miro Hrončok <mhroncok@redhat.com> - 1.10-15
- Correctly load .pyc mtime for Python 3.7+ bytecode cache
(RHEL-20900)
* Thu Jun 04 2020 Michal Domonkos <mdomonko@redhat.com> - 1.10-14
- Update crypto warnings in config file to reflect current Fedora policy
(RHBZ#1797545)