rpmlint/rpmlint-1.10-python3.7-pyc-mtime.patch

From 2bf599d7ed9d4b557b3806fa29439e740cae7f95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= <miro@hroncok.cz>
Date: Thu, 1 Mar 2018 17:39:10 +0100
Subject: [PATCH 1/2] Fix getting pyc mtime on Python 3.7

Fixes https://github.com/rpm-software-management/rpmlint/issues/128
---
 FilesCheck.py                      |  28 +++++++++++++++++++++++++---
 test/pyc/.gitignore                |   1 +
 test/pyc/__future__.cpython-36.pyc | Bin 0 -> 4169 bytes
 test/pyc/__future__.cpython-37.pyc | Bin 0 -> 4116 bytes
 test/test_files.py                 |  19 +++++++++++++++++++
 tools/Testing.py                   |   8 ++++++--
 6 files changed, 51 insertions(+), 5 deletions(-)
 create mode 100644 test/pyc/.gitignore
 create mode 100644 test/pyc/__future__.cpython-36.pyc
 create mode 100644 test/pyc/__future__.cpython-37.pyc

diff --git a/FilesCheck.py b/FilesCheck.py
index 232a91888..c8666b966 100644
--- a/FilesCheck.py
+++ b/FilesCheck.py
@@ -379,6 +379,27 @@ def py_demarshal_long(b):
     return (b[0] + (b[1] << 8) + (b[2] << 16) + (b[3] << 24))
 
 
+def pyc_magic_from_chunk(chunk):
+    """From given chunk (beginning of the file), return Python magic number"""
+    return py_demarshal_long(chunk[:4]) & 0xffff
+
+
+def pyc_mtime_from_chunk(chunk):
+    """From given chunk (beginning of the file), return mtime or None
+
+    From Python 3.7, mtime is not always present.
+
+    See https://www.python.org/dev/peps/pep-0552/#specification
+    """
+    magic = pyc_magic_from_chunk(chunk)
+    second = py_demarshal_long(chunk[4:8])
+    if magic >= _python_magic_values['3.7'][0]:
+        if second == 0:
+            return py_demarshal_long(chunk[8:12])
+        return None  # No mtime saved, TODO check hashes instead
+    return second
+
+
 def python_bytecode_to_script(path):
     """
     Given a python bytecode path, give the path of the .py file
@@ -731,7 +752,7 @@ def check(self, pkg):
                         if chunk:
                             # Verify that the magic ABI value embedded in the
                             # .pyc header is correct
-                            found_magic = py_demarshal_long(chunk[:4]) & 0xffff
+                            found_magic = pyc_magic_from_chunk(chunk)
                             exp_magic, exp_version = get_expected_pyc_magic(f)
                             if exp_magic and found_magic not in exp_magic:
                                 found_version = 'unknown'
@@ -754,13 +775,14 @@ def check(self, pkg):
 
                             # Verify that the timestamp embedded in the .pyc
                             # header matches the mtime of the .py file:
-                            pyc_timestamp = py_demarshal_long(chunk[4:8])
+                            pyc_timestamp = pyc_mtime_from_chunk(chunk)
                             # If it's a symlink, check target file mtime.
                             srcfile = pkg.readlink(files[source_file])
                             if not srcfile:
                                 printWarning(
                                     pkg, 'python-bytecode-without-source', f)
-                            elif pyc_timestamp != srcfile.mtime:
+                            elif (pyc_timestamp is not None and
+                                  pyc_timestamp != srcfile.mtime):
                                 cts = datetime.fromtimestamp(
                                     pyc_timestamp).isoformat()
                                 sts = datetime.fromtimestamp(
diff --git a/test/pyc/.gitignore b/test/pyc/.gitignore
new file mode 100644
index 000000000..0cc313a69
--- /dev/null
+++ b/test/pyc/.gitignore
@@ -0,0 +1 @@
+!*.pyc
diff --git a/test/test_files.py b/test/test_files.py
index 0192c13a5..a209a83a2 100644
--- a/test/test_files.py
+++ b/test/test_files.py
@@ -1,8 +1,10 @@
 import os
+import pytest
 
 import FilesCheck
 from FilesCheck import python_bytecode_to_script as pbts
 from FilesCheck import script_interpreter as se
+from FilesCheck import pyc_magic_from_chunk, pyc_mtime_from_chunk
 import Testing
 
 
@@ -33,6 +35,23 @@ def test_python_bytecode_magic(self):
             out = self._rpm_test_output(os.path.join("binary", package))
             assert "python-bytecode-wrong-magic-value" not in "\n".join(out)
 
+    @pytest.mark.parametrize('version, magic', ((36, 3379), (37, 3393)))
+    def test_pyc_magic_from_chunk(self, version, magic):
+        path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
+        with open(path, 'rb') as f:
+            chunk = f.read(16)
+        assert pyc_magic_from_chunk(chunk) == magic
+
+
+class TestPythonBytecodeMtime(object):
+
+    @pytest.mark.parametrize('version, mtime', ((36, 1513659236), (37, 1519778958)))
+    def test_pyc_mtime_from_chunk(self, version, mtime):
+        path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
+        with open(path, 'rb') as f:
+            chunk = f.read(16)
+        assert pyc_mtime_from_chunk(chunk) == mtime
+
 
 class TestDevelFiles(Testing.OutputTest):
 
diff --git a/tools/Testing.py b/tools/Testing.py
index 34b707931..2b9012701 100644
--- a/tools/Testing.py
+++ b/tools/Testing.py
@@ -43,13 +43,17 @@ def getOutput():
     return output
 
 
+def getTestedPath(path):
+    return os.path.join(_testpath(), path)
+
+
 def getTestedPackage(name):
-    pkg_path = glob.glob(os.path.join(_testpath(), name) + "-*.rpm")[0]
+    pkg_path = glob.glob(getTestedPath(name) + "-*.rpm")[0]
     return Pkg.Pkg(pkg_path, tempfile.gettempdir())
 
 
 def getTestedSpecPackage(name):
-    pkg_path = glob.glob(os.path.join(_testpath(), name) + ".spec")[0]
+    pkg_path = glob.glob(getTestedPath(name) + ".spec")[0]
     return Pkg.FakePkg(pkg_path)
 
 

From aba676f98dc3852cf39b459bd6d86bad0450746a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= <miro@hroncok.cz>
Date: Fri, 2 Mar 2018 10:46:30 +0100
Subject: [PATCH 2/2] pyc related tests: DRY

---
 test/test_files.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/test/test_files.py b/test/test_files.py
index a209a83a2..c6a162c5d 100644
--- a/test/test_files.py
+++ b/test/test_files.py
@@ -24,6 +24,13 @@ def test_pep0488(self):
         assert pbts("/usr/lib/python3.5/site-packages/__pycache__/pytest.cpython-35.pyc") == "/usr/lib/python3.5/site-packages/pytest.py"
 
 
+def chunk_from_pyc(version, size=16):
+    """Helper to get start of an example pyc file as bytes"""
+    path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
+    with open(path, 'rb') as f:
+        return f.read(size)
+
+
 class TestPythonBytecodeMagic(Testing.OutputTest):
 
     @classmethod
@@ -37,9 +44,7 @@ def test_python_bytecode_magic(self):
 
     @pytest.mark.parametrize('version, magic', ((36, 3379), (37, 3393)))
     def test_pyc_magic_from_chunk(self, version, magic):
-        path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
-        with open(path, 'rb') as f:
-            chunk = f.read(16)
+        chunk = chunk_from_pyc(version)
         assert pyc_magic_from_chunk(chunk) == magic
 
 
@@ -47,9 +52,7 @@ class TestPythonBytecodeMtime(object):
 
     @pytest.mark.parametrize('version, mtime', ((36, 1513659236), (37, 1519778958)))
     def test_pyc_mtime_from_chunk(self, version, mtime):
-        path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version))
-        with open(path, 'rb') as f:
-            chunk = f.read(16)
+        chunk = chunk_from_pyc(version)
         assert pyc_mtime_from_chunk(chunk) == mtime