From dd844654c3530a0fcae0ea4a80afbbc7d1482506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 5 Jan 2024 17:51:22 +0100 Subject: [PATCH] Correctly load .pyc mtime for Python 3.7+ bytecode cache Resolves: RHEL-20900 --- __future__.cpython-36.pyc | Bin 0 -> 4169 bytes __future__.cpython-37.pyc | Bin 0 -> 4116 bytes rpmlint-1.10-python3.7-pyc-mtime.patch | 196 +++++++++++++++++++++++++ rpmlint.spec | 13 +- 4 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 __future__.cpython-36.pyc create mode 100644 __future__.cpython-37.pyc create mode 100644 rpmlint-1.10-python3.7-pyc-mtime.patch diff --git a/__future__.cpython-36.pyc b/__future__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4923876278ff419d6715a6fc5d9d182fe8e4929 GIT binary patch literal 4169 zcmbtX+iu&)8Kw@FCB~;X$?mpI+gW9|wOU2apzF4PO+<_9^-VeS*G0AA+~N@+=EIVv^`zw~!Mq`xj=3+zecNTWYk89?59||1GKagPkdGv@#-OC%(Df#_#>}x@ z$!wo?24l&Fo|^nt*BPE8$%9bI4xdQ&J61K*@4YJPP6OL>xzm%5#9^#jud)~16+H0N z*}`DVhFD$y90Hx~NOk1!5v$g!x_&yAO4gYR5ZrOj>Q_$s^2#}NhJ)FO7mOKKlKuLA z5KbNG>#00Gp(mjj_nCj@H{M9)1H5UwLoXF{@8vhg-;uj7Lx* zd(?Q?c)%Y&!8!O05}9a6t8;b7!)ulu_PVQ#A6!}1YqpyQ{ifM&zG}AV{3>@~$!eWd zPh=4H76i4SV_9XMV%4Dy3sj#lKTx(in!(B#^_#ucK|2*>8GTLHS1qVw_wf@Ku$3P$ zmD&~>I33&8IPr<0@+QnQhaq`}X)=2<^;EFvDlj56%`Hn#11aETEjQ+Bk)yyUtA0Xf z@_>!`8wtAy>`VsCv#d}lDeC&_YBt5NZ3PL^Rg4Mz#BJHB?KtcNa=`(nTm_j&WPv~d z-~iK=BQjmU858Kklp~mtHxaWeTry@)bm%FC1L-lZ7#{{w#nRQ;#1kRB5)v}UL-;A? zN+~{Er-8>pV2vX*MS}ys4W&73p3L@Rt$>au%s$@5ByccJt2`05DCB;6bYl!vcD$KM zvkD-Q8cAMgO(6IYy(|v*AeIJ+x}is#I06%FXqq4By*pynMZGPf= z4umoj&x^KB?O<#|s6dk1qb-1T17wjz-O*0y+N3(uu@N%x$7t(>D|Kda-#ls`AGM>c z*+F3AO{1I8yP5i=2}4N#at9^-60c@!jGCv^{Dm4VM@^oZA~j{r!m`S~ zlz%GxxEOq=q4?u+fIhw;pkJd)0{W7up!DidIdgZbh|Tm+PaF^N`X8aW$X#frc9BP$ zyC|T|3k`d7omwF(dub(&DwC{!Mw?46QY%J9Uphl$J6oF;SYv0lB^^7Dw;t`^ z)&0;{`;L9`#jp0KXNWoXapQ~qdFpOV&ww6f&<;$qOJH$}fT~LPo!`vFYGf;{9L?gP zp59nK+H#@2j^#Q3YfqvoAd0T7AMjWTiYz&!Mo^v|x%PRSNwOP496DIYvr1miQtf}O zy%Ua|@r*?$x9-z_?Eq(@N(*n#p~2-3MskAE^eesY{EpFBGPk z2@#V2rwX7bRcH2a#%D2MQB6+MHrDuYu3ysGHspVg&Td_kj{h`M+WPt3MWwCg?Io?P zVOQ)opMVg#fP4qh@fxx5x-cxf!<8G6k?*KYLIcNFa;jo;D#920_j^FXm1(hQN-RuW zxtuF5yK=IkQ(Gs`r>tR`k#3qvaR$9%n%{=pNoLe-I`R%R6vm1$FSrv0nIfNx!FE9S zWi;hnd97S7SK`r2a;r;@%B+=i{lLW{ci4MNG0OmaSEvW>hw{!jL{OHQ#XkS=Ee3a zDtM>aA2gpEw?Tf@>@<4^1N_@KXt%pxV{{jzN5_NXUekQudex%;EXEJ!V+YUr-S+XI zX|`Ux?Dhu6C-bSVTZ2EC{lUSY`J&lbnt9o4bq3(tIUEqXAI|q2cUp(t=lE~Z8URPT zZ+twTdv?(4ePbT<&1cV#+KUIxLH7sUmvJ~KiUdV5p(p|rl}bhaugL!uxxS(pQWOc- qg<4epGKtmSKfntA&uEH;d~W$yD(IymB7fEl8wf_N^%2T%h literal 0 HcmV?d00001 diff --git a/__future__.cpython-37.pyc b/__future__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8bdba702db549bcad782b8e22d14d212e9fcb995 GIT binary patch literal 4116 zcmbtXTXWmS6$US&D9F0{B0F)C4V|Ro3_~SOFEe%8F)dRmJ2a_~lrzCYF(7tH!UF=# z0!Qo!c^*d*mwT%s3!}Hhw z{>}LJZB6?Z{YW30d5UlFPY|vd8qe@7&+$B8;RU|Ri+qjiL*2;m^Di z*1_Kc|1tkW+ycMHf6>*-`_`bKX&qtvk~4q6CL=2l{8Ptcj%WMhi4{71S1{M|hLJTC z%pQRgK~>l5mOWyw;|b;jv@;wDHt^;6x4O>o90@Coq-a@V!G6a|X7YQ##M+b4@jc7! z2vEF&@j-mC>;BJ zCmcA!>2eN+iz;Ki5sXvT;2oeTURh90L`#$Abt^>mB6%nc z51~_K1!I$_Dp3M?e{H!)H5*4L%&`?Br(}&anIIF=K}*mdae;;qC*rVezIpRT84!@? z4bOHXE)Eja)e%D3@@FYcPXK||Nd6*f9L107WvbjmONPp0cYr4M;f7fg4M1Tz7siL8j=avLZ9CBB-YF(OZi z{F#WBA(ADMCsHJ`t=U*Rd$Qo3;u};zgoXH02c_n)g=P&6}m{^UEpgt zjP+?TEdV=@&GgWlBp%`$+ylADTxh0tkp<0M{(zVHvYi!T9BqMSM>i(`t+>EsUln$am9=J~b*S|cNoS_Q6FRH&exN7Fr z$r-H2!RUmh`864fdj;ZEz~_7?b*q}LuroA^hu-v6{?V2T?UnN9{4X7Wa{_gBKb#NR^-jJJwH54AxVaC8XF|dbs^b-E<5g~udy7js zDkIyHn`8}uFU3TvbV{;M;rmC71e9s9X%4Y4fpRgEUxaeJqE%icIaiv%+uEj2_ToSM@sb1s;$TUya zJNZ7?@``+qh80nBE9wrY)-6p+Mr-HxUFK*Y-ly({Kedl^K;(z%7PU4ONfHummz18l zsd;Tw=yOwZyW~fUQ}en=!rZR8smry+sYIZ%@X>7Q1L6`hXzFrxHpNmd_3M*#K+f&D zq;4ErKpL)GZk6v(S51@qHgL0!Df&eRi)pOkKyB5#z4~+G4%n~it$L@{!{xu$Y_`9_ z=q^T&PI@Puy7|2EvO#|#j0aa^wP)RS^Q2cd8!wLAot|<3YU-Ot?+<3TSL@YZ)LRQP z$DKy22dS;Y9*KMBYR^flaoB#2zY2{W>}Yn4y{oxrwNB?-v(`1AJwIyBE4nhe+iM>i zcdxe8TCFw=fRe6)s8rY_luCst6)*(^m2wqu!nIThqyN}*@K@EK`fBmZxQKrL7%PGi bNIsKY`V?|{A&;WG_Q`HOq}7G&|3ZHNS6Enf literal 0 HcmV?d00001 diff --git a/rpmlint-1.10-python3.7-pyc-mtime.patch b/rpmlint-1.10-python3.7-pyc-mtime.patch new file mode 100644 index 0000000..a3ccfad --- /dev/null +++ b/rpmlint-1.10-python3.7-pyc-mtime.patch @@ -0,0 +1,196 @@ +From 2bf599d7ed9d4b557b3806fa29439e740cae7f95 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= +Date: Thu, 1 Mar 2018 17:39:10 +0100 +Subject: [PATCH 1/2] Fix getting pyc mtime on Python 3.7 + +Fixes https://github.com/rpm-software-management/rpmlint/issues/128 +--- + FilesCheck.py | 28 +++++++++++++++++++++++++--- + test/pyc/.gitignore | 1 + + test/pyc/__future__.cpython-36.pyc | Bin 0 -> 4169 bytes + test/pyc/__future__.cpython-37.pyc | Bin 0 -> 4116 bytes + test/test_files.py | 19 +++++++++++++++++++ + tools/Testing.py | 8 ++++++-- + 6 files changed, 51 insertions(+), 5 deletions(-) + create mode 100644 test/pyc/.gitignore + create mode 100644 test/pyc/__future__.cpython-36.pyc + create mode 100644 test/pyc/__future__.cpython-37.pyc + +diff --git a/FilesCheck.py b/FilesCheck.py +index 232a91888..c8666b966 100644 +--- a/FilesCheck.py ++++ b/FilesCheck.py +@@ -379,6 +379,27 @@ def py_demarshal_long(b): + return (b[0] + (b[1] << 8) + (b[2] << 16) + (b[3] << 24)) + + ++def pyc_magic_from_chunk(chunk): ++ """From given chunk (beginning of the file), return Python magic number""" ++ return py_demarshal_long(chunk[:4]) & 0xffff ++ ++ ++def pyc_mtime_from_chunk(chunk): ++ """From given chunk (beginning of the file), return mtime or None ++ ++ From Python 3.7, mtime is not always present. ++ ++ See https://www.python.org/dev/peps/pep-0552/#specification ++ """ ++ magic = pyc_magic_from_chunk(chunk) ++ second = py_demarshal_long(chunk[4:8]) ++ if magic >= _python_magic_values['3.7'][0]: ++ if second == 0: ++ return py_demarshal_long(chunk[8:12]) ++ return None # No mtime saved, TODO check hashes instead ++ return second ++ ++ + def python_bytecode_to_script(path): + """ + Given a python bytecode path, give the path of the .py file +@@ -731,7 +752,7 @@ def check(self, pkg): + if chunk: + # Verify that the magic ABI value embedded in the + # .pyc header is correct +- found_magic = py_demarshal_long(chunk[:4]) & 0xffff ++ found_magic = pyc_magic_from_chunk(chunk) + exp_magic, exp_version = get_expected_pyc_magic(f) + if exp_magic and found_magic not in exp_magic: + found_version = 'unknown' +@@ -754,13 +775,14 @@ def check(self, pkg): + + # Verify that the timestamp embedded in the .pyc + # header matches the mtime of the .py file: +- pyc_timestamp = py_demarshal_long(chunk[4:8]) ++ pyc_timestamp = pyc_mtime_from_chunk(chunk) + # If it's a symlink, check target file mtime. + srcfile = pkg.readlink(files[source_file]) + if not srcfile: + printWarning( + pkg, 'python-bytecode-without-source', f) +- elif pyc_timestamp != srcfile.mtime: ++ elif (pyc_timestamp is not None and ++ pyc_timestamp != srcfile.mtime): + cts = datetime.fromtimestamp( + pyc_timestamp).isoformat() + sts = datetime.fromtimestamp( +diff --git a/test/pyc/.gitignore b/test/pyc/.gitignore +new file mode 100644 +index 000000000..0cc313a69 +--- /dev/null ++++ b/test/pyc/.gitignore +@@ -0,0 +1 @@ ++!*.pyc +diff --git a/test/test_files.py b/test/test_files.py +index 0192c13a5..a209a83a2 100644 +--- a/test/test_files.py ++++ b/test/test_files.py +@@ -1,8 +1,10 @@ + import os ++import pytest + + import FilesCheck + from FilesCheck import python_bytecode_to_script as pbts + from FilesCheck import script_interpreter as se ++from FilesCheck import pyc_magic_from_chunk, pyc_mtime_from_chunk + import Testing + + +@@ -33,6 +35,23 @@ def test_python_bytecode_magic(self): + out = self._rpm_test_output(os.path.join("binary", package)) + assert "python-bytecode-wrong-magic-value" not in "\n".join(out) + ++ @pytest.mark.parametrize('version, magic', ((36, 3379), (37, 3393))) ++ def test_pyc_magic_from_chunk(self, version, magic): ++ path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) ++ with open(path, 'rb') as f: ++ chunk = f.read(16) ++ assert pyc_magic_from_chunk(chunk) == magic ++ ++ ++class TestPythonBytecodeMtime(object): ++ ++ @pytest.mark.parametrize('version, mtime', ((36, 1513659236), (37, 1519778958))) ++ def test_pyc_mtime_from_chunk(self, version, mtime): ++ path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) ++ with open(path, 'rb') as f: ++ chunk = f.read(16) ++ assert pyc_mtime_from_chunk(chunk) == mtime ++ + + class TestDevelFiles(Testing.OutputTest): + +diff --git a/tools/Testing.py b/tools/Testing.py +index 34b707931..2b9012701 100644 +--- a/tools/Testing.py ++++ b/tools/Testing.py +@@ -43,13 +43,17 @@ def getOutput(): + return output + + ++def getTestedPath(path): ++ return os.path.join(_testpath(), path) ++ ++ + def getTestedPackage(name): +- pkg_path = glob.glob(os.path.join(_testpath(), name) + "-*.rpm")[0] ++ pkg_path = glob.glob(getTestedPath(name) + "-*.rpm")[0] + return Pkg.Pkg(pkg_path, tempfile.gettempdir()) + + + def getTestedSpecPackage(name): +- pkg_path = glob.glob(os.path.join(_testpath(), name) + ".spec")[0] ++ pkg_path = glob.glob(getTestedPath(name) + ".spec")[0] + return Pkg.FakePkg(pkg_path) + + + +From aba676f98dc3852cf39b459bd6d86bad0450746a Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= +Date: Fri, 2 Mar 2018 10:46:30 +0100 +Subject: [PATCH 2/2] pyc related tests: DRY + +--- + test/test_files.py | 15 +++++++++------ + 1 file changed, 9 insertions(+), 6 deletions(-) + +diff --git a/test/test_files.py b/test/test_files.py +index a209a83a2..c6a162c5d 100644 +--- a/test/test_files.py ++++ b/test/test_files.py +@@ -24,6 +24,13 @@ def test_pep0488(self): + assert pbts("/usr/lib/python3.5/site-packages/__pycache__/pytest.cpython-35.pyc") == "/usr/lib/python3.5/site-packages/pytest.py" + + ++def chunk_from_pyc(version, size=16): ++ """Helper to get start of an example pyc file as bytes""" ++ path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) ++ with open(path, 'rb') as f: ++ return f.read(size) ++ ++ + class TestPythonBytecodeMagic(Testing.OutputTest): + + @classmethod +@@ -37,9 +44,7 @@ def test_python_bytecode_magic(self): + + @pytest.mark.parametrize('version, magic', ((36, 3379), (37, 3393))) + def test_pyc_magic_from_chunk(self, version, magic): +- path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) +- with open(path, 'rb') as f: +- chunk = f.read(16) ++ chunk = chunk_from_pyc(version) + assert pyc_magic_from_chunk(chunk) == magic + + +@@ -47,9 +52,7 @@ class TestPythonBytecodeMtime(object): + + @pytest.mark.parametrize('version, mtime', ((36, 1513659236), (37, 1519778958))) + def test_pyc_mtime_from_chunk(self, version, mtime): +- path = Testing.getTestedPath("pyc/__future__.cpython-{}.pyc".format(version)) +- with open(path, 'rb') as f: +- chunk = f.read(16) ++ chunk = chunk_from_pyc(version) + assert pyc_mtime_from_chunk(chunk) == mtime + + diff --git a/rpmlint.spec b/rpmlint.spec index a9ace5a..0001389 100644 --- a/rpmlint.spec +++ b/rpmlint.spec @@ -18,7 +18,7 @@ Name: rpmlint Version: 1.10 -Release: 14%{?dist} +Release: 15%{?dist} Summary: Tool for checking common errors in RPM packages Group: Development/Tools License: GPLv2 @@ -40,6 +40,11 @@ Patch2: rpmlint-1.10-no_python2.patch Patch3: rpmlint-1.10-fix_test.patch Patch4: rpmlint-1.10-rpm_surrogate_escaped_utf8.patch Patch5: rpmlint-1.10-update-crypto-example.patch +# https://github.com/rpm-software-management/rpmlint/commit/e52dcc73bab5c4310e9bb773e6aedea020e340ff +Patch6: rpmlint-1.10-python3.7-pyc-mtime.patch +# binary files from the above +Source61: __future__.cpython-36.pyc +Source62: __future__.cpython-37.pyc BuildArch: noarch %if %{with python3} %if 0%{?rhel} > 7 @@ -98,6 +103,8 @@ and source packages as well as spec files can be checked. %patch3 -p1 -b .fix_test %patch4 -p1 -b .rpm_surrogate_escaped_utf8 %patch5 -p1 -b .update_crypto_example +%patch6 -p1 -b .python3.7-pyc-mtime +cp -p %{SOURCE61} %{SOURCE62} test/pyc/ sed -i -e /MenuCheck/d Config.py cp -p config config.example install -pm 644 %{SOURCE3} config @@ -139,6 +146,10 @@ make check PYTHON=%{python} PYTEST=%{pytest} FLAKE8=%{flake8} %{_mandir}/man1/rpmlint.1* %changelog +* Fri Jan 05 2024 Miro Hrončok - 1.10-15 +- Correctly load .pyc mtime for Python 3.7+ bytecode cache + (RHEL-20900) + * Thu Jun 04 2020 Michal Domonkos - 1.10-14 - Update crypto warnings in config file to reflect current Fedora policy (RHBZ#1797545)