From 5169e0e34098129f657373704aa292c4ead27d87 Mon Sep 17 00:00:00 2001 From: Tomas Hrnciar Date: Tue, 20 Jul 2021 16:07:25 +0200 Subject: [PATCH] Automatically detect LICENSE files and mark them with %license macro --- README.md | 10 +++++---- pyproject-rpm-macros.spec | 1 + pyproject_save_files.py | 23 +++++++++++++++----- pyproject_save_files_test_data.yaml | 33 ++++++++++++++++++++++------- test_pyproject_save_files.py | 5 +++++ tests/python-setuptools.spec | 6 +++++- 6 files changed, 60 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e79e407..03c3fc0 100644 --- a/README.md +++ b/README.md @@ -184,11 +184,10 @@ For example, if a package provides the modules `requests` and `_requests`, write %pyproject_save_files requests _requests To add listed files to the `%files` section, use `%files -f %{pyproject_files}`. -Note that you still need to add any documentation and license manually (for now). +Note that you still need to add any documentation manually (for now). %files -n python3-requests -f %{pyproject_files} %doc README.rst - %license LICENSE You can use globs in the module names if listing them explicitly would be too tedious: @@ -214,10 +213,12 @@ However, in Fedora packages, always list executables explicitly to avoid uninten %files -n python3-requests -f %{pyproject_files} %doc README.rst - %license LICENSE %{_bindir}/downloader -`%pyproject_save_files` also automatically recognizes language (`*.mo`) files and marks them with `%lang` macro and appropriate language code. +`%pyproject_save_files` can automatically mark license files with `%license` macro +and language (`*.mo`) files with `%lang` macro and appropriate language code. +Only license files declared via [PEP 639] `License-Field` field are detected. +[PEP 639] is still a draft and can be changed in the future. Note that `%pyproject_save_files` uses data from the [RECORD file](https://www.python.org/dev/peps/pep-0627/). If you wish to rename, remove or otherwise change the installed files of a package @@ -303,6 +304,7 @@ so be prepared for problems. [PEP 508]: https://www.python.org/dev/peps/pep-0508/ [PEP 517]: https://www.python.org/dev/peps/pep-0517/ [PEP 518]: https://www.python.org/dev/peps/pep-0518/ +[PEP 639]: https://www.python.org/dev/peps/pep-0639/ [pip's documentation]: https://pip.pypa.io/en/stable/cli/pip_install/#vcs-support diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec index dfba40a..cd6b722 100644 --- a/pyproject-rpm-macros.spec +++ b/pyproject-rpm-macros.spec @@ -117,6 +117,7 @@ export HOSTNAME="rpmbuild" # to speedup tox in network-less mock, see rhbz#1856 - %%pyproject_buildrequires now fails when it encounters an invalid requirement - Fixes: rhbz#1983053 - Rename %%_pyproject_ghost_distinfo and %%_pyproject_record to indicate they are private +- Automatically detect LICENSE files and mark them with %%license macro * Fri Jul 23 2021 Fedora Release Engineering - 0-45 - Rebuilt for https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild diff --git a/pyproject_save_files.py b/pyproject_save_files.py index 04d1556..dac8b3f 100644 --- a/pyproject_save_files.py +++ b/pyproject_save_files.py @@ -5,6 +5,7 @@ import os from collections import defaultdict from pathlib import PosixPath, PurePosixPath +from importlib.metadata import Distribution # From RPM's build/files.c strtokWithQuotes delim argument @@ -144,7 +145,7 @@ def add_lang_to_module(paths, module_name, path): def classify_paths( - record_path, parsed_record_content, sitedirs, python_version + record_path, parsed_record_content, metadata, sitedirs, python_version ): """ For each BuildrootPath in parsed_record_content classify it to a dict structure @@ -160,7 +161,7 @@ def classify_paths( "files": [], # regular %file entries with dist-info content "dirs": [distinfo], # %dir %file entries with dist-info directory "docs": [], # to be used once there is upstream way to recognize READMEs - "licenses": [], # to be used once there is upstream way to recognize LICENSEs + "licenses": [], # %license entries parsed from dist-info METADATA file }, "lang": {}, # %lang entries: [module_name or None][language_code] lists of .mo files "modules": defaultdict(list), # each importable module (directory, .py, .so) @@ -170,6 +171,7 @@ def classify_paths( # In RECORDs generated by pip, there are no directories, only files. # The example RECORD from PEP 376 does not contain directories either. # Hence, we'll only assume files, but TODO get it officially documented. + license_files = metadata.get_all('License-File') for path in parsed_record_content: if path.suffix == ".pyc": # we handle bytecode separately @@ -180,8 +182,10 @@ def classify_paths( # RECORD and REQUESTED files are removed in %pyproject_install # See PEP 627 continue - # TODO is this a license/documentation? - paths["metadata"]["files"].append(path) + if license_files and path.name in license_files: + paths["metadata"]["licenses"].append(path) + else: + paths["metadata"]["files"].append(path) continue for sitedir in sitedirs: @@ -423,6 +427,14 @@ def load_parsed_record(pyproject_record): return parsed_record +def dist_metadata(buildroot, record_path): + """ + Returns distribution metadata (email.message.EmailMessage), possibly empty + """ + real_dist_path = record_path.parent.to_real(buildroot) + dist = Distribution.at(real_dist_path) + return dist.metadata + def pyproject_save_files(buildroot, sitelib, sitearch, python_version, pyproject_record, varargs): """ Takes arguments from the %{pyproject_save_files} macro @@ -439,8 +451,9 @@ def pyproject_save_files(buildroot, sitelib, sitearch, python_version, pyproject final_file_list = [] for record_path, files in parsed_records.items(): + metadata = dist_metadata(buildroot, record_path) paths_dict = classify_paths( - record_path, files, sitedirs, python_version + record_path, files, metadata, sitedirs, python_version ) final_file_list.extend( diff --git a/pyproject_save_files_test_data.yaml b/pyproject_save_files_test_data.yaml index 8173462..06ff207 100644 --- a/pyproject_save_files_test_data.yaml +++ b/pyproject_save_files_test_data.yaml @@ -50,11 +50,11 @@ classified: docs: [] files: - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/INSTALLER - - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/LICENSE - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/METADATA - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/WHEEL - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/top_level.txt - licenses: [] + licenses: + - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/LICENSE modules: requests: - files: @@ -415,13 +415,13 @@ classified: files: - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/AUTHORS - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/INSTALLER - - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE - - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE.python - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/METADATA - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/WHEEL - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/entry_points.txt - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/top_level.txt - licenses: [] + licenses: + - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE + - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE.python lang: django: af: @@ -7434,8 +7434,8 @@ dumped: - - '%dir /usr/lib/python3.7/site-packages/requests' - '%dir /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info' - '%dir /usr/lib/python3.7/site-packages/requests/__pycache__' + - '%license /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/LICENSE' - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/INSTALLER - - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/LICENSE - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/METADATA - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/WHEEL - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/top_level.txt @@ -11252,12 +11252,12 @@ dumped: - '%lang(zh) /usr/lib/python3.7/site-packages/django/contrib/sessions/locale/zh_Hant/LC_MESSAGES/django.mo' - '%lang(zh) /usr/lib/python3.7/site-packages/django/contrib/sites/locale/zh_Hans/LC_MESSAGES/django.mo' - '%lang(zh) /usr/lib/python3.7/site-packages/django/contrib/sites/locale/zh_Hant/LC_MESSAGES/django.mo' + - '%license /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE' + - '%license /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE.python' - /usr/bin/django-admin - /usr/bin/django-admin.py - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/AUTHORS - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/INSTALLER - - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE - - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/LICENSE.python - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/METADATA - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/WHEEL - /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/entry_points.txt @@ -14487,6 +14487,23 @@ dumped: - /usr/share/pronterface/zoom_in.png - /usr/share/pronterface/zoom_out.png +metadata: + requests: + path: /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/METADATA + content: | + Name: requests + Version: 2.22.0 + License-File: LICENSE + Whatever: False data + django: + path: /usr/lib/python3.7/site-packages/Django-3.0.7.dist-info/METADATA + content: | + Name: Django + Version: 3.0.7 + License-File: LICENSE + License-File: LICENSE.python + Whatever: False data + records: kerberos: path: /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/RECORD diff --git a/test_pyproject_save_files.py b/test_pyproject_save_files.py index 52e22a1..df2d05c 100755 --- a/test_pyproject_save_files.py +++ b/test_pyproject_save_files.py @@ -20,6 +20,7 @@ yaml_data = yaml.safe_load(yaml_file.read_text()) EXPECTED_DICT = yaml_data["classified"] EXPECTED_FILES = yaml_data["dumped"] TEST_RECORDS = yaml_data["records"] +TEST_METADATAS = yaml_data["metadata"] @pytest.fixture @@ -47,6 +48,10 @@ def prepare_pyproject_record(tmp_path, package=None, content=None): # Get test data and write dist-info/RECORD file record_path = BuildrootPath(TEST_RECORDS[package]["path"]) record_file.write_text(TEST_RECORDS[package]["content"]) + if package in TEST_METADATAS: + metadata_path = BuildrootPath(TEST_METADATAS[package]["path"]).to_real(tmp_path) + metadata_path.parent.mkdir(parents=True, exist_ok=True) + metadata_path.write_text(TEST_METADATAS[package]["content"]) # Parse RECORD file parsed_record = parse_record(record_path, read_record(record_file)) # Save JSON content to pyproject-record diff --git a/tests/python-setuptools.spec b/tests/python-setuptools.spec index 0e11620..2e73e74 100644 --- a/tests/python-setuptools.spec +++ b/tests/python-setuptools.spec @@ -67,8 +67,12 @@ rm pyproject.toml # We only run a subset of tests to speed things up and be less fragile PYTHONPATH=$(pwd) %pytest --ignore=pavement.py -k "sdist" +# Internal check that license file was recognized correctly +grep '^%%license' %{pyproject_files} > tested.license +echo '%%license %{python3_sitelib}/setuptools-%{version}.dist-info/LICENSE' > expected.license +diff tested.license expected.license + %files -n python3-setuptools -f %{pyproject_files} -%license LICENSE %doc docs/* CHANGES.rst README.rst %{python3_sitelib}/distutils-precedence.pth