%pyproject_save_files: List all files from RECORD and their %dirs

This is done to avoid troubles with %lang files listed as duplicated.

 1. It gets rid of a warning
 2. It fixes a problem described in:
    http://lists.rpm.org/pipermail/rpm-list/2020-November/002041.html

This is a backwards incompatible change,
packages that rename or remove the installed files after %pyproject_install
might no longer be compatible with %pyproject_save_files.

README.md

@@ -204,12 +204,12 @@ However, in Fedora packages, always list executables explicitly to avoid uninten
     %{_bindir}/downloader
 
 `%pyproject_save_files` also automatically recognizes language (`*.mo`) files and marks them with `%lang` macro and appropriate language code.
-Note that RPM might warn about such files listed twice:
-
-    warning: File listed twice: /usr/lib/python3.9/site-packages/django/conf/locale/af/LC_MESSAGES/django.mo
-
-The warning is harmless.
 
+Note that `%pyproject_save_files` uses data from the [RECORD file](https://www.python.org/dev/peps/pep-0627/).
+If you wish to rename, remove or otherwise change the installed files of a package
+*after* `%pyproject_install`, `%pyproject_save_files` might break.
+If possible, remove/rename such files in `%prep`.
+If not possible, avoid using `%pyproject_save_files` or edit/replace `%{pyproject_files}`.
 
 Generating Extras subpackages
 -----------------------------

pyproject-rpm-macros.spec

@@ -6,7 +6,7 @@ License:        MIT
 
 # Keep the version at zero and increment only release
 Version:        0
-Release:        33%{?dist}
+Release:        34%{?dist}
 
 # Macro files
 Source001:      macros.pyproject
@@ -97,6 +97,10 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 %license LICENSE
 
 %changelog
+* Fri Dec 04 2020 Miro Hrončok <miro@hroncok.cz> - 0-34
+- List all files in %%pyproject_files explicitly to avoid duplicate %%lang entries
+- If you amend the installed files after %%pyproject_install, %%pyproject_files might break
+
 * Fri Nov 27 2020 Miro Hrončok <mhroncok@redhat.com> - 0-33
 - Pass PYTHONDONTWRITEBYTECODE=1 to %%tox to avoid packaged PYTEST bytecode
 

pyproject_save_files.py

@@ -77,18 +77,18 @@ def pycached(script, python_version):
     return [script, pyc]
 
 
-def add_file_to_module(paths, module_name, module_type, *files):
+def add_file_to_module(paths, module_name, module_type, files_dirs, *files):
     """
     Helper procedure, adds given files to the module_name of a given module_type
     """
     for module in paths["modules"][module_name]:
         if module["type"] == module_type:
-            if files[0] not in module["files"]:
+            if files[0] not in module[files_dirs]:
-                module["files"].extend(files)
+                module[files_dirs].extend(files)
             break
     else:
         paths["modules"][module_name].append(
-            {"type": module_type, "files": list(files)}
+            {"type": module_type, "files": [], "dirs": [], files_dirs: list(files)}
         )
 
 
@@ -121,7 +121,7 @@ def classify_paths(
 
     For the dict structure, look at the beginning of this function's code.
 
-    Each "module" is a dict with "type" ("package", "script", "extension") and "files".
+    Each "module" is a dict with "type" ("package", "script", "extension"), and "files" and "dirs".
     """
     distinfo = record_path.parent
     paths = {
@@ -159,21 +159,26 @@ def classify_paths(
                     if path.suffix == ".so":
                         # extension modules can have 2 suffixes
                         name = BuildrootPath(path.stem).stem
-                        add_file_to_module(paths, name, "extension", path)
+                        add_file_to_module(paths, name, "extension", "files", path)
                     elif path.suffix == ".py":
                         name = path.stem
                         add_file_to_module(
-                            paths, name, "script", *pycached(path, python_version)
+                            paths, name, "script", "files", *pycached(path, python_version)
                         )
                     else:
                         paths["other"]["files"].append(path)
                 else:
-                    # this file is inside a dir, we classify that dir
+                    # this file is inside a dir, we add all dirs upwards until sitedir
                     index = path.parents.index(sitedir)
                     module_dir = path.parents[index - 1]
-                    add_file_to_module(paths, module_dir.name, "package", module_dir)
+                    for parent in list(path.parents)[:index]:  # no direct slice until Python 3.10
+                        add_file_to_module(paths, module_dir.name, "package", "dirs", parent)
+                    is_lang = False
                     if path.suffix == ".mo":
-                        add_lang_to_module(paths, module_dir.name, path)
+                        is_lang = add_lang_to_module(paths, module_dir.name, path)
+                    if not is_lang:
+                        path = pycached(path, python_version) if path.suffix == ".py" else [path]
+                        add_file_to_module(paths, module_dir.name, "package", "files", *path)
                 break
         else:
             if path.suffix == ".mo":
@@ -223,9 +228,7 @@ def generate_file_list(paths_dict, module_globs, include_others=False):
                     except KeyError:
                         pass
                     for module in modules[name]:
-                        if module["type"] == "package":
+                        files.update(f"%dir {p}" for p in module["dirs"])
-                            files.update(f"{p}/" for p in module["files"])
-                        else:
                         files.update(f"{p}" for p in module["files"])
                     done_modules.add(name)
                 done_globs.add(glob)

tests/python-django.spec

@@ -38,6 +38,9 @@ sed -i 's/asgiref ~= /asgiref >= /' setup.py
 
 
 %build
+# remove .po files (in ideal world, we would rebuild the .mo files first)
+find -name "*.po" | xargs rm -f
+
 %pyproject_wheel
 
 
@@ -45,9 +48,6 @@ sed -i 's/asgiref ~= /asgiref >= /' setup.py
 %pyproject_install
 %pyproject_save_files django
 
-# remove .po files
-find %{buildroot} -name "*.po" | xargs rm -f
-
 
 %check
 # Internal check if generated lang entries are same as