Add %pyproject_save_files macro

This macro save generates file section to %pyproject_files. It should
simplify %files section and allow to build by some automatic machinery

Supposed use case in Fedora:
    %install
    %pyproject_install
    %pyproject_save_files requests _requests

    %files -n python3-requests -f %{pyproject_files}
    %doc README.rst
    %license LICENSE

Automatic build of arbitrary packages (e.g. in Copr):
    %install
    %pyproject_install
    %pyproject_save_files * +bindir // save all modules with executables

    %files -n python3-requests -f %{pyproject_files}

Co-Authored-By: Miro Hrončok <miro@hroncok.cz>

README.md

@@ -130,6 +130,53 @@ in `%generate_buildrequires`. If not, you need to add:
 
     BuildRequires: python3dist(tox-current-env)
 
+
+Generating the %files section
+-----------------------------
+
+To generate the list of files in the `%files` section, you can use `%pyproject_save_files` after the `%pyproject_install` macro.
+It takes toplevel module names (i.e. the names used with `import` in Python) and stores paths for those modules and metadata for the package (dist-info directory) to a file stored at `%{pyproject_files}`.
+For example, if a package provides the modules `requests` and `_requests`, write:
+
+    %install
+    %pyproject_install
+    %pyproject_save_files requests _requests
+
+To add listed files to the `%files` section, use `%files -f %{pyproject_files}`.
+Note that you still need to add any documentation and license manually (for now).
+
+    %files -n python3-requests -f %{pyproject_files}
+    %doc README.rst
+    %license LICENSE
+
+You can use globs in the module names if listing them explicitly would be too tedious:
+
+    %install
+    %pyproject_install
+    %pyproject_save_files *requests
+
+In fully automated environmets, you can use the `*` glob to include all modules. In Fedora however, you should always use a more specific glob to avoid accidentally packaging unwanted files (for example, a top level module named `test`).
+
+Speaking about automated environments, it is possible to also list all executables in `/usr/bin` by adding a special `+bindir` argument.
+
+    %install
+    %pyproject_install
+    %pyproject_save_files * +bindir
+    
+    %files -n python3-requests -f %{pyproject_files}
+
+However, in Fedora packages, always list executables explicitly to avoid unintended collisions with other packages or accidental missing executables:
+
+    %install
+    %pyproject_install
+    %pyproject_save_files requests _requests
+    
+    %files -n python3-requests -f %{pyproject_files}
+    %doc README.rst
+    %license LICENSE
+    %{_bindir}/downloader
+
+
 Limitations
 -----------
 

macros.pyproject

@@ -1,5 +1,7 @@
 %_pyproject_wheeldir ./pyproject-macros-wheeldir
 
+%pyproject_files %{_builddir}/pyproject-files
+
 %pyproject_wheel() %{expand:\\\
 CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}" \\\
 %{__python3} -m pip wheel --wheel-dir %{_pyproject_wheeldir} --no-deps --use-pep517 --no-build-isolation --disable-pip-version-check --progress-bar off --verbose .
@@ -20,9 +22,23 @@ if [ -d %{buildroot}%{python3_sitearch} ]; then
 fi
 }
 
+
+%pyproject_save_files() %{expand:\\\
+%{__python3} %{_rpmconfigdir}/redhat/pyproject_save_files.py \\
+  --output "%{pyproject_files}" \\
+  --buildroot "%{buildroot}" \\
+  --sitelib "%{python3_sitelib}" \\
+  --sitearch "%{python3_sitearch}" \\
+  --bindir "%{_bindir}" \\
+  --python-version "%{python3_version}" \\
+  %{*}
+}
+
+
 %default_toxenv py%{python3_version_nodots}
 %toxenv %{default_toxenv}
 
+
 %pyproject_buildrequires(rxte:) %{expand:\\\
 %{-e:%{expand:%global toxenv %{-e*}}}
 echo 'python3-devel'
@@ -35,6 +51,7 @@ if [ -f %{__python3} ]; then
 fi
 }
 
+
 %tox(e:) %{expand:\\\
 TOX_TESTENV_PASSENV="${TOX_TESTENV_PASSENV:-*}" \\
 PATH="%{buildroot}%{_bindir}:$PATH" \\

pyproject-rpm-macros.spec

@@ -6,16 +6,27 @@ License:        MIT
 
 # Keep the version at zero and increment only release
 Version:        0
-Release:        13%{?dist}
+Release:        14%{?dist}
 
-Source0:        macros.pyproject
+# Macro files
-Source1:        pyproject_buildrequires.py
+Source001:      macros.pyproject
 
-Source8:        README.md
+# Implementation files
-Source9:        LICENSE
+Source101:      pyproject_buildrequires.py
+Source102:      pyproject_save_files.py
 
-Source10:       test_pyproject_buildrequires.py
+# Tests
-Source11:       testcases.yaml
+Source201:      test_pyproject_buildrequires.py
+Source202:      test_pyproject_save_files.py
+
+# Test data
+Source301:      pyproject_buildrequires_testcases.yaml
+Source302:      pyproject_save_files_test_data.yaml
+Source303:      test_RECORD
+
+# Metadata
+Source901:      README.md
+Source902:      LICENSE
 
 URL:            https://src.fedoraproject.org/rpms/pyproject-rpm-macros
 
@@ -72,21 +83,26 @@ mkdir -p %{buildroot}%{_rpmmacrodir}
 mkdir -p %{buildroot}%{_rpmconfigdir}/redhat
 install -m 644 macros.pyproject %{buildroot}%{_rpmmacrodir}/
 install -m 644 pyproject_buildrequires.py %{buildroot}%{_rpmconfigdir}/redhat/
+install -m 644 pyproject_save_files.py  %{buildroot}%{_rpmconfigdir}/redhat/
 
 %if %{with tests}
 %check
-%{__python3} -m pytest -vv
+%{python3} -m pytest -vv --doctest-modules
 %endif
 
 
 %files
 %{_rpmmacrodir}/macros.pyproject
 %{_rpmconfigdir}/redhat/pyproject_buildrequires.py
+%{_rpmconfigdir}/redhat/pyproject_save_files.py
 
 %doc README.md
 %license LICENSE
 
 %changelog
+* Wed Apr 15 2020 Patrik Kopkan <pkopkan@redhat.com> - 0-14
+- Add %%pyproject_save_file macro for generating file section
+
 * Mon Mar 02 2020 Miro Hrončok <mhroncok@redhat.com> - 0-13
 - Tox dependency generator: Handle deps read in from a text file (#1808601)
 

pyproject_save_files.py

@@ -0,0 +1,411 @@
+import argparse
+import csv
+import fnmatch
+import os
+import warnings
+
+from collections import defaultdict
+from pathlib import PosixPath, PurePosixPath
+
+
+class BuildrootPath(PurePosixPath):
+    """
+    This path represents a path in a buildroot.
+    When absolute, it is "relative" to a buildroot.
+
+    E.g. /usr/lib means %{buildroot}/usr/lib
+    The object carries no buildroot information.
+    """
+
+    @staticmethod
+    def from_real(realpath, *, root):
+        """
+        For a given real disk path, return a BuildrootPath in the given root.
+
+        For example::
+
+            >>> BuildrootPath.from_real(PosixPath('/tmp/buildroot/foo'), root=PosixPath('/tmp/buildroot'))
+            BuildrootPath('/foo')
+        """
+        return BuildrootPath("/") / realpath.relative_to(root)
+
+    def to_real(self, root):
+        """
+        Return a real PosixPath in the given root
+
+        For example::
+
+            >>> BuildrootPath('/foo').to_real(PosixPath('/tmp/buildroot'))
+            PosixPath('/tmp/buildroot/foo')
+        """
+        return root / self.relative_to("/")
+
+    def normpath(self):
+        """
+        Normalize all the potential /../ parts of the path without touching real files.
+
+        PurePaths don't have .resolve().
+        Paths have .resolve() but it touches real files.
+        This is an alternative. It assumes there are no symbolic links.
+
+        Example:
+
+            >>> BuildrootPath('/usr/lib/python/../pypy').normpath()
+            BuildrootPath('/usr/lib/pypy')
+        """
+        return type(self)(os.path.normpath(self))
+
+
+def locate_record(root, sitedirs):
+    """
+    Find a RECORD file in the given root.
+    sitedirs are BuildrootPaths.
+    Only RECORDs in dist-info dirs inside sitedirs are considered.
+    There can only be one RECORD file.
+
+    Returns a PosixPath of the RECORD file.
+    """
+    records = []
+    for sitedir in sitedirs:
+        records.extend(sitedir.to_real(root).glob("*.dist-info/RECORD"))
+
+    sitedirs_text = ", ".join(str(p) for p in sitedirs)
+    if len(records) == 0:
+        raise FileNotFoundError(f"There is no *.dist-info/RECORD in {sitedirs_text}")
+    if len(records) > 1:
+        raise FileExistsError(f"Multiple *.dist-info directories in {sitedirs_text}")
+
+    return records[0]
+
+
+def read_record(record_path):
+    """
+    A generator yielding individual RECORD triplets.
+
+    https://www.python.org/dev/peps/pep-0376/#record
+
+    The triplet is str-path, hash, size -- the last two optional.
+    We will later care only for the paths anyway.
+
+    Example:
+
+        >>> g = read_record(PosixPath('./test_RECORD'))
+        >>> next(g)
+        ['../../../bin/__pycache__/tldr.cpython-....pyc', '', '']
+        >>> next(g)
+        ['../../../bin/tldr', 'sha256=...', '12766']
+        >>> next(g)
+        ['../../../bin/tldr.py', 'sha256=...', '12766']
+    """
+    with open(record_path, newline="", encoding="utf-8") as f:
+        yield from csv.reader(
+            f, delimiter=",", quotechar='"', lineterminator=os.linesep
+        )
+
+
+def parse_record(record_path, record_content):
+    """
+    Returns a generator with BuildrootPaths parsed from record_content
+
+    params:
+    record_path: RECORD BuildrootPath
+    record_content: list of RECORD triplets
+                    first item is a str-path relative to directory where dist-info directory is
+                    (it can also be absolute according to the standard, but not from pip)
+
+    Examples:
+
+        >>> next(parse_record(BuildrootPath('/usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/RECORD'),
+        ...                   [('requests/sessions.py', 'sha256=xxx', '666'), ...]))
+        BuildrootPath('/usr/lib/python3.7/site-packages/requests/sessions.py')
+
+        >>> next(parse_record(BuildrootPath('/usr/lib/python3.7/site-packages/tldr-0.5.dist-info/RECORD'),
+        ...                   [('../../../bin/tldr', 'sha256=yyy', '777'), ...]))
+        BuildrootPath('/usr/bin/tldr')
+    """
+    sitedir = record_path.parent.parent  # trough the dist-info directory
+    # / with absolute right operand will remove the left operand
+    # any .. parts are resolved via normpath
+    return ((sitedir / row[0]).normpath() for row in record_content)
+
+
+def pycached(script, python_version):
+    """
+    For a script BuildrootPath, return a list with that path and its bytecode glob.
+    Like the %pycached macro.
+
+    The glob is represented as a BuildrootPath.
+
+    Examples:
+
+        >>> pycached(BuildrootPath('/whatever/bar.py'), '3.8')
+        [BuildrootPath('/whatever/bar.py'), BuildrootPath('/whatever/__pycache__/bar.cpython-38{,.opt-?}.pyc')]
+
+        >>> pycached(BuildrootPath('/opt/python3.10/foo.py'), '3.10')
+        [BuildrootPath('/opt/python3.10/foo.py'), BuildrootPath('/opt/python3.10/__pycache__/foo.cpython-310{,.opt-?}.pyc')]
+    """
+    assert script.suffix == ".py"
+    pyver = "".join(python_version.split(".")[:2])
+    pycname = f"{script.stem}.cpython-{pyver}{{,.opt-?}}.pyc"
+    pyc = script.parent / "__pycache__" / pycname
+    return [script, pyc]
+
+
+def add_file_to_module(paths, module_name, module_type, *files):
+    """
+    Helper procedure, adds given files to the module_name of a given module_type
+    """
+    for module in paths["modules"][module_name]:
+        if module["type"] == module_type:
+            if files[0] not in module["files"]:
+                module["files"].extend(files)
+            break
+    else:
+        paths["modules"][module_name].append(
+            {"type": module_type, "files": list(files)}
+        )
+
+
+def classify_paths(
+    record_path, parsed_record_content, sitedirs, bindir, python_version
+):
+    """
+    For each BuildrootPath in parsed_record_content classify it to a dict structure
+    that allows to filter the files for the %files section easier.
+
+    For the dict structure, look at the beginning of this function's code.
+
+    Each "module" is a dict with "type" ("package", "script", "extension") and "files".
+    """
+    distinfo = record_path.parent
+    paths = {
+        "metadata": {
+            "files": [],  # regular %file entries with dist-info content
+            "dirs": [distinfo],  # %dir %file entries with dist-info directory
+            "docs": [],  # to be used once there is upstream way to recognize READMEs
+            "licenses": [],  # to be used once there is upstream way to recognize LICENSEs
+        },
+        "modules": defaultdict(list),  # each importable module (directory, .py, .so)
+        "executables": {"files": []},  # regular %file entries in %{_bindir}
+        "other": {"files": []},  # regular %file entries we could not parse :(
+    }
+
+    # In RECORDs generated by pip, there are no directories, only files.
+    # The example RECORD from PEP 376 does not contain directories either.
+    # Hence, we'll only assume files, but TODO get it officially documented.
+    for path in parsed_record_content:
+        if path.suffix == ".pyc":
+            # we handle bytecode separately
+            continue
+
+        if path.parent == distinfo:
+            # TODO is this a license/documentation?
+            paths["metadata"]["files"].append(path)
+            continue
+
+        if path.parent == bindir:
+            paths["executables"]["files"].append(path)
+            continue
+
+        for sitedir in sitedirs:
+            if sitedir in path.parents:
+                if path.parent == sitedir:
+                    if path.suffix == ".so":
+                        # extension modules can have 2 suffixes
+                        name = BuildrootPath(path.stem).stem
+                        add_file_to_module(paths, name, "extension", path)
+                    elif path.suffix == ".py":
+                        name = path.stem
+                        add_file_to_module(
+                            paths, name, "script", *pycached(path, python_version)
+                        )
+                    else:
+                        # TODO classify .pth files
+                        warnings.warn(f"Unrecognized file: {path}")
+                        paths["other"]["files"].append(path)
+                else:
+                    # this file is inside a dir, we classify that dir
+                    index = path.parents.index(sitedir)
+                    module_dir = path.parents[index - 1]
+                    add_file_to_module(paths, module_dir.name, "package", module_dir)
+                break
+        else:
+            warnings.warn(f"Unrecognized file: {path}")
+            paths["other"]["files"].append(path)
+
+    return paths
+
+
+def generate_file_list(paths_dict, module_globs, include_executables=False):
+    """
+    This function takes the classified paths_dict and turns it into lines
+    for the %files section. Returns list with text lines, no Path objects.
+
+    Only includes files from modules that match module_globs, metadata and
+    optional executables.
+
+    It asserts that all globs match at least one module, raises ValueError otherwise.
+    Multiple globs matching identical module(s) are OK.
+    """
+    files = set()
+
+    if include_executables:
+        files.update(f"{p}" for p in paths_dict["executables"]["files"])
+
+    files.update(f"{p}" for p in paths_dict["metadata"]["files"])
+    for macro in "dir", "doc", "license":
+        files.update(f"%{macro} {p}" for p in paths_dict["metadata"][f"{macro}s"])
+
+    modules = paths_dict["modules"]
+    done_modules = set()
+    done_globs = set()
+
+    for glob in module_globs:
+        for name in modules:
+            if fnmatch.fnmatchcase(name, glob):
+                if name not in done_modules:
+                    for module in modules[name]:
+                        if module["type"] == "package":
+                            files.update(f"{p}/" for p in module["files"])
+                        else:
+                            files.update(f"{p}" for p in module["files"])
+                    done_modules.add(name)
+                done_globs.add(glob)
+
+    missed = module_globs - done_globs
+    if missed:
+        missed_text = ", ".join(sorted(missed))
+        raise ValueError(f"Globs did not match any module: {missed_text}")
+
+    return sorted(files)
+
+
+def parse_varargs(varargs):
+    """
+    Parse varargs from the %pyproject_save_files macro
+
+    Arguments starting with + are treated as a flags, everything else is a glob
+
+    Returns as set of globs, boolean flag whether to include executables from bindir
+
+    Raises ValueError for unknown flags and globs with dots (namespace packages).
+
+    Good examples:
+
+        >>> parse_varargs(['*'])
+        ({'*'}, False)
+
+        >>> mods, bindir = parse_varargs(['requests*', 'kerberos', '+bindir'])
+        >>> bindir
+        True
+        >>> sorted(mods)
+        ['kerberos', 'requests*']
+
+        >>> mods, bindir = parse_varargs(['tldr', 'tensorf*'])
+        >>> bindir
+        False
+        >>> sorted(mods)
+        ['tensorf*', 'tldr']
+
+        >>> parse_varargs(['+bindir'])
+        (set(), True)
+
+    Bad examples:
+
+        >>> parse_varargs(['+kinkdir'])
+        Traceback (most recent call last):
+          ...
+        ValueError: Invalid argument: +kinkdir
+
+        >>> parse_varargs(['good', '+bad', '*ugly*'])
+        Traceback (most recent call last):
+          ...
+        ValueError: Invalid argument: +bad
+
+        >>> parse_varargs(['+bad', 'my.bad'])
+        Traceback (most recent call last):
+          ...
+        ValueError: Invalid argument: +bad
+
+        >>> parse_varargs(['mod', 'mod.*'])
+        Traceback (most recent call last):
+          ...
+        ValueError: Attempted to use a namespaced package with dot in the glob: mod.*. ...
+
+        >>> parse_varargs(['my.bad', '+bad'])
+        Traceback (most recent call last):
+          ...
+        ValueError: Attempted to use a namespaced package with dot in the glob: my.bad. ...
+    """
+    include_bindir = False
+    globs = set()
+
+    for arg in varargs:
+        if arg.startswith("+"):
+            if arg == "+bindir":
+                include_bindir = True
+            else:
+                raise ValueError(f"Invalid argument: {arg}")
+        elif "." in arg:
+            top, *_ = arg.partition(".")
+            msg = (
+                f"Attempted to use a namespaced package with dot in the glob: {arg}. "
+                f"That is not (yet) supported. Use {top} instead and/or file a Bugzilla explaining your use case."
+            )
+            raise ValueError(msg)
+        else:
+            globs.add(arg)
+
+    return globs, include_bindir
+
+
+def pyproject_save_files(buildroot, sitelib, sitearch, bindir, python_version, varargs):
+    """
+    Takes arguments from the %{pyproject_save_files} macro
+
+    Returns list of paths for the %files section
+    """
+    # On 32 bit architectures, sitelib equals to sitearch
+    # This saves us browsing one directory twice
+    sitedirs = sorted({sitelib, sitearch})
+
+    globs, include_bindir = parse_varargs(varargs)
+    record_path_real = locate_record(buildroot, sitedirs)
+    record_path = BuildrootPath.from_real(record_path_real, root=buildroot)
+    parsed_record = parse_record(record_path, read_record(record_path_real))
+
+    paths_dict = classify_paths(
+        record_path, parsed_record, sitedirs, bindir, python_version
+    )
+    return generate_file_list(paths_dict, globs, include_bindir)
+
+
+def main(cli_args):
+    file_section = pyproject_save_files(
+        cli_args.buildroot,
+        cli_args.sitelib,
+        cli_args.sitearch,
+        cli_args.bindir,
+        cli_args.python_version,
+        cli_args.varargs,
+    )
+
+    cli_args.output.write_text("\n".join(file_section) + "\n", encoding="utf-8")
+
+
+def argparser():
+    parser = argparse.ArgumentParser()
+    r = parser.add_argument_group("required arguments")
+    r.add_argument("--output", type=PosixPath, required=True)
+    r.add_argument("--buildroot", type=PosixPath, required=True)
+    r.add_argument("--sitelib", type=BuildrootPath, required=True)
+    r.add_argument("--sitearch", type=BuildrootPath, required=True)
+    r.add_argument("--bindir", type=BuildrootPath, required=True)
+    r.add_argument("--python-version", type=str, required=True)
+    parser.add_argument("varargs", nargs="+")
+    return parser
+
+
+if __name__ == "__main__":
+    cli_args = argparser().parse_args()
+    main(cli_args)

pyproject_save_files_test_data.yaml

@@ -0,0 +1,321 @@
+classified:
+  kerberos:
+    executables:
+      files: []
+    metadata:
+      dirs:
+      - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info
+      docs: []
+      files:
+      - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/INSTALLER
+      - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/METADATA
+      - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/RECORD
+      - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/WHEEL
+      - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/top_level.txt
+      licenses: []
+    modules:
+      kerberos:
+      - files:
+        - /usr/lib64/python3.7/site-packages/kerberos.cpython-37m-x86_64-linux-gnu.so
+        type: extension
+    other:
+      files: []
+  mistune:
+    executables:
+      files: []
+    metadata:
+      dirs:
+      - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info
+      docs: []
+      files:
+      - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/INSTALLER
+      - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/LICENSE
+      - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/METADATA
+      - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/RECORD
+      - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/WHEEL
+      - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/top_level.txt
+      licenses: []
+    modules:
+      mistune:
+      - files:
+        - /usr/lib64/python3.7/site-packages/mistune.py
+        - /usr/lib64/python3.7/site-packages/__pycache__/mistune.cpython-37{,.opt-?}.pyc
+        type: script
+      - files:
+        - /usr/lib64/python3.7/site-packages/mistune.cpython-37m-x86_64-linux-gnu.so
+        type: extension
+    other:
+      files: []
+  requests:
+    executables:
+      files: []
+    metadata:
+      dirs:
+      - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info
+      docs: []
+      files:
+      - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/INSTALLER
+      - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/LICENSE
+      - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/METADATA
+      - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/RECORD
+      - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/WHEEL
+      - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/top_level.txt
+      licenses: []
+    modules:
+      requests:
+      - files:
+        - /usr/lib/python3.7/site-packages/requests
+        type: package
+    other:
+      files: []
+  tensorflow:
+    executables:
+      files:
+      - /usr/bin/estimator_ckpt_converter
+      - /usr/bin/saved_model_cli
+      - /usr/bin/tensorboard
+      - /usr/bin/tf_upgrade_v2
+      - /usr/bin/tflite_convert
+      - /usr/bin/toco
+      - /usr/bin/toco_from_protos
+    metadata:
+      dirs:
+      - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info
+      docs: []
+      files:
+      - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/INSTALLER
+      - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/METADATA
+      - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/RECORD
+      - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/WHEEL
+      - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/entry_points.txt
+      - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/top_level.txt
+      licenses: []
+    modules:
+      tensorflow:
+      - files:
+        - /usr/lib64/python3.7/site-packages/tensorflow
+        type: package
+      tensorflow_core:
+      - files:
+        - /usr/lib64/python3.7/site-packages/tensorflow_core
+        type: package
+      - files:
+        - /usr/lib/python3.7/site-packages/tensorflow_core
+        type: package
+    other:
+      files: []
+  tldr:
+    executables:
+      files:
+      - /usr/bin/tldr
+      - /usr/bin/tldr.py
+    metadata:
+      dirs:
+      - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info
+      docs: []
+      files:
+      - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/INSTALLER
+      - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/LICENSE
+      - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/METADATA
+      - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/RECORD
+      - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/WHEEL
+      - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/top_level.txt
+      licenses: []
+    modules:
+      tldr:
+      - files:
+        - /usr/lib/python3.7/site-packages/tldr.py
+        - /usr/lib/python3.7/site-packages/__pycache__/tldr.cpython-37{,.opt-?}.pyc
+        type: script
+    other:
+      files: []
+
+dumped:
+- - tensorflow
+  - tensorflow*
+  - - '%dir /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info'
+    - /usr/bin/estimator_ckpt_converter
+    - /usr/bin/saved_model_cli
+    - /usr/bin/tensorboard
+    - /usr/bin/tf_upgrade_v2
+    - /usr/bin/tflite_convert
+    - /usr/bin/toco
+    - /usr/bin/toco_from_protos
+    - /usr/lib/python3.7/site-packages/tensorflow_core/
+    - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/INSTALLER
+    - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/METADATA
+    - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/RECORD
+    - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/WHEEL
+    - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/entry_points.txt
+    - /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/top_level.txt
+    - /usr/lib64/python3.7/site-packages/tensorflow/
+    - /usr/lib64/python3.7/site-packages/tensorflow_core/
+- - kerberos
+  - ke?ber*
+  - - '%dir /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info'
+    - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/INSTALLER
+    - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/METADATA
+    - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/RECORD
+    - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/WHEEL
+    - /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/top_level.txt
+    - /usr/lib64/python3.7/site-packages/kerberos.cpython-37m-x86_64-linux-gnu.so
+- - requests
+  - requests
+  - - '%dir /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info'
+    - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/INSTALLER
+    - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/LICENSE
+    - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/METADATA
+    - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/RECORD
+    - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/WHEEL
+    - /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/top_level.txt
+    - /usr/lib/python3.7/site-packages/requests/
+- - tldr
+  - '*'
+  - - '%dir /usr/lib/python3.7/site-packages/tldr-0.5.dist-info'
+    - /usr/bin/tldr
+    - /usr/bin/tldr.py
+    - /usr/lib/python3.7/site-packages/__pycache__/tldr.cpython-37{,.opt-?}.pyc
+    - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/INSTALLER
+    - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/LICENSE
+    - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/METADATA
+    - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/RECORD
+    - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/WHEEL
+    - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/top_level.txt
+    - /usr/lib/python3.7/site-packages/tldr.py
+- - mistune
+  - mistune
+  - - '%dir /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info'
+    - /usr/lib64/python3.7/site-packages/__pycache__/mistune.cpython-37{,.opt-?}.pyc
+    - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/INSTALLER
+    - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/LICENSE
+    - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/METADATA
+    - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/RECORD
+    - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/WHEEL
+    - /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/top_level.txt
+    - /usr/lib64/python3.7/site-packages/mistune.cpython-37m-x86_64-linux-gnu.so
+    - /usr/lib64/python3.7/site-packages/mistune.py
+
+records:
+  kerberos:
+    path: /usr/lib64/python3.7/site-packages/kerberos-1.3.0.dist-info/RECORD
+    content: |
+      kerberos-1.3.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+      kerberos-1.3.0.dist-info/METADATA,sha256=ZLRjtEOsFUjO5gOL8XEZA9m-V1ayUeNz6ehNvCHf-00,5085
+      kerberos-1.3.0.dist-info/RECORD,,
+      kerberos-1.3.0.dist-info/WHEEL,sha256=ohybRue5bPR5MQUSq7c6AGl-iIAd0MXt_sfyYTZ1Rq8,104
+      kerberos-1.3.0.dist-info/top_level.txt,sha256=b07dCflqvOAEjUkeef-UGnR4feBslpNBJof69O7oA2s,9
+      kerberos.cpython-37m-x86_64-linux-gnu.so,sha256=EYqfkWOzHrj0kISjEAXCtGb7AWs4ZPLK8oWo32qwnQU,181784
+
+  requests:
+    path: /usr/lib/python3.7/site-packages/requests-2.22.0.dist-info/RECORD
+    content: |
+      requests-2.22.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+      requests-2.22.0.dist-info/LICENSE,sha256=vkGrrCxA-FMDB-jRcsWQtHb0pIi8amj43le3z2R4Zoc,582
+      requests-2.22.0.dist-info/METADATA,sha256=sJ1ZdIgF0uoV9U58VVoEZv1QTyMCpmc2MQnbkob3nsE,5523
+      requests-2.22.0.dist-info/RECORD,,
+      requests-2.22.0.dist-info/WHEEL,sha256=h_aVn5OB2IERUjMbi2pucmR_zzWJtk303YXvhh60NJ8,110
+      requests-2.22.0.dist-info/top_level.txt,sha256=fMSVmHfb5rbGOo6xv-O_tUX6j-WyixssE-SnwcDRxNQ,9
+      requests/__init__.py,sha256=PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA,3921
+      requests/__pycache__/__init__.cpython-37.pyc,,
+      requests/__pycache__/__version__.cpython-37.pyc,,
+      requests/__pycache__/_internal_utils.cpython-37.pyc,,
+      requests/__pycache__/adapters.cpython-37.pyc,,
+      requests/__pycache__/api.cpython-37.pyc,,
+      requests/__pycache__/auth.cpython-37.pyc,,
+      requests/__pycache__/certs.cpython-37.pyc,,
+      requests/__pycache__/compat.cpython-37.pyc,,
+      requests/__pycache__/cookies.cpython-37.pyc,,
+      requests/__pycache__/exceptions.cpython-37.pyc,,
+      requests/__pycache__/help.cpython-37.pyc,,
+      requests/__pycache__/hooks.cpython-37.pyc,,
+      requests/__pycache__/models.cpython-37.pyc,,
+      requests/__pycache__/packages.cpython-37.pyc,,
+      requests/__pycache__/sessions.cpython-37.pyc,,
+      requests/__pycache__/status_codes.cpython-37.pyc,,
+      requests/__pycache__/structures.cpython-37.pyc,,
+      requests/__pycache__/utils.cpython-37.pyc,,
+      requests/__version__.py,sha256=Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc,436
+      requests/_internal_utils.py,sha256=Zx3PnEUccyfsB-ie11nZVAW8qClJy0gx1qNME7rgT18,1096
+      requests/adapters.py,sha256=WelSM1BCQXdbjEuDsBxqKDADeY8BHmxlrwbNnLN2rr4,21344
+      requests/api.py,sha256=fbUo11QoLOoNgWU6FfvNz8vMj9bE_cMmICXBa7TZHJs,6271
+      requests/auth.py,sha256=QB2-cSUj1jrvWZfPXttsZpyAacQgtKLVk14vQW9TpSE,10206
+      requests/certs.py,sha256=dOB5rV2DZ13dEhq9BUa_4hd5kAqg59e_zUZB00faYz8,453
+      requests/compat.py,sha256=FVIeTOniQMHQkeE2JdJvar3OZ-b4IFh8aNezIn45zws,1678
+      requests/cookies.py,sha256=Y-bKX6TvW3FnYlE6Au0SXtVVWcaNdFvuAwQxw-G0iTI,18430
+      requests/exceptions.py,sha256=Q8YeWWxiHHXhkEynLpMgC_6_r_ZTYw2aITs9wCSAZNY,3185
+      requests/help.py,sha256=lLcBtKAar8T6T78e9Tc4Zfd_EEJFhntxgib1JHNctEI,3515
+      requests/hooks.py,sha256=QReGyy0bRcr5rkwCuObNakbYsc7EkiKeBwG4qHekr2Q,757
+      requests/models.py,sha256=bce6oORR26SY-dVPaqMpdBunD1zXzrgMSlH6jhfvuRA,34210
+      requests/packages.py,sha256=Q2rF0L5mc3wQAvc6q_lAVtPTDOaOeFgD-7kWSQLkjEQ,542
+      requests/sessions.py,sha256=DjbCotDW6xSAaBsjbW-L8l4N0UcwmrxVNgSrZgIjGWM,29332
+      requests/status_codes.py,sha256=XWlcpBjbCtq9sSqpH9_KKxgnLTf9Z__wCWolq21ySlg,4129
+      requests/structures.py,sha256=zoP8qly2Jak5e89HwpqjN1z2diztI-_gaqts1raJJBc,2981
+      requests/utils.py,sha256=LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A,30049
+
+  tensorflow:
+    path: /usr/lib64/python3.7/site-packages/tensorflow-2.1.0.dist-info/RECORD
+    content: |
+      ../../../bin/estimator_ckpt_converter,sha256=wXSBu157fEC7ahk8_qKuOequwuS_dIND41P1-xvyO0Y,263
+      ../../../bin/saved_model_cli,sha256=_LrDo0k33aBewRQW0IIdfA_JZLtm_eG7w8xmRKuANhA,238
+      ../../../bin/tensorboard,sha256=BTDa2F7MtGDw-qp5vQ5O9vcukALo06q9rfc3vkLHKYI,223
+      ../../../bin/tf_upgrade_v2,sha256=0ti0aqcaL7bn5QxVdan52Uwm4_m5GsYBT3zzWLULK8Q,248
+      ../../../bin/tflite_convert,sha256=LEBjgfefYbh8cUPJF8GGRtR7UcEVSIly68dx_qijKmE,236
+      ../../../bin/toco,sha256=LEBjgfefYbh8cUPJF8GGRtR7UcEVSIly68dx_qijKmE,236
+      ../../../bin/toco_from_protos,sha256=g1q1vsILK1ywu_BDkfvcmhHVO1AUM9dcw6Ru4hG4jLY,243
+      ../../../lib/python3.7/site-packages/tensorflow_core/include/tensorflow/core/common_runtime/allocator_retry.h,sha256=oDXeOvPmbf_zjO-gP30mux7mwmIPUXI5q7tFcORNKaw,2193
+      tensorflow-2.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+      tensorflow-2.1.0.dist-info/METADATA,sha256=g5W3QfLBbDHaqVmDvLXQIV2KfDFQe9zssq4fKz-Rah4,2859
+      tensorflow-2.1.0.dist-info/RECORD,,
+      tensorflow-2.1.0.dist-info/WHEEL,sha256=Wzr7ustLd3r5KN3AVdAm7EECHYuYQiejXMdpS694wLU,112
+      tensorflow-2.1.0.dist-info/entry_points.txt,sha256=6vsXW21b4Wj04oedqgNIbT1JLH-gSEunfyCWBeH243k,469
+      tensorflow-2.1.0.dist-info/top_level.txt,sha256=T9bRmG0NffG9XO9QPlpgQvIO7Em0gI1nAGFcs1GuHz0,27
+      tensorflow/__init__.py,sha256=W1FNzqxpmWEDlpxZdh395Jy5QY9U-s1mSgAKjztxHJM,4112
+      tensorflow/__pycache__/__init__.cpython-37.pyc,,
+      tensorflow_core/__init__.py,sha256=QYTltqjBzzTFwghHywspgxrUWap1uG5ofHscvUBvYFw,21681
+      tensorflow_core/__pycache__/__init__.cpython-37.pyc,,
+      tensorflow_core/_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+      tensorflow_core/_api/__pycache__/__init__.cpython-37.pyc,,
+      tensorflow_core/_api/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+      tensorflow_core/include/external/aws/aws-cpp-sdk-s3/include/aws/s3/model/InventoryConfiguration.h,sha256=k0DaoAWu79zxtS5kgBJoupQzyv5P1BKeb46CtaTS1as,10134
+      tensorflow_core/libtensorflow_framework.so.2,sha256=lVxbMz0TpmLqIWgMmfyZlZDxYzIQTFYUXfrTgw3LAFM,35339272
+      tensorflow_core/lite/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+      tensorflow_core/lite/__pycache__/__init__.cpython-37.pyc,,
+      tensorflow_core/lite/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+      tensorflow_core/lite/experimental/__pycache__/__init__.cpython-37.pyc,,
+      tensorflow_core/python/autograph/converters/arg_defaults.py,sha256=BC_45wSJNLEfRphB8Px148pKugW24mysmuDRdNt0oFc,3296
+      tensorflow_core/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+      tensorflow_core/tools/__pycache__/__init__.cpython-37.pyc,,
+      tensorflow_core/tools/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+      tensorflow_core/tools/docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+      tensorflow_core/tools/docs/__pycache__/__init__.cpython-37.pyc,,
+      tensorflow_core/tools/pip_package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+      tensorflow_core/tools/pip_package/__pycache__/__init__.cpython-37.pyc,,
+      tensorflow_core/tools/pip_package/setup.py,sha256=8gQDKPAps4bfO0mxijQzZpyaGm59WrbvMMN1kQmULGI,11261
+
+  tldr:
+    path: /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/RECORD
+    content: |
+      ../../../bin/__pycache__/tldr.cpython-37.pyc,,
+      ../../../bin/tldr,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766
+      ../../../bin/tldr.py,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766
+      __pycache__/tldr.cpython-37.pyc,,
+      tldr-0.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+      tldr-0.5.dist-info/LICENSE,sha256=q7quAfjDWCYKC_WRk_uaP6d2wwVpOpVjUSkv8l6H7xI,1075
+      tldr-0.5.dist-info/METADATA,sha256=AN5nYUVxo_zkVaMGKu34YDWWif84oA6uxKmTab213vM,3850
+      tldr-0.5.dist-info/RECORD,,
+      tldr-0.5.dist-info/WHEEL,sha256=S8S5VL-stOTSZDYxHyf0KP7eds0J72qrK0Evu3TfyAY,92
+      tldr-0.5.dist-info/top_level.txt,sha256=xHSI9WD6Y-_hONbi2b_9RIn9oiO7RBGHU3A8geJq3mI,5
+      tldr.py,sha256=aJlA3tIz4QYYy8e7DZUhPyLCqTwnfFjA7Nubwm9bPe0,12779
+
+  mistune:
+    path: /usr/lib64/python3.7/site-packages/mistune-0.8.3.dist-info/RECORD
+    content: |
+      __pycache__/mistune.cpython-37.pyc,,
+      mistune-0.8.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+      mistune-0.8.3.dist-info/LICENSE,sha256=DFJZw90KfEb0g1IhZF9ioGOMm5-qAq8IZ26AaeH_lks,1482
+      mistune-0.8.3.dist-info/METADATA,sha256=em5e2pPXINCvklOX9dEbh14XJjXyKIxv4ws7Gqvliyc,8390
+      mistune-0.8.3.dist-info/RECORD,,
+      mistune-0.8.3.dist-info/WHEEL,sha256=VhDzRVkjIQCHaI8B-spV-f4VqUney2V8tpBJUi2FE_Q,103
+      mistune-0.8.3.dist-info/top_level.txt,sha256=tjJTM65kAdwKAJ2mA769tnDGYYlfR8pqRsobKjVEfcg,8
+      mistune.cpython-37m-x86_64-linux-gnu.so,sha256=tclP68lWttoR8qJMooacURG12Q0Ij3I5yzbFo7xsNPI,3959336
+      mistune.py,sha256=1CU_A107jEtx78PjEtq6c4ZHtKdDonRSJODPtwIReVc,35484

test_RECORD

@@ -0,0 +1,11 @@
+../../../bin/__pycache__/tldr.cpython-37.pyc,,
+../../../bin/tldr,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766
+../../../bin/tldr.py,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766
+__pycache__/tldr.cpython-37.pyc,,
+tldr-0.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+tldr-0.5.dist-info/LICENSE,sha256=q7quAfjDWCYKC_WRk_uaP6d2wwVpOpVjUSkv8l6H7xI,1075
+tldr-0.5.dist-info/METADATA,sha256=AN5nYUVxo_zkVaMGKu34YDWWif84oA6uxKmTab213vM,3850
+tldr-0.5.dist-info/RECORD,,
+tldr-0.5.dist-info/WHEEL,sha256=S8S5VL-stOTSZDYxHyf0KP7eds0J72qrK0Evu3TfyAY,92
+tldr-0.5.dist-info/top_level.txt,sha256=xHSI9WD6Y-_hONbi2b_9RIn9oiO7RBGHU3A8geJq3mI,5
+tldr.py,sha256=aJlA3tIz4QYYy8e7DZUhPyLCqTwnfFjA7Nubwm9bPe0,12779

test_pyproject_buildrequires.py

@@ -12,7 +12,7 @@ except ImportError:
     import importlib_metadata
 
 testcases = {}
-with Path(__file__).parent.joinpath('testcases.yaml').open() as f:
+with Path(__file__).parent.joinpath('pyproject_buildrequires_testcases.yaml').open() as f:
     testcases = yaml.safe_load(f)
 
 

test_pyproject_save_files.py

@@ -0,0 +1,266 @@
+import pytest
+import yaml
+
+from pathlib import Path
+from pprint import pprint
+
+from pyproject_save_files import argparser, generate_file_list, main
+from pyproject_save_files import locate_record, parse_record, read_record
+from pyproject_save_files import BuildrootPath
+
+
+DIR = Path(__file__).parent
+BINDIR = BuildrootPath("/usr/bin")
+SITELIB = BuildrootPath("/usr/lib/python3.7/site-packages")
+SITEARCH = BuildrootPath("/usr/lib64/python3.7/site-packages")
+
+yaml_file = DIR / "pyproject_save_files_test_data.yaml"
+yaml_data = yaml.safe_load(yaml_file.read_text())
+EXPECTED_DICT = yaml_data["classified"]
+EXPECTED_FILES = yaml_data["dumped"]
+TEST_RECORDS = yaml_data["records"]
+
+
+def create_root(tmp_path, *records):
+    r"""
+    Create mock buildroot in tmp_path
+
+    parameters:
+    tmp_path: path where buildroot should be created
+    records: dicts with:
+      path: expected path found in buildroot
+      content: string content of the file
+
+    Example:
+
+        >>> record = {'path': '/usr/lib/python/tldr-0.5.dist-info/RECORD', 'content': '__pycache__/tldr.cpython-37.pyc,,\n...'}
+        >>> create_root(Path('tmp'), record)
+        PosixPath('tmp/buildroot')
+
+    The example creates ./tmp/buildroot/usr/lib/python/tldr-0.5.dist-info/RECORD with the content.
+
+        >>> import shutil
+        >>> shutil.rmtree(Path('./tmp'))
+    """
+    buildroot = tmp_path / "buildroot"
+    for record in records:
+        dest = buildroot / Path(record["path"]).relative_to("/")
+        dest.parent.mkdir(parents=True)
+        dest.write_text(record["content"])
+    return buildroot
+
+
+@pytest.fixture
+def tldr_root(tmp_path):
+    return create_root(tmp_path, TEST_RECORDS["tldr"])
+
+
+@pytest.fixture
+def output(tmp_path):
+    return tmp_path / "pyproject_files"
+
+
+def test_locate_record_good(tmp_path):
+    sitedir = tmp_path / "ha/ha/ha/site-packages"
+    distinfo = sitedir / "foo-0.6.dist-info"
+    distinfo.mkdir(parents=True)
+    record = distinfo / "RECORD"
+    record.write_text("\n")
+    sitedir = BuildrootPath.from_real(sitedir, root=tmp_path)
+    assert locate_record(tmp_path, {sitedir}) == record
+
+
+def test_locate_record_missing(tmp_path):
+    sitedir = tmp_path / "ha/ha/ha/site-packages"
+    distinfo = sitedir / "foo-0.6.dist-info"
+    distinfo.mkdir(parents=True)
+    sitedir = BuildrootPath.from_real(sitedir, root=tmp_path)
+    with pytest.raises(FileNotFoundError):
+        locate_record(tmp_path, {sitedir})
+
+
+def test_locate_record_misplaced(tmp_path):
+    sitedir = tmp_path / "ha/ha/ha/site-packages"
+    fakedir = tmp_path / "no/no/no/site-packages"
+    distinfo = fakedir / "foo-0.6.dist-info"
+    distinfo.mkdir(parents=True)
+    record = distinfo / "RECORD"
+    record.write_text("\n")
+    sitedir = BuildrootPath.from_real(sitedir, root=tmp_path)
+    with pytest.raises(FileNotFoundError):
+        locate_record(tmp_path, {sitedir})
+
+
+def test_locate_record_two_packages(tmp_path):
+    sitedir = tmp_path / "ha/ha/ha/site-packages"
+    for name in "foo-0.6.dist-info", "bar-1.8.dist-info":
+        distinfo = sitedir / name
+        distinfo.mkdir(parents=True)
+        record = distinfo / "RECORD"
+        record.write_text("\n")
+    sitedir = BuildrootPath.from_real(sitedir, root=tmp_path)
+    with pytest.raises(FileExistsError):
+        locate_record(tmp_path, {sitedir})
+
+
+def test_locate_record_two_sitedirs(tmp_path):
+    sitedirs = ["ha/ha/ha/site-packages", "ha/ha/ha64/site-packages"]
+    for idx, sitedir in enumerate(sitedirs):
+        sitedir = tmp_path / sitedir
+        distinfo = sitedir / "foo-0.6.dist-info"
+        distinfo.mkdir(parents=True)
+        record = distinfo / "RECORD"
+        record.write_text("\n")
+        sitedirs[idx] = BuildrootPath.from_real(sitedir, root=tmp_path)
+    with pytest.raises(FileExistsError):
+        locate_record(tmp_path, set(sitedirs))
+
+
+def test_parse_record_tldr():
+    record_path = BuildrootPath(TEST_RECORDS["tldr"]["path"])
+    record_content = read_record(DIR / "test_RECORD")
+    output = list(parse_record(record_path, record_content))
+    pprint(output)
+    expected = [
+        BINDIR / "__pycache__/tldr.cpython-37.pyc",
+        BINDIR / "tldr",
+        BINDIR / "tldr.py",
+        SITELIB / "__pycache__/tldr.cpython-37.pyc",
+        SITELIB / "tldr-0.5.dist-info/INSTALLER",
+        SITELIB / "tldr-0.5.dist-info/LICENSE",
+        SITELIB / "tldr-0.5.dist-info/METADATA",
+        SITELIB / "tldr-0.5.dist-info/RECORD",
+        SITELIB / "tldr-0.5.dist-info/WHEEL",
+        SITELIB / "tldr-0.5.dist-info/top_level.txt",
+        SITELIB / "tldr.py",
+    ]
+    assert output == expected
+
+
+def test_parse_record_tensorflow():
+    long = "tensorflow_core/include/tensorflow/core/common_runtime/base_collective_executor.h"
+    record_path = SITEARCH / "tensorflow-2.1.0.dist-info/RECORD"
+    record_content = [
+        ["../../../bin/toco_from_protos", "sha256=hello", "289"],
+        [f"../../../lib/python3.7/site-packages/{long}", "sha256=darkness", "1024"],
+        ["tensorflow-2.1.0.dist-info/METADATA", "sha256=friend", "2859"],
+    ]
+    output = list(parse_record(record_path, record_content))
+    pprint(output)
+    expected = [
+        BINDIR / "toco_from_protos",
+        SITELIB / long,
+        SITEARCH / "tensorflow-2.1.0.dist-info/METADATA",
+    ]
+    assert output == expected
+
+
+def remove_executables(expected):
+    return [p for p in expected if not p.startswith(str(BINDIR))]
+
+
+@pytest.mark.parametrize("include_executables", (True, False))
+@pytest.mark.parametrize("package, glob, expected", EXPECTED_FILES)
+def test_generate_file_list(package, glob, expected, include_executables):
+    paths_dict = EXPECTED_DICT[package]
+    modules_glob = {glob}
+    if not include_executables:
+        expected = remove_executables(expected)
+    tested = generate_file_list(paths_dict, modules_glob, include_executables)
+
+    assert tested == expected
+
+
+def test_generate_file_list_unused_glob():
+    paths_dict = EXPECTED_DICT["kerberos"]
+    modules_glob = {"kerberos", "unused_glob1", "unused_glob2", "kerb*"}
+    with pytest.raises(ValueError) as excinfo:
+        generate_file_list(paths_dict, modules_glob, True)
+
+    assert "unused_glob1, unused_glob2" in str(excinfo.value)
+    assert "kerb" not in str(excinfo.value)
+
+
+def default_options(output, mock_root):
+    return [
+        "--output",
+        str(output),
+        "--buildroot",
+        str(mock_root),
+        "--sitelib",
+        str(SITELIB),
+        "--sitearch",
+        str(SITEARCH),
+        "--bindir",
+        str(BINDIR),
+        "--python-version",
+        "3.7",  # test data are for 3.7
+    ]
+
+
+@pytest.mark.parametrize("include_executables", (True, False))
+@pytest.mark.parametrize("package, glob, expected", EXPECTED_FILES)
+def test_cli(tmp_path, package, glob, expected, include_executables):
+    mock_root = create_root(tmp_path, TEST_RECORDS[package])
+    output = tmp_path / "files"
+    globs = [glob, "+bindir"] if include_executables else [glob]
+    cli_args = argparser().parse_args([*default_options(output, mock_root), *globs])
+    main(cli_args)
+
+    if not include_executables:
+        expected = remove_executables(expected)
+    tested = output.read_text()
+    assert tested == "\n".join(expected) + "\n"
+
+
+def test_cli_no_RECORD(tmp_path):
+    mock_root = create_root(tmp_path)
+    output = tmp_path / "files"
+    cli_args = argparser().parse_args([*default_options(output, mock_root), "tldr*"])
+
+    with pytest.raises(FileNotFoundError):
+        main(cli_args)
+
+
+def test_cli_misplaced_RECORD(tmp_path, output):
+    record = {"path": "/usr/lib/", "content": TEST_RECORDS["tldr"]["content"]}
+    mock_root = create_root(tmp_path, record)
+    cli_args = argparser().parse_args([*default_options(output, mock_root), "tldr*"])
+
+    with pytest.raises(FileNotFoundError):
+        main(cli_args)
+
+
+def test_cli_find_too_many_RECORDS(tldr_root, output):
+    mock_root = create_root(tldr_root.parent, TEST_RECORDS["tensorflow"])
+    cli_args = argparser().parse_args([*default_options(output, mock_root), "tldr*"])
+
+    with pytest.raises(FileExistsError):
+        main(cli_args)
+
+
+def test_cli_bad_argument(tldr_root, output):
+    cli_args = argparser().parse_args(
+        [*default_options(output, tldr_root), "tldr*", "+foodir"]
+    )
+
+    with pytest.raises(ValueError):
+        main(cli_args)
+
+
+def test_cli_bad_option(tldr_root, output):
+    cli_args = argparser().parse_args(
+        [*default_options(output, tldr_root), "tldr*", "you_cannot_have_this"]
+    )
+
+    with pytest.raises(ValueError):
+        main(cli_args)
+
+
+def test_cli_bad_namespace(tldr_root, output):
+    cli_args = argparser().parse_args(
+        [*default_options(output, tldr_root), "tldr.didntread"]
+    )
+
+    with pytest.raises(ValueError):
+        main(cli_args)

tests/python-clikit.spec

@@ -12,12 +12,11 @@ BuildArch:      noarch
 BuildRequires:  pyproject-rpm-macros
 
 %description
-%{summary}.
+Tests building with the poetry build backend.
 
 
 %package -n python3-%{pypi_name}
 Summary:        %{summary}
-%{?python_provide:%python_provide python3-%{pypi_name}}
 
 %description -n python3-%{pypi_name}
 %{summary}.
@@ -37,10 +36,9 @@ Summary:        %{summary}
 
 %install
 %pyproject_install
+%pyproject_save_files clikit
 
 
-%files -n python3-%{pypi_name}
+%files -n python3-%{pypi_name} -f %{pyproject_files}
 %doc README.md
 %license LICENSE
-%{python3_sitelib}/%{pypi_name}/
-%{python3_sitelib}/%{pypi_name}-%{version}.dist-info/

tests/python-entrypoints.spec

@@ -11,15 +11,15 @@ BuildArch:      noarch
 BuildRequires:  pyproject-rpm-macros
 
 %description
-Discover and load entry points from installed packages.
+This package contains one .py module
+Building this tests the flit build backend.
 
 
 %package -n python3-%{pypi_name}
 Summary:        %{summary}
-%{?python_provide:%python_provide python3-%{pypi_name}}
 
 %description -n python3-%{pypi_name}
-Discover and load entry points from installed packages.
+%{summary}.
 
 
 %prep
@@ -36,11 +36,16 @@ Discover and load entry points from installed packages.
 
 %install
 %pyproject_install
+%pyproject_save_files entrypoints
 
 
-%files -n python3-%{pypi_name}
+%check
+# Internal check: Top level __pycache__ is never owned
+grep -vE '/__pycache__$' %{pyproject_files}
+grep -vE '/__pycache__/$' %{pyproject_files}
+grep -F '/__pycache__/' %{pyproject_files}
+
+
+%files -n python3-%{pypi_name} -f %{pyproject_files}
 %doc README.rst
 %license LICENSE
-%{python3_sitelib}/entrypoints-*.dist-info/
-%{python3_sitelib}/entrypoints.py
-%{python3_sitelib}/__pycache__/entrypoints.*

tests/python-isort.spec

@@ -0,0 +1,55 @@
+%global modname isort
+
+Name:               python-%{modname}
+Version:            4.3.21
+Release:            7%{?dist}
+Summary:            Python utility / library to sort Python imports
+
+License:            MIT
+URL:                https://github.com/timothycrosley/%{modname}
+Source0:            %{url}/archive/%{version}-2/%{modname}-%{version}-2.tar.gz
+BuildArch:          noarch
+BuildRequires:      pyproject-rpm-macros
+
+%description
+This package contains executables.
+Building this tests that executables are not listed when +bindir is not used
+with %%pyproject_save_files.
+
+%package -n python3-%{modname}
+Summary:            %{summary}
+
+%description -n python3-%{modname}
+%{summary}.
+
+
+%prep
+%autosetup -n %{modname}-%{version}-2
+
+
+%generate_buildrequires
+%pyproject_buildrequires -r
+
+
+%build
+%pyproject_wheel
+
+
+%install
+%pyproject_install
+%pyproject_save_files isort
+
+
+%check
+# Internal check if the instalation outputs expected result
+test -d %{buildroot}%{python3_sitelib}/%{modname}/
+test -d %{buildroot}%{python3_sitelib}/%{modname}-%{version}.dist-info/
+
+# Internal check that executables are not present when +bindir was not used with %%pyproject_save_files
+grep -vF %{buildroot}%{_bindir}/%{modname} %{pyproject_files}
+
+
+%files -n python3-%{modname} -f %{pyproject_files}
+%doc README.rst *.md
+%license LICENSE
+%{_bindir}/%{modname}

tests/python-ldap.spec

@@ -0,0 +1,88 @@
+# workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1806625
+%global debug_package %{nil}
+
+Name:           python-ldap
+Version:        3.1.0
+Release:        9%{?dist}
+License:        Python
+Summary:        An object-oriented API to access LDAP directory servers
+Source0:        %{pypi_source}
+
+BuildRequires:  pyproject-rpm-macros
+
+BuildRequires:  cyrus-sasl-devel
+BuildRequires:  gcc
+BuildRequires:  openldap-clients
+BuildRequires:  openldap-devel
+BuildRequires:  openldap-servers
+BuildRequires:  openssl-devel
+
+
+%description
+This package contains extension modules. Does not contain pyproject.toml.
+Has multiple files and directories.
+Building this tests:
+- the proper files are installed in the proper places
+- module glob in %%pyproject_save_files (some modules are included, some not)
+- combined manual and generated Buildrequires
+
+
+%package -n     python3-ldap
+Summary:        %{summary}
+
+%description -n python3-ldap
+%{summary}
+
+
+%prep
+%autosetup
+
+
+%generate_buildrequires
+%pyproject_buildrequires -t
+
+
+%build
+%pyproject_wheel
+
+
+%install
+%pyproject_install
+# We can pass multiple globs
+%pyproject_save_files ldap* *ldap
+
+
+%check
+# TODO: Upstream tox configuration calls setup.py test and rebuilds the extension module
+# But we want to test the installed one instead
+# This works but we are not testing what we ship
+# https://github.com/python-ldap/python-ldap/issues/326
+%tox
+
+# Internal check if the instalation outputs expected files
+test -d %{buildroot}%{python3_sitearch}/__pycache__/
+test -d %{buildroot}%{python3_sitearch}/python_ldap-%{version}.dist-info/
+test -d %{buildroot}%{python3_sitearch}/ldap/
+test -f %{buildroot}%{python3_sitearch}/ldapurl.py
+test -f %{buildroot}%{python3_sitearch}/ldif.py
+test -d %{buildroot}%{python3_sitearch}/slapdtest/
+test -f %{buildroot}%{python3_sitearch}/_ldap.cpython-*.so
+
+# Internal check: Unmatched modules are not supposed to be listed in %%{pyproject_files}
+# We'll list them explicitly
+grep -vF %{python3_sitearch}/ldif.py %{pyproject_files}
+grep -vF %{python3_sitearch}/__pycache__/ldif.cpython-%{python3_version_nodots}.pyc %{pyproject_files}
+grep -vF %{python3_sitearch}/__pycache__/ldif.cpython-%{python3_version_nodots}.opt-1.pyc %{pyproject_files}
+grep -vF %{python3_sitearch}/slapdtest/ %{pyproject_files}
+
+# Internal check: Top level __pycache__ is never owned
+grep -vE '/__pycache__$' %{pyproject_files}
+grep -vE '/__pycache__/$' %{pyproject_files}
+
+
+%files -n python3-ldap -f %{pyproject_files}
+%license LICENCE
+%doc CHANGES README TODO Demo
+# Explicitly listed files can be combined with automation
+%pycached %{python3_sitearch}/ldif.py
+%{python3_sitearch}/slapdtest/

tests/python-mistune.spec

@@ -0,0 +1,61 @@
+# workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1806625
+%global debug_package %{nil}
+
+Name:           python-mistune
+Version:        0.8.3
+Release:        11%{?dist}
+Summary:        Markdown parser for Python
+
+License:        BSD
+URL:            https://github.com/lepture/mistune
+Source0:        %{url}/archive/v%{version}.tar.gz
+
+BuildRequires:  gcc
+BuildRequires:  pyproject-rpm-macros
+
+# optional dependency, listed explicitly to have the extension module:
+BuildRequires:  python3-Cython
+
+%description
+This package contains an extension module. Does not contain pyproject.toml.
+Has a script (.py) and extension (.so) with identical name.
+Building this tests:
+- installing both a script and an extension with the same name
+- default build backend without pyproject.toml
+
+
+%package -n python3-mistune
+Summary:        %summary
+
+%description -n python3-mistune
+%{summary}
+
+
+%prep
+%autosetup -n mistune-%{version}
+
+
+%generate_buildrequires
+%pyproject_buildrequires
+
+
+%build
+%pyproject_wheel
+
+
+%install
+%pyproject_install
+%pyproject_save_files mistune
+
+
+%check
+# Internal check for our macros
+# making sure that pyproject_install outputs these files so that we can test behaviour of %%pyproject_save_files
+# when a package has multiple files with the same name (here script and extension)
+test -f %{buildroot}%{python3_sitearch}/mistune.py
+test -f %{buildroot}%{python3_sitearch}/mistune.cpython-*.so
+
+
+%files -n python3-mistune -f %{pyproject_files}
+%doc README.rst
+%license LICENSE

tests/python-openqa_client.spec

@@ -14,9 +14,9 @@ BuildRequires:  pyproject-rpm-macros
 %description
 This package uses tox.ini file with recursive deps (via the -r option).
 
+
 %package -n python3-%{pypi_name}
 Summary:        %{summary}
-%{?python_provide:%python_provide python3-%{pypi_name}}
 
 %description -n python3-%{pypi_name}
 %{summary}.
@@ -26,23 +26,27 @@ Summary:        %{summary}
 %autosetup -p1 -n %{pypi_name}-%{version}
 # setuptools-git is needed to build the source distribution, but not
 # for packaging, which *starts* from the source distribution
+# we sed it out to save ourselves a dependency, but that is not strictly required
 sed -i -e 's., "setuptools-git"..g' pyproject.toml
 
+
 %generate_buildrequires
 %pyproject_buildrequires -t
 
+
 %build
 %pyproject_wheel
 
+
 %install
 %pyproject_install
+%pyproject_save_files %{pypi_name}
+
 
 %check
 %tox
 
 
-%files -n python3-%{pypi_name}
+%files -n python3-%{pypi_name} -f %{pyproject_files}
 %doc README.*
 %license COPYING
-%{python3_sitelib}/%{pypi_name}/
-%{python3_sitelib}/%{pypi_name}-%{version}.dist-info/

tests/python-pluggy.spec

@@ -12,12 +12,16 @@ BuildArch:      noarch
 BuildRequires:  pyproject-rpm-macros
 
 %description
-%{summary}.
+A pure Python library. The package contains tox.ini. Does not contain executables.
+Building this tests:
+- generating runtime and testing dependencies
+- running tests with %%tox
+- the %%pyproject_save_files +bindir option works without actual executables
+- pyproject.toml with the setuptools backend and setuptools-scm
 
 
 %package -n python3-%{pypi_name}
 Summary:        %{summary}
-%{?python_provide:%python_provide python3-%{pypi_name}}
 
 %description -n python3-%{pypi_name}
 %{summary}.
@@ -37,14 +41,14 @@ Summary:        %{summary}
 
 %install
 %pyproject_install
+# There are no executables, but we are allowed to pass +bindir anyway
+%pyproject_save_files pluggy +bindir
 
 
 %check
 %tox
 
 
-%files -n python3-%{pypi_name}
+%files -n python3-%{pypi_name} -f %{pyproject_files}
 %doc README.rst
 %license LICENSE
-%{python3_sitelib}/%{pypi_name}/
-%{python3_sitelib}/%{pypi_name}-%{version}.dist-info/

tests/python-pytest.spec

@@ -11,15 +11,18 @@ BuildArch:      noarch
 BuildRequires:  pyproject-rpm-macros
 
 %description
-py.test provides simple, yet powerful testing for Python.
+This is a pure Python package with executables. It has a test suite in tox.ini
-
+and test dependencies specified via the [test] extra.
+Building this tests:
+- generating runtime and test dependencies by both tox.ini and extras
+- pyproject.toml with the setuptools backend and setuptools-scm
+- passing arguments into %%tox
 
 %package -n python3-%{pypi_name}
 Summary:        %{summary}
-%{?python_provide:%python_provide python3-%{pypi_name}}
 
 %description -n python3-%{pypi_name}
-py.test provides simple, yet powerful testing for Python.
+%{summary}.
 
 
 %prep
@@ -29,27 +32,21 @@ py.test provides simple, yet powerful testing for Python.
 %generate_buildrequires
 %pyproject_buildrequires -x testing -t
 
-
 %build
 %pyproject_wheel
 
 
 %install
 %pyproject_install
+%pyproject_save_files *pytest +bindir
 
 %check
-# Only run one test (which uses a test-only dependency, hypothesis).
+# Only run one test (which uses a test-only dependency, hypothesis)
-# (Unfortunately, some other tests still fail.)
+# See how to pass options trough the macro to tox, trough tox to pytest
 %tox -- -- -k metafunc
 
 
-%files -n python3-%{pypi_name}
+%files -n python3-%{pypi_name} -f %{pyproject_files}
 %doc README.rst
 %doc CHANGELOG.rst
 %license LICENSE
-%{_bindir}/pytest
-%{_bindir}/py.test
-%{python3_sitelib}/pytest-*.dist-info/
-%{python3_sitelib}/_pytest/
-%{python3_sitelib}/pytest.py
-%{python3_sitelib}/__pycache__/pytest.*

tests/tests.yml

@@ -31,6 +31,15 @@
     - openqa_client:
         dir: .
         run: ./mocktest.sh python-openqa_client
+    - ldap:
+        dir: .
+        run: ./mocktest.sh python-ldap
+    - isort:
+        dir: .
+        run: ./mocktest.sh python-isort
+    - mistune:
+        dir: .
+        run: ./mocktest.sh python-mistune
     required_packages:
     - mock
     - rpmdevtools

tests/tldr.spec

@@ -11,7 +11,10 @@ BuildArch:      noarch
 BuildRequires:  pyproject-rpm-macros
 
 %description
-%{summary}.
+A Python package containing executables.
+Building this tests:
+- there are no bytecompiled files in %%{_bindir}
+- the executable's shebang is adjusted properly
 
 %prep
 %autosetup -n %{name}-%{version}
@@ -24,16 +27,15 @@ BuildRequires:  pyproject-rpm-macros
 
 %install
 %pyproject_install
+%pyproject_save_files tldr +bindir
 
 %check
+# Internal check for our macros: tests we don't ship __pycache__ in bindir
 test ! -d %{buildroot}%{_bindir}/__pycache__
-head -n1 %{buildroot}%{_bindir}/%{name}.py | egrep '#!\s*%{python3}\s+%{py3_shbang_opts}\s*$'
 
-%files
+# Internal check for our macros: tests we have a proper shebang line
+head -n1 %{buildroot}%{_bindir}/%{name}.py | grep -E '#!\s*%{python3}\s+%{py3_shbang_opts}\s*$'
+
+%files -f %pyproject_files
 %license LICENSE
 %doc README.md
-%{_bindir}/%{name}
-%{_bindir}/%{name}.py
-%{python3_sitelib}/%{name}.py
-%{python3_sitelib}/__pycache__/*.pyc
-%{python3_sitelib}/%{name}-%{version}.dist-info/