Include compressed manpages correctly in the RPM package

Compressed manpages have different extension than those listed in the RECORD file,
so they were not recognized when %%pyproject_save_files '+auto' flag
was provided.
To enable the path recognition, if the manpage extension matches the one
listed in brp-compres, the extension is removed, and an asterisk is now added
to the manpages filenames.
Source: https://docs.fedoraproject.org/en-US/packaging-guidelines/#_manpages

Related: rhbz#1950291

compare_mandata.py

@@ -0,0 +1,83 @@
+'''Check whether the manpage extensions and directories list hardcoded in brp-compress
+are the same as the lists stored in pyproject_save_files.py.
+There is an open issue for RPM to provide them both as macros:
+https://github.com/rpm-software-management/rpm/issues/1865
+Once that happens, this script can be removed.
+'''
+
+import argparse
+import re
+import sys
+
+from pathlib import PosixPath
+
+from pyproject_buildrequires import print_err
+from pyproject_save_files import prepend_mandirs, MANPAGE_EXTENSIONS
+
+
+
+def read_brp_compress(filename):
+
+    contents = filename.read_text()
+    # To avoid duplicity of the manpage extensions which are listed a few times
+    # in the source file, they are stored in set and then retyped to a sorted list
+    manpage_exts = sorted(
+        set(re.findall(r'\(?(\w+)\\+\)?\$?', contents))
+    )
+
+    # Get rid of ${PREFIX} when extracting the manpage directories
+    mandirs = [
+        entry.replace('.${PREFIX}', '/PREFIX')
+        for entry in contents.split()
+        if entry.startswith('.${PREFIX}')
+    ]
+
+    return manpage_exts, sorted(mandirs)
+
+
+def compare_mandirs(brp_compress_mandirs):
+    '''
+    Check whether each of brp-compress mandirs entry is present in the list
+    stored in pyproject_save_files.py
+    '''
+
+    pyp_save_files_mandirs = sorted(prepend_mandirs(prefix='/PREFIX'))
+    if brp_compress_mandirs == pyp_save_files_mandirs:
+        return True
+    else:
+        print_err('Mandir lists don\'t match, update the list in pyproject_save_files.py')
+        print_err('brp-compress list:', brp_compress_mandirs)
+        print_err('pyproject_save_files list:', pyp_save_files_mandirs)
+        return False
+
+
+def compare_manpage_extensions(brp_compress_manpage_exts):
+    '''
+    Check whether each of brp-compress manpage extension is present in the list
+    stored in pyproject_save_files.py
+    '''
+
+    if brp_compress_manpage_exts == sorted(MANPAGE_EXTENSIONS):
+        return True
+    else:
+        print_err('Manpage extension lists don\'t match, update the list in pyproject_save_files.py')
+        print_err('brp-compress list:', brp_compress_manpage_exts)
+        print_err('pyproject_save_files list:', sorted(MANPAGE_EXTENSIONS))
+        return False
+
+
+def main(args):
+    src_manpage_exts, src_mandirs = read_brp_compress(args.filename)
+    extension_check_successful = compare_manpage_extensions(src_manpage_exts)
+    mandir_check_successful = compare_mandirs(src_mandirs)
+    if extension_check_successful and mandir_check_successful:
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--filename', type=PosixPath, required=True,
+                        help='Provide location of brp-compress file')
+    main(parser.parse_args())

macros.pyproject

@@ -97,6 +97,7 @@ fi
   --sitearch "%{python3_sitearch}" \\
   --python-version "%{python3_version}" \\
   --pyproject-record "%{_pyproject_record}" \\
+  --prefix "%{_prefix}" \\
   %{*}
 }
 

pyproject-rpm-macros.spec

@@ -12,7 +12,7 @@ License:        MIT
 # In other cases, such as backports, increment the point
 # release.
 Version:        0
-Release:        53%{?dist}
+Release:        54%{?dist}
 
 # Macro files
 Source001:      macros.pyproject
@@ -29,6 +29,7 @@ Source106:      pyproject_requirements_txt.py
 Source201:      test_pyproject_buildrequires.py
 Source202:      test_pyproject_save_files.py
 Source203:      test_pyproject_requirements_txt.py
+Source204:      compare_mandata.py
 
 # Test data
 Source301:      pyproject_buildrequires_testcases.yaml
@@ -106,6 +107,9 @@ install -m 644 pyproject_requirements_txt.py %{buildroot}%{_rpmconfigdir}/redhat
 %check
 export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856356
 %{python3} -m pytest -vv --doctest-modules
+
+# brp-compress is provided as an argument to get the right directory macro expansion
+%{python3} compare_mandata.py -f %{_rpmconfigdir}/brp-compress
 %endif
 
 
@@ -122,6 +126,9 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 %license LICENSE
 
 %changelog
+* Wed Jan 19 2022 Karolina Surma <ksurma@redhat.com> - 0-54
+- Include compressed manpages to the package if flag '+auto' is provided to %%pyproject_save_files
+
 * Fri Jan 14 2022 Miro Hrončok <mhroncok@redhat.com> - 0-53
 - %%pyproject_buildrequires: Make -r (include runtime) the default, use -R to opt-out
 

pyproject_save_files.py

@@ -12,6 +12,28 @@ from importlib.metadata import Distribution
 # From RPM's build/files.c strtokWithQuotes delim argument
 RPM_FILES_DELIMETERS = ' \n\t'
 
+# RPM hardcodes the lists of manpage extensions and directories,
+# so we have to maintain separate ones :(
+# There is an issue for RPM to provide the lists as macros:
+# https://github.com/rpm-software-management/rpm/issues/1865
+# The original lists can be found here:
+# https://github.com/rpm-software-management/rpm/blob/master/scripts/brp-compress
+MANPAGE_EXTENSIONS = ['gz', 'Z', 'bz2', 'xz', 'lzma', 'zst', 'zstd']
+MANDIRS = [
+    '/man/man*',
+    '/man/*/man*',
+    '/info',
+    '/share/man/man*',
+    '/share/man/*/man*',
+    '/share/info',
+    '/kerberos/man',
+    '/X11R6/man/man*',
+    '/lib/perl5/man/man*',
+    '/share/doc/*/man/man*',
+    '/lib/*/man/man*',
+    '/share/fish/man/man*',
+]
+
 
 class BuildrootPath(PurePosixPath):
     """
@@ -145,6 +167,56 @@ def add_lang_to_module(paths, module_name, path):
     return True
 
 
+def prepend_mandirs(prefix):
+    """
+    Return the list of man page directories prepended with the given prefix.
+    """
+    return [str(prefix) + mandir for mandir in MANDIRS]
+
+
+def normalize_manpage_filename(prefix, path):
+    """
+    If a path is processed by RPM's brp-compress script, strip it of the extension
+    (if the extension matches one of the listed by brp-compress),
+    append '*' to the filename and return it. If not, return the unchanged path.
+    Rationale: https://docs.fedoraproject.org/en-US/packaging-guidelines/#_manpages
+
+    Examples:
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/man/de/man1/linkchecker.1'))
+        BuildrootPath('/usr/share/man/de/man1/linkchecker.1*')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/doc/en/man/man1/getmac.1'))
+        BuildrootPath('/usr/share/doc/en/man/man1/getmac.1*')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/share/man/man8/abc.8.zstd'))
+        BuildrootPath('/usr/share/man/man8/abc.8*')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/kerberos/man/dir'))
+        BuildrootPath('/usr/kerberos/man/dir')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/kerberos/man/dir.1'))
+        BuildrootPath('/usr/kerberos/man/dir.1*')
+
+        >>> normalize_manpage_filename(PosixPath('/usr'), BuildrootPath('/usr/bin/getmac'))
+        BuildrootPath('/usr/bin/getmac')
+    """
+
+    prefixed_mandirs = prepend_mandirs(prefix)
+    for mandir in prefixed_mandirs:
+        # "dir" is explicitly excluded by RPM
+        # https://github.com/rpm-software-management/rpm/blob/rpm-4.17.0-release/scripts/brp-compress#L24
+        if fnmatch.fnmatch(str(path.parent), mandir) and path.name != "dir":
+            # "abc.1.gz2" -> "abc.1*"
+            if path.suffix[1:] in MANPAGE_EXTENSIONS:
+                return BuildrootPath(path.parent / (path.stem + "*"))
+            # "abc.1 -> abc.1*"
+            else:
+                return BuildrootPath(path.parent / (path.name + "*"))
+    else:
+        return path
+
+
 def is_valid_module_name(s):
     """Return True if a string is considered a valid module name and False otherwise.
 
@@ -215,7 +287,7 @@ def module_names_from_path(path):
 
 
 def classify_paths(
-    record_path, parsed_record_content, metadata, sitedirs, python_version
+    record_path, parsed_record_content, metadata, sitedirs, python_version, prefix
 ):
     """
     For each BuildrootPath in parsed_record_content classify it to a dict structure
@@ -301,6 +373,7 @@ def classify_paths(
             if path.suffix == ".mo":
                 add_lang_to_module(paths, None, path) or paths["other"]["files"].append(path)
             else:
+                path = normalize_manpage_filename(prefix, path)
                 paths["other"]["files"].append(path)
 
     return paths
@@ -528,7 +601,7 @@ def dist_metadata(buildroot, record_path):
     return dist.metadata
 
 
-def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_version, pyproject_record, varargs):
+def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_version, pyproject_record, prefix, varargs):
     """
     Takes arguments from the %{pyproject_save_files} macro
 
@@ -548,7 +621,7 @@ def pyproject_save_files_and_modules(buildroot, sitelib, sitearch, python_versio
     for record_path, files in parsed_records.items():
         metadata = dist_metadata(buildroot, record_path)
         paths_dict = classify_paths(
-            record_path, files, metadata, sitedirs, python_version
+            record_path, files, metadata, sitedirs, python_version, prefix
         )
 
         final_file_list.extend(
@@ -569,6 +642,7 @@ def main(cli_args):
         cli_args.sitearch,
         cli_args.python_version,
         cli_args.pyproject_record,
+        cli_args.prefix,
         cli_args.varargs,
     )
 
@@ -586,6 +660,7 @@ def argparser():
     r.add_argument("--sitearch", type=BuildrootPath, required=True)
     r.add_argument("--python-version", type=str, required=True)
     r.add_argument("--pyproject-record", type=PosixPath, required=True)
+    r.add_argument("--prefix", type=PosixPath, required=True)
     parser.add_argument("varargs", nargs="+")
     return parser
 

pyproject_save_files_test_data.yaml

@@ -217,6 +217,7 @@ classified:
       files:
       - /usr/bin/tldr
       - /usr/bin/tldr.py
+      - /usr/share/man/man1/tldr*
   ipykernel:
     metadata:
       dirs:
@@ -376,6 +377,7 @@ classified:
       - /usr/share/jupyter/kernels/python3/logo-64x64.png
       - /usr/share/jupyter/kernels/python3/logo-32x32.png
       - /usr/share/jupyter/kernels/python3/kernel.json
+      - /usr/man/man5/ipykernel.5*
   zope:
     metadata:
       dirs:
@@ -7517,6 +7519,7 @@ dumped:
     - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/WHEEL
     - /usr/lib/python3.7/site-packages/tldr-0.5.dist-info/top_level.txt
     - /usr/lib/python3.7/site-packages/tldr.py
+    - /usr/share/man/man1/tldr*
   - - tldr
 - - mistune
   - mistune
@@ -7675,6 +7678,7 @@ dumped:
     - /usr/lib/python3.7/site-packages/ipykernel/tests/utils.py
     - /usr/lib/python3.7/site-packages/ipykernel/trio_runner.py
     - /usr/lib/python3.7/site-packages/ipykernel/zmqshell.py
+    - /usr/man/man5/ipykernel.5*
     - /usr/share/jupyter/kernels/python3/kernel.json
     - /usr/share/jupyter/kernels/python3/logo-32x32.png
     - /usr/share/jupyter/kernels/python3/logo-64x64.png
@@ -15559,6 +15563,7 @@ records:
       ../../../bin/__pycache__/tldr.cpython-37.pyc,,
       ../../../bin/tldr,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766
       ../../../bin/tldr.py,sha256=6MUiLCWhldmV8OelT2dvPgS7q5GFwuhvd6th0Bb-LH4,12766
+      ../../../share/man/man1/tldr.bz2,sha256=xp_kqadh3PjDb4OaU8D8RQDcakrwl5AMmCnaOUV7ioo,10957
       __pycache__/tldr.cpython-37.pyc,,
       tldr-0.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
       tldr-0.5.dist-info/LICENSE,sha256=q7quAfjDWCYKC_WRk_uaP6d2wwVpOpVjUSkv8l6H7xI,1075
@@ -15587,6 +15592,7 @@ records:
       ../../../share/jupyter/kernels/python3/kernel.json,sha256=7o0-HNZRKjrk7Fqb71O3gptCssqWqfd_sxw5FNFeYO0,143
       ../../../share/jupyter/kernels/python3/logo-32x32.png,sha256=4ytcKCBy1xeIe2DacxeP3TWmXcPK6sunoCblpCVcyZc,1084
       ../../../share/jupyter/kernels/python3/logo-64x64.png,sha256=XJBjtDbO3wVnSA_kh-zg0UeeqVRfMQy6k_oYTMurKQ0,2180
+      ../../../man/man5/ipykernel.5,sha256=xp_kqadh3PjDb4OaU8D8RQDcakrwl5AMmCnaOUV7ioo,10957
       __pycache__/ipykernel_launcher.cpython-37.pyc,,
       ipykernel-5.2.1.dist-info/COPYING.md,sha256=YMWypaSJDUjGk7i5CKSWdbUkuErBWn7ByVY-Bea__ho,2835
       ipykernel-5.2.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4

test_pyproject_save_files.py

@@ -11,6 +11,7 @@ from pyproject_save_files import main as save_files_main
 from pyproject_save_files import module_names_from_path
 
 DIR = Path(__file__).parent
+PREFIX = Path("/usr")
 BINDIR = BuildrootPath("/usr/bin")
 DATADIR = BuildrootPath("/usr/share")
 SITELIB = BuildrootPath("/usr/lib/python3.7/site-packages")
@@ -110,7 +111,15 @@ def test_parse_record_tensorflow():
 
 
 def remove_others(expected):
-    return [p for p in expected if not (p.startswith(str(BINDIR)) or p.endswith(".pth") or p.rpartition(' ')[-1].startswith(str(DATADIR)))]
+    return [
+        p for p in expected
+        if not (
+            p.startswith(str(BINDIR)) or
+            p.endswith(".pth") or
+            p.endswith("*") or
+            p.rpartition(' ')[-1].startswith(str(DATADIR))
+        )
+    ]
 
 
 @pytest.mark.parametrize("include_auto", (True, False))
@@ -179,7 +188,9 @@ def default_options(output_files, output_modules, mock_root, pyproject_record):
         "--python-version",
         "3.7",  # test data are for 3.7,
         "--pyproject-record",
-        str(pyproject_record)
+        str(pyproject_record),
+        "--prefix",
+        str(PREFIX),
     ]
 
 

tests/python-getmac.spec

@@ -0,0 +1,50 @@
+Name:           python-getmac
+Version:        0.8.3
+Release:        0%{?dist}
+Summary:        Get MAC addresses of remote hosts and local interfaces
+License:        MIT
+URL:            https://github.com/GhostofGoes/getmac
+Source0:        %{pypi_source getmac}
+
+BuildArch:      noarch
+BuildRequires:  python3-devel
+BuildRequires:  pyproject-rpm-macros
+
+
+%global _description %{expand:
+Test that manpages are correctly processed by %%pyproject_save_files '*' +auto.}
+
+
+%description %_description
+
+%package -n     python3-getmac
+Summary:        %{summary}
+
+%description -n python3-getmac %_description
+
+
+%prep
+%autosetup -p1 -n getmac-%{version}
+
+
+%generate_buildrequires
+%pyproject_buildrequires -r
+
+
+%build
+%pyproject_wheel
+
+
+%install
+%pyproject_install
+%pyproject_save_files '*' +auto
+
+
+%check
+%pyproject_check_import
+# Internal check for our macros, assert there is a manpage:
+test -f %{buildroot}%{_mandir}/man1/getmac.1*
+
+
+%files -n python3-getmac -f %{pyproject_files}
+

tests/tests.yml

@@ -76,6 +76,9 @@
     - markupsafe:
         dir: .
         run: ./mocktest.sh python-markupsafe
+    - getmac:
+        dir: .
+        run: ./mocktest.sh python-getmac
     - double_install:
         dir: .
         run: ./mocktest.sh double-install