Add a script to generate Python bundled provides

See https://src.fedoraproject.org/rpms/python-setuptools/pull-request/40

Strictly speaking, this is not an RPM generator, but:

 - it generates provides
 - it is tighly coupled with pythondistdeps.py

Usage:

 1. Run `$ /usr/lib/rpm/pythonbundles.py .../vendored.txt`

 2. Copy the output into the spec as a macro definition:

    %global bundled %{expand:
    Provides: bundled(python3dist(appdirs)) = 1.4.3
    Provides: bundled(python3dist(packaging)) = 16.8
    Provides: bundled(python3dist(pyparsing)) = 2.2.1
    Provides: bundled(python3dist(six)) = 1.15
    }

 3. Use the macro to expand the provides
 4. Verify the macro contents in %check:

    %check
    ...
    %{_rpmconfigdir}/pythonbundles.py src/_vendor/vendored.txt --compare-with '%{bundled}'

python-rpm-generators.spec

@@ -12,6 +12,7 @@ Source1:        python.attr
 Source2:        pythondist.attr
 Source3:        pythonname.attr
 Source4:        pythondistdeps.py
+Source5:        pythonbundles.py
 
 BuildArch:      noarch
 
@@ -35,7 +36,7 @@ cp -a %{sources} .
 
 %install
 install -Dpm0644 -t %{buildroot}%{_fileattrsdir} *.attr
-install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} pythondistdeps.py
+install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} *.py
 
 %files -n python3-rpm-generators
 %license COPYING
@@ -43,10 +44,12 @@ install -Dpm0755 -t %{buildroot}%{_rpmconfigdir} pythondistdeps.py
 %{_fileattrsdir}/pythondist.attr
 %{_fileattrsdir}/pythonname.attr
 %{_rpmconfigdir}/pythondistdeps.py
+%{_rpmconfigdir}/pythonbundles.py
 
 %changelog
-* Wed Jun 17 2020 Miro Hrončok <mhroncok@redhat.com> - 11-8
+* Fri Jun 26 2020 Miro Hrončok <mhroncok@redhat.com> - 11-8
 - Fix python(abi) requires generator, it picked files from almost good directories
+- Add a script to generate Python bundled provides
 
 * Thu May 21 2020 Miro Hrončok <mhroncok@redhat.com> - 11-7
 - Use PEP 503 names for requires

pythonbundles.py

@@ -0,0 +1,90 @@
+#!/usr/bin/python3 -B
+#                  (imports pythondistdeps from /usr/lib/rpm, hence -B)
+#
+# This program is free software.
+#
+# It is placed in the public domain or under the CC0-1.0-Universal license,
+# whichever is more permissive.
+#
+# Alternatively, it may be redistributed and/or modified under the terms of
+# the LGPL version 2.1 (or later) or GPL version 2 (or later).
+#
+# Use this script to generate bundled provides, e.g.:
+# ./pythonbundles.py setuptools-47.1.1/pkg_resources/_vendor/vendored.txt
+
+import pathlib
+import sys
+
+# inject parse_version import to pythondistdeps
+# not the nicest API, but :/
+from pkg_resources import parse_version
+import pythondistdeps
+pythondistdeps.parse_version = parse_version
+
+
+def generate_bundled_provides(path, namespace):
+    provides = set()
+
+    for line in path.read_text().splitlines():
+        line, _, comment = line.partition('#')
+        if comment.startswith('egg='):
+            # not a real comment
+            # e.g. git+https://github.com/monty/spam.git@master#egg=spam&...
+            egg, *_ = comment.strip().partition(' ')
+            egg, *_ = egg.strip().partition('&')
+            name = pythondistdeps.normalize_name(egg[4:])
+            provides.add(f'Provides: bundled({namespace}({name}))')
+            continue
+        line = line.strip()
+        if line:
+            name, _, version = line.partition('==')
+            name = pythondistdeps.normalize_name(name)
+            bundled_name = f"bundled({namespace}({name}))"
+            python_provide = pythondistdeps.convert(bundled_name, '==', version)
+            provides.add(f'Provides: {python_provide}')
+
+    return provides
+
+
+def compare(expected, given):
+    stripped = (l.strip() for l in given)
+    no_comments = set(l for l in stripped if not l.startswith('#'))
+    no_comments.discard('')
+    if expected == no_comments:
+        return True
+    extra_expected = expected - no_comments
+    extra_given = no_comments - expected
+    if extra_expected:
+        print('Missing expected provides:', file=sys.stderr)
+        for provide in sorted(extra_expected):
+            print(f'    - {provide}', file=sys.stderr)
+    if extra_given:
+        print('Redundant unexpected provides:', file=sys.stderr)
+        for provide in sorted(extra_given):
+            print(f'    + {provide}', file=sys.stderr)
+    return False
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(prog=sys.argv[0],
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('vendored', metavar='VENDORED.TXT',
+                        help='Upstream information about vendored libraries')
+    parser.add_argument('-c', '--compare-with', action='store',
+                        help='A string value to compare with and verify')
+    parser.add_argument('-n', '--namespace', action='store',
+                        help='What namespace of provides will used', default='python3dist')
+    args = parser.parse_args()
+
+    provides = generate_bundled_provides(pathlib.Path(args.vendored), args.namespace)
+
+    if args.compare_with:
+        given = args.compare_with.splitlines()
+        same = compare(provides, given)
+        if not same:
+            sys.exit(1)
+    else:
+        for provide in sorted(provides):
+            print(provide)

tests/data/scripts_pythonbundles/pip.in

@@ -0,0 +1,24 @@
+appdirs==1.4.3
+CacheControl==0.12.6
+colorama==0.4.3
+contextlib2==0.6.0.post1
+distlib==0.3.0
+distro==1.5.0
+html5lib==1.0.1
+ipaddress==1.0.23  # Only needed on 2.6 and 2.7
+msgpack==1.0.0
+packaging==20.3
+pep517==0.8.2
+progress==1.5
+pyparsing==2.4.7
+requests==2.23.0
+    certifi==2020.04.05.1
+    chardet==3.0.4
+    idna==2.9
+    urllib3==1.25.8
+resolvelib==0.3.0
+retrying==1.3.3
+setuptools==44.0.0
+six==1.14.0
+toml==0.10.0
+webencodings==0.5.1

tests/data/scripts_pythonbundles/pip.out

@@ -0,0 +1,24 @@
+Provides: bundled(python3dist(appdirs)) = 1.4.3
+Provides: bundled(python3dist(cachecontrol)) = 0.12.6
+Provides: bundled(python3dist(certifi)) = 2020.4.5.1
+Provides: bundled(python3dist(chardet)) = 3.0.4
+Provides: bundled(python3dist(colorama)) = 0.4.3
+Provides: bundled(python3dist(contextlib2)) = 0.6^post1
+Provides: bundled(python3dist(distlib)) = 0.3
+Provides: bundled(python3dist(distro)) = 1.5
+Provides: bundled(python3dist(html5lib)) = 1.0.1
+Provides: bundled(python3dist(idna)) = 2.9
+Provides: bundled(python3dist(ipaddress)) = 1.0.23
+Provides: bundled(python3dist(msgpack)) = 1
+Provides: bundled(python3dist(packaging)) = 20.3
+Provides: bundled(python3dist(pep517)) = 0.8.2
+Provides: bundled(python3dist(progress)) = 1.5
+Provides: bundled(python3dist(pyparsing)) = 2.4.7
+Provides: bundled(python3dist(requests)) = 2.23
+Provides: bundled(python3dist(resolvelib)) = 0.3
+Provides: bundled(python3dist(retrying)) = 1.3.3
+Provides: bundled(python3dist(setuptools)) = 44
+Provides: bundled(python3dist(six)) = 1.14
+Provides: bundled(python3dist(toml)) = 0.10
+Provides: bundled(python3dist(urllib3)) = 1.25.8
+Provides: bundled(python3dist(webencodings)) = 0.5.1

tests/data/scripts_pythonbundles/pipenv.in

@@ -0,0 +1,59 @@
+appdirs==1.4.4
+backports.shutil_get_terminal_size==1.0.0
+backports.weakref==1.0.post1
+click==7.1.2
+click-completion==0.5.2
+click-didyoumean==0.0.3
+colorama==0.4.3
+delegator.py==0.1.1
+    pexpect==4.8.0
+        ptyprocess==0.6.0
+python-dotenv==0.10.3
+first==2.0.1
+iso8601==0.1.12
+jinja2==2.11.2
+markupsafe==1.1.1
+parse==1.15.0
+pathlib2==2.3.5
+    scandir==1.10
+pipdeptree==0.13.2
+pipreqs==0.4.10
+    docopt==0.6.2
+    yarg==0.1.9
+pythonfinder==1.2.4
+requests==2.23.0
+    chardet==3.0.4
+    idna==2.9
+    urllib3==1.25.9
+    certifi==2020.4.5.1
+requirementslib==1.5.11
+    attrs==19.3.0
+    distlib==0.3.0
+    packaging==20.3
+    pyparsing==2.4.7
+    plette==0.2.3
+        tomlkit==0.5.11
+shellingham==1.3.2
+six==1.14.0
+semver==2.9.0
+toml==0.10.1
+cached-property==1.5.1
+vistir==0.5.2
+pip-shims==0.5.2
+    contextlib2==0.6.0.post1
+    funcsigs==1.0.2
+enum34==1.1.10
+# yaspin==0.15.0
+yaspin==0.14.3
+cerberus==1.3.2
+resolvelib==0.3.0
+backports.functools_lru_cache==1.6.1
+pep517==0.8.2
+    zipp==0.6.0
+    importlib_metadata==1.6.0
+    importlib-resources==1.5.0
+    more-itertools==5.0.0
+git+https://github.com/sarugaku/passa.git@master#egg=passa
+orderedmultidict==1.0.1
+dparse==0.5.0
+python-dateutil==2.8.1

tests/data/scripts_pythonbundles/pipenv.out

@@ -0,0 +1,58 @@
+Provides: bundled(python3dist(appdirs)) = 1.4.4
+Provides: bundled(python3dist(attrs)) = 19.3
+Provides: bundled(python3dist(backports-functools-lru-cache)) = 1.6.1
+Provides: bundled(python3dist(backports-shutil-get-terminal-size)) = 1
+Provides: bundled(python3dist(backports-weakref)) = 1^post1
+Provides: bundled(python3dist(cached-property)) = 1.5.1
+Provides: bundled(python3dist(cerberus)) = 1.3.2
+Provides: bundled(python3dist(certifi)) = 2020.4.5.1
+Provides: bundled(python3dist(chardet)) = 3.0.4
+Provides: bundled(python3dist(click)) = 7.1.2
+Provides: bundled(python3dist(click-completion)) = 0.5.2
+Provides: bundled(python3dist(click-didyoumean)) = 0.0.3
+Provides: bundled(python3dist(colorama)) = 0.4.3
+Provides: bundled(python3dist(contextlib2)) = 0.6^post1
+Provides: bundled(python3dist(delegator-py)) = 0.1.1
+Provides: bundled(python3dist(distlib)) = 0.3
+Provides: bundled(python3dist(docopt)) = 0.6.2
+Provides: bundled(python3dist(dparse)) = 0.5
+Provides: bundled(python3dist(enum34)) = 1.1.10
+Provides: bundled(python3dist(first)) = 2.0.1
+Provides: bundled(python3dist(funcsigs)) = 1.0.2
+Provides: bundled(python3dist(idna)) = 2.9
+Provides: bundled(python3dist(importlib-metadata)) = 1.6
+Provides: bundled(python3dist(importlib-resources)) = 1.5
+Provides: bundled(python3dist(iso8601)) = 0.1.12
+Provides: bundled(python3dist(jinja2)) = 2.11.2
+Provides: bundled(python3dist(markupsafe)) = 1.1.1
+Provides: bundled(python3dist(more-itertools)) = 5
+Provides: bundled(python3dist(orderedmultidict)) = 1.0.1
+Provides: bundled(python3dist(packaging)) = 20.3
+Provides: bundled(python3dist(parse)) = 1.15
+Provides: bundled(python3dist(passa))
+Provides: bundled(python3dist(pathlib2)) = 2.3.5
+Provides: bundled(python3dist(pep517)) = 0.8.2
+Provides: bundled(python3dist(pexpect)) = 4.8
+Provides: bundled(python3dist(pip-shims)) = 0.5.2
+Provides: bundled(python3dist(pipdeptree)) = 0.13.2
+Provides: bundled(python3dist(pipreqs)) = 0.4.10
+Provides: bundled(python3dist(plette)) = 0.2.3
+Provides: bundled(python3dist(ptyprocess)) = 0.6
+Provides: bundled(python3dist(pyparsing)) = 2.4.7
+Provides: bundled(python3dist(python-dateutil)) = 2.8.1
+Provides: bundled(python3dist(python-dotenv)) = 0.10.3
+Provides: bundled(python3dist(pythonfinder)) = 1.2.4
+Provides: bundled(python3dist(requests)) = 2.23
+Provides: bundled(python3dist(requirementslib)) = 1.5.11
+Provides: bundled(python3dist(resolvelib)) = 0.3
+Provides: bundled(python3dist(scandir)) = 1.10
+Provides: bundled(python3dist(semver)) = 2.9
+Provides: bundled(python3dist(shellingham)) = 1.3.2
+Provides: bundled(python3dist(six)) = 1.14
+Provides: bundled(python3dist(toml)) = 0.10.1
+Provides: bundled(python3dist(tomlkit)) = 0.5.11
+Provides: bundled(python3dist(urllib3)) = 1.25.9
+Provides: bundled(python3dist(vistir)) = 0.5.2
+Provides: bundled(python3dist(yarg)) = 0.1.9
+Provides: bundled(python3dist(yaspin)) = 0.14.3
+Provides: bundled(python3dist(zipp)) = 0.6

tests/data/scripts_pythonbundles/pkg_resources.in

@@ -0,0 +1,4 @@
+packaging==16.8
+pyparsing==2.2.1
+six==1.10.0
+appdirs==1.4.3

tests/data/scripts_pythonbundles/pkg_resources.out

@@ -0,0 +1,4 @@
+Provides: bundled(python3dist(appdirs)) = 1.4.3
+Provides: bundled(python3dist(packaging)) = 16.8
+Provides: bundled(python3dist(pyparsing)) = 2.2.1
+Provides: bundled(python3dist(six)) = 1.10

tests/test_scripts_pythonbundles.py

@@ -0,0 +1,99 @@
+# Run tests using pytest, e.g. from the root directory
+#   $ python3 -m pytest --ignore tests/testing/ -vvv
+#
+# Requirements for this script:
+# - Python >= 3.6
+# - pytest
+import pathlib
+import pytest
+import random
+import sys
+import subprocess
+
+PYTHONBUNDLES = pathlib.Path(__file__).parent / '..' / 'pythonbundles.py'
+TEST_DATA = pathlib.Path(__file__).parent / 'data' / 'scripts_pythonbundles'
+
+
+def run_pythonbundles(*args, success=True):
+    """
+    Runs pythonbundles.py with given command line arguments
+
+    Arguments:
+      *args: Shell arguments passed to the script
+      success:
+       - true-ish: assert return code is 0 (default)
+       - false-ish (excluding None): assert return code is not 0
+       - None: don't assert return code value
+    """
+    cp = subprocess.run((sys.executable, PYTHONBUNDLES, *args), encoding='utf-8',
+                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if success:
+        assert cp.returncode == 0, cp.stderr
+    elif success is not None:
+        assert cp.returncode != 0, cp.stdout
+    return cp
+
+
+projects = pytest.mark.parametrize('project', ('pkg_resources', 'pip', 'pipenv'))
+
+
+@projects
+def test_output_consistency(project):
+    cp = run_pythonbundles(TEST_DATA / f'{project}.in')
+    expected = (TEST_DATA / f'{project}.out').read_text()
+    assert cp.stdout == expected, cp.stdout
+    assert cp.stderr == '', cp.stderr
+
+
+@pytest.mark.parametrize('namespace', ('python2dist', 'python3.11dist', 'pypy2.7dist'))
+@projects
+def test_namespace(project, namespace):
+    cp = run_pythonbundles(TEST_DATA / f'{project}.in', f'--namespace={namespace}')
+    expected = (TEST_DATA / f'{project}.out').read_text().replace('python3dist', namespace)
+    assert cp.stdout == expected, cp.stdout
+    assert cp.stderr == '', cp.stderr
+
+
+@projects
+def test_compare_with_identical(project):
+    expected = (TEST_DATA / f'{project}.out').read_text()
+    cp = run_pythonbundles(TEST_DATA / f'{project}.in', '--compare-with', expected)
+    assert cp.stdout == '', cp.stdout
+    assert cp.stderr == '', cp.stderr
+
+
+@projects
+def test_compare_with_shuffled(project):
+    expected = (TEST_DATA / f'{project}.out').read_text()
+    lines = expected.splitlines()
+    # some extra whitespace and comments
+    lines[0] = f'  {lines[0]}  '
+    lines.extend([''] * 3)
+    lines.append('# this is a comment on a single line')
+    random.shuffle(lines)
+    shuffled = '\n'.join(lines)
+    cp = run_pythonbundles(TEST_DATA / f'{project}.in', '--compare-with', shuffled)
+    assert cp.stdout == '', cp.stdout
+    assert cp.stderr == '', cp.stderr
+
+
+@projects
+def test_compare_with_missing(project):
+    expected = (TEST_DATA / f'{project}.out').read_text()
+    lines = expected.splitlines()
+    missing = lines[0]
+    del lines[0]
+    shorter = '\n'.join(lines)
+    cp = run_pythonbundles(TEST_DATA / f'{project}.in', '--compare-with', shorter, success=False)
+    assert cp.stdout == '', cp.stdout
+    assert cp.stderr == f'Missing expected provides:\n    - {missing}\n', cp.stderr
+
+
+@projects
+def test_compare_with_unexpected(project):
+    expected = (TEST_DATA / f'{project}.out').read_text()
+    unexpected = 'Provides: bundled(python3dist(brainfuck)) = 6.6.6'
+    longer = f'{expected}{unexpected}\n'
+    cp = run_pythonbundles(TEST_DATA / f'{project}.in', '--compare-with', longer, success=False)
+    assert cp.stdout == '', cp.stdout
+    assert cp.stderr == f'Redundant unexpected provides:\n    + {unexpected}\n', cp.stderr

tests/tests.yml

@@ -30,7 +30,7 @@
     - prepare-test-data:
         dir: .
         run: tar -xvf test-sources-*.tar.gz -C ./tests/data/scripts_pythondistdeps/
-    - pythondistdeps:
+    - pytest:
         dir: ./tests
         # Use update-test-sources.sh to update the test data
         run: python3 -m pytest --capture=no -vvv