Drop the rewheel patch, use RPM built wheels instead

00189-add-rewheel-module.patch

@@ -1,245 +0,0 @@
-diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
-index 7ff938b..8c3d062 100644
---- a/Lib/ensurepip/__init__.py
-+++ b/Lib/ensurepip/__init__.py
-@@ -1,8 +1,10 @@
- import os
- import os.path
- import pkgutil
-+import shutil
- import sys
- import tempfile
-+from ensurepip import rewheel
- 
- 
- __all__ = ["version", "bootstrap"]
-@@ -24,8 +26,15 @@ def _run_pip(args, additional_paths=None):
-         sys.path = additional_paths + sys.path
- 
-     # Install the bundled software
--    import pip._internal
--    return pip._internal.main(args)
-+    try:
-+        # pip 10
-+        from pip._internal import main
-+    except ImportError:
-+        # pip 9
-+        from pip import main
-+    if args[0] in ["install", "list", "wheel"]:
-+        args.append('--pre')
-+    return main(args)
- 
- 
- def version():
-@@ -88,20 +97,39 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
-         # omit pip and easy_install
-         os.environ["ENSUREPIP_OPTIONS"] = "install"
- 
-+    whls = []
-+    rewheel_dir = None
-+    # try to see if we have system-wide versions of _PROJECTS
-+    dep_records = rewheel.find_system_records([p[0] for p in _PROJECTS])
-+    # TODO: check if system-wide versions are the newest ones
-+    # if --upgrade is used?
-+    if all(dep_records):
-+        # if we have all _PROJECTS installed system-wide, we'll recreate
-+        # wheels from them and install those
-+        rewheel_dir = tempfile.TemporaryDirectory()
-+        for dr in dep_records:
-+            new_whl = rewheel.rewheel_from_record(dr, rewheel_dir.name)
-+            whls.append(os.path.join(rewheel_dir.name, new_whl))
-+    else:
-+        # if we don't have all the _PROJECTS installed system-wide,
-+        # let's just fall back to bundled wheels
-+        for project, version in _PROJECTS:
-+            whl = os.path.join(
-+                os.path.dirname(__file__),
-+                "_bundled",
-+                "{}-{}-py2.py3-none-any.whl".format(project, version)
-+            )
-+            whls.append(whl)
-+
-     with tempfile.TemporaryDirectory() as tmpdir:
-         # Put our bundled wheels into a temporary directory and construct the
-         # additional paths that need added to sys.path
-         additional_paths = []
--        for project, version in _PROJECTS:
--            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
--            whl = pkgutil.get_data(
--                "ensurepip",
--                "_bundled/{}".format(wheel_name),
--            )
--            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
--                fp.write(whl)
--
--            additional_paths.append(os.path.join(tmpdir, wheel_name))
-+        for whl in whls:
-+            shutil.copy(whl, tmpdir)
-+            additional_paths.append(os.path.join(tmpdir, os.path.basename(whl)))
-+        if rewheel_dir:
-+            rewheel_dir.cleanup()
- 
-         # Construct the arguments to be passed to the pip command
-         args = ["install", "--no-index", "--find-links", tmpdir]
-diff --git a/Lib/ensurepip/rewheel/__init__.py b/Lib/ensurepip/rewheel/__init__.py
-new file mode 100644
-index 0000000..753c764
---- /dev/null
-+++ b/Lib/ensurepip/rewheel/__init__.py
-@@ -0,0 +1,143 @@
-+import argparse
-+import codecs
-+import csv
-+import email.parser
-+import os
-+import io
-+import re
-+import site
-+import subprocess
-+import sys
-+import zipfile
-+
-+def run():
-+    parser = argparse.ArgumentParser(description='Recreate wheel of package with given RECORD.')
-+    parser.add_argument('record_path',
-+                        help='Path to RECORD file')
-+    parser.add_argument('-o', '--output-dir',
-+                        help='Dir where to place the wheel, defaults to current working dir.',
-+                        dest='outdir',
-+                        default=os.path.curdir)
-+
-+    ns = parser.parse_args()
-+    retcode = 0
-+    try:
-+        print(rewheel_from_record(**vars(ns)))
-+    except BaseException as e:
-+        print('Failed: {}'.format(e))
-+        retcode = 1
-+    sys.exit(1)
-+
-+def find_system_records(projects):
-+    """Return list of paths to RECORD files for system-installed projects.
-+
-+    If a project is not installed, the resulting list contains None instead
-+    of a path to its RECORD
-+    """
-+    records = []
-+    # get system site-packages dirs
-+    sys_sitepack = site.getsitepackages([sys.base_prefix, sys.base_exec_prefix])
-+    sys_sitepack = [sp for sp in sys_sitepack if os.path.exists(sp)]
-+    # try to find all projects in all system site-packages
-+    for project in projects:
-+        path = None
-+        for sp in sys_sitepack:
-+            dist_info_re = os.path.join(sp, project) + r'-[^\{0}]+\.dist-info'.format(os.sep)
-+            candidates = [os.path.join(sp, p) for p in os.listdir(sp)]
-+            # filter out candidate dirs based on the above regexp
-+            filtered = [c for c in candidates if re.match(dist_info_re, c)]
-+            # if we have 0 or 2 or more dirs, something is wrong...
-+            if len(filtered) == 1:
-+                path = filtered[0]
-+        if path is not None:
-+            records.append(os.path.join(path, 'RECORD'))
-+        else:
-+            records.append(None)
-+    return records
-+
-+def rewheel_from_record(record_path, outdir):
-+    """Recreates a whee of package with given record_path and returns path
-+    to the newly created wheel."""
-+    site_dir = os.path.dirname(os.path.dirname(record_path))
-+    record_relpath = record_path[len(site_dir):].strip(os.path.sep)
-+    to_write, to_omit = get_records_to_pack(site_dir, record_relpath)
-+    new_wheel_name = get_wheel_name(record_path)
-+    new_wheel_path = os.path.join(outdir, new_wheel_name + '.whl')
-+
-+    new_wheel = zipfile.ZipFile(new_wheel_path, mode='w', compression=zipfile.ZIP_DEFLATED)
-+    # we need to write a new record with just the files that we will write,
-+    # e.g. not binaries and *.pyc/*.pyo files
-+    new_record = io.StringIO()
-+    writer = csv.writer(new_record)
-+
-+    # handle files that we can write straight away
-+    for f, sha_hash, size in to_write:
-+        new_wheel.write(os.path.join(site_dir, f), arcname=f)
-+        writer.writerow([f, sha_hash,size])
-+
-+    # rewrite the old wheel file with a new computed one
-+    writer.writerow([record_relpath, '', ''])
-+    new_wheel.writestr(record_relpath, new_record.getvalue())
-+
-+    new_wheel.close()
-+
-+    return new_wheel.filename
-+
-+def get_wheel_name(record_path):
-+    """Return proper name of the wheel, without .whl."""
-+
-+    wheel_info_path = os.path.join(os.path.dirname(record_path), 'WHEEL')
-+    with codecs.open(wheel_info_path, encoding='utf-8') as wheel_info_file:
-+        wheel_info = email.parser.Parser().parsestr(wheel_info_file.read())
-+
-+    metadata_path = os.path.join(os.path.dirname(record_path), 'METADATA')
-+    with codecs.open(metadata_path, encoding='utf-8') as metadata_file:
-+        metadata = email.parser.Parser().parsestr(metadata_file.read())
-+
-+    # construct name parts according to wheel spec
-+    distribution = metadata.get('Name')
-+    version = metadata.get('Version')
-+    build_tag = '' # nothing for now
-+    lang_tag = []
-+    for t in wheel_info.get_all('Tag'):
-+        lang_tag.append(t.split('-')[0])
-+    lang_tag = '.'.join(lang_tag)
-+    abi_tag, plat_tag = wheel_info.get('Tag').split('-')[1:3]
-+    # leave out build tag, if it is empty
-+    to_join = filter(None, [distribution, version, build_tag, lang_tag, abi_tag, plat_tag])
-+    return '-'.join(list(to_join))
-+
-+def get_records_to_pack(site_dir, record_relpath):
-+    """Accepts path of sitedir and path of RECORD file relative to it.
-+    Returns two lists:
-+    - list of files that can be written to new RECORD straight away
-+    - list of files that shouldn't be written or need some processing
-+      (pyc and pyo files, scripts)
-+    """
-+    record_file_path = os.path.join(site_dir, record_relpath)
-+    with codecs.open(record_file_path, encoding='utf-8') as record_file:
-+        record_contents = record_file.read()
-+    # temporary fix for https://github.com/pypa/pip/issues/1376
-+    # we need to ignore files under ".data" directory
-+    data_dir = os.path.dirname(record_relpath).strip(os.path.sep)
-+    data_dir = data_dir[:-len('dist-info')] + 'data'
-+
-+    to_write = []
-+    to_omit = []
-+    for l in record_contents.splitlines():
-+        spl = l.split(',')
-+        if len(spl) == 3:
-+            # new record will omit (or write differently):
-+            # - abs paths, paths with ".." (entry points),
-+            # - pyc+pyo files
-+            # - the old RECORD file
-+            # TODO: is there any better way to recognize an entry point?
-+            if os.path.isabs(spl[0]) or spl[0].startswith('..') or \
-+               spl[0].endswith('.pyc') or spl[0].endswith('.pyo') or \
-+               spl[0] == record_relpath or spl[0].startswith(data_dir):
-+                to_omit.append(spl)
-+            else:
-+                to_write.append(spl)
-+        else:
-+            pass # bad RECORD or empty line
-+    return to_write, to_omit
-diff --git a/Makefile.pre.in b/Makefile.pre.in
-index d07b312..1c6720e 100644
---- a/Makefile.pre.in
-+++ b/Makefile.pre.in
-@@ -1301,7 +1301,7 @@ LIBSUBDIRS=	tkinter tkinter/test tkinter/test/test_tkinter \
- 		test/test_asyncio \
- 		collections concurrent concurrent/futures encodings \
- 		email email/mime test/test_email test/test_email/data \
--		ensurepip ensurepip/_bundled \
-+		ensurepip ensurepip/_bundled ensurepip/rewheel \
- 		html json test/test_json http dbm xmlrpc \
- 		sqlite3 sqlite3/test \
- 		logging csv wsgiref urllib \

00189-use-rpm-wheels.patch

@@ -0,0 +1,51 @@
+diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
+index 4748ba4..fc02255 100644
+--- a/Lib/ensurepip/__init__.py
++++ b/Lib/ensurepip/__init__.py
+@@ -1,16 +1,27 @@
++import distutils.version
++import glob
+ import os
+ import os.path
+-import pkgutil
+ import sys
+ import tempfile
+ 
+ 
+ __all__ = ["version", "bootstrap"]
+ 
++_WHEEL_DIR = "/usr/share/python-wheels/"
+ 
+-_SETUPTOOLS_VERSION = "39.0.1"
+ 
+-_PIP_VERSION = "10.0.1"
++def _get_most_recent_wheel_version(pkg):
++    prefix = os.path.join(_WHEEL_DIR, "{}-".format(pkg))
++    suffix = "-py2.py3-none-any.whl"
++    pattern = "{}*{}".format(prefix, suffix)
++    versions = (p[len(prefix):-len(suffix)] for p in glob.glob(pattern))
++    return str(max(versions, key=distutils.version.LooseVersion))
++
++
++_SETUPTOOLS_VERSION = _get_most_recent_wheel_version("setuptools")
++
++_PIP_VERSION = _get_most_recent_wheel_version("pip")
+ 
+ _PROJECTS = [
+     ("setuptools", _SETUPTOOLS_VERSION),
+@@ -94,12 +105,9 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
+         additional_paths = []
+         for project, version in _PROJECTS:
+             wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+-            whl = pkgutil.get_data(
+-                "ensurepip",
+-                "_bundled/{}".format(wheel_name),
+-            )
+-            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+-                fp.write(whl)
++            with open(os.path.join(_WHEEL_DIR, wheel_name), "rb") as sfp:
++                with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
++                    fp.write(sfp.read())
+ 
+             additional_paths.append(os.path.join(tmpdir, wheel_name))
+ 

python3.spec

@@ -14,7 +14,7 @@ URL: https://www.python.org/
 #  WARNING  When rebasing to a new Python version,
 #           remember to update the python3-docs package as well
 Version: %{pybasever}.0
-Release: 5%{?dist}
+Release: 6%{?dist}
 License: Python
 
 
@@ -33,6 +33,9 @@ License: Python
 # WARNING: This does not change the package name and summary above
 %bcond_with flatpackage
 
+# Whether to use RPM build wheels from the python-{pip,setuptools}-wheel package
+# Uses upstream bundled prebuilt wheels otherwise
+%bcond_without rpmwheels
 
 # Expensive optimizations (mainly, profile-guided optimizations)
 %ifarch %{ix86} x86_64
@@ -46,13 +49,6 @@ License: Python
 # Run the test suite in %%check
 %bcond_without tests
 
-# Ability to reuse RPM-installed pip using rewheel
-%if %{with flatpackage}
-%bcond_with rewheel
-%else
-%bcond_without rewheel
-%endif
-
 # Extra build for debugging the interpreter or C-API extensions
 # (the -debug subpackages)
 %if %{with flatpackage}
@@ -190,9 +186,9 @@ BuildRequires: /usr/bin/dtrace
 # workaround http://bugs.python.org/issue19804 (test_uuid requires ifconfig)
 BuildRequires: /usr/sbin/ifconfig
 
-%if %{with rewheel}
+%if %{with rpmwheels}
-BuildRequires: python3-setuptools
+BuildRequires: python-setuptools-wheel
-BuildRequires: python3-pip
+BuildRequires: python-pip-wheel
 %endif
 
 
@@ -283,10 +279,9 @@ Patch170: 00170-gc-assertions.patch
 Patch178: 00178-dont-duplicate-flags-in-sysconfig.patch
 
 # 00189 #
-# Add the rewheel module, allowing to recreate wheels from already installed
+# Instead of bundled wheels, use our RPM packaged wheels from
-# ones
+# /usr/share/python-wheels
-# https://github.com/bkabrda/rewheel
+Patch189: 00189-use-rpm-wheels.patch
-Patch189: 00189-add-rewheel-module.patch
 
 # 00205 #
 # LIBPL variable in makefile takes LIBPL from configure.ac
@@ -363,11 +358,6 @@ Obsoletes: python%{pyshortver}
 %global platpyver 3.6.2-20
 Obsoletes: platform-python < %{platpyver}
 
-%if %{with rewheel}
-Requires: python3-setuptools
-Requires: python3-pip
-%endif
-
 # This prevents ALL subpackages built from this spec to require
 # /usr/bin/python3*. Granularity per subpackage is impossible.
 # It's intended for the libs package not to drag in the interpreter, see
@@ -414,6 +404,14 @@ Requires: glibc%{?_isa} >= 2.24.90-26
 Requires: gdbm-libs%{?_isa} >= 1:1.13
 %endif
 
+%if %{with rpmwheels}
+Requires: python-setuptools-wheel
+Requires: python-pip-wheel
+%else
+Provides: bundled(python3-pip) = 10.0.1
+Provides: bundled(python3-setuptools) = 39.0.1
+%endif
+
 # There are files in the standard library that have python shebang.
 # We've filtered the automatic requirement out so libs are installable without
 # the main package. This however makes it pulled in by default.
@@ -443,12 +441,7 @@ Requires: %{name}-libs%{?_isa} = %{version}-%{release}
 BuildRequires: python-rpm-macros
 Requires: python-rpm-macros
 Requires: python3-rpm-macros
-
-%if %{with rewheel}
-# without rewheel is used to bootstrap setuptools+pip
-# python3-rpm-generators needs python3-setuptools, so we cannot have it yet
 Requires: python3-rpm-generators
-%endif
 
 # https://bugzilla.redhat.com/show_bug.cgi?id=1217376
 # https://bugzilla.redhat.com/show_bug.cgi?id=1496757
@@ -579,9 +572,13 @@ Requires: redhat-rpm-config
 %global __requires_exclude ^python\\(abi\\) = 3\\..$
 %global __provides_exclude ^python\\(abi\\) = 3\\..$
 
-# We keep those inside on purpose
+%if %{with rpmwheels}
+Requires: python-setuptools-wheel
+Requires: python-pip-wheel
+%else
 Provides: bundled(python3-pip) = 10.0.1
 Provides: bundled(python3-setuptools) = 39.0.1
+%endif
 
 # The description for the flat package
 %description
@@ -604,11 +601,6 @@ version once Python %{pybasever} is stable.
 # Remove bundled libraries to ensure that we're using the system copy.
 rm -r Modules/expat
 
-%if %{with rewheel}
-%global pip_version %(pip3 --version | cut -d' ' -f2)
-sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/ensurepip/__init__.py
-%endif
-
 #
 # Apply patches:
 #
@@ -625,8 +617,9 @@ sed -r -i s/'_PIP_VERSION = "[0-9.]+"'/'_PIP_VERSION = "%{pip_version}"'/ Lib/en
 %patch170 -p1
 %patch178 -p1
 
-%if %{with rewheel}
+%if %{with rpmwheels}
 %patch189 -p1
+rm Lib/ensurepip/_bundled/*.whl
 %endif
 
 %patch205 -p1
@@ -1107,20 +1100,13 @@ CheckPython optimized
 %{pylibdir}/ensurepip/*.py
 %{pylibdir}/ensurepip/__pycache__/*%{bytecode_suffixes}
 
-%if %{without flatpackage}
+%if %{with rpmwheels}
 %exclude %{pylibdir}/ensurepip/_bundled
 %else
 %dir %{pylibdir}/ensurepip/_bundled
 %{pylibdir}/ensurepip/_bundled/*.whl
 %endif
 
-%if %{with rewheel}
-%dir %{pylibdir}/ensurepip/rewheel/
-%dir %{pylibdir}/ensurepip/rewheel/__pycache__/
-%{pylibdir}/ensurepip/rewheel/*.py
-%{pylibdir}/ensurepip/rewheel/__pycache__/*%{bytecode_suffixes}
-%endif
-
 %dir %{pylibdir}/test/
 %dir %{pylibdir}/test/__pycache__/
 %dir %{pylibdir}/test/support/
@@ -1520,6 +1506,9 @@ CheckPython optimized
 # ======================================================
 
 %changelog
+* Wed Aug 15 2018 Miro Hrončok <mhroncok@redhat.com> - 3.7.0-6
+- Use RPM built wheels of pip and setuptools in ensurepip instead of our rewheel patch
+
 * Fri Aug 10 2018 Igor Gnatenko <ignatenkobrain@fedoraproject.org> - 3.7.0-5
 - Fix wrong requirement on gdbm