pkgset: Cherry-pick packages from Koji when we know already what packages will end up in compose

Merges: https://pagure.io/pungi/pull-request/763
Signed-off-by: Jan Kaluza <jkaluza@redhat.com>

pungi/phases/gather/__init__.py

@@ -335,12 +335,13 @@ def write_prepopulate_file(compose):
         shutil.rmtree(tmp_dir)
 
 
-def get_prepopulate_packages(compose, arch, variant):
+def get_prepopulate_packages(compose, arch, variant, include_arch=True):
     """Read prepopulate file and return list of packages for given tree.
 
     If ``variant`` is ``None``, all variants in the file are considered. The
     result of this function is a set of strings of format
-    ``package_name.arch``.
+    ``package_name.arch``. If ``include_arch`` is False, the ".arch" suffix
+    is not included in packages in returned list.
     """
     result = set()
 
@@ -361,7 +362,10 @@ def get_prepopulate_packages(compose, arch, variant):
                     raise ValueError(
                         "Incompatible package arch '%s' for tree arch '%s' in prepopulate package '%s'"
                         % (pkg_arch, arch, pkg_name))
-                result.add(i)
+                if include_arch:
+                    result.add(i)
+                else:
+                    result.add(pkg_name)
     return result
 
 
@@ -491,3 +495,51 @@ def get_system_release_packages(compose, arch, variant, package_sets):
             filter_packages.add((pkg.name, None))
 
     return packages, filter_packages
+
+
+def get_packages_to_gather(compose, arch=None, variant=None, include_arch=True,
+                           include_prepopulated=False):
+    """
+    Returns the list of names of packages and list of names of groups which
+    would be included in a compose as GATHER phase result.
+    This works only for "comps" or "json" gather_source. For "module"
+    gather_source, this always return an empty list, because it is not clear
+    what packages will end up in a compose before the gather phase is run.
+
+    :param str arch: Arch to return packages for. If not set, returns packages
+        for all arches.
+    :param Variant variant: Variant to return packages for, If not set, returns
+        packages for all variants of a compose.
+    :param include_arch: When True, the arch of package will be included in
+        returned list as ["pkg_name.arch", ...]. Otherwise only
+        ["pkg_name", ...] is returned.
+    :param include_prepopulated: When True, the prepopulated packages will
+        be included in a list of packages.
+    """
+    if compose.conf["gather_source"] == "module":
+        return []
+
+    arches = [arch] if arch else compose.get_arches()
+
+    GatherSource = get_gather_source(compose.conf["gather_source"])
+    src = GatherSource(compose)
+
+    packages = set([])
+    groups = set([])
+    for arch in arches:
+        pkgs, grps = src(arch, variant)
+        groups = groups.union(set(grps))
+
+        additional_packages = get_additional_packages(compose, arch, None)
+        for pkg_name, pkg_arch in pkgs | additional_packages:
+            if not include_arch or pkg_arch is None:
+                packages.add(pkg_name)
+            else:
+                packages.add("%s.%s" % (pkg_name, pkg_arch))
+
+        if include_prepopulated:
+            prepopulated = get_prepopulate_packages(
+                compose, arch, variant, include_arch)
+            packages = packages.union(prepopulated)
+
+    return list(packages), list(groups)

pungi/phases/pkgset/pkgsets.py

@@ -208,10 +208,13 @@ class FilelistPackageSet(PackageSetBase):
 
 
 class KojiPackageSet(PackageSetBase):
-    def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None):
+    def __init__(self, koji_wrapper, sigkey_ordering, arches=None, logger=None,
+                 packages=None):
         super(KojiPackageSet, self).__init__(sigkey_ordering=sigkey_ordering,
                                              arches=arches, logger=logger)
         self.koji_wrapper = koji_wrapper
+        # Names of packages to look for in the Koji tag.
+        self.packages = packages
 
     def __getstate__(self):
         result = self.__dict__.copy()
@@ -281,6 +284,7 @@ class KojiPackageSet(PackageSetBase):
             builds_by_id.setdefault(build_info["build_id"], build_info)
 
         skipped_arches = []
+        skipped_packages_count = 0
         for rpm_info in rpms:
             if self.arches and rpm_info["arch"] not in self.arches:
                 if rpm_info["arch"] not in skipped_arches:
@@ -288,11 +292,20 @@ class KojiPackageSet(PackageSetBase):
                     skipped_arches.append(rpm_info["arch"])
                 continue
 
+            if self.packages and rpm_info['name'] not in self.packages:
+                skipped_packages_count += 1
+                continue
+
             build_info = builds_by_id[rpm_info["build_id"]]
             if rpm_info["arch"] in ("src", "nosrc"):
                 result_srpms.append((rpm_info, build_info))
             else:
                 result_rpms.append((rpm_info, build_info))
+
+        if skipped_packages_count:
+            self.log_debug("Skipped %d packages, not marked as to be "
+                           "included in a compose." % skipped_packages_count)
+
         result = self.read_packages(result_rpms, result_srpms)
 
         # Create a log with package NEVRAs and the tag they are coming from

pungi/phases/pkgset/sources/source_koji.py

@@ -21,11 +21,13 @@ import re
 from kobo.shortcuts import force_list
 
 import pungi.wrappers.kojiwrapper
+from pungi.wrappers.comps import CompsWrapper
 import pungi.phases.pkgset.pkgsets
 from pungi.arch import get_valid_arches
 from pungi.util import is_arch_multilib, retry
 
 from pungi.phases.pkgset.common import create_arch_repos, create_global_repo, populate_arch_pkgsets
+from pungi.phases.gather import get_packages_to_gather
 
 
 import pungi.phases.pkgset.source
@@ -163,6 +165,20 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
     # List of compose_tags per variant
     variant_tags = {}
 
+    # In case we use "nodeps" gather_method, we might now the final list of
+    # packages which will end up in the compose even now, so instead of
+    # reading all the packages from Koji tag, we can just cherry-pick the ones
+    # which are really needed to do the compose and safe lot of time and
+    # resources here.
+    packages_to_gather = []
+    if compose.conf["gather_method"] == "nodeps":
+        packages_to_gather, groups = get_packages_to_gather(
+            compose, include_arch=False, include_prepopulated=True)
+        if groups:
+            comps = CompsWrapper(compose.paths.work.comps())
+            for group in groups:
+                packages_to_gather += comps.get_packages(group)
+
     session = get_pdc_client_session(compose)
     for variant in compose.all_variants.values():
         variant.pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
@@ -219,7 +235,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
                              "'%s'" % compose_tag)
             pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
                 koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
-                arches=all_arches)
+                arches=all_arches, packages=packages_to_gather)
             # Create a filename for log with package-to-tag mapping. The tag
             # name is included in filename, so any slashes in it are replaced
             # with underscores just to be safe.

pungi/phases/pkgset/sources/source_repos.py

@@ -25,7 +25,7 @@ from pungi.util import makedirs, is_arch_multilib
 from pungi.wrappers.pungi import PungiWrapper
 
 from pungi.phases.pkgset.common import create_global_repo, create_arch_repos, populate_arch_pkgsets
-from pungi.phases.gather import get_prepopulate_packages, get_additional_packages
+from pungi.phases.gather import get_prepopulate_packages, get_packages_to_gather
 from pungi.linker import LinkerThread, LinkerPool
 
 
@@ -160,24 +160,7 @@ def write_pungi_config(compose, arch, variant, repos=None, comps_repo=None, pack
         return
 
     compose.log_info(msg)
-
-    # TODO move to a function
-    gather_source = "GatherSource%s" % compose.conf["gather_source"]
-    from pungi.phases.gather.source import GatherSourceContainer
-    import pungi.phases.gather.sources
-    GatherSourceContainer.register_module(pungi.phases.gather.sources)
-    container = GatherSourceContainer()
-    SourceClass = container[gather_source]
-    src = SourceClass(compose)
-
-    packages = []
-    pkgs, grps = src(arch, variant)
-    additional_packages = get_additional_packages(compose, arch, None)
-    for pkg_name, pkg_arch in pkgs | additional_packages:
-        if pkg_arch is None:
-            packages.append(pkg_name)
-        else:
-            packages.append("%s.%s" % (pkg_name, pkg_arch))
+    packages, grps = get_packages_to_gather(compose, arch, variant)
 
     # include *all* packages providing system-release
     if "system-release" not in packages:

tests/test_gather_phase.py

@@ -755,6 +755,17 @@ class TestGetPrepopulate(helpers.PungiTestCase):
              "bar.x86_64"]
         )
 
+    def test_for_all_variants_include_arch_set_to_false(self):
+        helpers.copy_fixture('prepopulate.json',
+                             os.path.join(self.topdir, 'work', 'global', 'prepopulate.json'))
+        self.assertItemsEqual(
+            gather.get_prepopulate_packages(self.compose, 'x86_64', None,
+                                            include_arch=False),
+            ["foo-common",
+             "foo",
+             "bar"]
+        )
+
 
 class TestGatherPhase(helpers.PungiTestCase):
     @mock.patch('pungi.phases.gather.link_files')
@@ -797,3 +808,74 @@ class TestGatherPhase(helpers.PungiTestCase):
 
         self.assertEqual(gather_wrapper.call_args_list, [])
         self.assertTrue(os.path.isfile(os.path.join(self.topdir, 'compose', 'metadata', 'rpms.json')))
+
+
+class TestGetPackagesToGather(helpers.PungiTestCase):
+    def setUp(self):
+        super(TestGetPackagesToGather, self).setUp()
+        self.compose = helpers.DummyCompose(self.topdir, {
+            'additional_packages': [
+                ('.*', {'*': ['pkg', 'foo2.x86_64']}),
+            ]
+        })
+        helpers.copy_fixture('prepopulate.json',
+                             os.path.join(self.topdir, 'work', 'global', 'prepopulate.json'))
+
+    @mock.patch('pungi.phases.gather.get_gather_source')
+    def test_all_arches(self, get_gather_source):
+        get_gather_source.return_value = mock.Mock(
+            return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
+        )
+
+        packages, groups = gather.get_packages_to_gather(self.compose)
+
+        self.assertItemsEqual(packages, ["foo", "foo2.x86_64", "pkg"])
+        self.assertItemsEqual(groups, ["core"])
+
+    @mock.patch('pungi.phases.gather.get_gather_source')
+    def test_all_include_arch_set_to_false(self, get_gather_source):
+        get_gather_source.return_value = mock.Mock(
+            return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
+        )
+
+        packages, groups = gather.get_packages_to_gather(self.compose, include_arch=False)
+
+        self.assertItemsEqual(packages, ["foo", "foo2", "pkg"])
+        self.assertItemsEqual(groups, ["core"])
+
+    @mock.patch('pungi.phases.gather.get_gather_source')
+    def test_all_include_prepopulated(self, get_gather_source):
+        get_gather_source.return_value = mock.Mock(
+            return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
+        )
+
+        packages, groups = gather.get_packages_to_gather(self.compose, include_prepopulated=True)
+
+        self.assertItemsEqual(packages, ["foo", "pkg", "foo-common.noarch",
+                                         "foo.x86_64", "foo.i686", "foo2.x86_64",
+                                         "bar.x86_64"])
+        self.assertItemsEqual(groups, ["core"])
+
+    @mock.patch('pungi.phases.gather.get_gather_source')
+    def test_all_include_prepopulated_no_include_arch(self, get_gather_source):
+        get_gather_source.return_value = mock.Mock(
+            return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
+        )
+
+        packages, groups = gather.get_packages_to_gather(self.compose, include_prepopulated=True,
+                                                         include_arch=False)
+
+        self.assertItemsEqual(packages, ["foo", "pkg", "foo-common",
+                                         "foo2", "bar"])
+        self.assertItemsEqual(groups, ["core"])
+
+    @mock.patch('pungi.phases.gather.get_gather_source')
+    def test_all_one_arch(self, get_gather_source):
+        get_gather_source.return_value = mock.Mock(
+            return_value=mock.Mock(return_value=(set([('foo', None)]), set(['core'])))
+        )
+
+        packages, groups = gather.get_packages_to_gather(self.compose, "x86_64")
+
+        self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"])
+        self.assertItemsEqual(groups, ["core"])

tests/test_pkgset_pkgsets.py

@@ -275,6 +275,31 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
         self.assertRegexpMatches(str(ctx.exception),
                                  r'^RPM\(s\) not found for sigs: .+Check log for details.+')
 
+    def test_packages_attribute(self):
+        self._touch_files([
+            'rpms/pungi@4.1.3@3.fc25@noarch',
+            'rpms/pungi@4.1.3@3.fc25@src',
+            'rpms/bash@4.3.42@4.fc24@i686',
+            'rpms/bash@4.3.42@4.fc24@x86_64',
+            'rpms/bash@4.3.42@4.fc24@src',
+            'rpms/bash-debuginfo@4.3.42@4.fc24@i686',
+            'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64',
+        ])
+
+        pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None],
+                                        packages=["bash"])
+
+        result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log')
+
+        self.assertEqual(
+            self.koji_wrapper.koji_proxy.mock_calls,
+            [mock.call.listTaggedRPMS('f25', event=None, inherit=True, latest=True)])
+
+        self.assertPkgsetEqual(result,
+                               {'src': ['rpms/bash@4.3.42@4.fc24@src'],
+                                'i686': ['rpms/bash@4.3.42@4.fc24@i686'],
+                                'x86_64': ['rpms/bash@4.3.42@4.fc24@x86_64']})
+
 
 @mock.patch('kobo.pkgset.FileCache', new=MockFileCache)
 class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase):

tests/test_pkgset_source_koji.py

@@ -167,6 +167,27 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
             [mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf',
                                       remove_path_prefix='/prefix')])
 
+    @mock.patch('cPickle.dumps')
+    @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.populate')
+    @mock.patch('pungi.phases.pkgset.pkgsets.KojiPackageSet.save_file_list')
+    def test_populate_packages_to_gather(self, save_file_list, popuplate,
+                                         pickle_dumps):
+        self.compose = helpers.DummyCompose(self.topdir, {
+            'gather_method': 'nodeps',
+            'gather_source': 'none',
+            'pkgset_koji_tag': 'f25',
+            'sigkeys': mock.Mock(),
+            'additional_packages': [
+                ('.*', {'*': ['pkg', 'foo.x86_64']}),
+            ]
+        })
+        self.compose.DEBUG = False
+        pickle_dumps.return_value = 'DATA'
+
+        pkgset = source_koji.populate_global_pkgset(
+            self.compose, self.koji_wrapper, '/prefix', 123456)
+        self.assertItemsEqual(pkgset.packages, ["pkg", "foo"])
+
 
 class TestGetPackageSetFromKoji(helpers.PungiTestCase):
     def setUp(self):