gather: Log tag from which we pulled a package

For each tag we ask Koji about (there might be more than one in
modularity case), we create a log file with list of RPMs and details
about which tag they were pulled from. This makes it easier to find out
where the package is inherited from.

Fixes: #547
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/pkgset/pkgsets.py

@@ -259,7 +259,13 @@ class KojiPackageSet(PackageSetBase):
                        % (rpm_info, self.sigkey_ordering, paths))
         return None
 
-    def populate(self, tag, event=None, inherit=True):
+    def populate(self, tag, event=None, inherit=True, logfile=None):
+        """Populate the package set with packages from given tag.
+
+        :param event: the Koji event to query at (or latest if not given)
+        :param inherit: whether to enable tag inheritance
+        :param logfile: path to file where package source tags should be logged
+        """
         result_rpms = []
         result_srpms = []
 
@@ -288,5 +294,14 @@ class KojiPackageSet(PackageSetBase):
             else:
                 result_rpms.append((rpm_info, build_info))
         result = self.read_packages(result_rpms, result_srpms)
+
+        # Create a log with package NEVRAs and the tag they are coming from
+        if logfile:
+            with open(logfile, 'w') as f:
+                for rpm in rpms:
+                    build = builds_by_id[rpm['build_id']]
+                    f.write('{name}-{ep}:{version}-{release}.{arch}: {tag} [{tag_id}]\n'.format(
+                        tag=build['tag_name'], tag_id=build['tag_id'], ep=rpm['epoch'] or 0, **rpm))
+
         self.log_info("[DONE ] %s" % msg)
         return result

pungi/phases/pkgset/sources/source_koji.py

@@ -204,7 +204,12 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
             pkgset = pungi.phases.pkgset.pkgsets.KojiPackageSet(
                 koji_wrapper, compose.conf["sigkeys"], logger=compose._logger,
                 arches=all_arches)
-            pkgset.populate(compose_tag, event_id, inherit=inherit)
+            # Create a filename for log with package-to-tag mapping. The tag
+            # name is included in filename, so any slashes in it are replaced
+            # with underscores just to be safe.
+            logfile = compose.paths.log.log_file(
+                None, 'packages_from_%s' % compose_tag.replace('/', '_'))
+            pkgset.populate(compose_tag, event_id, inherit=inherit, logfile=logfile)
             for variant in compose.all_variants.values():
                 if compose_tag in variant_tags[variant]:
                     # Optimization for case where we have just single compose

tests/test_pkgset_pkgsets.py

@@ -167,7 +167,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
 
         pkgset = pkgsets.KojiPackageSet(self.koji_wrapper, [None])
 
-        result = pkgset.populate('f25')
+        result = pkgset.populate('f25', logfile=self.topdir + '/pkgset.log')
 
         self.assertEqual(
             self.koji_wrapper.koji_proxy.mock_calls,
@@ -182,6 +182,18 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
                                 'x86_64': ['rpms/bash@4.3.42@4.fc24@x86_64',
                                            'rpms/bash-debuginfo@4.3.42@4.fc24@x86_64']})
 
+        self.maxDiff = None
+        with open(self.topdir + '/pkgset.log') as f:
+            self.assertEqual(
+                sorted(f.read().strip().splitlines()),
+                sorted(['pungi-0:4.1.3-3.fc25.noarch: f25 [335]',
+                        'pungi-0:4.1.3-3.fc25.src: f25 [335]',
+                        'bash-0:4.3.42-4.fc24.i686: f25 [335]',
+                        'bash-0:4.3.42-4.fc24.x86_64: f25 [335]',
+                        'bash-0:4.3.42-4.fc24.src: f25 [335]',
+                        'bash-debuginfo-0:4.3.42-4.fc24.i686: f25 [335]',
+                        'bash-debuginfo-0:4.3.42-4.fc24.x86_64: f25 [335]']))
+
     def test_only_one_arch(self):
         self._touch_files([
             'rpms/bash@4.3.42@4.fc24@x86_64',

tests/test_pkgset_source_koji.py

@@ -81,6 +81,7 @@ class TestGetKojiEvent(helpers.PungiTestCase):
         with open(self.event_file) as f:
             self.assertEqual(json.load(f), EVENT_INFO)
 
+
 class TestPopulateGlobalPkgset(helpers.PungiTestCase):
     def setUp(self):
         super(TestPopulateGlobalPkgset, self).setUp()
@@ -106,7 +107,8 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
         self.assertIs(pkgset, orig_pkgset)
         self.assertEqual(
             pkgset.mock_calls,
-            [mock.call.populate('f25', 123456, inherit=True),
+            [mock.call.populate('f25', 123456, inherit=True,
+                                logfile=self.topdir + '/logs/global/packages_from_f25.global.log'),
              mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf',
                                       remove_path_prefix='/prefix')])
         self.assertItemsEqual(pickle_dumps.call_args_list,