pkgset: Add ability to wait for signed packages

If packages are appearing quickly in Koji, and signing them is triggered
by automation, there may be a delay between the package being signed and
compose running. In such case it may be preferable to wait for the
signed copy rather than fail the compose.

JIRA: RHELCMP-3932
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

doc/configuration.rst

@@ -581,6 +581,17 @@ Options
     (for example) between composes, then Pungi may not respect those changes
     in your new compose.
 
+**signed_packages_retries** = 1
+    (*int*) -- In automated workflows a compose may start before signed
+    packages are written to disk. In such case it may make sense to wait for
+    the package to appear on storage. This option controls how many times to
+    try to look for the signed copy.
+
+**signed_packages_wait** = 30
+    (*int*) -- Interval in seconds for how long to wait between attemts to find
+    signed packages. This option only makes sense when
+    ``signed_packages_retries`` is set higher than to 1.
+
 
 Example
 -------

pungi/checks.py

@@ -722,6 +722,8 @@ def make_schema():
                 "minItems": 1,
                 "default": [None],
             },
+            "signed_packages_retries": {"type": "number", "default": 1},
+            "signed_packages_wait": {"type": "number", "default": 30},
             "variants_file": {"$ref": "#/definitions/str_or_scm_dict"},
             "comps_file": {"$ref": "#/definitions/str_or_scm_dict"},
             "comps_filter_environments": {"type": "boolean", "default": True},

pungi/phases/pkgset/pkgsets.py

@@ -22,6 +22,7 @@ It automatically finds a signed copies according to *sigkey_ordering*.
 import itertools
 import json
 import os
+import time
 from six.moves import cPickle as pickle
 
 import kobo.log
@@ -332,6 +333,8 @@ class KojiPackageSet(PackageSetBase):
         cache_region=None,
         extra_builds=None,
         extra_tasks=None,
+        signed_packages_retries=1,
+        signed_packages_wait=30,
     ):
         """
         Creates new KojiPackageSet.
@@ -364,6 +367,9 @@ class KojiPackageSet(PackageSetBase):
         :param list extra_tasks: Extra RPMs defined as Koji task IDs to get from Koji
             and include in the package set. Useful when building testing compose
             with RPM scratch builds.
+        :param int signed_packages_retries: How many times should a search for
+            signed package be repeated.
+        :param int signed_packages_wait: How long to wait between search attemts.
         """
         super(KojiPackageSet, self).__init__(
             name,
@@ -380,6 +386,8 @@ class KojiPackageSet(PackageSetBase):
         self.extra_builds = extra_builds or []
         self.extra_tasks = extra_tasks or []
         self.reuse = None
+        self.signed_packages_retries = signed_packages_retries
+        self.signed_packages_wait = signed_packages_wait
 
     def __getstate__(self):
         result = self.__dict__.copy()
@@ -506,6 +514,9 @@ class KojiPackageSet(PackageSetBase):
 
         pathinfo = self.koji_wrapper.koji_module.pathinfo
         paths = []
+
+        retries = self.signed_packages_retries
+        while retries > 0:
             for sigkey in self.sigkey_ordering:
                 if not sigkey:
                     # we're looking for *signed* copies here
@@ -514,10 +525,18 @@ class KojiPackageSet(PackageSetBase):
                 rpm_path = os.path.join(
                     pathinfo.build(build_info), pathinfo.signed(rpm_info, sigkey)
                 )
+                if rpm_path not in paths:
                     paths.append(rpm_path)
                 if os.path.isfile(rpm_path):
                     return rpm_path
 
+            # No signed copy was found, wait a little and try again.
+            retries -= 1
+            if retries > 0:
+                nvr = "%(name)s-%(version)s-%(release)s" % rpm_info
+                self.log_debug("Waiting for signed package to appear for %s", nvr)
+                time.sleep(self.signed_packages_wait)
+
         if None in self.sigkey_ordering or "" in self.sigkey_ordering:
             # use an unsigned copy (if allowed)
             rpm_path = os.path.join(pathinfo.build(build_info), pathinfo.rpm(rpm_info))

pungi/phases/pkgset/sources/source_koji.py

@@ -811,6 +811,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
             cache_region=compose.cache_region,
             extra_builds=extra_builds,
             extra_tasks=extra_tasks,
+            signed_packages_retries=compose.conf["signed_packages_retries"],
+            signed_packages_wait=compose.conf["signed_packages_wait"],
         )
 
         # Check if we have cache for this tag from previous compose. If so, use

tests/test_pkgset_pkgsets.py

@@ -303,6 +303,58 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
         )
         self.assertRegex(str(ctx.exception), figure)
 
+    @mock.patch("os.path.isfile")
+    @mock.patch("time.sleep")
+    def test_find_signed_after_wait(self, sleep, isfile):
+        checked_files = set()
+
+        def check_file(path):
+            """First check for any path will fail, second and further will succeed."""
+            if path in checked_files:
+                return True
+            checked_files.add(path)
+            return False
+
+        isfile.side_effect = check_file
+
+        fst_key, snd_key = ["cafebabe", "deadbeef"]
+        pkgset = pkgsets.KojiPackageSet(
+            "pkgset",
+            self.koji_wrapper,
+            [fst_key, snd_key],
+            arches=["x86_64"],
+            signed_packages_retries=3,
+            signed_packages_wait=5,
+        )
+
+        result = pkgset.populate("f25")
+
+        self.assertEqual(
+            self.koji_wrapper.koji_proxy.mock_calls,
+            [mock.call.listTaggedRPMS("f25", event=None, inherit=True, latest=True)],
+        )
+
+        fst_pkg = "signed/%s/bash-debuginfo@4.3.42@4.fc24@x86_64"
+        snd_pkg = "signed/%s/bash@4.3.42@4.fc24@x86_64"
+
+        self.assertPkgsetEqual(
+            result, {"x86_64": [fst_pkg % "cafebabe", snd_pkg % "cafebabe"]}
+        )
+        # Wait once for each of the two packages
+        self.assertEqual(sleep.call_args_list, [mock.call(5)] * 2)
+        # Each file will be checked three times
+        self.assertEqual(
+            isfile.call_args_list,
+            [
+                mock.call(os.path.join(self.topdir, fst_pkg % fst_key)),
+                mock.call(os.path.join(self.topdir, fst_pkg % snd_key)),
+                mock.call(os.path.join(self.topdir, fst_pkg % fst_key)),
+                mock.call(os.path.join(self.topdir, snd_pkg % fst_key)),
+                mock.call(os.path.join(self.topdir, snd_pkg % snd_key)),
+                mock.call(os.path.join(self.topdir, snd_pkg % fst_key)),
+            ],
+        )
+
     def test_can_not_find_signed_package_allow_invalid_sigkeys(self):
         pkgset = pkgsets.KojiPackageSet(
             "pkgset",
@@ -346,6 +398,32 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
             r"^RPM\(s\) not found for sigs: .+Check log for details.+",
         )
 
+    @mock.patch("time.sleep")
+    def test_can_not_find_signed_package_with_retries(self, time):
+        pkgset = pkgsets.KojiPackageSet(
+            "pkgset",
+            self.koji_wrapper,
+            ["cafebabe"],
+            arches=["x86_64"],
+            signed_packages_retries=3,
+            signed_packages_wait=5,
+        )
+
+        with self.assertRaises(RuntimeError) as ctx:
+            pkgset.populate("f25")
+
+        self.assertEqual(
+            self.koji_wrapper.koji_proxy.mock_calls,
+            [mock.call.listTaggedRPMS("f25", event=None, inherit=True, latest=True)],
+        )
+
+        self.assertRegex(
+            str(ctx.exception),
+            r"^RPM\(s\) not found for sigs: .+Check log for details.+",
+        )
+        # Two packages making three attempts each, so two waits per package.
+        self.assertEqual(time.call_args_list, [mock.call(5)] * 4)
+
     def test_packages_attribute(self):
         self._touch_files(
             [