osbs: Reuse images from old compose

JIRA: RHELCMP-5972
Signed-off-by: Haibo Lin <hlin@redhat.com>

pungi/checks.py

@@ -1202,6 +1202,7 @@ def make_schema():
                 "anyOf": [{"type": "string"}, {"type": "number"}],
                 "default": 10 * 1024 * 1024,
             },
+            "osbs_allow_reuse": {"type": "boolean", "default": False},
             "osbs": {
                 "type": "object",
                 "patternProperties": {

pungi/phases/osbs.py

@@ -1,23 +1,29 @@
 # -*- coding: utf-8 -*-
 
+import copy
 import fnmatch
 import json
 import os
 from kobo.threads import ThreadPool, WorkerThread
 from kobo import shortcuts
+from productmd.rpms import Rpms
+from six.moves import configparser
 
 from .base import ConfigGuardedPhase, PhaseLoggerMixin
 from .. import util
 from ..wrappers import kojiwrapper
+from ..wrappers.scm import get_file_from_scm
 
 
 class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase):
     name = "osbs"
 
-    def __init__(self, compose):
+    def __init__(self, compose, pkgset_phase, buildinstall_phase):
         super(OSBSPhase, self).__init__(compose)
         self.pool = ThreadPool(logger=self.logger)
         self.pool.registries = {}
+        self.pool.pkgset_phase = pkgset_phase
+        self.pool.buildinstall_phase = buildinstall_phase
 
     def run(self):
         for variant in self.compose.get_variants():
@@ -77,8 +83,8 @@ class OSBSThread(WorkerThread):
     def worker(self, compose, variant, config):
         msg = "OSBS task for variant %s" % variant.uid
         self.pool.log_info("[BEGIN] %s" % msg)
-        koji = kojiwrapper.KojiWrapper(compose)
+
-        koji.login()
+        original_config = copy.deepcopy(config)
 
         # Start task
         source = config.pop("url")
@@ -94,33 +100,99 @@ class OSBSThread(WorkerThread):
 
         config["yum_repourls"] = repos
 
-        task_id = koji.koji_proxy.buildContainer(
-            source, target, config, priority=priority
-        )
-
-        koji.save_task_id(task_id)
-
-        # Wait for it to finish and capture the output into log file (even
-        # though there is not much there).
         log_dir = os.path.join(compose.paths.log.topdir(), "osbs")
         util.makedirs(log_dir)
         log_file = os.path.join(
             log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
         )
+        reuse_file = log_file[:-4] + ".reuse.json"
+
+        try:
+            image_conf = self._get_image_conf(compose, original_config)
+        except Exception as e:
+            image_conf = None
+            self.pool.log_info(
+                "Can't get image-build.conf for variant: %s source: %s - %s"
+                % (variant.uid, source, str(e))
+            )
+
+        koji = kojiwrapper.KojiWrapper(compose)
+        koji.login()
+
+        task_id = self._try_to_reuse(
+            compose, variant, original_config, image_conf, reuse_file
+        )
+
+        if not task_id:
+            task_id = koji.koji_proxy.buildContainer(
+                source, target, config, priority=priority
+            )
+
+        koji.save_task_id(task_id)
+
+        # Wait for it to finish and capture the output into log file (even
+        # though there is not much there).
         if koji.watch_task(task_id, log_file) != 0:
             raise RuntimeError(
                 "OSBS: task %s failed: see %s for details" % (task_id, log_file)
             )
 
         scratch = config.get("scratch", False)
-        nvr = add_metadata(variant, task_id, compose, scratch)
+        nvr, archive_ids = add_metadata(variant, task_id, compose, scratch)
         if nvr:
             registry = get_registry(compose, nvr, registry)
             if registry:
                 self.pool.registries[nvr] = registry
 
+        self._write_reuse_metadata(
+            compose,
+            variant,
+            original_config,
+            image_conf,
+            task_id,
+            archive_ids,
+            reuse_file,
+        )
+
         self.pool.log_info("[DONE ] %s" % msg)
 
+    def _get_image_conf(self, compose, config):
+        """Get image-build.conf from git repo.
+
+        :param Compose compose: Current compose.
+        :param dict config: One osbs config item of compose.conf["osbs"][$variant]
+        """
+        tmp_dir = compose.mkdtemp(prefix="osbs_")
+
+        url = config["url"].split("#")
+        if len(url) == 1:
+            url.append(config["git_branch"])
+
+        filename = "image-build.conf"
+        get_file_from_scm(
+            {
+                "scm": "git",
+                "repo": url[0],
+                "branch": url[1],
+                "file": [filename],
+            },
+            tmp_dir,
+        )
+
+        c = configparser.ConfigParser()
+        c.read(os.path.join(tmp_dir, filename))
+        return c
+
+    def _get_ksurl(self, image_conf):
+        """Get ksurl from image-build.conf"""
+        ksurl = image_conf.get("image-build", "ksurl")
+
+        if ksurl:
+            resolver = util.GitUrlResolver(offline=False)
+            return resolver(ksurl)
+        else:
+            return None
+
     def _get_repo(self, compose, repo, gpgkey=None):
         """
         Return repo file URL of repo, if repo contains "://", it's already a
@@ -177,6 +249,151 @@ class OSBSThread(WorkerThread):
 
         return util.translate_path(compose, repo_file)
 
+    def _try_to_reuse(self, compose, variant, config, image_conf, reuse_file):
+        """Try to reuse results of old compose.
+
+        :param Compose compose: Current compose.
+        :param Variant variant: Current variant.
+        :param dict config: One osbs config item of compose.conf["osbs"][$variant]
+        :param ConfigParser image_conf: ConfigParser obj of image-build.conf.
+        :param str reuse_file: Path to reuse metadata file
+        """
+        log_msg = "Cannot reuse old osbs phase results - %s"
+
+        if not compose.conf["osbs_allow_reuse"]:
+            self.pool.log_info(log_msg % "reuse of old osbs results is disabled.")
+            return False
+
+        old_reuse_file = compose.paths.old_compose_path(reuse_file)
+        if not old_reuse_file:
+            self.pool.log_info(log_msg % "Can't find old reuse metadata file")
+            return False
+
+        try:
+            with open(old_reuse_file) as f:
+                old_reuse_metadata = json.load(f)
+        except Exception as e:
+            self.pool.log_info(
+                log_msg % "Can't load old reuse metadata file: %s" % str(e)
+            )
+            return False
+
+        if old_reuse_metadata["config"] != config:
+            self.pool.log_info(log_msg % "osbs config changed")
+            return False
+
+        if not image_conf:
+            self.pool.log_info(log_msg % "Can't get image-build.conf")
+            return False
+
+        # Make sure ksurl not change
+        try:
+            ksurl = self._get_ksurl(image_conf)
+        except Exception as e:
+            self.pool.log_info(
+                log_msg % "Can't get ksurl from image-build.conf - %s" % str(e)
+            )
+            return False
+
+        if not old_reuse_metadata["ksurl"]:
+            self.pool.log_info(
+                log_msg % "Can't get ksurl from old compose reuse metadata."
+            )
+            return False
+
+        if ksurl != old_reuse_metadata["ksurl"]:
+            self.pool.log_info(log_msg % "ksurl changed")
+            return False
+
+        # Make sure buildinstall phase is reused
+        try:
+            arches = image_conf.get("image-build", "arches").split(",")
+        except Exception as e:
+            self.pool.log_info(
+                log_msg % "Can't get arches from image-build.conf - %s" % str(e)
+            )
+        for arch in arches:
+            if not self.pool.buildinstall_phase.reused(variant, arch):
+                self.pool.log_info(
+                    log_msg % "buildinstall phase changed %s.%s" % (variant, arch)
+                )
+                return False
+
+        # Make sure rpms installed in image exists in current compose
+        rpm_manifest_file = compose.paths.compose.metadata("rpms.json")
+        rpm_manifest = Rpms()
+        rpm_manifest.load(rpm_manifest_file)
+        rpms = set()
+        for variant in rpm_manifest.rpms:
+            for arch in rpm_manifest.rpms[variant]:
+                for src in rpm_manifest.rpms[variant][arch]:
+                    for nevra in rpm_manifest.rpms[variant][arch][src]:
+                        rpms.add(nevra)
+
+        for nevra in old_reuse_metadata["rpmlist"]:
+            if nevra not in rpms:
+                self.pool.log_info(
+                    log_msg % "%s does not exist in current compose" % nevra
+                )
+                return False
+
+        self.pool.log_info(
+            "Reusing old OSBS task %d result" % old_reuse_file["task_id"]
+        )
+        return old_reuse_file["task_id"]
+
+    def _write_reuse_metadata(
+        self, compose, variant, config, image_conf, task_id, archive_ids, reuse_file
+    ):
+        """Write metadata to file for reusing.
+
+        :param Compose compose: Current compose.
+        :param Variant variant: Current variant.
+        :param dict config: One osbs config item of compose.conf["osbs"][$variant]
+        :param ConfigParser image_conf: ConfigParser obj of image-build.conf.
+        :param int task_id: Koji task id of osbs task.
+        :param list archive_ids: List of koji archive id
+        :param str reuse_file: Path to reuse metadata file.
+        """
+        msg = "Writing reuse metadata file %s" % reuse_file
+        compose.log_info(msg)
+
+        rpmlist = set()
+        koji = kojiwrapper.KojiWrapper(compose)
+        for archive_id in archive_ids:
+            rpms = koji.koji_proxy.listRPMs(imageID=archive_id)
+            for item in rpms:
+                if item["epoch"]:
+                    rpmlist.add(
+                        "%s:%s-%s-%s.%s"
+                        % (
+                            item["name"],
+                            item["epoch"],
+                            item["version"],
+                            item["release"],
+                            item["arch"],
+                        )
+                    )
+                else:
+                    rpmlist.add("%s.%s" % (item["nvr"], item["arch"]))
+
+        try:
+            ksurl = self._get_ksurl(image_conf)
+        except Exception:
+            ksurl = None
+
+        data = {
+            "config": config,
+            "ksurl": ksurl,
+            "rpmlist": sorted(rpmlist),
+            "task_id": task_id,
+        }
+        try:
+            with open(reuse_file, "w") as f:
+                json.dump(data, f, indent=4)
+        except Exception as e:
+            compose.log_info(msg + " failed - %s" % str(e))
+
 
 def add_metadata(variant, task_id, compose, is_scratch):
     """Given a task ID, find details about the container and add it to global
@@ -200,7 +417,7 @@ def add_metadata(variant, task_id, compose, is_scratch):
         compose.containers_metadata.setdefault(variant.uid, {}).setdefault(
             "scratch", []
         ).append(metadata)
-        return None
+        return None, []
 
     else:
         build_id = int(result["koji_builds"][0])
@@ -218,6 +435,7 @@ def add_metadata(variant, task_id, compose, is_scratch):
                 "creation_time": buildinfo["creation_time"],
             }
         )
+        archive_ids = []
         for archive in archives:
             data = {
                 "filename": archive["filename"],
@@ -234,4 +452,5 @@ def add_metadata(variant, task_id, compose, is_scratch):
             compose.containers_metadata.setdefault(variant.uid, {}).setdefault(
                 arch, []
             ).append(data)
-        return nvr
+            archive_ids.append(archive["id"])
+        return nvr, archive_ids

pungi/scripts/pungi_koji.py

@@ -408,7 +408,7 @@ def run_compose(
     livemedia_phase = pungi.phases.LiveMediaPhase(compose)
     image_build_phase = pungi.phases.ImageBuildPhase(compose, buildinstall_phase)
     osbuild_phase = pungi.phases.OSBuildPhase(compose)
-    osbs_phase = pungi.phases.OSBSPhase(compose)
+    osbs_phase = pungi.phases.OSBSPhase(compose, pkgset_phase, buildinstall_phase)
     image_container_phase = pungi.phases.ImageContainerPhase(compose)
     image_checksum_phase = pungi.phases.ImageChecksumPhase(compose)
     repoclosure_phase = pungi.phases.RepoclosurePhase(compose)

tests/test_osbs_phase.py

@@ -19,7 +19,7 @@ class OSBSPhaseTest(helpers.PungiTestCase):
 
         pool = ThreadPool.return_value
 
-        phase = osbs.OSBSPhase(compose)
+        phase = osbs.OSBSPhase(compose, None, None)
         phase.run()
 
         self.assertEqual(len(pool.add.call_args_list), 1)
@@ -33,7 +33,7 @@ class OSBSPhaseTest(helpers.PungiTestCase):
         compose = helpers.DummyCompose(self.topdir, {})
         compose.just_phases = None
         compose.skip_phases = []
-        phase = osbs.OSBSPhase(compose)
+        phase = osbs.OSBSPhase(compose, None, None)
         self.assertTrue(phase.skip())
 
     @mock.patch("pungi.phases.osbs.ThreadPool")
@@ -42,7 +42,7 @@ class OSBSPhaseTest(helpers.PungiTestCase):
         compose.just_phases = None
         compose.skip_phases = []
         compose.notifier = mock.Mock()
-        phase = osbs.OSBSPhase(compose)
+        phase = osbs.OSBSPhase(compose, None, None)
         phase.start()
         phase.stop()
         phase.pool.registries = {"foo": "bar"}
@@ -139,6 +139,8 @@ METADATA = {
     }
 }
 
+RPMS = []
+
 SCRATCH_TASK_RESULT = {
     "koji_builds": [],
     "repositories": [
@@ -182,6 +184,7 @@ class OSBSThreadTest(helpers.PungiTestCase):
             self.wrapper.koji_proxy.getTaskResult.return_value = TASK_RESULT
             self.wrapper.koji_proxy.getBuild.return_value = BUILD_INFO
             self.wrapper.koji_proxy.listArchives.return_value = ARCHIVES
+            self.wrapper.koji_proxy.listRPMs.return_value = RPMS
         self.wrapper.koji_proxy.getLatestBuilds.return_value = [
             mock.Mock(),
             mock.Mock(),
@@ -233,6 +236,7 @@ class OSBSThreadTest(helpers.PungiTestCase):
                 [
                     mock.call.koji_proxy.getBuild(54321),
                     mock.call.koji_proxy.listArchives(54321),
+                    mock.call.koji_proxy.listRPMs(imageID=1436049),
                 ]
             )
         self.assertEqual(self.wrapper.mock_calls, expect_calls)
@@ -269,8 +273,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         self.assertIn("    Possible reason: %r is a required property" % key, errors)
         self.assertEqual([], warnings)
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_minimal_run(self, KojiWrapper):
+    def test_minimal_run(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "f24-docker-candidate",
@@ -285,8 +290,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         self._assertCorrectMetadata()
         self._assertRepoFile()
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_run_failable(self, KojiWrapper):
+    def test_run_failable(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "f24-docker-candidate",
@@ -302,8 +308,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         self._assertCorrectMetadata()
         self._assertRepoFile()
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_run_with_more_args(self, KojiWrapper):
+    def test_run_with_more_args(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "f24-docker-candidate",
@@ -322,8 +329,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         self._assertCorrectMetadata()
         self._assertRepoFile()
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_run_with_extra_repos(self, KojiWrapper):
+    def test_run_with_extra_repos(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "f24-docker-candidate",
@@ -395,8 +403,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         self._assertCorrectCalls(options)
         self._assertCorrectMetadata()
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_run_with_deprecated_registry(self, KojiWrapper):
+    def test_run_with_deprecated_registry(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "f24-docker-candidate",
@@ -426,8 +435,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         self._assertRepoFile(["Server", "Everything"])
         self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": {"foo": "bar"}})
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_run_with_registry(self, KojiWrapper):
+    def test_run_with_registry(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "f24-docker-candidate",
@@ -457,8 +467,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         self._assertRepoFile(["Server", "Everything"])
         self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": [{"foo": "bar"}]})
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_run_with_extra_repos_in_list(self, KojiWrapper):
+    def test_run_with_extra_repos_in_list(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "f24-docker-candidate",
@@ -487,8 +498,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         self._assertCorrectMetadata()
         self._assertRepoFile(["Server", "Everything", "Client"])
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_run_with_gpgkey_enabled(self, KojiWrapper):
+    def test_run_with_gpgkey_enabled(self, KojiWrapper, get_file_from_scm):
         gpgkey = "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release"
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
@@ -547,8 +559,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
         }
         self._assertConfigMissing(cfg, "git_branch")
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_failing_task(self, KojiWrapper):
+    def test_failing_task(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "fedora-24-docker-candidate",
@@ -563,8 +576,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
 
         self.assertRegex(str(ctx.exception), r"task 12345 failed: see .+ for details")
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_failing_task_with_failable(self, KojiWrapper):
+    def test_failing_task_with_failable(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "fedora-24-docker-candidate",
@@ -577,8 +591,9 @@ class OSBSThreadTest(helpers.PungiTestCase):
 
         self.t.process((self.compose, self.compose.variants["Server"], cfg), 1)
 
+    @mock.patch("pungi.phases.osbs.get_file_from_scm")
     @mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
-    def test_scratch_metadata(self, KojiWrapper):
+    def test_scratch_metadata(self, KojiWrapper, get_file_from_scm):
         cfg = {
             "url": "git://example.com/repo?#BEEFCAFE",
             "target": "f24-docker-candidate",