pkgset: Reuse pkgset repos

JIRA: COMPOSE-4158
Signed-off-by: Haibo Lin <hlin@redhat.com>

pungi/paths.py

@@ -504,6 +504,14 @@ class WorkPaths(object):
         filename = "pkgset_%s_file_cache.pickle" % pkgset_name
         return os.path.join(self.topdir(arch="global"), filename)
 
+    def pkgset_reuse_file(self, pkgset_name):
+        """
+        Example:
+            work/global/pkgset_f30-compose_reuse.pickle
+        """
+        filename = "pkgset_%s_reuse.pickle" % pkgset_name
+        return os.path.join(self.topdir(arch="global", create_dir=False), filename)
+
 
 class ComposePaths(object):
     def __init__(self, compose):

pungi/phases/pkgset/common.py

@@ -48,12 +48,6 @@ def get_create_global_repo_cmd(compose, path_prefix, repo_dir_global, pkgset):
     createrepo_checksum = compose.conf["createrepo_checksum"]
     repo = CreaterepoWrapper(createrepo_c=createrepo_c)
 
-    pkgset.save_file_list(
-        compose.paths.work.package_list(arch="global", pkgset=pkgset),
-        remove_path_prefix=path_prefix,
-    )
-    pkgset.save_file_cache(compose.paths.work.pkgset_file_cache(pkgset.name))
-
     # find an old compose suitable for repodata reuse
     update_md_path = None
     old_repo_dir = compose.paths.old_compose_path(
@@ -196,21 +190,32 @@ class MaterializedPackageSet(object):
             pkgset_global.name, arch="global"
         )
         paths = {"global": repo_dir_global}
-        cmd = get_create_global_repo_cmd(
+
-            compose, path_prefix, repo_dir_global, pkgset_global
+        pkgset_global.save_file_list(
+            compose.paths.work.package_list(arch="global", pkgset=pkgset_global),
+            remove_path_prefix=path_prefix,
         )
-        logfile = compose.paths.log.log_file(
+        pkgset_global.save_file_cache(
-            "global", "arch_repo.%s" % pkgset_global.name
+            compose.paths.work.pkgset_file_cache(pkgset_global.name)
         )
-        t = threading.Thread(
+
-            target=run_create_global_repo, args=(compose, cmd, logfile)
+        if getattr(pkgset_global, "reuse", None) is None:
-        )
+            cmd = get_create_global_repo_cmd(
-        t.start()
+                compose, path_prefix, repo_dir_global, pkgset_global
+            )
+            logfile = compose.paths.log.log_file(
+                "global", "arch_repo.%s" % pkgset_global.name
+            )
+            t = threading.Thread(
+                target=run_create_global_repo, args=(compose, cmd, logfile)
+            )
+            t.start()
 
         package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global)
         package_sets["global"] = pkgset_global
 
-        t.join()
+        if getattr(pkgset_global, "reuse", None) is None:
+            t.join()
 
         create_arch_repos(compose, path_prefix, paths, pkgset_global, mmd)
 

pungi/phases/pkgset/pkgsets.py

@@ -20,6 +20,7 @@ It automatically finds a signed copies according to *sigkey_ordering*.
 """
 
 import itertools
+import json
 import os
 from six.moves import cPickle as pickle
 
@@ -30,7 +31,7 @@ import kobo.rpmlib
 from kobo.threads import WorkerThread, ThreadPool
 
 import pungi.wrappers.kojiwrapper
-from pungi.util import pkg_is_srpm
+from pungi.util import pkg_is_srpm, copy_all
 from pungi.arch import get_valid_arches, is_excluded
 
 
@@ -370,6 +371,7 @@ class KojiPackageSet(PackageSetBase):
         self.populate_only_packages = populate_only_packages
         self.cache_region = cache_region
         self.extra_builds = extra_builds or []
+        self.reuse = None
 
     def __getstate__(self):
         result = self.__dict__.copy()
@@ -583,6 +585,127 @@ class KojiPackageSet(PackageSetBase):
         self.log_info("[DONE ] %s" % msg)
         return result
 
+    def write_reuse_file(self, compose, include_packages):
+        """Write data to files for reusing in future.
+
+        :param compose: compose object
+        :param include_packages: an iterable of tuples (package name, arch) that should
+                                 be included.
+        """
+        reuse_file = compose.paths.work.pkgset_reuse_file(self.name)
+        self.log_info("Writing pkgset reuse file: %s" % reuse_file)
+        try:
+            with open(reuse_file, "wb") as f:
+                pickle.dump(
+                    {
+                        "name": self.name,
+                        "allow_invalid_sigkeys": self._allow_invalid_sigkeys,
+                        "arches": self.arches,
+                        "sigkeys": self.sigkey_ordering,
+                        "packages": self.packages,
+                        "populate_only_packages": self.populate_only_packages,
+                        "rpms_by_arch": self.rpms_by_arch,
+                        "srpms_by_name": self.srpms_by_name,
+                        "extra_builds": self.extra_builds,
+                        "include_packages": include_packages,
+                    },
+                    f,
+                    protocol=pickle.HIGHEST_PROTOCOL,
+                )
+        except Exception as e:
+            self.log_warning("Writing pkgset reuse file failed: %s" % str(e))
+
+    def _get_koji_event_from_file(self, event_file):
+        with open(event_file, "r") as f:
+            return json.load(f)["id"]
+
+    def try_to_reuse(self, compose, tag, inherit=True, include_packages=None):
+        """Try to reuse pkgset data of old compose.
+        :param compose: compose object
+        :param str tag: koji tag name
+        :param inherit: whether to enable tag inheritance
+        :param include_packages: an iterable of tuples (package name, arch) that should
+                                 be included.
+        """
+        self.log_info("Trying to reuse pkgset data of old compose")
+        if not compose.paths.get_old_compose_topdir():
+            self.log_debug("No old compose found. Nothing to reuse.")
+            return False
+
+        event_file = os.path.join(
+            compose.paths.work.topdir(arch="global", create_dir=False), "koji-event"
+        )
+        old_event_file = compose.paths.old_compose_path(event_file)
+
+        try:
+            koji_event = self._get_koji_event_from_file(event_file)
+            old_koji_event = self._get_koji_event_from_file(old_event_file)
+        except Exception as e:
+            self.log_debug("Can't read koji event from file: %s" % str(e))
+            return False
+
+        if koji_event != old_koji_event:
+            self.log_debug(
+                "Koji event doesn't match, querying changes between event %d and %d"
+                % (old_koji_event, koji_event)
+            )
+            changed = self.koji_proxy.queryHistory(
+                tables=["tag_listing"], tag=tag, afterEvent=old_koji_event
+            )
+            if changed["tag_listing"]:
+                self.log_debug("Builds under tag %s changed. Can't reuse." % tag)
+                return False
+
+            if inherit:
+                inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event)
+                for t in inherit_tags:
+                    changed = self.koji_proxy.queryHistory(
+                        tables=["tag_listing"],
+                        tag=t["name"],
+                        afterEvent=old_koji_event,
+                        beforeEvent=koji_event + 1,
+                    )
+                    if changed["tag_listing"]:
+                        self.log_debug(
+                            "Builds under inherited tag %s changed. Can't reuse."
+                            % t["name"]
+                        )
+                        return False
+
+        repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False)
+        old_repo_dir = compose.paths.old_compose_path(repo_dir)
+
+        old_reuse_file = compose.paths.old_compose_path(
+            compose.paths.work.pkgset_reuse_file(tag)
+        )
+
+        try:
+            self.log_debug("Loading reuse file: %s" % old_reuse_file)
+            reuse_data = self.load_old_file_cache(old_reuse_file)
+        except Exception as e:
+            self.log_debug("Failed to load reuse file: %s" % str(e))
+            return False
+
+        if (
+            reuse_data["allow_invalid_sigkeys"] == self._allow_invalid_sigkeys
+            and reuse_data["packages"] == self.packages
+            and reuse_data["populate_only_packages"] == self.populate_only_packages
+            and reuse_data["extra_builds"] == self.extra_builds
+            and reuse_data["sigkeys"] == self.sigkey_ordering
+            and reuse_data["include_packages"] == include_packages
+        ):
+            self.log_info("Copying repo data for reuse: %s" % old_repo_dir)
+            copy_all(old_repo_dir, repo_dir)
+            self.reuse = old_repo_dir
+            self.rpms_by_arch = reuse_data["rpms_by_arch"]
+            self.srpms_by_name = reuse_data["srpms_by_name"]
+            if self.old_file_cache:
+                self.file_cache = self.old_file_cache
+            return True
+        else:
+            self.log_info("Criteria does not match. Nothing to reuse.")
+            return False
+
 
 def _is_src(rpm_info):
     """Check if rpm info object returned by Koji refers to source packages."""

pungi/phases/pkgset/sources/source_koji.py

@@ -696,12 +696,20 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
                         nevra = parse_nvra(rpm_nevra)
                         modular_packages.add((nevra["name"], nevra["arch"]))
 
-        pkgset.populate(
+        pkgset.try_to_reuse(
+            compose,
             compose_tag,
-            event,
             inherit=should_inherit,
             include_packages=modular_packages,
         )
+
+        if pkgset.reuse is None:
+            pkgset.populate(
+                compose_tag,
+                event,
+                inherit=should_inherit,
+                include_packages=modular_packages,
+            )
         for variant in compose.all_variants.values():
             if compose_tag in variant_tags[variant]:
 
@@ -721,6 +729,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
             ),
         )
 
+        pkgset.write_reuse_file(compose, include_packages=modular_packages)
+
     return pkgsets
 
 

tests/test_pkgset_common.py

@@ -45,6 +45,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
     def _make_pkgset(self, name):
         pkgset = mock.Mock()
         pkgset.name = name
+        pkgset.reuse = None
 
         def mock_subset(primary, arch_list, exclusive_noarch):
             self.subsets[primary] = mock.Mock()

tests/test_pkgset_pkgsets.py

@@ -514,6 +514,203 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
         )
 
 
+class TestReuseKojiPkgset(helpers.PungiTestCase):
+    def setUp(self):
+        super(TestReuseKojiPkgset, self).setUp()
+        self.old_compose_dir = tempfile.mkdtemp()
+        self.old_compose = helpers.DummyCompose(self.old_compose_dir, {})
+        self.compose = helpers.DummyCompose(
+            self.topdir, {"old_composes": os.path.dirname(self.old_compose_dir)}
+        )
+
+        self.koji_wrapper = mock.Mock()
+
+        self.tag = "test-tag"
+        self.inherited_tag = "inherited-test-tag"
+        self.pkgset = pkgsets.KojiPackageSet(
+            self.tag, self.koji_wrapper, [None], arches=["x86_64"]
+        )
+        self.pkgset.log_debug = mock.Mock()
+        self.pkgset.log_info = mock.Mock()
+
+    def assert_not_reuse(self):
+        self.assertIsNone(getattr(self.pkgset, "reuse", None))
+
+    def test_resue_no_old_compose_found(self):
+        self.pkgset.try_to_reuse(self.compose, self.tag)
+        self.pkgset.log_info.assert_called_once_with(
+            "Trying to reuse pkgset data of old compose"
+        )
+        self.pkgset.log_debug.assert_called_once_with(
+            "No old compose found. Nothing to reuse."
+        )
+        self.assert_not_reuse()
+
+    @mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
+    def test_reuse_read_koji_event_file_failed(self, mock_old_topdir):
+        mock_old_topdir.return_value = self.old_compose_dir
+        self.pkgset._get_koji_event_from_file = mock.Mock(
+            side_effect=Exception("unknown error")
+        )
+        self.pkgset.try_to_reuse(self.compose, self.tag)
+        self.pkgset.log_debug.assert_called_once_with(
+            "Can't read koji event from file: unknown error"
+        )
+        self.assert_not_reuse()
+
+    @mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
+    def test_reuse_build_under_tag_changed(self, mock_old_topdir):
+        mock_old_topdir.return_value = self.old_compose_dir
+        self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
+        self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": [{}]}
+
+        self.pkgset.try_to_reuse(self.compose, self.tag)
+
+        self.assertEqual(
+            self.pkgset.log_debug.mock_calls,
+            [
+                mock.call(
+                    "Koji event doesn't match, querying changes between event 1 and 3"
+                ),
+                mock.call("Builds under tag %s changed. Can't reuse." % self.tag),
+            ],
+        )
+        self.assert_not_reuse()
+
+    @mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
+    def test_reuse_build_under_inherited_tag_changed(self, mock_old_topdir):
+        mock_old_topdir.return_value = self.old_compose_dir
+        self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
+        self.koji_wrapper.koji_proxy.queryHistory.side_effect = [
+            {"tag_listing": []},
+            {"tag_listing": [{}]},
+        ]
+        self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [
+            {"name": self.inherited_tag}
+        ]
+
+        self.pkgset.try_to_reuse(self.compose, self.tag)
+
+        self.assertEqual(
+            self.pkgset.log_debug.mock_calls,
+            [
+                mock.call(
+                    "Koji event doesn't match, querying changes between event 1 and 3"
+                ),
+                mock.call(
+                    "Builds under inherited tag %s changed. Can't reuse."
+                    % self.inherited_tag
+                ),
+            ],
+        )
+        self.assert_not_reuse()
+
+    @mock.patch("pungi.paths.os.path.exists", return_value=True)
+    @mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
+    def test_reuse_failed_load_reuse_file(self, mock_old_topdir, mock_exists):
+        mock_old_topdir.return_value = self.old_compose_dir
+        self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
+        self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
+        self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
+        self.pkgset.load_old_file_cache = mock.Mock(
+            side_effect=Exception("unknown error")
+        )
+
+        self.pkgset.try_to_reuse(self.compose, self.tag)
+
+        self.assertEqual(
+            self.pkgset.log_debug.mock_calls,
+            [
+                mock.call(
+                    "Koji event doesn't match, querying changes between event 1 and 3"
+                ),
+                mock.call(
+                    "Loading reuse file: %s"
+                    % os.path.join(
+                        self.old_compose_dir,
+                        "work/global",
+                        "pkgset_%s_reuse.pickle" % self.tag,
+                    )
+                ),
+                mock.call("Failed to load reuse file: unknown error"),
+            ],
+        )
+        self.assert_not_reuse()
+
+    @mock.patch("pungi.paths.os.path.exists", return_value=True)
+    @mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
+    def test_reuse_criteria_not_match(self, mock_old_topdir, mock_exists):
+        mock_old_topdir.return_value = self.old_compose_dir
+        self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
+        self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
+        self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
+        self.pkgset.load_old_file_cache = mock.Mock(
+            return_value={"allow_invalid_sigkeys": True}
+        )
+
+        self.pkgset.try_to_reuse(self.compose, self.tag)
+
+        self.assertEqual(
+            self.pkgset.log_debug.mock_calls,
+            [
+                mock.call(
+                    "Koji event doesn't match, querying changes between event 1 and 3"
+                ),
+                mock.call(
+                    "Loading reuse file: %s"
+                    % os.path.join(
+                        self.old_compose_dir,
+                        "work/global",
+                        "pkgset_%s_reuse.pickle" % self.tag,
+                    )
+                ),
+            ],
+        )
+        self.assertEqual(
+            self.pkgset.log_info.mock_calls,
+            [
+                mock.call("Trying to reuse pkgset data of old compose"),
+                mock.call("Criteria does not match. Nothing to reuse."),
+            ],
+        )
+        self.assert_not_reuse()
+
+    @mock.patch("pungi.phases.pkgset.pkgsets.copy_all")
+    @mock.patch("pungi.paths.os.path.exists", return_value=True)
+    @mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
+    def test_reuse_pkgset(self, mock_old_topdir, mock_exists, mock_copy_all):
+        mock_old_topdir.return_value = self.old_compose_dir
+        self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
+        self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
+        self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
+        self.pkgset.load_old_file_cache = mock.Mock(
+            return_value={
+                "allow_invalid_sigkeys": self.pkgset._allow_invalid_sigkeys,
+                "packages": self.pkgset.packages,
+                "populate_only_packages": self.pkgset.populate_only_packages,
+                "extra_builds": self.pkgset.extra_builds,
+                "sigkeys": self.pkgset.sigkey_ordering,
+                "include_packages": None,
+                "rpms_by_arch": mock.Mock(),
+                "srpms_by_name": mock.Mock(),
+            }
+        )
+        self.pkgset.old_file_cache = mock.Mock()
+
+        self.pkgset.try_to_reuse(self.compose, self.tag)
+
+        old_repo_dir = os.path.join(self.old_compose_dir, "work/global/repo", self.tag)
+        self.assertEqual(
+            self.pkgset.log_info.mock_calls,
+            [
+                mock.call("Trying to reuse pkgset data of old compose"),
+                mock.call("Copying repo data for reuse: %s" % old_repo_dir),
+            ],
+        )
+        self.assertEqual(old_repo_dir, self.pkgset.reuse)
+        self.assertEqual(self.pkgset.file_cache, self.pkgset.old_file_cache)
+
+
 @mock.patch("kobo.pkgset.FileCache", new=MockFileCache)
 class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase):
     def test_merge_in_another_arch(self):

tests/test_pkgset_source_koji.py

@@ -86,6 +86,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
     def test_populate(self, KojiPackageSet, materialize):
         materialize.side_effect = self.mock_materialize
 
+        KojiPackageSet.return_value.reuse = None
         orig_pkgset = KojiPackageSet.return_value
 
         pkgsets = source_koji.populate_global_pkgset(
@@ -113,6 +114,8 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
 
         materialize.side_effect = self.mock_materialize
 
+        KojiPackageSet.return_value.reuse = None
+
         pkgsets = source_koji.populate_global_pkgset(
             self.compose, self.koji_wrapper, "/prefix", 123456
         )