almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

pkgset: Reuse pkgset repos

Browse Source 
JIRA: COMPOSE-4158
Signed-off-by: Haibo Lin <hlin@redhat.com>
This commit is contained in:
Haibo Lin 2020-02-25 11:02:39 +08:00
parent 169fa5b453
commit 3543f8fb3e
7 changed files with 365 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pungi/paths.py
View File

@ -504,6 +504,14 @@ class WorkPaths(object):
filename = "pkgset_%s_file_cache.pickle" % pkgset_name
return os.path.join(self.topdir(arch="global"), filename)
def pkgset_reuse_file(self, pkgset_name):
"""
Example:
work/global/pkgset_f30-compose_reuse.pickle
"""
filename = "pkgset_%s_reuse.pickle" % pkgset_name
return os.path.join(self.topdir(arch="global", create_dir=False), filename)
class ComposePaths(object):
def __init__(self, compose):

35
pungi/phases/pkgset/common.py
View File

@ -48,12 +48,6 @@ def get_create_global_repo_cmd(compose, path_prefix, repo_dir_global, pkgset):
createrepo_checksum = compose.conf["createrepo_checksum"]
repo = CreaterepoWrapper(createrepo_c=createrepo_c)
pkgset.save_file_list(
compose.paths.work.package_list(arch="global", pkgset=pkgset),
remove_path_prefix=path_prefix,
)
pkgset.save_file_cache(compose.paths.work.pkgset_file_cache(pkgset.name))
# find an old compose suitable for repodata reuse
update_md_path = None
old_repo_dir = compose.paths.old_compose_path(
@ -196,21 +190,32 @@ class MaterializedPackageSet(object):
pkgset_global.name, arch="global"
)
paths = {"global": repo_dir_global}
cmd = get_create_global_repo_cmd(
compose, path_prefix, repo_dir_global, pkgset_global
pkgset_global.save_file_list(
compose.paths.work.package_list(arch="global", pkgset=pkgset_global),
remove_path_prefix=path_prefix,
)
logfile = compose.paths.log.log_file(
"global", "arch_repo.%s" % pkgset_global.name
pkgset_global.save_file_cache(
compose.paths.work.pkgset_file_cache(pkgset_global.name)
)
t = threading.Thread(
target=run_create_global_repo, args=(compose, cmd, logfile)
)
t.start()
if getattr(pkgset_global, "reuse", None) is None:
cmd = get_create_global_repo_cmd(
compose, path_prefix, repo_dir_global, pkgset_global
)
logfile = compose.paths.log.log_file(
"global", "arch_repo.%s" % pkgset_global.name
)
t = threading.Thread(
target=run_create_global_repo, args=(compose, cmd, logfile)
)
t.start()
package_sets = populate_arch_pkgsets(compose, path_prefix, pkgset_global)
package_sets["global"] = pkgset_global
t.join()
if getattr(pkgset_global, "reuse", None) is None:
t.join()
create_arch_repos(compose, path_prefix, paths, pkgset_global, mmd)

125
pungi/phases/pkgset/pkgsets.py
View File

@ -20,6 +20,7 @@ It automatically finds a signed copies according to *sigkey_ordering*.
"""
import itertools
import json
import os
from six.moves import cPickle as pickle
@ -30,7 +31,7 @@ import kobo.rpmlib
from kobo.threads import WorkerThread, ThreadPool
import pungi.wrappers.kojiwrapper
from pungi.util import pkg_is_srpm
from pungi.util import pkg_is_srpm, copy_all
from pungi.arch import get_valid_arches, is_excluded
@ -370,6 +371,7 @@ class KojiPackageSet(PackageSetBase):
self.populate_only_packages = populate_only_packages
self.cache_region = cache_region
self.extra_builds = extra_builds or []
self.reuse = None
def __getstate__(self):
result = self.__dict__.copy()
@ -583,6 +585,127 @@ class KojiPackageSet(PackageSetBase):
self.log_info("[DONE ] %s" % msg)
return result
def write_reuse_file(self, compose, include_packages):
"""Write data to files for reusing in future.
:param compose: compose object
:param include_packages: an iterable of tuples (package name, arch) that should
be included.
"""
reuse_file = compose.paths.work.pkgset_reuse_file(self.name)
self.log_info("Writing pkgset reuse file: %s" % reuse_file)
try:
with open(reuse_file, "wb") as f:
pickle.dump(
{
"name": self.name,
"allow_invalid_sigkeys": self._allow_invalid_sigkeys,
"arches": self.arches,
"sigkeys": self.sigkey_ordering,
"packages": self.packages,
"populate_only_packages": self.populate_only_packages,
"rpms_by_arch": self.rpms_by_arch,
"srpms_by_name": self.srpms_by_name,
"extra_builds": self.extra_builds,
"include_packages": include_packages,
},
f,
protocol=pickle.HIGHEST_PROTOCOL,
)
except Exception as e:
self.log_warning("Writing pkgset reuse file failed: %s" % str(e))
def _get_koji_event_from_file(self, event_file):
with open(event_file, "r") as f:
return json.load(f)["id"]
def try_to_reuse(self, compose, tag, inherit=True, include_packages=None):
"""Try to reuse pkgset data of old compose.
:param compose: compose object
:param str tag: koji tag name
:param inherit: whether to enable tag inheritance
:param include_packages: an iterable of tuples (package name, arch) that should
be included.
"""
self.log_info("Trying to reuse pkgset data of old compose")
if not compose.paths.get_old_compose_topdir():
self.log_debug("No old compose found. Nothing to reuse.")
return False
event_file = os.path.join(
compose.paths.work.topdir(arch="global", create_dir=False), "koji-event"
)
old_event_file = compose.paths.old_compose_path(event_file)
try:
koji_event = self._get_koji_event_from_file(event_file)
old_koji_event = self._get_koji_event_from_file(old_event_file)
except Exception as e:
self.log_debug("Can't read koji event from file: %s" % str(e))
return False
if koji_event != old_koji_event:
self.log_debug(
"Koji event doesn't match, querying changes between event %d and %d"
% (old_koji_event, koji_event)
)
changed = self.koji_proxy.queryHistory(
tables=["tag_listing"], tag=tag, afterEvent=old_koji_event
)
if changed["tag_listing"]:
self.log_debug("Builds under tag %s changed. Can't reuse." % tag)
return False
if inherit:
inherit_tags = self.koji_proxy.getFullInheritance(tag, koji_event)
for t in inherit_tags:
changed = self.koji_proxy.queryHistory(
tables=["tag_listing"],
tag=t["name"],
afterEvent=old_koji_event,
beforeEvent=koji_event + 1,
)
if changed["tag_listing"]:
self.log_debug(
"Builds under inherited tag %s changed. Can't reuse."
% t["name"]
)
return False
repo_dir = compose.paths.work.pkgset_repo(tag, create_dir=False)
old_repo_dir = compose.paths.old_compose_path(repo_dir)
old_reuse_file = compose.paths.old_compose_path(
compose.paths.work.pkgset_reuse_file(tag)
)
try:
self.log_debug("Loading reuse file: %s" % old_reuse_file)
reuse_data = self.load_old_file_cache(old_reuse_file)
except Exception as e:
self.log_debug("Failed to load reuse file: %s" % str(e))
return False
if (
reuse_data["allow_invalid_sigkeys"] == self._allow_invalid_sigkeys
and reuse_data["packages"] == self.packages
and reuse_data["populate_only_packages"] == self.populate_only_packages
and reuse_data["extra_builds"] == self.extra_builds
and reuse_data["sigkeys"] == self.sigkey_ordering
and reuse_data["include_packages"] == include_packages
):
self.log_info("Copying repo data for reuse: %s" % old_repo_dir)
copy_all(old_repo_dir, repo_dir)
self.reuse = old_repo_dir
self.rpms_by_arch = reuse_data["rpms_by_arch"]
self.srpms_by_name = reuse_data["srpms_by_name"]
if self.old_file_cache:
self.file_cache = self.old_file_cache
return True
else:
self.log_info("Criteria does not match. Nothing to reuse.")
return False
def _is_src(rpm_info):
"""Check if rpm info object returned by Koji refers to source packages."""

14
pungi/phases/pkgset/sources/source_koji.py
View File

@ -696,12 +696,20 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
nevra = parse_nvra(rpm_nevra)
modular_packages.add((nevra["name"], nevra["arch"]))
pkgset.populate(
pkgset.try_to_reuse(
compose,
compose_tag,
event,
inherit=should_inherit,
include_packages=modular_packages,
)
if pkgset.reuse is None:
pkgset.populate(
compose_tag,
event,
inherit=should_inherit,
include_packages=modular_packages,
)
for variant in compose.all_variants.values():
if compose_tag in variant_tags[variant]:
@ -721,6 +729,8 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event):
),
)
pkgset.write_reuse_file(compose, include_packages=modular_packages)
return pkgsets

1
tests/test_pkgset_common.py
View File

@ -45,6 +45,7 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
def _make_pkgset(self, name):
pkgset = mock.Mock()
pkgset.name = name
pkgset.reuse = None
def mock_subset(primary, arch_list, exclusive_noarch):
self.subsets[primary] = mock.Mock()

197
tests/test_pkgset_pkgsets.py
View File

@ -514,6 +514,203 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
)
class TestReuseKojiPkgset(helpers.PungiTestCase):
def setUp(self):
super(TestReuseKojiPkgset, self).setUp()
self.old_compose_dir = tempfile.mkdtemp()
self.old_compose = helpers.DummyCompose(self.old_compose_dir, {})
self.compose = helpers.DummyCompose(
self.topdir, {"old_composes": os.path.dirname(self.old_compose_dir)}
)
self.koji_wrapper = mock.Mock()
self.tag = "test-tag"
self.inherited_tag = "inherited-test-tag"
self.pkgset = pkgsets.KojiPackageSet(
self.tag, self.koji_wrapper, [None], arches=["x86_64"]
)
self.pkgset.log_debug = mock.Mock()
self.pkgset.log_info = mock.Mock()
def assert_not_reuse(self):
self.assertIsNone(getattr(self.pkgset, "reuse", None))
def test_resue_no_old_compose_found(self):
self.pkgset.try_to_reuse(self.compose, self.tag)
self.pkgset.log_info.assert_called_once_with(
"Trying to reuse pkgset data of old compose"
)
self.pkgset.log_debug.assert_called_once_with(
"No old compose found. Nothing to reuse."
)
self.assert_not_reuse()
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
def test_reuse_read_koji_event_file_failed(self, mock_old_topdir):
mock_old_topdir.return_value = self.old_compose_dir
self.pkgset._get_koji_event_from_file = mock.Mock(
side_effect=Exception("unknown error")
)
self.pkgset.try_to_reuse(self.compose, self.tag)
self.pkgset.log_debug.assert_called_once_with(
"Can't read koji event from file: unknown error"
)
self.assert_not_reuse()
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
def test_reuse_build_under_tag_changed(self, mock_old_topdir):
mock_old_topdir.return_value = self.old_compose_dir
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": [{}]}
self.pkgset.try_to_reuse(self.compose, self.tag)
self.assertEqual(
self.pkgset.log_debug.mock_calls,
[
mock.call(
"Koji event doesn't match, querying changes between event 1 and 3"
),
mock.call("Builds under tag %s changed. Can't reuse." % self.tag),
],
)
self.assert_not_reuse()
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
def test_reuse_build_under_inherited_tag_changed(self, mock_old_topdir):
mock_old_topdir.return_value = self.old_compose_dir
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
self.koji_wrapper.koji_proxy.queryHistory.side_effect = [
{"tag_listing": []},
{"tag_listing": [{}]},
]
self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [
{"name": self.inherited_tag}
]
self.pkgset.try_to_reuse(self.compose, self.tag)
self.assertEqual(
self.pkgset.log_debug.mock_calls,
[
mock.call(
"Koji event doesn't match, querying changes between event 1 and 3"
),
mock.call(
"Builds under inherited tag %s changed. Can't reuse."
% self.inherited_tag
),
],
)
self.assert_not_reuse()
@mock.patch("pungi.paths.os.path.exists", return_value=True)
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
def test_reuse_failed_load_reuse_file(self, mock_old_topdir, mock_exists):
mock_old_topdir.return_value = self.old_compose_dir
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
self.pkgset.load_old_file_cache = mock.Mock(
side_effect=Exception("unknown error")
)
self.pkgset.try_to_reuse(self.compose, self.tag)
self.assertEqual(
self.pkgset.log_debug.mock_calls,
[
mock.call(
"Koji event doesn't match, querying changes between event 1 and 3"
),
mock.call(
"Loading reuse file: %s"
% os.path.join(
self.old_compose_dir,
"work/global",
"pkgset_%s_reuse.pickle" % self.tag,
)
),
mock.call("Failed to load reuse file: unknown error"),
],
)
self.assert_not_reuse()
@mock.patch("pungi.paths.os.path.exists", return_value=True)
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
def test_reuse_criteria_not_match(self, mock_old_topdir, mock_exists):
mock_old_topdir.return_value = self.old_compose_dir
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
self.pkgset.load_old_file_cache = mock.Mock(
return_value={"allow_invalid_sigkeys": True}
)
self.pkgset.try_to_reuse(self.compose, self.tag)
self.assertEqual(
self.pkgset.log_debug.mock_calls,
[
mock.call(
"Koji event doesn't match, querying changes between event 1 and 3"
),
mock.call(
"Loading reuse file: %s"
% os.path.join(
self.old_compose_dir,
"work/global",
"pkgset_%s_reuse.pickle" % self.tag,
)
),
],
)
self.assertEqual(
self.pkgset.log_info.mock_calls,
[
mock.call("Trying to reuse pkgset data of old compose"),
mock.call("Criteria does not match. Nothing to reuse."),
],
)
self.assert_not_reuse()
@mock.patch("pungi.phases.pkgset.pkgsets.copy_all")
@mock.patch("pungi.paths.os.path.exists", return_value=True)
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
def test_reuse_pkgset(self, mock_old_topdir, mock_exists, mock_copy_all):
mock_old_topdir.return_value = self.old_compose_dir
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": []}
self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
self.pkgset.load_old_file_cache = mock.Mock(
return_value={
"allow_invalid_sigkeys": self.pkgset._allow_invalid_sigkeys,
"packages": self.pkgset.packages,
"populate_only_packages": self.pkgset.populate_only_packages,
"extra_builds": self.pkgset.extra_builds,
"sigkeys": self.pkgset.sigkey_ordering,
"include_packages": None,
"rpms_by_arch": mock.Mock(),
"srpms_by_name": mock.Mock(),
}
)
self.pkgset.old_file_cache = mock.Mock()
self.pkgset.try_to_reuse(self.compose, self.tag)
old_repo_dir = os.path.join(self.old_compose_dir, "work/global/repo", self.tag)
self.assertEqual(
self.pkgset.log_info.mock_calls,
[
mock.call("Trying to reuse pkgset data of old compose"),
mock.call("Copying repo data for reuse: %s" % old_repo_dir),
],
)
self.assertEqual(old_repo_dir, self.pkgset.reuse)
self.assertEqual(self.pkgset.file_cache, self.pkgset.old_file_cache)
@mock.patch("kobo.pkgset.FileCache", new=MockFileCache)
class TestMergePackageSets(PkgsetCompareMixin, unittest.TestCase):
def test_merge_in_another_arch(self):

3
tests/test_pkgset_source_koji.py
View File

@ -86,6 +86,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
def test_populate(self, KojiPackageSet, materialize):
materialize.side_effect = self.mock_materialize
KojiPackageSet.return_value.reuse = None
orig_pkgset = KojiPackageSet.return_value
pkgsets = source_koji.populate_global_pkgset(
@ -113,6 +114,8 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
materialize.side_effect = self.mock_materialize
KojiPackageSet.return_value.reuse = None
pkgsets = source_koji.populate_global_pkgset(
self.compose, self.koji_wrapper, "/prefix", 123456
)