almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Backport patches

Browse Source 
and rebase them to avoid conflicts
This commit is contained in:
Lubomír Sedlář 2018-03-08 15:19:20 +01:00
parent 27f8aada0f
commit ccb3abf912
9 changed files with 695 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
830.patch → 0001-Support-multiple-sources-in-one-variant.patch
View File

@ -1,8 +1,10 @@
From 364d7f5229dd36dd9c5f778f3575133be538cc47 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Feb 01 2018 13:31:09 +0000
Subject: Support multiple sources in one variant
From 659eb0215a7a53628533c195cb6dc6e461d8be27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Tue, 7 Nov 2017 14:16:37 +0100
Subject: [PATCH 1/8] Support multiple sources in one variant
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
With this patch the gather_source option is no longer used. Instead, all
sources are always used. If they return at least some input packages,
@ -18,11 +20,23 @@ Each gathering step is logged separately. All the logs are preserved for
later inspection.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
doc/configuration.rst | 36 +++++---
doc/gathering.rst | 6 +-
pungi/checks.py | 39 ++++----
pungi/paths.py | 21 +++--
pungi/phases/gather/__init__.py | 127 ++++++++++++++++++---------
pungi/phases/gather/methods/method_deps.py | 15 ++--
pungi/phases/gather/methods/method_nodeps.py | 5 +-
tests/data/dummy-pungi.conf | 8 +-
tests/helpers.py | 1 -
tests/test_config.py | 25 ------
tests/test_gather_phase.py | 51 +++++++----
tests/test_pkgset_source_koji.py | 1 -
12 files changed, 199 insertions(+), 136 deletions(-)
diff --git a/doc/configuration.rst b/doc/configuration.rst
index a140831..dbe5ba0 100644
index a1408317..dbe5ba08 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -35,7 +35,6 @@ Minimal Config Example
@ -96,7 +110,7 @@ index a140831..dbe5ba0 100644
# bz#123456
('^(Workstation|Server)$', {
diff --git a/doc/gathering.rst b/doc/gathering.rst
index 5111f61..08b348b 100644
index 5111f617..08b348b7 100644
--- a/doc/gathering.rst
+++ b/doc/gathering.rst
@@ -8,9 +8,9 @@ a subset of the content targeted at a particular use case.
@ -113,7 +127,7 @@ index 5111f61..08b348b 100644
.. note::
The inputs for both explicit package list and comps file are interpreted as
diff --git a/pungi/checks.py b/pungi/checks.py
index faf18a2..9b317ab 100644
index faf18a24..9b317ab6 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -554,12 +554,30 @@ def make_schema():
@ -186,7 +200,7 @@ index faf18a2..9b317ab 100644
"requires": (
(lambda x: bool(x), ["productimg_install_class"]),
diff --git a/pungi/paths.py b/pungi/paths.py
index 42d0e3e..0e50679 100644
index 42d0e3eb..0e506795 100644
--- a/pungi/paths.py
+++ b/pungi/paths.py
@@ -107,32 +107,37 @@ class WorkPaths(object):
@ -236,7 +250,7 @@ index 42d0e3e..0e50679 100644
def pungi_cache_dir(self, arch, variant=None, create_dir=True):
"""
diff --git a/pungi/phases/gather/__init__.py b/pungi/phases/gather/__init__.py
index 23b9079..bc056e2 100644
index 23b9079b..bc056e25 100644
--- a/pungi/phases/gather/__init__.py
+++ b/pungi/phases/gather/__init__.py
@@ -24,7 +24,7 @@ from productmd.rpms import Rpms
@ -450,7 +464,7 @@ index 23b9079..bc056e2 100644
return list(packages), list(groups)
diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py
index f4d0d6f..7c9e8fb 100644
index f4d0d6f8..7c9e8fb6 100644
--- a/pungi/phases/gather/methods/method_deps.py
+++ b/pungi/phases/gather/methods/method_deps.py
@@ -40,8 +40,9 @@ class GatherMethodDeps(pungi.phases.gather.method.GatherMethodBase):
@ -500,7 +514,7 @@ index f4d0d6f..7c9e8fb 100644
multilib_methods = get_arch_variant_data(compose.conf, 'multilib', arch, variant)
diff --git a/pungi/phases/gather/methods/method_nodeps.py b/pungi/phases/gather/methods/method_nodeps.py
index 69249da..ffc8e46 100644
index 69249da2..ffc8e460 100644
--- a/pungi/phases/gather/methods/method_nodeps.py
+++ b/pungi/phases/gather/methods/method_nodeps.py
@@ -28,7 +28,10 @@ class GatherMethodNodeps(pungi.phases.gather.method.GatherMethodBase):
@ -516,7 +530,7 @@ index 69249da..ffc8e46 100644
return self.worker(log, arch, variant, *args, **kwargs)
diff --git a/tests/data/dummy-pungi.conf b/tests/data/dummy-pungi.conf
index faaebb5..2d1f21f 100644
index faaebb5b..2d1f21f1 100644
--- a/tests/data/dummy-pungi.conf
+++ b/tests/data/dummy-pungi.conf
@@ -36,8 +36,12 @@ createrepo_checksum = "sha256"
@ -535,7 +549,7 @@ index faaebb5..2d1f21f 100644
check_deps = False
hashed_directories = True
diff --git a/tests/helpers.py b/tests/helpers.py
index 3f258ba..f069635 100644
index 3f258ba9..f069635d 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -186,7 +186,6 @@ BASE_CONFIG = dict(
@ -547,7 +561,7 @@ index 3f258ba..f069635 100644
)
diff --git a/tests/test_config.py b/tests/test_config.py
index 5cfdcbb..b2c1c8f 100644
index 5cfdcbb0..b2c1c8fb 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -213,31 +213,6 @@ class CreaterepoConfigTestCase(ConfigTestCase):
@ -583,7 +597,7 @@ index 5cfdcbb..b2c1c8f 100644
cfg = load_config(
pkgset_source='koji',
diff --git a/tests/test_gather_phase.py b/tests/test_gather_phase.py
index 28598fa..1630ef3 100644
index 28598fac..1630ef33 100644
--- a/tests/test_gather_phase.py
+++ b/tests/test_gather_phase.py
@@ -460,7 +460,7 @@ class TestGetVariantPackages(helpers.PungiTestCase):
@ -724,7 +738,7 @@ index 28598fa..1630ef3 100644
class TestWritePrepopulate(helpers.PungiTestCase):
def test_without_config(self):
diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py
index 6ea1716..b1e1308 100644
index 6ea17166..b1e1308d 100644
--- a/tests/test_pkgset_source_koji.py
+++ b/tests/test_pkgset_source_koji.py
@@ -204,7 +204,6 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
@ -735,4 +749,6 @@ index 6ea1716..b1e1308 100644
'pkgset_koji_tag': 'f25',
'sigkeys': mock.Mock(),
'additional_packages': [
--
2.13.6

25
859.patch → 0002-Remove-comps-groups-from-purely-modular-variants.patch
View File

@ -1,8 +1,10 @@
From bd852f4059b2061b30f335eef7e8734492a307f2 Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Mar 02 2018 07:46:23 +0000
Subject: Remove comps groups from purely modular variants
From de5dcc7e9ebb3c5b7201b404f302b9ac7dcab722 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Fri, 2 Mar 2018 08:33:37 +0100
Subject: [PATCH 2/8] Remove comps groups from purely modular variants
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The comps source should not return all groups when there are only
modules defined. This fixes part of the problem: non-modular packages
@ -13,11 +15,14 @@ filtered to not contain any groups (because packages from there will not
be in the repo).
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/phases/gather/sources/source_comps.py | 3 ++-
pungi/phases/init.py | 6 +++++-
tests/test_initphase.py | 1 +
3 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/pungi/phases/gather/sources/source_comps.py b/pungi/phases/gather/sources/source_comps.py
index 655c767..447e0a4 100644
index 655c7673..447e0a47 100644
--- a/pungi/phases/gather/sources/source_comps.py
+++ b/pungi/phases/gather/sources/source_comps.py
@@ -39,7 +39,8 @@ class GatherSourceComps(pungi.phases.gather.source.GatherSourceBase):
@ -31,7 +36,7 @@ index 655c767..447e0a4 100644
# filtering if the variant is top-level and has no groups (to use
# all of them).
diff --git a/pungi/phases/init.py b/pungi/phases/init.py
index cbda494..a01168a 100644
index cbda4949..a01168a9 100644
--- a/pungi/phases/init.py
+++ b/pungi/phases/init.py
@@ -45,12 +45,16 @@ class InitPhase(PhaseBase):
@ -53,7 +58,7 @@ index cbda494..a01168a 100644
else:
# The variant does not mention any groups, copy
diff --git a/tests/test_initphase.py b/tests/test_initphase.py
index 7d8b639..ae6c4e5 100644
index 7d8b639a..ae6c4e52 100644
--- a/tests/test_initphase.py
+++ b/tests/test_initphase.py
@@ -53,6 +53,7 @@ class TestInitPhase(PungiTestCase):
@ -64,4 +69,6 @@ index 7d8b639..ae6c4e5 100644
phase = init.InitPhase(compose)
phase.run()
--
2.13.6

21
861.patch → 0003-pkgset-Correctly-detect-single-tag-for-variant.patch
View File

@ -1,8 +1,10 @@
From 6514dc85f31853e261cb009f1f2f0e13e30e651c Mon Sep 17 00:00:00 2001
From: Lubomír Sedlář <lsedlar@redhat.com>
Date: Mar 06 2018 08:56:31 +0000
Subject: pkgset: Correctly detect single tag for variant
From adcb2e23312914535dd71b15d4705c8101055836 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Tue, 6 Mar 2018 08:47:17 +0100
Subject: [PATCH 3/8] pkgset: Correctly detect single tag for variant
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
We need to check tags for the variant, not for the whole compose. This
results in merge always being done even if there is a single tag. For
@ -11,14 +13,15 @@ f29 tag in Fedora this takes about 2 hours for each variant.
Relates: https://pagure.io/pungi/issue/860
Relates: https://bugzilla.redhat.com/show_bug.cgi?id=1551653
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/phases/pkgset/sources/source_koji.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 7d22290..534a77c 100644
index 98eed625..17d66773 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -320,7 +320,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
@@ -305,7 +305,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
if compose_tag in variant_tags[variant]:
# Optimization for case where we have just single compose
# tag - we do not have to merge in this case...
@ -27,4 +30,6 @@ index 7d22290..534a77c 100644
variant.pkgset = pkgset
else:
variant.pkgset.merge(pkgset, None, list(all_arches))
--
2.13.6

78
0004-image-build-Accept-tar.xz-extension-for-docker-image.patch Normal file
View File

@ -0,0 +1,78 @@
From fde41452c0bb030eb3467a87eaf25d7f789cba52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Thu, 8 Mar 2018 09:07:48 +0100
Subject: [PATCH 4/8] image-build: Accept tar.xz extension for docker images
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Fixes: https://pagure.io/pungi/issue/863
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/phases/image_build.py | 43 +++++++++++++++++++++++--------------------
1 file changed, 23 insertions(+), 20 deletions(-)
diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py
index ef4242c7..ea517b4e 100644
--- a/pungi/phases/image_build.py
+++ b/pungi/phases/image_build.py
@@ -20,22 +20,22 @@ from productmd.images import Image
# name will be ending with. The extensions are used to filter out which task
# results will be pulled into the compose.
EXTENSIONS = {
- 'docker': 'tar.gz',
- 'liveimg-squashfs': 'liveimg.squashfs',
- 'qcow': 'qcow',
- 'qcow2': 'qcow2',
- 'raw': 'raw',
- 'raw-xz': 'raw.xz',
- 'rhevm-ova': 'rhevm.ova',
- 'tar-gz': 'tar.gz',
- 'vagrant-hyperv': 'vagrant-hyperv.box',
- 'vagrant-libvirt': 'vagrant-libvirt.box',
- 'vagrant-virtualbox': 'vagrant-virtualbox.box',
- 'vagrant-vmware-fusion': 'vagrant-vmware-fusion.box',
- 'vdi': 'vdi',
- 'vmdk': 'vdmk',
- 'vpc': 'vhd',
- 'vsphere-ova': 'vsphere.ova',
+ 'docker': ['tar.gz', 'tar.xz'],
+ 'liveimg-squashfs': ['liveimg.squashfs'],
+ 'qcow': ['qcow'],
+ 'qcow2': ['qcow2'],
+ 'raw': ['raw'],
+ 'raw-xz': ['raw.xz'],
+ 'rhevm-ova': ['rhevm.ova'],
+ 'tar-gz': ['tar.gz'],
+ 'vagrant-hyperv': ['vagrant-hyperv.box'],
+ 'vagrant-libvirt': ['vagrant-libvirt.box'],
+ 'vagrant-virtualbox': ['vagrant-virtualbox.box'],
+ 'vagrant-vmware-fusion': ['vagrant-vmware-fusion.box'],
+ 'vdi': ['vdi'],
+ 'vmdk': ['vdmk'],
+ 'vpc': ['vhd'],
+ 'vsphere-ova': ['vsphere.ova'],
}
@@ -216,10 +216,13 @@ class CreateImageBuildThread(WorkerThread):
for arch, paths in paths.items():
for path in paths:
for format in cmd['image_conf']['image-build']['format']:
- suffix = EXTENSIONS[format]
- if path.endswith(suffix):
- image_infos.append({'path': path, 'suffix': suffix, 'type': format, 'arch': arch})
- break
+ for suffix in EXTENSIONS[format]:
+ if path.endswith(suffix):
+ image_infos.append({'path': path,
+ 'suffix': suffix,
+ 'type': format,
+ 'arch': arch})
+ break
# The usecase here is that you can run koji image-build with multiple --format
# It's ok to do it serialized since we're talking about max 2 images per single
--
2.13.6

225
0005-Write-package-whitelist-for-each-variant.patch Normal file
View File

@ -0,0 +1,225 @@
From 63329d48c3bc1c72a7bacd654a3ce6e93f6041e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 7 Mar 2018 12:35:33 +0100
Subject: [PATCH 5/8] Write package whitelist for each variant
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
If we have a package set for the variant (which happens if there are
modules), include a list of all NEVRAs in the pungi kickstart.
This can be used to make sure only packages from correct tag get into
the compose. If two packages with same name but different version get
into the compose, this can help get even older version into a particular
variant.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/ks.py | 15 +++++++++
pungi/phases/gather/methods/method_deps.py | 7 +++-
pungi/wrappers/pungi.py | 51 +++++++++++-------------------
tests/helpers.py | 1 +
tests/test_gather_method_deps.py | 29 +++++++++++++++--
5 files changed, 68 insertions(+), 35 deletions(-)
diff --git a/pungi/ks.py b/pungi/ks.py
index ecb8821f..517242aa 100644
--- a/pungi/ks.py
+++ b/pungi/ks.py
@@ -127,6 +127,19 @@ class PrepopulateSection(pykickstart.sections.Section):
self.handler.prepopulate.add(line)
+class PackageWhitelistSection(pykickstart.sections.Section):
+ sectionOpen = "%package-whitelist"
+
+ def handleLine(self, line):
+ if not self.handler:
+ return
+
+ (h, s, t) = line.partition('#')
+ line = h.rstrip()
+
+ self.handler.package_whitelist.add(line)
+
+
class KickstartParser(pykickstart.parser.KickstartParser):
def setupSections(self):
pykickstart.parser.KickstartParser.setupSections(self)
@@ -134,6 +147,7 @@ class KickstartParser(pykickstart.parser.KickstartParser):
self.registerSection(MultilibBlacklistSection(self.handler))
self.registerSection(MultilibWhitelistSection(self.handler))
self.registerSection(PrepopulateSection(self.handler))
+ self.registerSection(PackageWhitelistSection(self.handler))
def get_packages(self, dnf_obj):
packages = set()
@@ -194,6 +208,7 @@ class PungiHandler(HandlerClass):
self.multilib_blacklist = set()
self.multilib_whitelist = set()
self.prepopulate = set()
+ self.package_whitelist = set()
def get_ksparser(ks_path=None):
diff --git a/pungi/phases/gather/methods/method_deps.py b/pungi/phases/gather/methods/method_deps.py
index 7c9e8fb6..d38343f3 100644
--- a/pungi/phases/gather/methods/method_deps.py
+++ b/pungi/phases/gather/methods/method_deps.py
@@ -88,12 +88,17 @@ def write_pungi_config(compose, arch, variant, packages, groups, filter_packages
'No packages included in %s.%s (no comps groups, no input packages, no prepopulate)'
% (variant.uid, arch))
+ package_whitelist = set()
+ if variant.pkgset:
+ for rpm_obj in variant.pkgset.rpms_by_arch.get(arch, []):
+ package_whitelist.add(rpm_obj.nevra)
+
pungi_wrapper.write_kickstart(
ks_path=pungi_cfg, repos=repos, groups=groups, packages=packages_str,
exclude_packages=filter_packages_str,
lookaside_repos=lookaside_repos, fulltree_excludes=fulltree_excludes,
multilib_whitelist=multilib_whitelist, multilib_blacklist=multilib_blacklist,
- prepopulate=prepopulate)
+ prepopulate=prepopulate, package_whitelist=package_whitelist)
def resolve_deps(compose, arch, variant, source_name=None):
diff --git a/pungi/wrappers/pungi.py b/pungi/wrappers/pungi.py
index 501d6f57..04110f47 100644
--- a/pungi/wrappers/pungi.py
+++ b/pungi/wrappers/pungi.py
@@ -32,9 +32,22 @@ UNRESOLVED_DEPENDENCY_RE = re.compile(r"^.*Unresolvable dependency (.+) in ([^ ]
MISSING_COMPS_PACKAGE_RE = re.compile(r"^.*Could not find a match for (.+) in any configured repo")
+def _write_ks_section(f, section, lines):
+ if lines:
+ f.write("\n%%%s\n" % section)
+ for i in sorted(lines):
+ f.write("%s\n" % i)
+
+ f.write("%end\n")
+
+
class PungiWrapper(object):
- def write_kickstart(self, ks_path, repos, groups, packages, exclude_packages=None, comps_repo=None, lookaside_repos=None, fulltree_excludes=None, multilib_blacklist=None, multilib_whitelist=None, prepopulate=None):
+ def write_kickstart(self, ks_path, repos, groups, packages,
+ exclude_packages=None, comps_repo=None,
+ lookaside_repos=None, fulltree_excludes=None,
+ multilib_blacklist=None, multilib_whitelist=None,
+ prepopulate=None, package_whitelist=None):
groups = groups or []
exclude_packages = exclude_packages or {}
lookaside_repos = lookaside_repos or {}
@@ -75,37 +88,11 @@ class PungiWrapper(object):
kickstart.write("%end\n")
- # %fulltree-excludes
- if fulltree_excludes:
- kickstart.write("\n")
- kickstart.write("%fulltree-excludes\n")
- for i in sorted(fulltree_excludes):
- kickstart.write("%s\n" % i)
- kickstart.write("%end\n")
-
- # %multilib-blacklist
- if multilib_blacklist:
- kickstart.write("\n")
- kickstart.write("%multilib-blacklist\n")
- for i in sorted(multilib_blacklist):
- kickstart.write("%s\n" % i)
- kickstart.write("%end\n")
-
- # %multilib-whitelist
- if multilib_whitelist:
- kickstart.write("\n")
- kickstart.write("%multilib-whitelist\n")
- for i in sorted(multilib_whitelist):
- kickstart.write("%s\n" % i)
- kickstart.write("%end\n")
-
- # %prepopulate
- if prepopulate:
- kickstart.write("\n")
- kickstart.write("%prepopulate\n")
- for i in sorted(prepopulate):
- kickstart.write("%s\n" % i)
- kickstart.write("%end\n")
+ _write_ks_section(kickstart, "fulltree-excludes", fulltree_excludes)
+ _write_ks_section(kickstart, "multilib-blacklist", multilib_blacklist)
+ _write_ks_section(kickstart, "multilib-whitelist", multilib_whitelist)
+ _write_ks_section(kickstart, "prepopulate", prepopulate)
+ _write_ks_section(kickstart, "package-whitelist", package_whitelist)
kickstart.close()
diff --git a/tests/helpers.py b/tests/helpers.py
index f069635d..b82de42f 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -38,6 +38,7 @@ class MockVariant(mock.Mock):
self.mmds = []
self.arch_mmds = {}
self.variants = {}
+ self.pkgset = mock.Mock(rpms_by_arch={})
def __str__(self):
return self.uid
diff --git a/tests/test_gather_method_deps.py b/tests/test_gather_method_deps.py
index bd93a185..31bf82b7 100644
--- a/tests/test_gather_method_deps.py
+++ b/tests/test_gather_method_deps.py
@@ -39,7 +39,7 @@ class TestWritePungiConfig(helpers.PungiTestCase):
groups=['grp1'], prepopulate=prepopulate,
repos={'pungi-repo': self.topdir + '/work/x86_64/repo'},
exclude_packages=['pkg3', 'pkg4.x86_64'],
- fulltree_excludes=fulltree)
+ fulltree_excludes=fulltree, package_whitelist=set())
@mock.patch('pungi.phases.gather.get_lookaside_repos')
@mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
@@ -54,11 +54,36 @@ class TestWritePungiConfig(helpers.PungiTestCase):
multilib_whitelist=[], multilib_blacklist=[],
groups=[], prepopulate=None,
repos={'pungi-repo': self.topdir + '/work/x86_64/repo'},
- exclude_packages=[], fulltree_excludes=None)
+ exclude_packages=[], fulltree_excludes=None,
+ package_whitelist=set())
self.assertEqual(glr.call_args_list,
[mock.call(self.compose, 'x86_64', self.compose.variants['Server'])])
@mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
+ def test_with_whitelist(self, PungiWrapper):
+ pkgs = [('pkg1', None), ('pkg2', 'x86_64')]
+ grps = ['grp1']
+ filter = [('pkg3', None), ('pkg4', 'x86_64')]
+ self.compose.variants['Server'].pkgset.rpms_by_arch['x86_64'] = [
+ mock.Mock(nevra='pkg-1.0.0-1')
+ ]
+ white = mock.Mock()
+ black = mock.Mock()
+ prepopulate = mock.Mock()
+ fulltree = mock.Mock()
+ deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'],
+ pkgs, grps, filter, white, black,
+ prepopulate=prepopulate, fulltree_excludes=fulltree)
+ self.assertWritten(PungiWrapper, packages=['pkg1', 'pkg2.x86_64'],
+ ks_path=self.topdir + '/work/x86_64/pungi/Server.x86_64.conf',
+ lookaside_repos={}, multilib_whitelist=white, multilib_blacklist=black,
+ groups=['grp1'], prepopulate=prepopulate,
+ repos={'pungi-repo': self.topdir + '/work/x86_64/repo'},
+ exclude_packages=['pkg3', 'pkg4.x86_64'],
+ fulltree_excludes=fulltree,
+ package_whitelist=set(['pkg-1.0.0-1']))
+
+ @mock.patch('pungi.phases.gather.methods.method_deps.PungiWrapper')
def test_without_input(self, PungiWrapper):
with self.assertRaises(RuntimeError) as ctx:
deps.write_pungi_config(self.compose, 'x86_64', self.compose.variants['Server'],
--
2.13.6

158
0006-gather-Honor-package-whitelist.patch Normal file
View File

@ -0,0 +1,158 @@
From 1bfea4523b803917e37f81f83519721848012674 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 7 Mar 2018 13:42:09 +0100
Subject: [PATCH 6/8] gather: Honor package whitelist
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Basically everything not on the list is excluded. This has to be applied
before we filter only the latest versions (otherwise we could lose
packages that are on the whitelist).
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
bin/pungi-gather | 1 +
pungi/gather_dnf.py | 29 +++++++++++++++++++-----
tests/test_gather.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 88 insertions(+), 5 deletions(-)
diff --git a/bin/pungi-gather b/bin/pungi-gather
index daf80f2c..aa70977b 100755
--- a/bin/pungi-gather
+++ b/bin/pungi-gather
@@ -125,6 +125,7 @@ def main(persistdir, cachedir):
gather_opts.multilib_whitelist = ksparser.handler.multilib_whitelist
gather_opts.prepopulate = ksparser.handler.prepopulate
gather_opts.fulltree_excludes = ksparser.handler.fulltree_excludes
+ gather_opts.package_whitelist = ksparser.handler.package_whitelist
g = Gather(dnf_obj, gather_opts)
diff --git a/pungi/gather_dnf.py b/pungi/gather_dnf.py
index 1a47eea6..1023c57d 100644
--- a/pungi/gather_dnf.py
+++ b/pungi/gather_dnf.py
@@ -69,6 +69,8 @@ class GatherOptions(pungi.common.OptionsBase):
# lookaside repos; packages will be flagged accordingly
self.lookaside_repos = []
+ self.package_whitelist = set()
+
self.merge_options(**kwargs)
@@ -363,12 +365,29 @@ class Gather(GatherBase):
self.logger.debug("EXCLUDED by %s: %s", pattern, [str(p) for p in pkgs])
self.dnf._sack.add_excludes(pkgs)
+ all_queues = ['q_binary_packages', 'q_native_binary_packages',
+ 'q_multilib_binary_packages', 'q_noarch_binary_packages',
+ 'q_source_packages', 'q_native_debug_packages',
+ 'q_multilib_debug_packages']
+
+ if self.opts.package_whitelist:
+ with Profiler("Gather._apply_excludes():apply-package-whitelist'"):
+ to_keep = []
+ for pattern in self.opts.package_whitelist:
+ nvra = parse_nvra(pattern)
+ nvra.pop('src')
+ try:
+ nvra['epoch'] = int(nvra.pop('epoch'))
+ except ValueError:
+ pass
+ to_keep.extend(self._query.filter(**nvra).run())
+
+ for queue in all_queues:
+ setattr(self, queue, getattr(self, queue).filter(pkg=to_keep).latest().apply())
+
with Profiler("Gather._apply_excludes():exclude-queries"):
- self._filter_queue('q_binary_packages', exclude)
- self._filter_queue('q_native_binary_packages', exclude)
- self._filter_queue('q_multilib_binary_packages', exclude)
- self._filter_queue('q_noarch_binary_packages', exclude)
- self._filter_queue('q_source_packages', exclude)
+ for queue in all_queues:
+ self._filter_queue(queue, exclude)
@Profiler("Gather.add_initial_packages()")
def add_initial_packages(self, pattern_list):
diff --git a/tests/test_gather.py b/tests/test_gather.py
index 0b015abe..e73ae9c3 100644
--- a/tests/test_gather.py
+++ b/tests/test_gather.py
@@ -1791,8 +1791,71 @@ class DNFDepsolvingTestCase(DepsolvingBase, unittest.TestCase):
def test_bash_older(self):
pass
+ def test_whitelist_old_version(self):
+ # There are two version of dummy-bash in the package set; let's
+ # whitelist only the older one and its dependencies.
+ packages = [
+ "dummy-bash",
+ ]
+ package_whitelist = [
+ "dummy-basesystem-10.0-6.noarch",
+ "dummy-basesystem-10.0-6.src",
+ "dummy-bash-debuginfo-4.2.37-5.x86_64",
+ "dummy-bash-4.2.37-5.x86_64",
+ "dummy-bash-4.2.37-5.src",
+ "dummy-filesystem-4.2.37-6.x86_64",
+ "dummy-filesystem-4.2.37-6.src",
+ "dummy-glibc-common-2.14-5.x86_64",
+ "dummy-glibc-debuginfo-common-2.14-5.x86_64",
+ "dummy-glibc-debuginfo-2.14-5.x86_64",
+ "dummy-glibc-2.14-5.x86_64",
+ "dummy-glibc-2.14-5.src",
+ ]
+ pkg_map = self.go(packages, None, greedy="none", package_whitelist=package_whitelist)
+
+ self.assertNotIn("dummy-bash-4.2.37-5.i686.rpm", pkg_map["rpm"])
+ self.assertNotIn("dummy-bash-4.2.37-6.i686.rpm", pkg_map["rpm"])
+ self.assertNotIn("dummy-bash-4.2.37-6.x86_64.rpm", pkg_map["rpm"])
+
+ self.assertItemsEqual(pkg_map["rpm"], [
+ "dummy-basesystem-10.0-6.noarch.rpm",
+ "dummy-bash-4.2.37-5.x86_64.rpm",
+ "dummy-filesystem-4.2.37-6.x86_64.rpm",
+ "dummy-glibc-2.14-5.x86_64.rpm",
+ "dummy-glibc-common-2.14-5.x86_64.rpm",
+ ])
+ self.assertItemsEqual(pkg_map["srpm"], [
+ "dummy-basesystem-10.0-6.src.rpm",
+ "dummy-bash-4.2.37-5.src.rpm",
+ "dummy-filesystem-4.2.37-6.src.rpm",
+ "dummy-glibc-2.14-5.src.rpm",
+ ])
+ self.assertItemsEqual(pkg_map["debuginfo"], [
+ "dummy-bash-debuginfo-4.2.37-5.x86_64.rpm",
+ "dummy-glibc-debuginfo-2.14-5.x86_64.rpm",
+ "dummy-glibc-debuginfo-common-2.14-5.x86_64.rpm",
+ ])
+
def test_firefox_selfhosting_with_krb5_lookaside(self):
super(DNFDepsolvingTestCase, self).test_firefox_selfhosting_with_krb5_lookaside()
self.assertFlags("dummy-krb5-devel-1.10-5.x86_64", [PkgFlag.lookaside])
self.assertFlags("dummy-krb5-1.10-5.src", [PkgFlag.lookaside])
self.assertFlags("dummy-krb5-debuginfo-1.10-5.x86_64", [PkgFlag.lookaside])
+
+ def test_package_whitelist(self):
+ packages = ['*']
+ whitelist = [
+ 'dummy-bash-4.2.37-6.x86_64',
+ 'dummy-bash-4.2.37-6.src',
+ ]
+
+ pkg_map = self.go(packages, None, package_whitelist=whitelist)
+
+ self.assertItemsEqual(pkg_map["rpm"], [
+ 'dummy-bash-4.2.37-6.x86_64.rpm',
+ ])
+ self.assertItemsEqual(pkg_map["srpm"], [
+ 'dummy-bash-4.2.37-6.src.rpm',
+ ])
+ self.assertItemsEqual(pkg_map["debuginfo"], [
+ ])
--
2.13.6

78
0007-pkgset-Remove-check-for-unique-name.patch Normal file
View File

@ -0,0 +1,78 @@
From 74b0d14095733c66c54d47edaac69ef056f55332 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 7 Mar 2018 13:58:53 +0100
Subject: [PATCH 7/8] pkgset: Remove check for unique name
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
We now have a way to select even older version of package (since the
newer one can be left out of the whitelist), so we can include multiple
versions of the same package into global package set.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/phases/pkgset/pkgsets.py | 9 +--------
pungi/phases/pkgset/sources/source_koji.py | 5 ++---
2 files changed, 3 insertions(+), 11 deletions(-)
diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
index 4d902f4f..9038f1b1 100644
--- a/pungi/phases/pkgset/pkgsets.py
+++ b/pungi/phases/pkgset/pkgsets.py
@@ -134,12 +134,9 @@ class PackageSetBase(kobo.log.LoggingBase):
return self.rpms_by_arch
- def merge(self, other, primary_arch, arch_list, unique_name=False):
+ def merge(self, other, primary_arch, arch_list):
"""
Merge ``other`` package set into this instance.
-
- With ``unique_name=True`` a package will be added only if there is not
- a package with the same name already.
"""
msg = "Merging package sets for %s: %s" % (primary_arch, arch_list)
self.log_debug("[BEGIN] %s" % msg)
@@ -163,15 +160,11 @@ class PackageSetBase(kobo.log.LoggingBase):
else:
exclusivearch_list = None
for arch in arch_list:
- known_packages = set(pkg.name for pkg in self.rpms_by_arch.get(arch, []))
self.rpms_by_arch.setdefault(arch, [])
for i in other.rpms_by_arch.get(arch, []):
if i.file_path in self.file_cache:
# TODO: test if it really works
continue
- if unique_name and i.name in known_packages:
- self.log_debug('Not merging in %r' % i)
- continue
if exclusivearch_list and arch == "noarch":
if is_excluded(i, exclusivearch_list, logger=self._logger):
continue
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 17d66773..94ee79df 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -263,7 +263,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
if not variant_tags[variant]:
variant_tags[variant].extend(force_list(compose.conf["pkgset_koji_tag"]))
- # Add global tag if supplied.
+ # Add global tag(s) if supplied.
if 'pkgset_koji_tag' in compose.conf:
if compose.conf["pkgset_koji_tag"] == "not-used":
# The magic value is used for modular composes to avoid errors
@@ -314,8 +314,7 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
if len(compose_tags) == 1:
global_pkgset = pkgset
else:
- global_pkgset.merge(pkgset, None, list(all_arches),
- unique_name=compose_tag in compose.conf['pkgset_koji_tag'])
+ global_pkgset.merge(pkgset, None, list(all_arches))
with open(global_pkgset_path, 'wb') as f:
data = pickle.dumps(global_pkgset)
f.write(data)
--
2.13.6

80
0008-pkgset-Merge-initial-package-set-without-checks.patch Normal file
View File

@ -0,0 +1,80 @@
From 40c8f95b2bba62f454c7f996409e8bf1d775eee4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= <lsedlar@redhat.com>
Date: Wed, 7 Mar 2018 13:59:12 +0100
Subject: [PATCH 8/8] pkgset: Merge initial package set without checks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
For the first pass we don't need to filter out exclusive architectures,
and we don't need to exclude source packages without any binary
packages. We just want to merge the two package sets as fast as
possible.
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
---
pungi/phases/pkgset/pkgsets.py | 11 +++++++++++
pungi/phases/pkgset/sources/source_koji.py | 4 ++--
tests/test_pkgset_source_koji.py | 4 ++--
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
index 9038f1b1..d53c6af9 100644
--- a/pungi/phases/pkgset/pkgsets.py
+++ b/pungi/phases/pkgset/pkgsets.py
@@ -182,6 +182,17 @@ class PackageSetBase(kobo.log.LoggingBase):
self.log_debug("[DONE ] %s" % msg)
+ def fast_merge(self, other):
+ """
+ Merge two package sets together without any filtering of packages. All
+ packages from `other` package set are taken.
+ """
+ for arch in other.rpms_by_arch.keys():
+ self.rpms_by_arch.setdefault(arch, [])
+ for i in other.rpms_by_arch.get(arch, []):
+ self.file_cache.file_cache[i.file_path] = i
+ self.rpms_by_arch[arch].append(i)
+
def save_file_list(self, file_path, remove_path_prefix=None):
with open(file_path, "w") as f:
for arch in sorted(self.rpms_by_arch):
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 94ee79df..2ce14be6 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -308,13 +308,13 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
if len(variant_tags[variant]) == 1:
variant.pkgset = pkgset
else:
- variant.pkgset.merge(pkgset, None, list(all_arches))
+ variant.pkgset.fast_merge(pkgset)
# Optimization for case where we have just single compose
# tag - we do not have to merge in this case...
if len(compose_tags) == 1:
global_pkgset = pkgset
else:
- global_pkgset.merge(pkgset, None, list(all_arches))
+ global_pkgset.fast_merge(pkgset)
with open(global_pkgset_path, 'wb') as f:
data = pickle.dumps(global_pkgset)
f.write(data)
diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py
index b1e1308d..53670843 100644
--- a/tests/test_pkgset_source_koji.py
+++ b/tests/test_pkgset_source_koji.py
@@ -170,8 +170,8 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
logfile=self.topdir + '/logs/global/packages_from_f25-extra.global.log')])
pkgset.assert_has_calls([mock.call.save_file_list(self.topdir + '/work/global/package_list/global.conf',
remove_path_prefix='/prefix')])
- # for each tag, call pkgset.merge once for each variant and once for global pkgset
- self.assertEqual(pkgset.merge.call_count, 2 * (len(self.compose.all_variants.values()) + 1))
+ # for each tag, call pkgset.fast_merge once for each variant and once for global pkgset
+ self.assertEqual(pkgset.fast_merge.call_count, 2 * (len(self.compose.all_variants.values()) + 1))
self.assertItemsEqual(pickle_dumps.call_args_list,
[mock.call(orig_pkgset)])
with open(self.pkgset_path) as f:
--
2.13.6

17
pungi.spec
View File

@ -1,15 +1,20 @@
Name: pungi
Version: 4.1.22
Release: 6%{?dist}
Release: 7%{?dist}
Summary: Distribution compose tool
Group: Development/Tools
License: GPLv2
URL: https://pagure.io/pungi
Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2
Patch0: https://pagure.io/pungi/pull-request/830.patch
Patch1: https://pagure.io/pungi/pull-request/859.patch
Patch2: https://pagure.io/pungi/pull-request/861.patch
Patch0: 0001-Support-multiple-sources-in-one-variant.patch
Patch1: 0002-Remove-comps-groups-from-purely-modular-variants.patch
Patch2: 0003-pkgset-Correctly-detect-single-tag-for-variant.patch
Patch3: 0004-image-build-Accept-tar.xz-extension-for-docker-image.patch
Patch4: 0005-Write-package-whitelist-for-each-variant.patch
Patch5: 0006-gather-Honor-package-whitelist.patch
Patch6: 0007-pkgset-Remove-check-for-unique-name.patch
Patch7: 0008-pkgset-Merge-initial-package-set-without-checks.patch
BuildRequires: python3-nose
BuildRequires: python3-mock
BuildRequires: python2-devel
@ -170,6 +175,10 @@ rm -rf %{buildroot}%{python2_sitelib}/%{name}_utils
%{_bindir}/%{name}-wait-for-signed-ostree-handler
%changelog
* Thu Mar 08 2018 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.22-7
- image-build: Accept tar.xz extension for docker images
- Allow multiple versions of the same package in package set
* Tue Mar 06 2018 Lubomír Sedlář <lsedlar@redhat.com> - 4.1.22-6
- Speed up compose with modules