From a726c6a0b326e9a3dc44b096ec0721b82b8c140a Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 22 Nov 2017 12:39:21 +0000 Subject: [PATCH] Backport patches for PR#790,791,796 Signed-off-by: Patrick Uiterwijk --- 790.patch | 120 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 791.patch | 66 +++++++++++++++++++++++++++++ 796.patch | 96 ++++++++++++++++++++++++++++++++++++++++++ pungi.spec | 12 +++++- 4 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 790.patch create mode 100644 791.patch create mode 100644 796.patch diff --git a/790.patch b/790.patch new file mode 100644 index 00000000..01b7aeec --- /dev/null +++ b/790.patch @@ -0,0 +1,120 @@ +From 1a10a1fe8333e16fa448d96377baff0aab0ee780 Mon Sep 17 00:00:00 2001 +From: Patrick Uiterwijk +Date: Nov 06 2017 14:17:33 +0000 +Subject: Optionally do old_compose per release type + + +This would make sure that e.g. "updates" composes don't try to use "updates-testing" as an +old_compose_path, which would create practically useless deltarpms and for no repodata +reuse at all. + +Signed-off-by: Patrick Uiterwijk + +--- + +diff --git a/pungi/checks.py b/pungi/checks.py +index 1f31fd9..cd1940c 100644 +--- a/pungi/checks.py ++++ b/pungi/checks.py +@@ -626,6 +626,10 @@ def make_schema(): + "enum": ["yum", "dnf"], + }, + ++ "old_composes_per_release_type": { ++ "type": "boolean", ++ "default": False, ++ }, + "hashed_directories": { + "type": "boolean", + "default": False, +diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py +index 2860823..9b8984b 100644 +--- a/pungi/phases/createrepo.py ++++ b/pungi/phases/createrepo.py +@@ -275,6 +275,7 @@ def _get_old_package_dirs(compose, repo_dir): + compose.old_composes, + compose.ci_base.release.short, + compose.ci_base.release.version, ++ compose.ci_base.release.type_suffix if compose.conf['old_composes_per_release_type'] else None, + compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None, + compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None, + allowed_statuses=['FINISHED', 'FINISHED_INCOMPLETE'], +diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py +index c6c4f25..0ed76e8 100644 +--- a/pungi/phases/pkgset/common.py ++++ b/pungi/phases/pkgset/common.py +@@ -55,7 +55,14 @@ def create_global_repo(compose, path_prefix): + old_compose_path = None + update_md_path = None + if compose.old_composes: +- old_compose_path = find_old_compose(compose.old_composes, compose.conf["release_short"], compose.conf["release_version"], compose.conf.get("base_product_short"), compose.conf.get("base_product_version")) ++ old_compose_path = find_old_compose( ++ compose.old_composes, ++ compose.ci_base.release.short, ++ compose.ci_base.release.version, ++ compose.ci_base.release.type_suffix if compose.conf['old_composes_per_release_type'] else None, ++ compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None, ++ compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None, ++ ) + if old_compose_path is None: + compose.log_info("No suitable old compose found in: %s" % compose.old_composes) + else: +diff --git a/pungi/util.py b/pungi/util.py +index 678590a..0162e17 100644 +--- a/pungi/util.py ++++ b/pungi/util.py +@@ -395,8 +395,8 @@ def get_file_size(path): + + + def find_old_compose(old_compose_dirs, release_short, release_version, +- base_product_short=None, base_product_version=None, +- allowed_statuses=None): ++ release_type_suffix=None, base_product_short=None, ++ base_product_version=None, allowed_statuses=None): + allowed_statuses = allowed_statuses or ("FINISHED", "FINISHED_INCOMPLETE", "DOOMED") + composes = [] + +@@ -417,6 +417,8 @@ def find_old_compose(old_compose_dirs, release_short, release_version, + # TODO: read .composeinfo + + pattern = "%s-%s" % (release_short, release_version) ++ if release_type_suffix: ++ pattern += release_type_suffix + if base_product_short: + pattern += "-%s" % base_product_short + if base_product_version: +@@ -425,6 +427,12 @@ def find_old_compose(old_compose_dirs, release_short, release_version, + if not i.startswith(pattern): + continue + ++ suffix = i[len(pattern):] ++ if release_type_suffix and (len(suffix) < 2 or not suffix[1].isdigit()): ++ # This covers the case where we are looking for -updates, but there ++ # is an updates-testing as well. ++ continue ++ + path = os.path.join(compose_dir, i) + if not os.path.isdir(path): + continue +diff --git a/tests/test_util.py b/tests/test_util.py +index f88eafd..05804ac 100644 +--- a/tests/test_util.py ++++ b/tests/test_util.py +@@ -253,6 +253,16 @@ class TestFindOldCompose(unittest.TestCase): + old = util.find_old_compose(self.tmp_dir, 'Fedora', 'Rawhide') + self.assertEqual(old, self.tmp_dir + '/Fedora-Rawhide-20160229.1') + ++ def test_find_correct_type(self): ++ touch(self.tmp_dir + '/Fedora-26-updates-20160229.0/STATUS', 'FINISHED') ++ touch(self.tmp_dir + '/Fedora-26-updates-testing-20160229.0/STATUS', 'FINISHED') ++ old = util.find_old_compose(self.tmp_dir, 'Fedora', '26', '-updates') ++ self.assertEqual(old, self.tmp_dir + '/Fedora-26-updates-20160229.0') ++ old = util.find_old_compose(self.tmp_dir, 'Fedora', '26', '-updates-testing') ++ self.assertEqual(old, self.tmp_dir + '/Fedora-26-updates-testing-20160229.0') ++ old = util.find_old_compose(self.tmp_dir, 'Fedora', '26') ++ self.assertEqual(old, self.tmp_dir + '/Fedora-26-updates-testing-20160229.0') ++ + def test_find_latest_with_two_digit_respin(self): + touch(self.tmp_dir + '/Fedora-Rawhide-20160228.n.9/STATUS', 'FINISHED') + touch(self.tmp_dir + '/Fedora-Rawhide-20160228.n.10/STATUS', 'FINISHED') + diff --git a/791.patch b/791.patch new file mode 100644 index 00000000..ce65d738 --- /dev/null +++ b/791.patch @@ -0,0 +1,66 @@ +From 1dbd0248d4ff9b8417b51d9692f8705022740e18 Mon Sep 17 00:00:00 2001 +From: Patrick Uiterwijk +Date: Nov 06 2017 15:14:44 +0000 +Subject: Implement version.compose_id version generator + + +Signed-off-by: Patrick Uiterwijk + +--- + +diff --git a/pungi/util.py b/pungi/util.py +index 0162e17..0607b6c 100644 +--- a/pungi/util.py ++++ b/pungi/util.py +@@ -782,9 +782,12 @@ def version_generator(compose, gen): + """ + if gen == '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN': + return '%s.%s' % (compose.image_version, compose.image_release) +- if gen == '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN': ++ elif gen == '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN': + return compose.image_release +- if gen and gen[0] == '!': ++ elif gen == '!RELEASE_FROM_VERSION_COMPOSE_ID': ++ return '%s.%s' % (compose.ci_base.release.version, ++ compose.ci_base.id) ++ elif gen and gen[0] == '!': + raise RuntimeError("Unknown version generator '%s'" % gen) + return gen + +diff --git a/tests/test_util.py b/tests/test_util.py +index 05804ac..3209676 100644 +--- a/tests/test_util.py ++++ b/tests/test_util.py +@@ -601,6 +601,20 @@ class GetRepoFuncsTestCase(unittest.TestCase): + + + class TestVersionGenerator(unittest.TestCase): ++ def setUp(self): ++ ci = mock.MagicMock() ++ ci.respin = 0 ++ ci.id = 'RHEL-8.0-20180101.0' ++ ci.release.version = '8' ++ ci.date = '20160101' ++ ci.type = 'nightly' ++ ci.type_suffix = '' ++ ci.label = 'RC-1.0' ++ ci.label_major_version = '1' ++ ++ self.compose = mock.MagicMock() ++ self.compose.ci_base = ci ++ + def test_unknown_generator(self): + compose = mock.Mock() + with self.assertRaises(RuntimeError) as ctx: +@@ -617,6 +631,10 @@ class TestVersionGenerator(unittest.TestCase): + compose = mock.Mock() + self.assertEqual(util.version_generator(compose, None), None) + ++ def test_release_from_version_compose_id(self): ++ self.assertEqual(util.version_generator(self.compose, '!RELEASE_FROM_VERSION_COMPOSE_ID'), ++ '8.RHEL-8.0-20180101.0') ++ + + class TestTZOffset(unittest.TestCase): + @mock.patch('time.daylight', new=False) + diff --git a/796.patch b/796.patch new file mode 100644 index 00000000..1f10ba7c --- /dev/null +++ b/796.patch @@ -0,0 +1,96 @@ +From 8181c5be48c736dadb1d2733306ab8edb8a2d05e Mon Sep 17 00:00:00 2001 +From: Patrick Uiterwijk +Date: Nov 10 2017 10:14:11 +0000 +Subject: Turn COMPOSE_ID version generator into DATE_RESPIN + + +Signed-off-by: Patrick Uiterwijk + +--- + +diff --git a/doc/configuration.rst b/doc/configuration.rst +index fd5b67b..077457c 100644 +--- a/doc/configuration.rst ++++ b/doc/configuration.rst +@@ -950,17 +950,17 @@ Version and release values for certain artifacts can be generated automatically + based on release version, compose label, date, type and respin. This can be + used to shorten the config and keep it the same for multiple uses. + +-+----------------------------+-------------------+--------------+------------------+ +-| Compose ID | Label | Version | Release | +-+============================+===================+==============+==================+ +-| ``F-Rawhide-20170406.n.0`` | ``-`` | ``Rawhide`` | ``20170406.n.0`` | +-+----------------------------+-------------------+--------------+------------------+ +-| ``F-26-20170329.1`` | ``Alpha-1.6`` | ``26_Alpha`` | ``1.6`` | +-+----------------------------+-------------------+--------------+------------------+ +-| ``F-Atomic-25-20170407.0`` | ``RC-20170407.0`` | ``25`` | ``20170407.0`` | +-+----------------------------+-------------------+--------------+------------------+ +-| ``F-Atomic-25-20170407.0`` | ``-`` | ``25`` | ``20170407.0`` | +-+----------------------------+-------------------+--------------+------------------+ +++----------------------------+-------------------+--------------+--------------+--------+------------------+ ++| Compose ID | Label | Version | Date | Respin | Release | +++============================+===================+==============+==============+========+==================+ ++| ``F-Rawhide-20170406.n.0`` | ``-`` | ``Rawhide`` | ``20170406`` | ``0`` | ``20170406.n.0`` | +++----------------------------+-------------------+--------------+--------------+--------+------------------+ ++| ``F-26-20170329.1`` | ``Alpha-1.6`` | ``26_Alpha`` | ``20170329`` | ``1`` | ``1.6`` | +++----------------------------+-------------------+--------------+--------------+--------+------------------+ ++| ``F-Atomic-25-20170407.0`` | ``RC-20170407.0`` | ``25`` | ``20170407`` | ``0`` | ``20170407.0`` | +++----------------------------+-------------------+--------------+--------------+--------+------------------+ ++| ``F-Atomic-25-20170407.0`` | ``-`` | ``25`` | ``20170407`` | ``0`` | ``20170407.0`` | +++----------------------------+-------------------+--------------+--------------+--------+------------------+ + + All non-``RC`` milestones from label get appended to the version. For release + either label is used or date, type and respin. +@@ -1230,6 +1230,8 @@ repository with a new commit. + * ``version`` -- (*str*) Version string to be added as versioning metadata. + If this option is set to ``!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN``, + a value will be generated automatically as ``$VERSION.$RELEASE``. ++ If this option is set to ``!VERSION_FROM_VERSION_DATE_RESPIN``, ++ a value will be generated automatically as ``$VERSION.$DATE.$RESPIN``. + :ref:`See how those values are created `. + * ``tag_ref`` -- (*bool*, default ``True``) If set to ``False``, a git + reference will not be created. +diff --git a/pungi/util.py b/pungi/util.py +index 0607b6c..2d2df82 100644 +--- a/pungi/util.py ++++ b/pungi/util.py +@@ -784,9 +784,10 @@ def version_generator(compose, gen): + return '%s.%s' % (compose.image_version, compose.image_release) + elif gen == '!RELEASE_FROM_LABEL_DATE_TYPE_RESPIN': + return compose.image_release +- elif gen == '!RELEASE_FROM_VERSION_COMPOSE_ID': +- return '%s.%s' % (compose.ci_base.release.version, +- compose.ci_base.id) ++ elif gen == '!VERSION_FROM_VERSION_DATE_RESPIN': ++ return '%s.%s.%s' % (compose.ci_base.release.version, ++ compose.ci_base.date, ++ compose.compose_respin) + elif gen and gen[0] == '!': + raise RuntimeError("Unknown version generator '%s'" % gen) + return gen +diff --git a/tests/test_util.py b/tests/test_util.py +index 3209676..e0dd2b1 100644 +--- a/tests/test_util.py ++++ b/tests/test_util.py +@@ -614,6 +614,7 @@ class TestVersionGenerator(unittest.TestCase): + + self.compose = mock.MagicMock() + self.compose.ci_base = ci ++ self.compose.compose_respin = 0 + + def test_unknown_generator(self): + compose = mock.Mock() +@@ -631,9 +632,9 @@ class TestVersionGenerator(unittest.TestCase): + compose = mock.Mock() + self.assertEqual(util.version_generator(compose, None), None) + +- def test_release_from_version_compose_id(self): +- self.assertEqual(util.version_generator(self.compose, '!RELEASE_FROM_VERSION_COMPOSE_ID'), +- '8.RHEL-8.0-20180101.0') ++ def test_release_from_version_date_respin(self): ++ self.assertEqual(util.version_generator(self.compose, '!VERSION_FROM_VERSION_DATE_RESPIN'), ++ '8.20160101.0') + + + class TestTZOffset(unittest.TestCase): + diff --git a/pungi.spec b/pungi.spec index f840c196..a6db9851 100644 --- a/pungi.spec +++ b/pungi.spec @@ -1,6 +1,6 @@ Name: pungi Version: 4.1.20 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Distribution compose tool Group: Development/Tools @@ -8,6 +8,9 @@ License: GPLv2 URL: https://pagure.io/pungi Source0: https://pagure.io/releases/%{name}/%{name}-%{version}.tar.bz2 Patch0: 0001-gather-get_packages_to_gather-returns-a-tuple.patch +Patch1: https://pagure.io/pungi/pull-request/790.patch +Patch2: https://pagure.io/pungi/pull-request/791.patch +Patch3: https://pagure.io/pungi/pull-request/796.patch BuildRequires: python-nose, python-mock BuildRequires: python-devel, python-setuptools, python2-productmd >= 1.3 BuildRequires: python-lockfile, kobo-rpmlib, createrepo_c @@ -101,6 +104,9 @@ notification to Fedora Message Bus. %prep %setup -q %patch0 -p1 +%patch1 -p1 +%patch2 -p1 +%patch3 -p1 %build %{__python} setup.py build @@ -155,6 +161,10 @@ cd tests && ./test_compose.sh %{_bindir}/%{name}-wait-for-signed-ostree-handler %changelog +* Wed Nov 22 2017 Patrick Uiterwijk - 4.1.20-3 +- Backport patch for PR#790 - old_composes per release type +- Backport patch for PR#791,796 - implement DATE_RESPIN version generator + * Tue Nov 21 2017 Lubomír Sedlář - 4.1.20-2 - Fix crash in modular compose