From 910f816be4e9d3f24f0e345c70bcbd877af63540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 14 Jul 2017 10:39:47 +0200 Subject: [PATCH] createrepo: Use correct paths for old package dirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Createrepo expects to be pointed to a directory with the actual RPM files, not the previous repo. This means that when hashed directories are used, we need to pass in a lot of directories. Fixes: https://pagure.io/pungi/issue/344 Signed-off-by: Lubomír Sedlář --- pungi/phases/createrepo.py | 61 ++++++++++++++++++------- tests/test_createrepophase.py | 86 +++++++++++++++++++++++++++++++++-- 2 files changed, 127 insertions(+), 20 deletions(-) diff --git a/pungi/phases/createrepo.py b/pungi/phases/createrepo.py index 5ab65f49..ac769b67 100644 --- a/pungi/phases/createrepo.py +++ b/pungi/phases/createrepo.py @@ -93,7 +93,6 @@ def create_variant_repo(compose, arch, variant, pkg_type): createrepo_c = compose.conf["createrepo_c"] createrepo_checksum = compose.conf["createrepo_checksum"] - createrepo_deltas = compose.conf["createrepo_deltas"] repo = CreaterepoWrapper(createrepo_c=createrepo_c) repo_dir_arch = compose.paths.work.arch_repo(arch='global' if pkg_type == 'srpm' else arch) @@ -145,20 +144,7 @@ def create_variant_repo(compose, arch, variant, pkg_type): for rel_path in sorted(rpms): f.write("%s\n" % rel_path) - old_packages_dir = None - if createrepo_deltas: - old_compose_path = find_old_compose( - compose.old_composes, - compose.ci_base.release.short, - compose.ci_base.release.version, - compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None, - compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None - ) - if not old_compose_path: - compose.log_info("No suitable old compose found in: %s" % compose.old_composes) - else: - rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/') - old_packages_dir = os.path.join(old_compose_path, rel_dir) + old_package_dirs = _get_old_package_dirs(compose, repo_dir) comps_path = None if compose.has_comps and pkg_type == "rpm": @@ -166,8 +152,9 @@ def create_variant_repo(compose, arch, variant, pkg_type): cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True, pkglist=file_list, outputdir=repo_dir, workers=3, groupfile=comps_path, update_md_path=repo_dir_arch, - checksum=createrepo_checksum, deltas=createrepo_deltas, - oldpackagedirs=old_packages_dir, + checksum=createrepo_checksum, + deltas=compose.conf['createrepo_deltas'], + oldpackagedirs=old_package_dirs, use_xz=compose.conf['createrepo_use_xz']) log_file = compose.paths.log.log_file(arch, "createrepo-%s.%s" % (variant, pkg_type)) run(cmd, logfile=log_file, show_cmd=True) @@ -262,3 +249,43 @@ def get_productids_from_scm(compose): shutil.rmtree(tmp_dir) compose.log_info("[DONE ] %s" % msg) + + +def _get_old_package_dirs(compose, repo_dir): + """Given a compose and a path to a repo in it, try to find corresponging + repo in an older compose and return a list of paths to directories with + packages in it. + """ + if not compose.conf['createrepo_deltas']: + return None + old_compose_path = find_old_compose( + compose.old_composes, + compose.ci_base.release.short, + compose.ci_base.release.version, + compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None, + compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None + ) + if not old_compose_path: + compose.log_info("No suitable old compose found in: %s" % compose.old_composes) + return None + rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/') + old_package_dirs = os.path.join(old_compose_path, rel_dir, 'Packages') + if compose.conf['hashed_directories']: + old_package_dirs = _find_package_dirs(old_package_dirs) + return old_package_dirs + + +def _find_package_dirs(base): + """Assuming the packages are in directories hashed by first letter, find + all the buckets in given base. + """ + buckets = set() + try: + for subdir in os.listdir(base): + bucket = os.path.join(base, subdir) + if os.path.isdir(bucket): + buckets.add(bucket) + except OSError: + # The directory does not exist, so no drpms for you! + pass + return sorted(buckets) diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py index fe571514..ea101256 100644 --- a/tests/test_createrepophase.py +++ b/tests/test_createrepophase.py @@ -309,7 +309,87 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/os', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, - oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os', + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages', + use_xz=False)]) + self.assertItemsEqual( + repo.get_modifyrepo_cmd.mock_calls, + []) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_rpms_with_deltas_hashed_dirs(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': True, + 'hashed_directories': True, + }) + compose.DEBUG = False + compose.has_comps = False + compose.old_composes = [self.topdir + '/old'] + touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED') + self.maxDiff = None + + for f in ['a/a.rpm', 'b/b.rpm', 'foo']: + touch(self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/' + f) + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/os', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, + oldpackagedirs=[ + self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/a', + self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/b', + ], + use_xz=False)]) + self.assertItemsEqual( + repo.get_modifyrepo_cmd.mock_calls, + []) + with open(list_file) as f: + self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n') + + @mock.patch('pungi.phases.createrepo.run') + @mock.patch('pungi.phases.createrepo.CreaterepoWrapper') + def test_variant_repo_rpms_with_deltas_hashed_dirs_but_old_doesnt_exist(self, CreaterepoWrapperCls, run): + compose = DummyCompose(self.topdir, { + 'createrepo_checksum': 'sha256', + 'createrepo_deltas': True, + 'hashed_directories': True, + }) + compose.DEBUG = False + compose.has_comps = False + compose.old_composes = [self.topdir + '/old'] + touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED') + self.maxDiff = None + + repo = CreaterepoWrapperCls.return_value + copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json')) + + create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm') + + list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf' + self.assertEqual(CreaterepoWrapperCls.mock_calls[0], + mock.call(createrepo_c=True)) + self.assertItemsEqual( + repo.get_createrepo_cmd.mock_calls, + [mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256', + database=True, groupfile=None, workers=3, + outputdir=self.topdir + '/compose/Server/x86_64/os', + pkglist=list_file, skip_stat=True, update=True, + update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, + oldpackagedirs=[], use_xz=False)]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, @@ -344,7 +424,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/source/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/global/repo', deltas=True, - oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree', + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree/Packages', use_xz=False)]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls, @@ -381,7 +461,7 @@ class TestCreateVariantRepo(PungiTestCase): outputdir=self.topdir + '/compose/Server/x86_64/debug/tree', pkglist=list_file, skip_stat=True, update=True, update_md_path=self.topdir + '/work/x86_64/repo', deltas=True, - oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree', + oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree/Packages', use_xz=False)]) self.assertItemsEqual( repo.get_modifyrepo_cmd.mock_calls,