almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

createrepo: Use correct paths for old package dirs

Browse Source 
Createrepo expects to be pointed to a directory with the actual RPM
files, not the previous repo. This means that when hashed directories
are used, we need to pass in a lot of directories.

Fixes: https://pagure.io/pungi/issue/344
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2017-07-14 10:39:47 +02:00
parent 5c5708afe3
commit 910f816be4
2 changed files with 127 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
pungi/phases/createrepo.py
View File

@ -93,7 +93,6 @@ def create_variant_repo(compose, arch, variant, pkg_type):
createrepo_c = compose.conf["createrepo_c"]
createrepo_checksum = compose.conf["createrepo_checksum"]
createrepo_deltas = compose.conf["createrepo_deltas"]
repo = CreaterepoWrapper(createrepo_c=createrepo_c)
repo_dir_arch = compose.paths.work.arch_repo(arch='global' if pkg_type == 'srpm' else arch)
@ -145,20 +144,7 @@ def create_variant_repo(compose, arch, variant, pkg_type):
for rel_path in sorted(rpms):
f.write("%s\n" % rel_path)
old_packages_dir = None
if createrepo_deltas:
old_compose_path = find_old_compose(
compose.old_composes,
compose.ci_base.release.short,
compose.ci_base.release.version,
compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None,
compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None
)
if not old_compose_path:
compose.log_info("No suitable old compose found in: %s" % compose.old_composes)
else:
rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/')
old_packages_dir = os.path.join(old_compose_path, rel_dir)
old_package_dirs = _get_old_package_dirs(compose, repo_dir)
comps_path = None
if compose.has_comps and pkg_type == "rpm":
@ -166,8 +152,9 @@ def create_variant_repo(compose, arch, variant, pkg_type):
cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True,
pkglist=file_list, outputdir=repo_dir, workers=3,
groupfile=comps_path, update_md_path=repo_dir_arch,
checksum=createrepo_checksum, deltas=createrepo_deltas,
oldpackagedirs=old_packages_dir,
checksum=createrepo_checksum,
deltas=compose.conf['createrepo_deltas'],
oldpackagedirs=old_package_dirs,
use_xz=compose.conf['createrepo_use_xz'])
log_file = compose.paths.log.log_file(arch, "createrepo-%s.%s" % (variant, pkg_type))
run(cmd, logfile=log_file, show_cmd=True)
@ -262,3 +249,43 @@ def get_productids_from_scm(compose):
shutil.rmtree(tmp_dir)
compose.log_info("[DONE ] %s" % msg)
def _get_old_package_dirs(compose, repo_dir):
"""Given a compose and a path to a repo in it, try to find corresponging
repo in an older compose and return a list of paths to directories with
packages in it.
"""
if not compose.conf['createrepo_deltas']:
return None
old_compose_path = find_old_compose(
compose.old_composes,
compose.ci_base.release.short,
compose.ci_base.release.version,
compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None,
compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None
)
if not old_compose_path:
compose.log_info("No suitable old compose found in: %s" % compose.old_composes)
return None
rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/')
old_package_dirs = os.path.join(old_compose_path, rel_dir, 'Packages')
if compose.conf['hashed_directories']:
old_package_dirs = _find_package_dirs(old_package_dirs)
return old_package_dirs
def _find_package_dirs(base):
"""Assuming the packages are in directories hashed by first letter, find
all the buckets in given base.
"""
buckets = set()
try:
for subdir in os.listdir(base):
bucket = os.path.join(base, subdir)
if os.path.isdir(bucket):
buckets.add(bucket)
except OSError:
# The directory does not exist, so no drpms for you!
pass
return sorted(buckets)

86
tests/test_createrepophase.py
View File

@ -309,7 +309,87 @@ class TestCreateVariantRepo(PungiTestCase):
outputdir=self.topdir + '/compose/Server/x86_64/os',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os',
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages',
use_xz=False)])
self.assertItemsEqual(
repo.get_modifyrepo_cmd.mock_calls,
[])
with open(list_file) as f:
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
@mock.patch('pungi.phases.createrepo.run')
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
def test_variant_repo_rpms_with_deltas_hashed_dirs(self, CreaterepoWrapperCls, run):
compose = DummyCompose(self.topdir, {
'createrepo_checksum': 'sha256',
'createrepo_deltas': True,
'hashed_directories': True,
})
compose.DEBUG = False
compose.has_comps = False
compose.old_composes = [self.topdir + '/old']
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
self.maxDiff = None
for f in ['a/a.rpm', 'b/b.rpm', 'foo']:
touch(self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/' + f)
repo = CreaterepoWrapperCls.return_value
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
mock.call(createrepo_c=True))
self.assertItemsEqual(
repo.get_createrepo_cmd.mock_calls,
[mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/x86_64/os',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
oldpackagedirs=[
self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/a',
self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/b',
],
use_xz=False)])
self.assertItemsEqual(
repo.get_modifyrepo_cmd.mock_calls,
[])
with open(list_file) as f:
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
@mock.patch('pungi.phases.createrepo.run')
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
def test_variant_repo_rpms_with_deltas_hashed_dirs_but_old_doesnt_exist(self, CreaterepoWrapperCls, run):
compose = DummyCompose(self.topdir, {
'createrepo_checksum': 'sha256',
'createrepo_deltas': True,
'hashed_directories': True,
})
compose.DEBUG = False
compose.has_comps = False
compose.old_composes = [self.topdir + '/old']
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
self.maxDiff = None
repo = CreaterepoWrapperCls.return_value
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
mock.call(createrepo_c=True))
self.assertItemsEqual(
repo.get_createrepo_cmd.mock_calls,
[mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
database=True, groupfile=None, workers=3,
outputdir=self.topdir + '/compose/Server/x86_64/os',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
oldpackagedirs=[],
use_xz=False)])
self.assertItemsEqual(
repo.get_modifyrepo_cmd.mock_calls,
@ -344,7 +424,7 @@ class TestCreateVariantRepo(PungiTestCase):
outputdir=self.topdir + '/compose/Server/source/tree',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/global/repo', deltas=True,
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree',
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree/Packages',
use_xz=False)])
self.assertItemsEqual(
repo.get_modifyrepo_cmd.mock_calls,
@ -381,7 +461,7 @@ class TestCreateVariantRepo(PungiTestCase):
outputdir=self.topdir + '/compose/Server/x86_64/debug/tree',
pkglist=list_file, skip_stat=True, update=True,
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree',
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree/Packages',
use_xz=False)])
self.assertItemsEqual(
repo.get_modifyrepo_cmd.mock_calls,