createrepo: Use correct paths for old package dirs
Createrepo expects to be pointed to a directory with the actual RPM files, not the previous repo. This means that when hashed directories are used, we need to pass in a lot of directories. Fixes: https://pagure.io/pungi/issue/344 Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář
parent
5c5708afe3
commit
910f816be4
|
|
@ -93,7 +93,6 @@ def create_variant_repo(compose, arch, variant, pkg_type):
|
|||
|
||||
createrepo_c = compose.conf["createrepo_c"]
|
||||
createrepo_checksum = compose.conf["createrepo_checksum"]
|
||||
createrepo_deltas = compose.conf["createrepo_deltas"]
|
||||
repo = CreaterepoWrapper(createrepo_c=createrepo_c)
|
||||
repo_dir_arch = compose.paths.work.arch_repo(arch='global' if pkg_type == 'srpm' else arch)
|
||||
|
||||
|
|
@ -145,20 +144,7 @@ def create_variant_repo(compose, arch, variant, pkg_type):
|
|||
for rel_path in sorted(rpms):
|
||||
f.write("%s\n" % rel_path)
|
||||
|
||||
old_packages_dir = None
|
||||
if createrepo_deltas:
|
||||
old_compose_path = find_old_compose(
|
||||
compose.old_composes,
|
||||
compose.ci_base.release.short,
|
||||
compose.ci_base.release.version,
|
||||
compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None,
|
||||
compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None
|
||||
)
|
||||
if not old_compose_path:
|
||||
compose.log_info("No suitable old compose found in: %s" % compose.old_composes)
|
||||
else:
|
||||
rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/')
|
||||
old_packages_dir = os.path.join(old_compose_path, rel_dir)
|
||||
old_package_dirs = _get_old_package_dirs(compose, repo_dir)
|
||||
|
||||
comps_path = None
|
||||
if compose.has_comps and pkg_type == "rpm":
|
||||
|
|
@ -166,8 +152,9 @@ def create_variant_repo(compose, arch, variant, pkg_type):
|
|||
cmd = repo.get_createrepo_cmd(repo_dir, update=True, database=True, skip_stat=True,
|
||||
pkglist=file_list, outputdir=repo_dir, workers=3,
|
||||
groupfile=comps_path, update_md_path=repo_dir_arch,
|
||||
checksum=createrepo_checksum, deltas=createrepo_deltas,
|
||||
oldpackagedirs=old_packages_dir,
|
||||
checksum=createrepo_checksum,
|
||||
deltas=compose.conf['createrepo_deltas'],
|
||||
oldpackagedirs=old_package_dirs,
|
||||
use_xz=compose.conf['createrepo_use_xz'])
|
||||
log_file = compose.paths.log.log_file(arch, "createrepo-%s.%s" % (variant, pkg_type))
|
||||
run(cmd, logfile=log_file, show_cmd=True)
|
||||
|
|
@ -262,3 +249,43 @@ def get_productids_from_scm(compose):
|
|||
|
||||
shutil.rmtree(tmp_dir)
|
||||
compose.log_info("[DONE ] %s" % msg)
|
||||
|
||||
|
||||
def _get_old_package_dirs(compose, repo_dir):
|
||||
"""Given a compose and a path to a repo in it, try to find corresponging
|
||||
repo in an older compose and return a list of paths to directories with
|
||||
packages in it.
|
||||
"""
|
||||
if not compose.conf['createrepo_deltas']:
|
||||
return None
|
||||
old_compose_path = find_old_compose(
|
||||
compose.old_composes,
|
||||
compose.ci_base.release.short,
|
||||
compose.ci_base.release.version,
|
||||
compose.ci_base.base_product.short if compose.ci_base.release.is_layered else None,
|
||||
compose.ci_base.base_product.version if compose.ci_base.release.is_layered else None
|
||||
)
|
||||
if not old_compose_path:
|
||||
compose.log_info("No suitable old compose found in: %s" % compose.old_composes)
|
||||
return None
|
||||
rel_dir = relative_path(repo_dir, compose.topdir.rstrip('/') + '/')
|
||||
old_package_dirs = os.path.join(old_compose_path, rel_dir, 'Packages')
|
||||
if compose.conf['hashed_directories']:
|
||||
old_package_dirs = _find_package_dirs(old_package_dirs)
|
||||
return old_package_dirs
|
||||
|
||||
|
||||
def _find_package_dirs(base):
|
||||
"""Assuming the packages are in directories hashed by first letter, find
|
||||
all the buckets in given base.
|
||||
"""
|
||||
buckets = set()
|
||||
try:
|
||||
for subdir in os.listdir(base):
|
||||
bucket = os.path.join(base, subdir)
|
||||
if os.path.isdir(bucket):
|
||||
buckets.add(bucket)
|
||||
except OSError:
|
||||
# The directory does not exist, so no drpms for you!
|
||||
pass
|
||||
return sorted(buckets)
|
||||
|
|
|
|||
|
|
@ -309,7 +309,87 @@ class TestCreateVariantRepo(PungiTestCase):
|
|||
outputdir=self.topdir + '/compose/Server/x86_64/os',
|
||||
pkglist=list_file, skip_stat=True, update=True,
|
||||
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
|
||||
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os',
|
||||
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages',
|
||||
use_xz=False)])
|
||||
self.assertItemsEqual(
|
||||
repo.get_modifyrepo_cmd.mock_calls,
|
||||
[])
|
||||
with open(list_file) as f:
|
||||
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.run')
|
||||
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
|
||||
def test_variant_repo_rpms_with_deltas_hashed_dirs(self, CreaterepoWrapperCls, run):
|
||||
compose = DummyCompose(self.topdir, {
|
||||
'createrepo_checksum': 'sha256',
|
||||
'createrepo_deltas': True,
|
||||
'hashed_directories': True,
|
||||
})
|
||||
compose.DEBUG = False
|
||||
compose.has_comps = False
|
||||
compose.old_composes = [self.topdir + '/old']
|
||||
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
|
||||
self.maxDiff = None
|
||||
|
||||
for f in ['a/a.rpm', 'b/b.rpm', 'foo']:
|
||||
touch(self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/' + f)
|
||||
|
||||
repo = CreaterepoWrapperCls.return_value
|
||||
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
|
||||
|
||||
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
|
||||
|
||||
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
|
||||
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
|
||||
mock.call(createrepo_c=True))
|
||||
self.assertItemsEqual(
|
||||
repo.get_createrepo_cmd.mock_calls,
|
||||
[mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
|
||||
database=True, groupfile=None, workers=3,
|
||||
outputdir=self.topdir + '/compose/Server/x86_64/os',
|
||||
pkglist=list_file, skip_stat=True, update=True,
|
||||
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
|
||||
oldpackagedirs=[
|
||||
self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/a',
|
||||
self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/os/Packages/b',
|
||||
],
|
||||
use_xz=False)])
|
||||
self.assertItemsEqual(
|
||||
repo.get_modifyrepo_cmd.mock_calls,
|
||||
[])
|
||||
with open(list_file) as f:
|
||||
self.assertEqual(f.read(), 'Packages/b/bash-4.3.30-2.fc21.x86_64.rpm\n')
|
||||
|
||||
@mock.patch('pungi.phases.createrepo.run')
|
||||
@mock.patch('pungi.phases.createrepo.CreaterepoWrapper')
|
||||
def test_variant_repo_rpms_with_deltas_hashed_dirs_but_old_doesnt_exist(self, CreaterepoWrapperCls, run):
|
||||
compose = DummyCompose(self.topdir, {
|
||||
'createrepo_checksum': 'sha256',
|
||||
'createrepo_deltas': True,
|
||||
'hashed_directories': True,
|
||||
})
|
||||
compose.DEBUG = False
|
||||
compose.has_comps = False
|
||||
compose.old_composes = [self.topdir + '/old']
|
||||
touch(os.path.join(self.topdir, 'old', 'test-1.0-20151203.0', 'STATUS'), 'FINISHED')
|
||||
self.maxDiff = None
|
||||
|
||||
repo = CreaterepoWrapperCls.return_value
|
||||
copy_fixture('server-rpms.json', compose.paths.compose.metadata('rpms.json'))
|
||||
|
||||
create_variant_repo(compose, 'x86_64', compose.variants['Server'], 'rpm')
|
||||
|
||||
list_file = self.topdir + '/work/x86_64/repo_package_list/Server.x86_64.rpm.conf'
|
||||
self.assertEqual(CreaterepoWrapperCls.mock_calls[0],
|
||||
mock.call(createrepo_c=True))
|
||||
self.assertItemsEqual(
|
||||
repo.get_createrepo_cmd.mock_calls,
|
||||
[mock.call(self.topdir + '/compose/Server/x86_64/os', checksum='sha256',
|
||||
database=True, groupfile=None, workers=3,
|
||||
outputdir=self.topdir + '/compose/Server/x86_64/os',
|
||||
pkglist=list_file, skip_stat=True, update=True,
|
||||
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
|
||||
oldpackagedirs=[],
|
||||
use_xz=False)])
|
||||
self.assertItemsEqual(
|
||||
repo.get_modifyrepo_cmd.mock_calls,
|
||||
|
|
@ -344,7 +424,7 @@ class TestCreateVariantRepo(PungiTestCase):
|
|||
outputdir=self.topdir + '/compose/Server/source/tree',
|
||||
pkglist=list_file, skip_stat=True, update=True,
|
||||
update_md_path=self.topdir + '/work/global/repo', deltas=True,
|
||||
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree',
|
||||
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/source/tree/Packages',
|
||||
use_xz=False)])
|
||||
self.assertItemsEqual(
|
||||
repo.get_modifyrepo_cmd.mock_calls,
|
||||
|
|
@ -381,7 +461,7 @@ class TestCreateVariantRepo(PungiTestCase):
|
|||
outputdir=self.topdir + '/compose/Server/x86_64/debug/tree',
|
||||
pkglist=list_file, skip_stat=True, update=True,
|
||||
update_md_path=self.topdir + '/work/x86_64/repo', deltas=True,
|
||||
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree',
|
||||
oldpackagedirs=self.topdir + '/old/test-1.0-20151203.0/compose/Server/x86_64/debug/tree/Packages',
|
||||
use_xz=False)])
|
||||
self.assertItemsEqual(
|
||||
repo.get_modifyrepo_cmd.mock_calls,
|
||||
|
|
|
|||
Loading…
Reference in New Issue