gather: Use another variant as lookaside

Create a temporary repository and add it as another lookaside in the
compose.

JIRA: COMPOSE-2426
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/paths.py

@@ -179,6 +179,17 @@ class WorkPaths(object):
             makedirs(path)
         return path
 
+    def lookaside_repo(self, arch, variant, create_dir=True):
+        """
+        Examples:
+            work/x86_64/Server/lookaside_repo
+        """
+        path = os.path.join(self.topdir(arch, create_dir=create_dir),
+                            variant.uid, "lookaside_repo")
+        if create_dir:
+            makedirs(path)
+        return path
+
     def package_list(self, arch=None, variant=None, pkg_type=None, create_dir=True):
         """
         Examples:
@@ -200,6 +211,13 @@ class WorkPaths(object):
         path = os.path.join(path, file_name)
         return path
 
+    def lookaside_package_list(self, arch, variant, create_dir=True):
+        """
+        Examples:
+            work/x86_64/package_list/Server.x86_64.lookaside.conf
+        """
+        return self.package_list(arch, variant, pkg_type='lookaside', create_dir=create_dir)
+
     def pungi_download_dir(self, arch, create_dir=True):
         """
         Examples:

pungi/phases/gather/__init__.py

@@ -19,16 +19,20 @@ import os
 import shutil
 
 from kobo.rpmlib import parse_nvra
+from kobo.shortcuts import run
 from productmd.rpms import Rpms
 
 from pungi.wrappers.scm import get_file_from_scm
 from .link import link_files
+from ...wrappers.createrepo import CreaterepoWrapper
+import pungi.wrappers.kojiwrapper
 
 from pungi import Modulemd
 from pungi.arch import get_compatible_arches, split_name_arch
 from pungi.graph import SimpleAcyclicOrientedGraph
 from pungi.phases.base import PhaseBase
-from pungi.util import get_arch_data, get_arch_variant_data, get_variant_data
+from pungi.util import (get_arch_data, get_arch_variant_data, get_variant_data,
+                        makedirs)
 
 
 def get_gather_source(name):
@@ -308,6 +312,77 @@ def _prepare_variant_as_lookaside(compose):
     return list(variant_processing_order)
 
 
+def _make_lookaside_repo(compose, variant, arch, pkg_map):
+    """
+    Create variant lookaside repo for given variant and architecture with
+    packages from the map. If the repo repo already exists, then nothing will
+    happen. This could happen if multiple variants depend on this one.
+    """
+    repo = compose.paths.work.lookaside_repo(arch, variant, create_dir=False)
+    if os.path.exists(repo):
+        # We have already generated this, nothing to do.
+        return repo
+
+    makedirs(repo)
+    msg = 'Generating lookaside repo from %s.%s' % (variant.uid, arch)
+    compose.log_info('[BEGIN] %s', msg)
+
+    prefixes = {
+        'repos': lambda: os.path.join(compose.paths.work.topdir(
+            arch="global"), "download") + "/",
+        'koji': lambda: pungi.wrappers.kojiwrapper.KojiWrapper(
+            compose.conf['koji_profile']).koji_module.config.topdir.rstrip("/") + "/"
+    }
+    path_prefix = prefixes[compose.conf['pkgset_source']]()
+    pkglist = compose.paths.work.lookaside_package_list(arch=arch, variant=variant)
+    with open(pkglist, 'w') as f:
+        for packages in pkg_map[arch][variant.uid].values():
+            for pkg in packages:
+                pkg = pkg['path']
+                if path_prefix and pkg.startswith(path_prefix):
+                    pkg = pkg[len(path_prefix):]
+                f.write('%s\n' % pkg)
+
+    cr = CreaterepoWrapper(compose.conf['createrepo_c'])
+    cmd = cr.get_createrepo_cmd(path_prefix, update=True, database=True, skip_stat=True,
+                                pkglist=pkglist,
+                                outputdir=repo,
+                                baseurl="file://%s" % path_prefix,
+                                workers=compose.conf["createrepo_num_workers"],
+                                update_md_path=compose.paths.work.arch_repo(arch))
+    run(cmd,
+        logfile=compose.paths.log.log_file(arch, "lookaside_repo_%s" % (variant.uid)),
+        show_cmd=True)
+    compose.log_info('[DONE ] %s', msg)
+
+    return repo
+
+
+def _update_config(compose, variant_uid, arch, repo):
+    """
+    Add the variant lookaside repository into the configuration.
+    """
+    lookasides = compose.conf.setdefault('gather_lookaside_repos', [])
+    lookasides.append(('^%s$' % variant_uid, {arch: repo}))
+
+
+def _update_lookaside_config(compose, variant, arch, pkg_map):
+    """
+    Make sure lookaside repo for all variants that the given one depends on
+    exist, and that configuration is updated to use those repos.
+    """
+    for dest, lookaside_variant_uid in compose.conf.get('variant_as_lookaside', []):
+        lookaside_variant = compose.all_variants[lookaside_variant_uid]
+        if dest != variant.uid:
+            continue
+        if arch not in lookaside_variant.arches:
+            compose.log_warning('[SKIP] Skipping lookaside from %s for %s.%s due to arch mismatch',
+                                lookaside_variant.uid, variant.uid, arch)
+            continue
+        repo = _make_lookaside_repo(compose, lookaside_variant, arch, pkg_map)
+        _update_config(compose, variant.uid, arch, repo)
+
+
 def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltree=False):
     """Run gathering on all arches of all variants of given type.
 
@@ -330,6 +405,12 @@ def _gather_variants(result, compose, variant_type, package_sets, exclude_fulltr
             if exclude_fulltree:
                 for pkg_name, pkg_arch in get_parent_pkgs(arch, variant, result)["srpm"]:
                     fulltree_excludes.add(pkg_name)
+
+            # Get lookaside repos for this variant from other variants. Based
+            # on the ordering we already know that we have the packages from
+            # there.
+            _update_lookaside_config(compose, variant, arch, result)
+
             pkg_map = gather_packages(compose, arch, variant, package_sets, fulltree_excludes=fulltree_excludes)
             result.setdefault(arch, {})[variant.uid] = pkg_map
 

tests/test_gather_phase.py

@@ -901,3 +901,176 @@ class TestGetPackagesToGather(helpers.PungiTestCase):
 
         self.assertItemsEqual(packages, ["foo", "pkg", "foo2.x86_64"])
         self.assertItemsEqual(groups, ["core"])
+
+
+class TestUpdateConfig(unittest.TestCase):
+
+    def test_add_to_empty(self):
+        compose = mock.Mock(conf={})
+        gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo')
+        self.assertEqual(compose.conf,
+                         {'gather_lookaside_repos': [
+                             ('^Server$', {'x86_64': '/tmp/foo'})
+                         ]})
+
+    def test_add_to_existing(self):
+        compose = mock.Mock(conf={'gather_lookaside_repos': [
+            ('^Server$', {'x86_64': '/tmp/bar'}),
+        ]})
+        gather._update_config(compose, 'Server', 'x86_64', '/tmp/foo')
+        self.assertEqual(compose.conf,
+                         {'gather_lookaside_repos': [
+                             ('^Server$', {'x86_64': '/tmp/bar'}),
+                             ('^Server$', {'x86_64': '/tmp/foo'})
+                         ]})
+
+
+class TestUpdateLookasideConfig(helpers.PungiTestCase):
+
+    def setUp(self):
+        super(TestUpdateLookasideConfig, self).setUp()
+        self.compose = helpers.DummyCompose(self.topdir, {})
+        self.pkg_map = mock.Mock()
+
+    @mock.patch('pungi.phases.gather._update_config')
+    @mock.patch('pungi.phases.gather._make_lookaside_repo')
+    def test_no_config(self, mock_make_repo, mock_update_config):
+        gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
+                                        'x86_64', self.pkg_map)
+        self.assertEqual(mock_make_repo.call_args_list, [])
+        self.assertEqual(mock_update_config.call_args_list, [])
+
+    @mock.patch('pungi.phases.gather._update_config')
+    @mock.patch('pungi.phases.gather._make_lookaside_repo')
+    def test_no_matching_config(self, mock_make_repo, mock_update_config):
+        self.compose.conf['variant_as_lookaside'] = [('Everything', 'Client')]
+        gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
+                                        'x86_64', self.pkg_map)
+        self.assertEqual(mock_make_repo.call_args_list, [])
+        self.assertEqual(mock_update_config.call_args_list, [])
+
+    @mock.patch('pungi.phases.gather._update_config')
+    @mock.patch('pungi.phases.gather._make_lookaside_repo')
+    def test_missing_arch(self, mock_make_repo, mock_update_config):
+        # Client only has amd64
+        self.compose.conf['variant_as_lookaside'] = [('Server', 'Client')]
+        gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
+                                        'x86_64', self.pkg_map)
+        self.assertEqual(len(self.compose.log_warning.call_args_list), 1)
+        self.assertEqual(mock_make_repo.call_args_list, [])
+        self.assertEqual(mock_update_config.call_args_list, [])
+
+    @mock.patch('pungi.phases.gather._update_config')
+    @mock.patch('pungi.phases.gather._make_lookaside_repo')
+    def test_match(self, mock_make_repo, mock_update_config):
+        self.compose.conf['variant_as_lookaside'] = [('Server', 'Everything')]
+        gather._update_lookaside_config(self.compose, self.compose.variants['Server'],
+                                        'x86_64', self.pkg_map)
+        self.assertEqual(len(self.compose.log_warning.call_args_list), 0)
+        self.assertEqual(mock_make_repo.call_args_list,
+                         [mock.call(self.compose,
+                                    self.compose.variants['Everything'],
+                                    'x86_64',
+                                    self.pkg_map)])
+        self.assertEqual(mock_update_config.call_args_list,
+                         [mock.call(self.compose, 'Server', 'x86_64',
+                                    mock_make_repo.return_value)])
+
+
+class TestMakeLookasideRepo(helpers.PungiTestCase):
+
+    def setUp(self):
+        super(TestMakeLookasideRepo, self).setUp()
+        self.compose = helpers.DummyCompose(self.topdir, {})
+        self.variant = self.compose.variants['Server']
+        self.arch = 'x86_64'
+        self.repodir = self.compose.paths.work.lookaside_repo(self.arch, self.variant, create_dir=False)
+        self.pkglist = self.compose.paths.work.lookaside_package_list(self.arch, self.variant)
+
+    @mock.patch('pungi.phases.gather.run')
+    def test_existing_repo(self, mock_run):
+        helpers.touch(os.path.join(self.repodir, 'repodata', 'primary.xml'))
+        repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, {})
+        self.assertEqual(self.repodir, repopath)
+        self.assertFalse(os.path.exists(self.pkglist))
+        self.assertEqual(mock_run.call_args_list, [])
+
+    def assertCorrect(self, repopath, path_prefix, MockCR, mock_run):
+        with open(self.pkglist) as f:
+            packages = f.read().splitlines()
+        self.assertItemsEqual(packages,
+                              ['pkg/pkg-1.0-1.x86_64.rpm',
+                               'pkg/pkg-debuginfo-1.0-1.x86_64.rpm',
+                               'pkg/pkg-1.0-1.src.rpm'])
+
+        self.assertEqual(self.repodir, repopath)
+        print(MockCR.return_value.get_createrepo_cmd.call_args_list)
+        print([mock.call(path_prefix, update=True, database=True, skip_stat=True,
+                         pkglist=self.pkglist, outputdir=repopath,
+                         baseurl="file://%s" % path_prefix, workers=3,
+                         update_md_path=self.compose.paths.work.arch_repo(self.arch))])
+        self.assertEqual(MockCR.return_value.get_createrepo_cmd.call_args_list,
+                         [mock.call(path_prefix, update=True, database=True, skip_stat=True,
+                                    pkglist=self.pkglist, outputdir=repopath,
+                                    baseurl="file://%s" % path_prefix, workers=3,
+                                    update_md_path=self.compose.paths.work.arch_repo(self.arch))])
+        self.assertEqual(mock_run.call_args_list,
+                         [mock.call(MockCR.return_value.get_createrepo_cmd.return_value,
+                                    logfile=os.path.join(
+                                        self.topdir, 'logs', self.arch,
+                                        'lookaside_repo_Server.%s.log' % self.arch),
+                                    show_cmd=True)])
+
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    @mock.patch('pungi.phases.gather.CreaterepoWrapper')
+    @mock.patch('pungi.phases.gather.run')
+    def test_create_repo_koji_pkgset(self, mock_run, MockCR, MockKW):
+        self.compose.conf.update({
+            'pkgset_source': 'koji',
+            'koji_profile': 'koji',
+        })
+
+        pkg_map = {
+            self.arch: {
+                self.variant.uid: {
+                    'rpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.x86_64.rpm'}],
+                    'debuginfo': [{'path': '/tmp/packages/pkg/pkg-debuginfo-1.0-1.x86_64.rpm'}],
+                    'srpm': [{'path': '/tmp/packages/pkg/pkg-1.0-1.src.rpm'}],
+                }
+            }
+        }
+
+        MockKW.return_value.koji_module.config.topdir = '/tmp/packages'
+
+        repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map)
+
+        self.assertCorrect(repopath, '/tmp/packages/', MockCR, mock_run)
+
+    @mock.patch('pungi.phases.gather.CreaterepoWrapper')
+    @mock.patch('pungi.phases.gather.run')
+    def test_create_repo_repos_pkgset(self, mock_run, MockCR):
+        self.compose.conf.update({
+            'pkgset_source': 'repos',
+        })
+
+        dl_dir = self.compose.paths.work.topdir('global')
+
+        pkg_map = {
+            self.arch: {
+                self.variant.uid: {
+                    'rpm': [
+                        {'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.x86_64.rpm')}
+                    ],
+                    'debuginfo': [
+                        {'path': os.path.join(dl_dir, 'download/pkg/pkg-debuginfo-1.0-1.x86_64.rpm')}
+                    ],
+                    'srpm': [
+                        {'path': os.path.join(dl_dir, 'download/pkg/pkg-1.0-1.src.rpm')}
+                    ],
+                }
+            }
+        }
+
+        repopath = gather._make_lookaside_repo(self.compose, self.variant, self.arch, pkg_map)
+
+        self.assertCorrect(repopath, dl_dir + '/download/', MockCR, mock_run)