Ostree can use pkgset repos

Related: https://pagure.io/pungi/issue/778

Signed-off-by: Ondrej Nosek <onosek@redhat.com>

pungi/phases/ostree.py

@@ -58,7 +58,8 @@ class OSTreeThread(WorkerThread):
         repodir = os.path.join(workdir, 'config_repo')
         self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
 
-        repos = get_repo_dicts(compose, shortcuts.force_list(config['repo']))
+        repo_baseurl = compose.paths.work.arch_repo('$basearch', create_dir=False)
+        repos = get_repo_dicts(shortcuts.force_list(config['repo']) + shortcuts.force_list(translate_path(compose, repo_baseurl)), logger=self.pool)
 
         # copy the original config and update before save to a json file
         new_config = copy.copy(config)

pungi/util.py

@@ -715,19 +715,16 @@ def _translate_url_to_repo_id(url):
     return ''.join([s if s in list(_REPOID_CHARS) else '_' for s in url])
 
 
-def get_repo_dict(compose, repo, arch='$basearch'):
+def get_repo_dict(repo):
     """
     Convert repo to a dict of repo options.
 
-    If repo is a string, translate it to repo url if necessary (when it's
+    If repo is a string that represents url, set it as 'baseurl' in result dict,
-    not a url), and set it as 'baseurl' in result dict, also generate
+    also generate a repo id/name as 'name' key in result dict.
-    a repo id/name as 'name' key in result dict.
+    If repo is a dict, and if 'name' key is missing in the dict, generate one for it.
-    If repo is a dict, translate value of 'baseurl' key to url if necessary,
+    Repo (str or dict) that has not url format is no longer processed.
-    if 'name' key is missing in the dict, generate one for it.
 
-    @param compose - required for access to variants
     @param repo - A string or dict, if it is a dict, key 'baseurl' is required
-    @param arch - string to be used as arch in repo url
     """
     repo_dict = {}
     if isinstance(repo, dict):
@@ -737,10 +734,8 @@ def get_repo_dict(compose, repo, arch='$basearch'):
             if name is None:
                 name = _translate_url_to_repo_id(url)
         else:
-            # url is variant uid
+            # url is variant uid - this possibility is now discontinued
-            if name is None:
+            return {}
-                name = '%s-%s' % (compose.compose_id, url)
-            url = get_repo_url(compose, url, arch=arch)
         repo['name'] = name
         repo['baseurl'] = url
         return repo
@@ -751,23 +746,23 @@ def get_repo_dict(compose, repo, arch='$basearch'):
             repo_dict['name'] = _translate_url_to_repo_id(repo)
             repo_dict['baseurl'] = repo
         else:
-            repo_dict['name'] = '%s-%s' % (compose.compose_id, repo)
+            return {}
-            repo_dict['baseurl'] = get_repo_url(compose, repo)
-
     return repo_dict
 
 
-def get_repo_dicts(compose, repos, arch='$basearch'):
+def get_repo_dicts(repos, logger=None):
     """
     Convert repos to a list of repo dicts.
 
-    @param compose - required for access to variants
     @param repo - A list of string or dict, if item is a dict, key 'baseurl' is required
-    @param arch - string to be used as arch in repo url
     """
     repo_dicts = []
     for repo in repos:
-        repo_dict = get_repo_dict(compose, repo, arch=arch)
+        repo_dict = get_repo_dict(repo)
+        if repo_dict == {}:
+            if logger:
+                logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'ostree' phase: %s" % (repo))
+        else:
             repo_dicts.append(repo_dict)
     return repo_dicts
 

tests/test_config_validate_script.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 
-import mock
 import os
 import subprocess
 import sys
@@ -20,8 +19,7 @@ from tests import helpers
 
 class ConfigValidateScriptTest(helpers.PungiTestCase):
 
-    @mock.patch('kobo.shortcuts.run')
+    def test_validate_dummy_config(self):
-    def test_validate_dummy_config(self, run):
         DUMMY_CONFIG = os.path.join(HERE, 'data/dummy-pungi.conf')
         interp = 'python2' if six.PY2 else 'python3'
         p = subprocess.Popen([interp, PUNGI_CONFIG_VALIDATE, DUMMY_CONFIG],

tests/test_ostree_phase.py

@@ -118,7 +118,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
             'koji_profile': 'koji',
             'runroot_tag': 'rrt',
             'translate_paths': [
-                (self.topdir + '/compose', 'http://example.com')
+                (self.topdir, 'http://example.com')
             ]
         })
         self.pool = mock.Mock()
@@ -148,6 +148,28 @@ class OSTreeThreadTest(helpers.PungiTestCase):
             return {'task_id': 1234, 'retcode': retcode, 'output': 'Foo bar\n'}
         return fake_runroot
 
+    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
+    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
+    def test_extra_config_content(self, KojiWrapper, get_dir_from_scm):
+        get_dir_from_scm.side_effect = self._dummy_config_repo
+        self.compose.conf['runroot_weights'] = {'ostree': 123}
+
+        koji = KojiWrapper.return_value
+        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
+
+        t = ostree.OSTreeThread(self.pool)
+
+        extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json')
+        self.assertFalse(os.path.isfile(extra_config_file))
+
+        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)
+
+        self.assertTrue(os.path.isfile(extra_config_file))
+        with open(extra_config_file, 'r') as f:
+            extraconf_content = json.load(f)
+        proper_extraconf_content = json.loads('{"repo": [{"name": "http:__example.com_work__basearch_repo", "baseurl": "http://example.com/work/$basearch/repo"}]}')
+        self.assertEqual(proper_extraconf_content, extraconf_content)
+
     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
     @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
     def test_run(self, KojiWrapper, get_dir_from_scm):
@@ -269,7 +291,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
                                     arch='x86_64',
                                     ref='fedora-atomic/25/x86_64',
                                     commitid=None,
-                                    repo_path=self.repo,
+                                    repo_path='http://example.com/place/for/atomic',
                                     local_repo_path=self.repo)])
 
     @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@@ -402,14 +424,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
         koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
 
         cfg = {
-            'repo': [
+            'repo': [  # Variant type repos will not be included into extra_config. This part of the config is deprecated
-                'Everything',
+                'Everything',  # do not include
                 {
                     'name': 'repo_a',
                     'baseurl': 'http://url/to/repo/a',
                     'exclude': 'systemd-container'
                 },
-                {
+                {  # do not include
                     'name': 'Server',
                     'baseurl': 'Server',
                     'exclude': 'systemd-container'
@@ -428,12 +450,13 @@ class OSTreeThreadTest(helpers.PungiTestCase):
 
         extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json')
         self.assertTrue(os.path.isfile(extra_config_file))
-        extra_config = json.load(open(extra_config_file, 'r'))
+        with open(extra_config_file, 'r') as extra_config_fd:
+            extra_config = json.load(extra_config_fd)
         self.assertTrue(extra_config.get('keep_original_sources', False))
-        self.assertEqual(len(extra_config.get('repo', [])), len(cfg['repo']))
+        self.assertEqual(len(extra_config.get('repo', [])), 2)  # should equal to number of valid repositories in cfg['repo'] + default repository
-        self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/Server/$basearch/os')
+        self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/work/$basearch/repo')
         self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://url/to/repo/a')
-        self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/Everything/$basearch/os')
+
 
 if __name__ == '__main__':
     unittest.main()

tests/test_util.py

@@ -560,43 +560,35 @@ class GetRepoFuncsTestCase(unittest.TestCase):
         self.assertEqual(util.get_repo_urls(self.compose, repos), expect)
 
     def test_get_repo_dict_from_normal_url(self):
-        repo_dict = util.get_repo_dict(self.compose, 'http://example.com/repo')
+        repo_dict = util.get_repo_dict('http://example.com/repo')
         expect = {'name': 'http:__example.com_repo', 'baseurl': 'http://example.com/repo'}
         self.assertEqual(repo_dict, expect)
 
     def test_get_repo_dict_from_variant_uid(self):
-        repo_dict = util.get_repo_dict(self.compose, 'Server')
+        repo_dict = util.get_repo_dict('Server')  # this repo format is deprecated
-        expect = {
+        expect = {}
-            'name': "%s-%s" % (self.compose.compose_id, 'Server'),
-            'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os',
-        }
         self.assertEqual(repo_dict, expect)
 
     def test_get_repo_dict_from_repo_dict(self):
-        repo = {'baseurl': 'Server'}
+        repo = {'baseurl': 'Server'}  # this repo format is deprecated
-        expect = {
+        expect = {}
-            'name': '%s-%s' % (self.compose.compose_id, 'Server'),
+        repo_dict = util.get_repo_dict(repo)
-            'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os'
-        }
-        repo_dict = util.get_repo_dict(self.compose, repo)
         self.assertEqual(repo_dict, expect)
 
     def test_get_repo_dicts(self):
         repos = [
             'http://example.com/repo',
-            'Server',
+            'Server',  # this repo format is deprecated (and will not be included into final repo_dict)
-            {'baseurl': 'Client'},
+            {'baseurl': 'Client'},  # this repo format is deprecated
             {'baseurl': 'ftp://example.com/linux/repo'},
             {'name': 'testrepo', 'baseurl': 'ftp://example.com/linux/repo'},
         ]
         expect = [
             {'name': 'http:__example.com_repo', 'baseurl': 'http://example.com/repo'},
-            {'name': '%s-%s' % (self.compose.compose_id, 'Server'), 'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os'},
-            {'name': '%s-%s' % (self.compose.compose_id, 'Client'), 'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Client/$basearch/os'},
             {'name': 'ftp:__example.com_linux_repo', 'baseurl': 'ftp://example.com/linux/repo'},
             {'name': 'testrepo', 'baseurl': 'ftp://example.com/linux/repo'},
         ]
-        repos = util.get_repo_dicts(self.compose, repos)
+        repos = util.get_repo_dicts(repos)
         self.assertEqual(repos, expect)