almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

Ostree can use pkgset repos

Browse Source 
Related: https://pagure.io/pungi/issue/778

Signed-off-by: Ondrej Nosek <onosek@redhat.com>
This commit is contained in:
Ondrej Nosek 2018-02-06 22:29:22 +01:00
parent 364d7f5229
commit c7cc200246
5 changed files with 59 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pungi/phases/ostree.py
View File

@ -58,7 +58,8 @@ class OSTreeThread(WorkerThread):
repodir = os.path.join(workdir, 'config_repo') repodir = os.path.join(workdir, 'config_repo')
self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master')) self._clone_repo(repodir, config['config_url'], config.get('config_branch', 'master'))
repos = get_repo_dicts(compose, shortcuts.force_list(config['repo'])) repo_baseurl = compose.paths.work.arch_repo('$basearch', create_dir=False)
repos = get_repo_dicts(shortcuts.force_list(config['repo']) + shortcuts.force_list(translate_path(compose, repo_baseurl)), logger=self.pool)
# copy the original config and update before save to a json file # copy the original config and update before save to a json file
new_config = copy.copy(config) new_config = copy.copy(config)

33
pungi/util.py
View File

@ -715,19 +715,16 @@ def _translate_url_to_repo_id(url):
return ''.join([s if s in list(_REPOID_CHARS) else '_' for s in url]) return ''.join([s if s in list(_REPOID_CHARS) else '_' for s in url])
def get_repo_dict(compose, repo, arch='$basearch'): def get_repo_dict(repo):
""" """
Convert repo to a dict of repo options. Convert repo to a dict of repo options.
If repo is a string, translate it to repo url if necessary (when it's If repo is a string that represents url, set it as 'baseurl' in result dict,
not a url), and set it as 'baseurl' in result dict, also generate also generate a repo id/name as 'name' key in result dict.
a repo id/name as 'name' key in result dict. If repo is a dict, and if 'name' key is missing in the dict, generate one for it.
If repo is a dict, translate value of 'baseurl' key to url if necessary, Repo (str or dict) that has not url format is no longer processed.
if 'name' key is missing in the dict, generate one for it.
@param compose - required for access to variants
@param repo - A string or dict, if it is a dict, key 'baseurl' is required @param repo - A string or dict, if it is a dict, key 'baseurl' is required
@param arch - string to be used as arch in repo url
""" """
repo_dict = {} repo_dict = {}
if isinstance(repo, dict): if isinstance(repo, dict):
@ -737,10 +734,8 @@ def get_repo_dict(compose, repo, arch='$basearch'):
if name is None: if name is None:
name = _translate_url_to_repo_id(url) name = _translate_url_to_repo_id(url)
else: else:
# url is variant uid # url is variant uid - this possibility is now discontinued
if name is None: return {}
name = '%s-%s' % (compose.compose_id, url)
url = get_repo_url(compose, url, arch=arch)
repo['name'] = name repo['name'] = name
repo['baseurl'] = url repo['baseurl'] = url
return repo return repo
@ -751,23 +746,23 @@ def get_repo_dict(compose, repo, arch='$basearch'):
repo_dict['name'] = _translate_url_to_repo_id(repo) repo_dict['name'] = _translate_url_to_repo_id(repo)
repo_dict['baseurl'] = repo repo_dict['baseurl'] = repo
else: else:
repo_dict['name'] = '%s-%s' % (compose.compose_id, repo) return {}
repo_dict['baseurl'] = get_repo_url(compose, repo)
return repo_dict return repo_dict
def get_repo_dicts(compose, repos, arch='$basearch'): def get_repo_dicts(repos, logger=None):
""" """
Convert repos to a list of repo dicts. Convert repos to a list of repo dicts.
@param compose - required for access to variants
@param repo - A list of string or dict, if item is a dict, key 'baseurl' is required @param repo - A list of string or dict, if item is a dict, key 'baseurl' is required
@param arch - string to be used as arch in repo url
""" """
repo_dicts = [] repo_dicts = []
for repo in repos: for repo in repos:
repo_dict = get_repo_dict(compose, repo, arch=arch) repo_dict = get_repo_dict(repo)
if repo_dict == {}:
if logger:
logger.log_warning("Variant-type source repository is deprecated and will be ignored during 'ostree' phase: %s" % (repo))
else:
repo_dicts.append(repo_dict) repo_dicts.append(repo_dict)
return repo_dicts return repo_dicts

4
tests/test_config_validate_script.py
View File

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import mock
import os import os
import subprocess import subprocess
import sys import sys
@ -20,8 +19,7 @@ from tests import helpers
class ConfigValidateScriptTest(helpers.PungiTestCase): class ConfigValidateScriptTest(helpers.PungiTestCase):
@mock.patch('kobo.shortcuts.run') def test_validate_dummy_config(self):
def test_validate_dummy_config(self, run):
DUMMY_CONFIG = os.path.join(HERE, 'data/dummy-pungi.conf') DUMMY_CONFIG = os.path.join(HERE, 'data/dummy-pungi.conf')
interp = 'python2' if six.PY2 else 'python3' interp = 'python2' if six.PY2 else 'python3'
p = subprocess.Popen([interp, PUNGI_CONFIG_VALIDATE, DUMMY_CONFIG], p = subprocess.Popen([interp, PUNGI_CONFIG_VALIDATE, DUMMY_CONFIG],

41
tests/test_ostree_phase.py
View File

@ -118,7 +118,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
'koji_profile': 'koji', 'koji_profile': 'koji',
'runroot_tag': 'rrt', 'runroot_tag': 'rrt',
'translate_paths': [ 'translate_paths': [
(self.topdir + '/compose', 'http://example.com') (self.topdir, 'http://example.com')
] ]
}) })
self.pool = mock.Mock() self.pool = mock.Mock()
@ -148,6 +148,28 @@ class OSTreeThreadTest(helpers.PungiTestCase):
return {'task_id': 1234, 'retcode': retcode, 'output': 'Foo bar\n'} return {'task_id': 1234, 'retcode': retcode, 'output': 'Foo bar\n'}
return fake_runroot return fake_runroot
@mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_extra_config_content(self, KojiWrapper, get_dir_from_scm):
get_dir_from_scm.side_effect = self._dummy_config_repo
self.compose.conf['runroot_weights'] = {'ostree': 123}
koji = KojiWrapper.return_value
koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
t = ostree.OSTreeThread(self.pool)
extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json')
self.assertFalse(os.path.isfile(extra_config_file))
t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)
self.assertTrue(os.path.isfile(extra_config_file))
with open(extra_config_file, 'r') as f:
extraconf_content = json.load(f)
proper_extraconf_content = json.loads('{"repo": [{"name": "http:__example.com_work__basearch_repo", "baseurl": "http://example.com/work/$basearch/repo"}]}')
self.assertEqual(proper_extraconf_content, extraconf_content)
@mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper') @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
def test_run(self, KojiWrapper, get_dir_from_scm): def test_run(self, KojiWrapper, get_dir_from_scm):
@ -269,7 +291,7 @@ class OSTreeThreadTest(helpers.PungiTestCase):
arch='x86_64', arch='x86_64',
ref='fedora-atomic/25/x86_64', ref='fedora-atomic/25/x86_64',
commitid=None, commitid=None,
repo_path=self.repo, repo_path='http://example.com/place/for/atomic',
local_repo_path=self.repo)]) local_repo_path=self.repo)])
@mock.patch('pungi.wrappers.scm.get_dir_from_scm') @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
@ -402,14 +424,14 @@ class OSTreeThreadTest(helpers.PungiTestCase):
koji.run_runroot_cmd.side_effect = self._mock_runroot(0) koji.run_runroot_cmd.side_effect = self._mock_runroot(0)
cfg = { cfg = {
'repo': [ 'repo': [ # Variant type repos will not be included into extra_config. This part of the config is deprecated
'Everything', 'Everything', # do not include
{ {
'name': 'repo_a', 'name': 'repo_a',
'baseurl': 'http://url/to/repo/a', 'baseurl': 'http://url/to/repo/a',
'exclude': 'systemd-container' 'exclude': 'systemd-container'
}, },
{ { # do not include
'name': 'Server', 'name': 'Server',
'baseurl': 'Server', 'baseurl': 'Server',
'exclude': 'systemd-container' 'exclude': 'systemd-container'
@ -428,12 +450,13 @@ class OSTreeThreadTest(helpers.PungiTestCase):
extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json') extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json')
self.assertTrue(os.path.isfile(extra_config_file)) self.assertTrue(os.path.isfile(extra_config_file))
extra_config = json.load(open(extra_config_file, 'r')) with open(extra_config_file, 'r') as extra_config_fd:
extra_config = json.load(extra_config_fd)
self.assertTrue(extra_config.get('keep_original_sources', False)) self.assertTrue(extra_config.get('keep_original_sources', False))
self.assertEqual(len(extra_config.get('repo', [])), len(cfg['repo'])) self.assertEqual(len(extra_config.get('repo', [])), 2) # should equal to number of valid repositories in cfg['repo'] + default repository
self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/Server/$basearch/os') self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/work/$basearch/repo')
self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://url/to/repo/a') self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://url/to/repo/a')
self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://example.com/Everything/$basearch/os')
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

26
tests/test_util.py
View File

@ -560,43 +560,35 @@ class GetRepoFuncsTestCase(unittest.TestCase):
self.assertEqual(util.get_repo_urls(self.compose, repos), expect) self.assertEqual(util.get_repo_urls(self.compose, repos), expect)
def test_get_repo_dict_from_normal_url(self): def test_get_repo_dict_from_normal_url(self):
repo_dict = util.get_repo_dict(self.compose, 'http://example.com/repo') repo_dict = util.get_repo_dict('http://example.com/repo')
expect = {'name': 'http:__example.com_repo', 'baseurl': 'http://example.com/repo'} expect = {'name': 'http:__example.com_repo', 'baseurl': 'http://example.com/repo'}
self.assertEqual(repo_dict, expect) self.assertEqual(repo_dict, expect)
def test_get_repo_dict_from_variant_uid(self): def test_get_repo_dict_from_variant_uid(self):
repo_dict = util.get_repo_dict(self.compose, 'Server') repo_dict = util.get_repo_dict('Server') # this repo format is deprecated
expect = { expect = {}
'name': "%s-%s" % (self.compose.compose_id, 'Server'),
'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os',
}
self.assertEqual(repo_dict, expect) self.assertEqual(repo_dict, expect)
def test_get_repo_dict_from_repo_dict(self): def test_get_repo_dict_from_repo_dict(self):
repo = {'baseurl': 'Server'} repo = {'baseurl': 'Server'} # this repo format is deprecated
expect = { expect = {}
'name': '%s-%s' % (self.compose.compose_id, 'Server'), repo_dict = util.get_repo_dict(repo)
'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os'
}
repo_dict = util.get_repo_dict(self.compose, repo)
self.assertEqual(repo_dict, expect) self.assertEqual(repo_dict, expect)
def test_get_repo_dicts(self): def test_get_repo_dicts(self):
repos = [ repos = [
'http://example.com/repo', 'http://example.com/repo',
'Server', 'Server', # this repo format is deprecated (and will not be included into final repo_dict)
{'baseurl': 'Client'}, {'baseurl': 'Client'}, # this repo format is deprecated
{'baseurl': 'ftp://example.com/linux/repo'}, {'baseurl': 'ftp://example.com/linux/repo'},
{'name': 'testrepo', 'baseurl': 'ftp://example.com/linux/repo'}, {'name': 'testrepo', 'baseurl': 'ftp://example.com/linux/repo'},
] ]
expect = [ expect = [
{'name': 'http:__example.com_repo', 'baseurl': 'http://example.com/repo'}, {'name': 'http:__example.com_repo', 'baseurl': 'http://example.com/repo'},
{'name': '%s-%s' % (self.compose.compose_id, 'Server'), 'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Server/$basearch/os'},
{'name': '%s-%s' % (self.compose.compose_id, 'Client'), 'baseurl': 'http://example.com/RHEL-8.0-20180101.n.0/compose/Client/$basearch/os'},
{'name': 'ftp:__example.com_linux_repo', 'baseurl': 'ftp://example.com/linux/repo'}, {'name': 'ftp:__example.com_linux_repo', 'baseurl': 'ftp://example.com/linux/repo'},
{'name': 'testrepo', 'baseurl': 'ftp://example.com/linux/repo'}, {'name': 'testrepo', 'baseurl': 'ftp://example.com/linux/repo'},
] ]
repos = util.get_repo_dicts(self.compose, repos) repos = util.get_repo_dicts(repos)
self.assertEqual(repos, expect) self.assertEqual(repos, expect)