pungi/tests/test_ostree_phase.py

# -*- coding: utf-8 -*-


import json
import mock

import os
import sys

from tests import helpers
from pungi.phases import ostree


class OSTreePhaseTest(helpers.PungiTestCase):

    @mock.patch('pungi.phases.ostree.ThreadPool')
    def test_run(self, ThreadPool):
        cfg = helpers.IterableMock()
        compose = helpers.DummyCompose(self.topdir, {
            'ostree': [
                ('^Everything$', {'x86_64': cfg})
            ],
            'runroot': True,
            "translate_paths": [
                (self.topdir, "http://example.com")
            ],
        })

        pool = ThreadPool.return_value

        phase = ostree.OSTreePhase(compose, self._make_pkgset_phase(["p1", "p2"]))
        phase.run()

        self.assertEqual(len(pool.add.call_args_list), 1)
        self.assertEqual(
            pool.add.call_args_list[0][0][0].repos,
            [
                "http://example.com/work/$basearch/repo/p1",
                "http://example.com/work/$basearch/repo/p2",
            ],
        )
        self.assertEqual(pool.queue_put.call_args_list,
                         [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg))])

    @mock.patch('pungi.phases.ostree.ThreadPool')
    def test_skip_without_config(self, ThreadPool):
        compose = helpers.DummyCompose(self.topdir, {})
        compose.just_phases = None
        compose.skip_phases = []
        phase = ostree.OSTreePhase(compose)
        self.assertTrue(phase.skip())

    @mock.patch('pungi.phases.ostree.ThreadPool')
    def test_run_with_simple_config(self, ThreadPool):
        cfg = helpers.IterableMock(get=lambda x, y: None)
        compose = helpers.DummyCompose(self.topdir, {
            'ostree': {
                '^Everything$': cfg
            }
        })

        pool = ThreadPool.return_value

        phase = ostree.OSTreePhase(compose, self._make_pkgset_phase(["p1"]))
        phase.run()

        self.assertEqual(len(pool.add.call_args_list), 2)
        self.assertEqual(pool.queue_put.call_args_list,
                         [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg)),
                          mock.call((compose, compose.variants['Everything'], 'amd64', cfg))])

    @mock.patch('pungi.phases.ostree.ThreadPool')
    def test_run_with_simple_config_limit_arches(self, ThreadPool):
        cfg = helpers.IterableMock(get=lambda x, y: ['x86_64'])
        compose = helpers.DummyCompose(self.topdir, {
            'ostree': {
                '^Everything$': cfg
            }
        })

        pool = ThreadPool.return_value

        phase = ostree.OSTreePhase(compose, self._make_pkgset_phase(["p1"]))
        phase.run()

        self.assertEqual(len(pool.add.call_args_list), 1)
        self.assertEqual(pool.queue_put.call_args_list,
                         [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg))])

    @mock.patch('pungi.phases.ostree.ThreadPool')
    def test_run_with_simple_config_limit_arches_two_blocks(self, ThreadPool):
        cfg1 = helpers.IterableMock(get=lambda x, y: ['x86_64'])
        cfg2 = helpers.IterableMock(get=lambda x, y: ['s390x'])
        compose = helpers.DummyCompose(self.topdir, {
            'ostree': {
                '^Everything$': [cfg1, cfg2],
            }
        })

        pool = ThreadPool.return_value

        phase = ostree.OSTreePhase(compose, self._make_pkgset_phase(["p1"]))
        phase.run()

        self.assertEqual(len(pool.add.call_args_list), 2)
        self.assertEqual(pool.queue_put.call_args_list,
                         [mock.call((compose, compose.variants['Everything'], 'x86_64', cfg1)),
                          mock.call((compose, compose.variants['Everything'], 's390x', cfg2))])


class OSTreeThreadTest(helpers.PungiTestCase):

    def setUp(self):
        super(OSTreeThreadTest, self).setUp()
        self.repo = os.path.join(self.topdir, 'place/for/atomic')
        os.makedirs(os.path.join(self.repo, 'refs', 'heads'))
        self.cfg = {
            'repo': 'Everything',
            'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
            'config_branch': 'f24',
            'treefile': 'fedora-atomic-docker-host.json',
            'ostree_repo': self.repo,
        }
        self.compose = helpers.DummyCompose(self.topdir, {
            'koji_profile': 'koji',
            'runroot_tag': 'rrt',
            'translate_paths': [
                (self.topdir, 'http://example.com')
            ],
        })
        self.pool = mock.Mock()

    def _dummy_config_repo(self, scm_dict, target, logger=None):
        os.makedirs(target)
        helpers.touch(os.path.join(target, 'fedora-atomic-docker-host.json'),
                      json.dumps({'ref': 'fedora-atomic/25/x86_64',
                                  'repos': ['fedora-rawhide', 'fedora-24', 'fedora-23']}))
        helpers.touch(os.path.join(target, 'fedora-rawhide.repo'),
                      '[fedora-rawhide]\nmirrorlist=mirror-mirror-on-the-wall')
        helpers.touch(os.path.join(target, 'fedora-24.repo'),
                      '[fedora-24]\nmetalink=who-is-the-fairest-of-them-all')
        helpers.touch(os.path.join(target, 'fedora-23.repo'),
                      '[fedora-23]\nbaseurl=why-not-zoidberg?')

    def _mock_runroot(self, retcode, writefiles=None):
        """Pretend to run a task in runroot, creating a log file with given line

        Also allows for writing other files of requested"""
        def fake_runroot(self, log_file, **kwargs):
            if writefiles:
                logdir = os.path.dirname(log_file)
                for filename in writefiles:
                    helpers.touch(os.path.join(logdir, filename),
                                  '\n'.join(writefiles[filename]))
                    helpers.touch(os.path.join(logdir, filename + ".stamp"))
            return {'task_id': 1234, 'retcode': retcode, 'output': 'Foo bar\n'}
        return fake_runroot

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_extra_config_content(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo
        self.compose.conf['runroot_weights'] = {'ostree': 123}

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)

        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json')
        self.assertFalse(os.path.isfile(extra_config_file))

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.assertTrue(os.path.isfile(extra_config_file))
        with open(extra_config_file, 'r') as f:
            extraconf_content = json.load(f)

        proper_extraconf_content = {
            "repo": [
                {"name": "http:__example.com_repo_1",
                 "baseurl": "http://example.com/repo/1"},
                {"name": "http:__example.com_work__basearch_comps_repo_Everything",
                 "baseurl": "http://example.com/work/$basearch/comps_repo_Everything"}
            ]
        }
        self.assertEqual(proper_extraconf_content, extraconf_content)

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo
        self.compose.conf['runroot_weights'] = {'ostree': 123}

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)

        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.assertEqual(get_dir_from_scm.call_args_list,
                         [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
                                     'branch': 'f24', 'dir': '.'},
                                    self.topdir + '/work/ostree-1/config_repo', logger=self.pool._logger)])
        self.assertEqual(koji.get_runroot_cmd.call_args_list,
                         [mock.call('rrt', 'x86_64',
                                    ['pungi-make-ostree',
                                     'tree',
                                     '--repo=%s' % self.repo,
                                     '--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir,
                                     '--treefile=%s/fedora-atomic-docker-host.json' % (
                                         self.topdir + '/work/ostree-1/config_repo'),
                                     '--extra-config=%s/extra_config.json' % (self.topdir + '/work/ostree-1')],
                                    channel=None, mounts=[self.topdir, self.repo],
                                    packages=['pungi', 'ostree', 'rpm-ostree'],
                                    task_id=True, use_shell=True, new_chroot=True, weight=123)])
        self.assertEqual(koji.run_runroot_cmd.call_args_list,
                         [mock.call(koji.get_runroot_cmd.return_value,
                                    log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])

        self.assertTrue(os.path.isfile(os.path.join(self.topdir, 'work/ostree-1/extra_config.json')))
        self.assertTrue(os.path.isdir(self.repo))

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_fail(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo

        self.cfg['failable'] = ['*']
        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(1)

        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.compose._logger.error.assert_has_calls([
            mock.call('[FAIL] Ostree (variant Everything, arch x86_64) failed, but going on anyway.'),
            mock.call('Runroot task failed: 1234. See %s for more details.'
                      % (self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log'))
        ])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_handle_exception(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo

        self.cfg['failable'] = ['*']
        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = helpers.boom

        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.compose._logger.error.assert_has_calls([
            mock.call('[FAIL] Ostree (variant Everything, arch x86_64) failed, but going on anyway.'),
            mock.call('BOOM')
        ])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_send_message(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo

        self.compose.notifier = mock.Mock()
        self.compose.conf['translate_paths'] = [(self.topdir, 'http://example.com/')]

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(
            0,
            {'commitid.log': 'fca3465861a',
             'create-ostree-repo.log':
                ['Doing work', 'fedora-atomic/25/x86_64 -> fca3465861a']})
        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.assertEqual(self.compose.notifier.send.mock_calls,
                         [mock.call('ostree',
                                    variant='Everything',
                                    arch='x86_64',
                                    ref='fedora-atomic/25/x86_64',
                                    commitid='fca3465861a',
                                    repo_path='http://example.com/place/for/atomic',
                                    local_repo_path=self.repo)])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_send_message_custom_ref(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo
        self.cfg["ostree_ref"] = "my/${basearch}"

        self.compose.notifier = mock.Mock()
        self.compose.conf['translate_paths'] = [(self.topdir, 'http://example.com/')]

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(
            0,
            {'commitid.log': 'fca3465861a',
             'create-ostree-repo.log':
                ['Doing work', 'fedora-atomic/25/x86_64 -> fca3465861a']})
        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.assertEqual(self.compose.notifier.send.mock_calls,
                         [mock.call('ostree',
                                    variant='Everything',
                                    arch='x86_64',
                                    ref='my/x86_64',
                                    commitid='fca3465861a',
                                    repo_path='http://example.com/place/for/atomic',
                                    local_repo_path=self.repo)])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_send_message_without_commit_id(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo

        self.compose.notifier = mock.Mock()

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(
            0,
            {'create-ostree-repo.log': ['Doing work', 'Weird output']})
        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.assertEqual(self.compose.notifier.send.mock_calls,
                         [mock.call('ostree',
                                    variant='Everything',
                                    arch='x86_64',
                                    ref='fedora-atomic/25/x86_64',
                                    commitid=None,
                                    repo_path='http://example.com/place/for/atomic',
                                    local_repo_path=self.repo)])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_send_no_message_on_failure(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo

        self.compose.notifier = mock.Mock()

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(1)
        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        self.assertRaises(RuntimeError, t.process,
                          (self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg),
                          1)
        self.assertEqual(self.compose.notifier.send.mock_calls, [])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_with_update_summary(self, KojiWrapper, get_dir_from_scm):
        self.cfg['update_summary'] = True

        get_dir_from_scm.side_effect = self._dummy_config_repo

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)

        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.assertEqual(get_dir_from_scm.call_args_list,
                         [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
                                     'branch': 'f24', 'dir': '.'},
                                    self.topdir + '/work/ostree-1/config_repo', logger=self.pool._logger)])
        self.assertEqual(koji.get_runroot_cmd.call_args_list,
                         [mock.call('rrt', 'x86_64',
                                    ['pungi-make-ostree',
                                     'tree',
                                     '--repo=%s' % self.repo,
                                     '--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir,
                                     '--treefile=%s/fedora-atomic-docker-host.json' % (
                                         self.topdir + '/work/ostree-1/config_repo'),
                                     '--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir,
                                     '--update-summary'],
                                    channel=None, mounts=[self.topdir, self.repo],
                                    packages=['pungi', 'ostree', 'rpm-ostree'],
                                    task_id=True, use_shell=True, new_chroot=True, weight=None)])
        self.assertEqual(koji.run_runroot_cmd.call_args_list,
                         [mock.call(koji.get_runroot_cmd.return_value,
                                    log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_with_versioning_metadata(self, KojiWrapper, get_dir_from_scm):
        self.cfg['version'] = '24'

        get_dir_from_scm.side_effect = self._dummy_config_repo

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)

        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.assertEqual(get_dir_from_scm.call_args_list,
                         [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
                                     'branch': 'f24', 'dir': '.'},
                                    self.topdir + '/work/ostree-1/config_repo', logger=self.pool._logger)])
        self.assertEqual(koji.get_runroot_cmd.call_args_list,
                         [mock.call('rrt', 'x86_64',
                                    ['pungi-make-ostree',
                                     'tree',
                                     '--repo=%s' % self.repo,
                                     '--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir,
                                     '--treefile=%s/fedora-atomic-docker-host.json' % (
                                         self.topdir + '/work/ostree-1/config_repo'),
                                     '--version=24',
                                     '--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir],
                                    channel=None, mounts=[self.topdir, self.repo],
                                    packages=['pungi', 'ostree', 'rpm-ostree'],
                                    task_id=True, use_shell=True, new_chroot=True, weight=None)])
        self.assertEqual(koji.run_runroot_cmd.call_args_list,
                         [mock.call(koji.get_runroot_cmd.return_value,
                                    log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_run_with_generated_versioning_metadata(self, KojiWrapper, get_dir_from_scm):
        self.cfg['version'] = '!OSTREE_VERSION_FROM_LABEL_DATE_TYPE_RESPIN'

        get_dir_from_scm.side_effect = self._dummy_config_repo

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)

        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', self.cfg), 1)

        self.assertEqual(get_dir_from_scm.call_args_list,
                         [mock.call({'scm': 'git', 'repo': 'https://git.fedorahosted.org/git/fedora-atomic.git',
                                     'branch': 'f24', 'dir': '.'},
                                    self.topdir + '/work/ostree-1/config_repo', logger=self.pool._logger)])
        self.assertEqual(koji.get_runroot_cmd.call_args_list,
                         [mock.call('rrt', 'x86_64',
                                    ['pungi-make-ostree',
                                     'tree',
                                     '--repo=%s' % self.repo,
                                     '--log-dir=%s/logs/x86_64/Everything/ostree-1' % self.topdir,
                                     '--treefile=%s/fedora-atomic-docker-host.json' % (
                                         self.topdir + '/work/ostree-1/config_repo'),
                                     '--version=25.20151203.t.0',
                                     '--extra-config=%s/work/ostree-1/extra_config.json' % self.topdir],
                                    channel=None, mounts=[self.topdir, self.repo],
                                    packages=['pungi', 'ostree', 'rpm-ostree'],
                                    task_id=True, use_shell=True, new_chroot=True, weight=None)])
        self.assertEqual(koji.run_runroot_cmd.call_args_list,
                         [mock.call(koji.get_runroot_cmd.return_value,
                                    log_file=self.topdir + '/logs/x86_64/Everything/ostree-1/runroot.log')])

    @mock.patch('pungi.wrappers.scm.get_dir_from_scm')
    @mock.patch('pungi.wrappers.kojiwrapper.KojiWrapper')
    def test_write_extra_config_file(self, KojiWrapper, get_dir_from_scm):
        get_dir_from_scm.side_effect = self._dummy_config_repo

        koji = KojiWrapper.return_value
        koji.run_runroot_cmd.side_effect = self._mock_runroot(0)

        cfg = {
            'repo': [  # Variant type repos will not be included into extra_config. This part of the config is deprecated
                'Everything',  # do not include
                {
                    'name': 'repo_a',
                    'baseurl': 'http://url/to/repo/a',
                    'exclude': 'systemd-container'
                },
                {  # do not include
                    'name': 'Server',
                    'baseurl': 'Server',
                    'exclude': 'systemd-container'
                }
            ],
            'keep_original_sources': True,
            'config_url': 'https://git.fedorahosted.org/git/fedora-atomic.git',
            'config_branch': 'f24',
            'treefile': 'fedora-atomic-docker-host.json',
            'ostree_repo': self.repo
        }

        t = ostree.OSTreeThread(self.pool, ["http://example.com/repo/1"])

        t.process((self.compose, self.compose.variants['Everything'], 'x86_64', cfg), 1)

        extra_config_file = os.path.join(self.topdir, 'work/ostree-1/extra_config.json')
        self.assertTrue(os.path.isfile(extra_config_file))
        with open(extra_config_file, 'r') as extra_config_fd:
            extra_config = json.load(extra_config_fd)
        self.assertTrue(extra_config.get('keep_original_sources', False))
        # should equal to number of valid repositories in cfg['repo'] + default repository + comps repository
        self.assertEqual(len(extra_config.get('repo', [])), 3)
        self.assertEqual(extra_config.get('repo').pop()['baseurl'],
                         'http://example.com/work/$basearch/comps_repo_Everything')
        self.assertEqual(extra_config.get("repo").pop()["baseurl"], "http://example.com/repo/1")
        self.assertEqual(extra_config.get('repo').pop()['baseurl'], 'http://url/to/repo/a')