pungi/pungi/phases/osbs.py

# -*- coding: utf-8 -*-

import json
import os
from kobo.threads import ThreadPool, WorkerThread
from kobo import shortcuts

from .base import ConfigGuardedPhase, PhaseLoggerMixin
from .. import util
from ..wrappers import kojiwrapper


class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase):
    name = 'osbs'

    def __init__(self, compose):
        super(OSBSPhase, self).__init__(compose)
        self.pool = ThreadPool(logger=self.logger)
        self.pool.metadata = {}
        self.pool.registries = {}

    def run(self):
        for variant in self.compose.get_variants():
            for conf in self.get_config_block(variant):
                self.pool.add(OSBSThread(self.pool))
                self.pool.queue_put((self.compose, variant, conf))

        self.pool.start()

    def dump_metadata(self):
        """Create a file with image metadata if the phase actually ran."""
        if self._skipped:
            return
        with open(self.compose.paths.compose.metadata('osbs.json'), 'w') as f:
            json.dump(self.pool.metadata, f, indent=4, sort_keys=True,
                      separators=(',', ': '))

    def request_push(self):
        """Store configuration data about where to push the created images and
        then send the same data to message bus.
        """
        if not self.pool.registries:
            return

        # Write the data into a file.
        registry_file = os.path.join(
            self.compose.paths.log.topdir(), "osbs-registries.json"
        )
        with open(registry_file, "w") as fh:
            json.dump(self.pool.registries, fh)

        # Send a message with the data
        if self.compose.notifier:
            self.compose.notifier.send(
                "osbs-request-push",
                config_location=util.translate_path(self.compose, registry_file),
                config=self.pool.registries,
            )


class OSBSThread(WorkerThread):
    def process(self, item, num):
        compose, variant, config = item
        self.num = num
        with util.failable(compose, bool(config.pop('failable', None)), variant, '*', 'osbs',
                           logger=self.pool._logger):
            self.worker(compose, variant, config)

    def worker(self, compose, variant, config):
        msg = 'OSBS task for variant %s' % variant.uid
        self.pool.log_info('[BEGIN] %s' % msg)
        koji = kojiwrapper.KojiWrapper(compose.conf['koji_profile'])
        koji.login()

        # Start task
        source = config.pop('url')
        target = config.pop('target')
        priority = config.pop('priority', None)
        gpgkey = config.pop('gpgkey', None)
        repos = [self._get_repo(compose, v, gpgkey=gpgkey)
                 for v in [variant.uid] + shortcuts.force_list(config.pop('repo', []))]
        registry = config.pop("registry", None)

        config['yum_repourls'] = repos

        task_id = koji.koji_proxy.buildContainer(source, target, config,
                                                 priority=priority)

        # Wait for it to finish and capture the output into log file (even
        # though there is not much there).
        log_dir = os.path.join(compose.paths.log.topdir(), 'osbs')
        util.makedirs(log_dir)
        log_file = os.path.join(log_dir, '%s-%s-watch-task.log'
                                % (variant.uid, self.num))
        if koji.watch_task(task_id, log_file) != 0:
            raise RuntimeError('OSBS: task %s failed: see %s for details'
                               % (task_id, log_file))

        scratch = config.get('scratch', False)
        nvr = self._add_metadata(variant, task_id, compose, scratch)
        if nvr and registry:
            self.pool.registries[nvr] = registry

        self.pool.log_info('[DONE ] %s' % msg)

    def _add_metadata(self, variant, task_id, compose, is_scratch):
        # Create new Koji session. The task could take so long to finish that
        # our session will expire. This second session does not need to be
        # authenticated since it will only do reading operations.
        koji = kojiwrapper.KojiWrapper(compose.conf['koji_profile'])

        # Create metadata
        metadata = {
            'compose_id': compose.compose_id,
            'koji_task': task_id,
        }

        result = koji.koji_proxy.getTaskResult(task_id)
        if is_scratch:
            metadata.update({
                'repositories': result['repositories'],
            })
            # add a fake arch of 'scratch', so we can construct the metadata
            # in same data structure as real builds.
            self.pool.metadata.setdefault(
                variant.uid, {}).setdefault('scratch', []).append(metadata)
            return None

        else:
            build_id = int(result['koji_builds'][0])
            buildinfo = koji.koji_proxy.getBuild(build_id)
            archives = koji.koji_proxy.listArchives(build_id)

            nvr = "%(name)s-%(version)s-%(release)s" % buildinfo

            metadata.update({
                'name': buildinfo['name'],
                'version': buildinfo['version'],
                'release': buildinfo['release'],
                'nvr': nvr,
                'creation_time': buildinfo['creation_time'],
            })
            for archive in archives:
                data = {
                    'filename': archive['filename'],
                    'size': archive['size'],
                    'checksum': archive['checksum'],
                }
                data.update(archive['extra'])
                data.update(metadata)
                arch = archive['extra']['image']['arch']
                self.pool.log_debug('Created Docker base image %s-%s-%s.%s' % (
                    metadata['name'], metadata['version'], metadata['release'], arch))
                self.pool.metadata.setdefault(
                    variant.uid, {}).setdefault(arch, []).append(data)
            return nvr

    def _get_repo(self, compose, repo, gpgkey=None):
        """
        Return repo file URL of repo, if repo contains "://", it's already a
        URL of repo file. Or it's a variant UID or local path, then write a .repo
        file pointing to that location and return the URL to .repo file.
        """
        if "://" in repo:
            return repo

        if repo.startswith("/"):
            # The repo is an absolute path on the filesystem
            repo_path = repo
            variant = "local"
            repo_file = os.path.join(
                compose.paths.work.tmp_dir(None, None),
                "compose-rpms-%s-%s.repo" % (variant, self.num),
            )

        else:
            # We got a variant name and have to find the repository for that variant.
            try:
                variant = compose.all_variants[repo]
            except KeyError:
                raise RuntimeError(
                    "There is no variant %s to get repo from to pass to OSBS." % repo
                )
            repo_path = compose.paths.compose.repository(
                "$basearch", variant, create_dir=False
            )

            repo_file = os.path.join(
                compose.paths.work.tmp_dir(None, variant),
                'compose-rpms-%s-%s.repo' % (variant, self.num),
            )

        gpgcheck = 1 if gpgkey else 0
        with open(repo_file, 'w') as f:
            f.write('[%s-%s-%s]\n' % (compose.compose_id, variant, self.num))
            f.write('name=Compose %s (RPMs) - %s\n' % (compose.compose_id, variant))
            f.write('baseurl=%s\n' % util.translate_path(compose, repo_path))
            f.write('enabled=1\n')
            f.write('gpgcheck=%s\n' % gpgcheck)
            if gpgcheck:
                f.write('gpgkey=%s\n' % gpgkey)

        return util.translate_path(compose, repo_file)