almalinux/pungi
14
1
Fork 0
Code Issues Pull Requests Releases Activity

osbs: Process data about pushing images to registries

Browse Source 
This patch does not do any actual pushing. It will only extract data
about push targets from the main configuration and store it together
with exact Koji NVR in a well-defined location, and also send the data
to message bus for another service to handle.

JIRA: COMPOSE-3228
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
This commit is contained in:
Lubomír Sedlář 2019-02-07 14:17:09 +01:00
parent 23bf01bb45
commit 7693e562b1
5 changed files with 102 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/pungi-koji
View File

@ -430,6 +430,7 @@ def run_compose(compose, create_latest_link=True, latest_link_status=None):
test_phase.stop()
compose.write_status("FINISHED")
osbs_phase.request_push()
latest_link = False
if create_latest_link:
if latest_link_status is None:

15
doc/configuration.rst
View File

@ -1455,12 +1455,12 @@ Example config
OSBS Settings
=============
*Pungi* can build docker images in OSBS. The build is initiated through Koji
*Pungi* can build container images in OSBS. The build is initiated through Koji
``container-build`` plugin. The base image will be using RPMs from the current
compose and a ``Dockerfile`` from specified Git repository.
Please note that the image is uploaded to a Docker v2 registry and not exported
into compose directory. There will be a metadata file in
Please note that the image is uploaded to a registry and not exported into
compose directory. There will be a metadata file in
``compose/metadata/osbs.json`` with details about the built images (assuming
they are not scratch builds).
@ -1487,6 +1487,15 @@ they are not scratch builds).
option will most likely change to list architectures that are allowed
to fail.
It is possible to configure extra information about where to push the image
(unless it is a scratch build). Pungi will take any value in ``registry``
key in the configuration and collect them across all built images. The data
will be saved into ``logs/global/osbs-registries.json`` as a mapping from
Koji NVR to the registry data. The same data is also sent to the message
bus on ``osbs-request-push`` topic once the compose finishes successfully.
Handling the message and performing the actual push is outside of scope for
Pungi.
The configuration will pass other attributes directly to the Koji task.
This includes ``name``, ``version``, ``scratch`` and ``priority``.

1
pungi/checks.py
View File

@ -497,6 +497,7 @@ def make_schema():
},
"gpgkey": {"type": "string"},
"git_branch": {"type": "string"},
"registry": {"type": "object"},
},
"required": ["url", "target", "git_branch"]
},

35
pungi/phases/osbs.py
View File

@ -17,6 +17,7 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase):
super(OSBSPhase, self).__init__(compose)
self.pool = ThreadPool(logger=self.logger)
self.pool.metadata = {}
self.pool.registries = {}
def run(self):
for variant in self.compose.get_variants():
@ -34,6 +35,28 @@ class OSBSPhase(PhaseLoggerMixin, ConfigGuardedPhase):
json.dump(self.pool.metadata, f, indent=4, sort_keys=True,
separators=(',', ': '))
def request_push(self):
"""Store configuration data about where to push the created images and
then send the same data to message bus.
"""
if not self.pool.registries:
return
# Write the data into a file.
registry_file = os.path.join(
self.compose.paths.log.topdir(), "osbs-registries.json"
)
with open(registry_file, "w") as fh:
json.dump(self.pool.registries, fh)
# Send a message with the data
if self.compose.notifier:
self.compose.notifier.send(
"osbs-request-push",
config_location=util.translate_path(self.compose, registry_file),
config=self.pool.registries,
)
class OSBSThread(WorkerThread):
def process(self, item, num):
@ -56,6 +79,7 @@ class OSBSThread(WorkerThread):
gpgkey = config.pop('gpgkey', None)
repos = [self._get_repo(compose, v, gpgkey=gpgkey)
for v in [variant.uid] + shortcuts.force_list(config.pop('repo', []))]
registry = config.pop("registry", None)
config['yum_repourls'] = repos
@ -73,7 +97,9 @@ class OSBSThread(WorkerThread):
% (task_id, log_file))
scratch = config.get('scratch', False)
self._add_metadata(variant, task_id, compose, scratch)
nvr = self._add_metadata(variant, task_id, compose, scratch)
if nvr and registry:
self.pool.registries[nvr] = registry
self.pool.log_info('[DONE ] %s' % msg)
@ -98,16 +124,20 @@ class OSBSThread(WorkerThread):
# in same data structure as real builds.
self.pool.metadata.setdefault(
variant.uid, {}).setdefault('scratch', []).append(metadata)
return None
else:
build_id = int(result['koji_builds'][0])
buildinfo = koji.koji_proxy.getBuild(build_id)
archives = koji.koji_proxy.listArchives(build_id)
nvr = "%(name)s-%(version)s-%(release)s" % buildinfo
metadata.update({
'name': buildinfo['name'],
'version': buildinfo['version'],
'release': buildinfo['release'],
'nvr': '%(name)s-%(version)s-%(release)s' % buildinfo,
'nvr': nvr,
'creation_time': buildinfo['creation_time'],
})
for archive in archives:
@ -123,6 +153,7 @@ class OSBSThread(WorkerThread):
metadata['name'], metadata['version'], metadata['release'], arch))
self.pool.metadata.setdefault(
variant.uid, {}).setdefault(arch, []).append(data)
return nvr
def _get_repo(self, compose, repo, gpgkey=None):
"""

56
tests/test_osbs_phase.py
View File

@ -75,6 +75,29 @@ class OSBSPhaseTest(helpers.PungiTestCase):
self.assertFalse(os.path.isfile(self.topdir + '/compose/metadata/osbs.json'))
@mock.patch("pungi.phases.osbs.ThreadPool")
def test_request_push(self, ThreadPool):
compose = helpers.DummyCompose(self.topdir, {
"osbs": {"^Everything$": {}}
})
compose.just_phases = None
compose.skip_phases = []
compose.notifier = mock.Mock()
phase = osbs.OSBSPhase(compose)
phase.start()
phase.stop()
phase.pool.registries = {"foo": "bar"}
phase.request_push()
with open(os.path.join(self.topdir, "logs/global/osbs-registries.json")) as f:
data = json.load(f)
self.assertEqual(data, phase.pool.registries)
self.assertEqual(
compose.notifier.call_args_list,
[],
)
TASK_RESULT = {
'koji_builds': ['54321'],
@ -169,7 +192,7 @@ class OSBSThreadTest(helpers.PungiTestCase):
def setUp(self):
super(OSBSThreadTest, self).setUp()
self.pool = mock.Mock(metadata={})
self.pool = mock.Mock(metadata={}, registries={})
self.t = osbs.OSBSThread(self.pool)
self.compose = helpers.DummyCompose(self.topdir, {
'koji_profile': 'koji',
@ -331,6 +354,37 @@ class OSBSThreadTest(helpers.PungiTestCase):
self._assertCorrectMetadata()
self._assertRepoFile(['Server', 'Everything'])
@mock.patch("pungi.phases.osbs.kojiwrapper.KojiWrapper")
def test_run_with_registry(self, KojiWrapper):
cfg = {
"url": "git://example.com/repo?#BEEFCAFE",
"target": "f24-docker-candidate",
"git_branch": "f24-docker",
"name": "my-name",
"version": "1.0",
"repo": ["Everything", "http://pkgs.example.com/my.repo"],
"registry": {"foo": "bar"},
}
self._setupMock(KojiWrapper)
self._assertConfigCorrect(cfg)
self.t.process((self.compose, self.compose.variants["Server"], cfg), 1)
options = {
"name": "my-name",
"version": "1.0",
"git_branch": "f24-docker",
"yum_repourls": [
"http://root/work/global/tmp-Server/compose-rpms-Server-1.repo",
"http://root/work/global/tmp-Everything/compose-rpms-Everything-1.repo",
"http://pkgs.example.com/my.repo",
]
}
self._assertCorrectCalls(options)
self._assertCorrectMetadata()
self._assertRepoFile(["Server", "Everything"])
self.assertEqual(self.t.pool.registries, {"my-name-1.0-1": {"foo": "bar"}})
@mock.patch('pungi.phases.osbs.kojiwrapper.KojiWrapper')
def test_run_with_extra_repos_in_list(self, KojiWrapper):
cfg = {