Initial commit

.gitignore

@@ -0,0 +1,2 @@
+SOURCES/assetcache-9267281b4e7ec98515f7d6e680c5b17d1944e29a.tar.xz
+SOURCES/openQA-9267281b4e7ec98515f7d6e680c5b17d1944e29a.tar.gz

.openqa.metadata

@@ -0,0 +1,2 @@
+ab9254c6b67e392274d9afe2043a89e97e1c97a5 SOURCES/assetcache-9267281b4e7ec98515f7d6e680c5b17d1944e29a.tar.xz
+81fa866caef7e69e2d4c10ccb823fb5e2d0929cf SOURCES/openQA-9267281b4e7ec98515f7d6e680c5b17d1944e29a.tar.gz

SOURCES/23-fedora-messaging.t

@@ -0,0 +1,317 @@
+#! /usr/bin/perl
+
+# Copyright (C) 2016-2019 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+BEGIN {
+    unshift @INC, 'lib';
+}
+
+use Mojo::Base;
+use Mojo::IOLoop;
+
+use FindBin;
+use lib "$FindBin::Bin/lib";
+use OpenQA::Client;
+use OpenQA::Jobs::Constants;
+use OpenQA::Test::Database;
+use Test::MockModule;
+use Test::More;
+use Test::Mojo;
+use Test::Warnings;
+use Mojo::File qw(tempdir path);
+use OpenQA::WebAPI::Plugin::AMQP;
+
+my %published;
+my $plugin_mock_callcount;
+
+my $uuid_mock = Test::MockModule->new('UUID::URandom');
+$uuid_mock->mock(
+    create_uuid_string => sub {
+        return '9fbba7a5-2402-4f6b-a20e-af478eee05f5';
+    }
+);
+
+# we mock the parent class here so we can test the child class
+my $plugin_mock = Test::MockModule->new('OpenQA::WebAPI::Plugin::AMQP');
+$plugin_mock->mock(
+    publish_amqp => sub {
+        my ($self, $topic, $body, $headerframe) = @_;
+        # ignore the non-fedoraci messages, makes it easier to
+        # understand the expected call counts
+        if ($topic =~ /\.ci\./) {
+            $plugin_mock_callcount++;
+            # strip the time-based bits, at least till
+            # https://github.com/cho45/Test-Time/issues/14 is done
+            delete $body->{'generated_at'};
+            delete $headerframe->{'headers'}->{'sent-at'};
+            $published{$topic} = [$body, $headerframe];
+        }
+    }
+);
+
+OpenQA::Test::Database->new->create(fixtures_glob => '01-jobs.pl 03-users.pl 05-job_modules.pl');
+
+# this test also serves to test plugin loading via config file
+my $conf = << 'EOF';
+[global]
+plugins=FedoraMessaging
+base_url=https://openqa.stg.fedoraproject.org
+[amqp]
+topic_prefix=org.fedoraproject.stg
+EOF
+
+my $tempdir = tempdir;
+$ENV{OPENQA_CONFIG} = $tempdir;
+path($ENV{OPENQA_CONFIG})->make_path->child("openqa.ini")->spurt($conf);
+
+my $t = Test::Mojo->new('OpenQA::WebAPI');
+
+# XXX: Test::Mojo loses its app when setting a new ua
+# https://github.com/kraih/mojo/issues/598
+my $app = $t->app;
+$t->ua(
+    OpenQA::Client->new(apikey => 'PERCIVALKEY02', apisecret => 'PERCIVALSECRET02')->ioloop(Mojo::IOLoop->singleton));
+$t->app($app);
+
+my $settings = {
+    DISTRI      => 'Unicorn',
+    FLAVOR      => 'pink',
+    VERSION     => '42',
+    BUILD       => 'Fedora-Rawhide-20180129.n.0',
+    TEST        => 'rainbow',
+    ISO         => 'whatever.iso',
+    DESKTOP     => 'DESKTOP',
+    KVM         => 'KVM',
+    ISO_MAXSIZE => 1,
+    MACHINE     => "RainbowPC",
+    ARCH        => 'x86_64',
+    SUBVARIANT  => 'workstation',
+    TEST_TARGET => 'ISO',
+};
+
+my $expected_artifact = {
+    id           => 'Fedora-Rawhide-20180129.n.0',
+    type         => 'productmd-compose',
+    compose_type => 'nightly',
+};
+
+my $expected_contact = {
+    name  => 'Fedora openQA',
+    team  => 'Fedora QA',
+    url   => 'https://openqa.stg.fedoraproject.org',
+    docs  => 'https://fedoraproject.org/wiki/OpenQA',
+    irc   => '#fedora-qa',
+    email => 'qa-devel@lists.fedoraproject.org',
+};
+
+my $expected_image = {
+    id  => 'whatever.iso',
+    name  => 'whatever.iso',
+    type  => 'ISO',
+};
+
+my $expected_pipeline = {
+    id   => 'openqa.Fedora-Rawhide-20180129.n.0.rainbow.RainbowPC.pink.x86_64',
+    name => 'openqa.Fedora-Rawhide-20180129.n.0.rainbow.RainbowPC.pink.x86_64',
+};
+
+my $expected_run = {
+    url => '',
+    log => '',
+    id  => '',
+};
+
+my $expected_system = {
+    os           => 'fedora-42',
+    provider     => 'openqa',
+    architecture => 'x86_64',
+    variant      => 'workstation',
+};
+
+my $expected_test = {
+    category  => 'validation',
+    type      => 'rainbow RainbowPC pink x86_64',
+    namespace => 'compose',
+    lifetime  => 240,
+};
+
+my $expected_error;
+
+my $expected_version = '0.2.1';
+
+sub get_expected {
+    my $expected = {
+        artifact => $expected_artifact,
+        contact  => $expected_contact,
+        image    => $expected_image,
+        pipeline => $expected_pipeline,
+        run      => $expected_run,
+        system   => [$expected_system],
+        test     => $expected_test,
+        version  => $expected_version,
+    };
+    $expected->{'error'} = $expected_error if ($expected_error);
+    return $expected;
+}
+
+# create a job via API
+my $job;
+my $newjob;
+subtest 'create job' => sub {
+    # reset the call count
+    $plugin_mock_callcount = 0;
+    $t->post_ok("/api/v1/jobs" => form => $settings)->status_is(200);
+    ok($job = $t->tx->res->json->{id}, 'got ID of new job');
+    ok($plugin_mock_callcount == 1,    'plugin mock was called');
+    my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.queued'}};
+    $expected_run = {
+        url => "https://openqa.stg.fedoraproject.org/tests/$job",
+        log => "https://openqa.stg.fedoraproject.org/tests/$job/file/autoinst-log.txt",
+        id  => $job,
+    };
+    my $expected = get_expected;
+    my $expected_headerframe = {
+        headers => {
+            fedora_messaging_severity   => 20,
+            fedora_messaging_schema     => 'base.message',
+        },
+        content_encoding => 'utf-8',
+        delivery_mode => 2,
+        # if this fails, probably means mock broke
+        message_id => '9fbba7a5-2402-4f6b-a20e-af478eee05f5',
+    };
+    is_deeply($body, $expected, 'job create triggers standardized amqp');
+    is_deeply($headerframe, $expected_headerframe, 'amqp header frame is as expected');
+};
+
+subtest 'mark job as done' => sub {
+    $plugin_mock_callcount = 0;
+    $t->post_ok("/api/v1/jobs/$job/set_done")->status_is(200);
+    ok($plugin_mock_callcount == 1, 'mock was called');
+    my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.error'}};
+    $expected_error = {reason => INCOMPLETE,};
+    delete $expected_test->{'lifetime'};
+    my $expected = get_expected;
+    is_deeply($body, $expected, 'job done (failed) triggers standardized amqp');
+};
+
+subtest 'duplicate and cancel job' => sub {
+    $plugin_mock_callcount = 0;
+    $t->post_ok("/api/v1/jobs/$job/duplicate")->status_is(200);
+    $newjob = $t->tx->res->json->{id};
+    ok($plugin_mock_callcount == 1, 'mock was called');
+    my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.queued'}};
+    $expected_run = {
+        clone_of => $job,
+        url      => "https://openqa.stg.fedoraproject.org/tests/$newjob",
+        log      => "https://openqa.stg.fedoraproject.org/tests/$newjob/file/autoinst-log.txt",
+        id       => $newjob,
+    };
+    $expected_test->{'lifetime'} = 240;
+    $expected_error = '';
+    delete $expected_test->{'result'};
+    my $expected = get_expected;
+    is_deeply($body, $expected, 'job duplicate triggers standardized amqp');
+
+    $plugin_mock_callcount = 0;
+    $t->post_ok("/api/v1/jobs/$newjob/cancel")->status_is(200);
+    ok($plugin_mock_callcount == 1, 'mock was called');
+    my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.error'}};
+    $expected_error = {reason => 'user_cancelled',};
+    delete $expected_test->{'lifetime'};
+    delete $expected_run->{'clone_of'};
+    my $expected = get_expected;
+    is_deeply($body, $expected, 'job cancel triggers standardized amqp');
+};
+
+subtest 'duplicate and pass job' => sub {
+    $plugin_mock_callcount = 0;
+    $t->post_ok("/api/v1/jobs/$newjob/duplicate")->status_is(200);
+    my $newerjob = $t->tx->res->json->{id};
+    # explicitly set job as passed
+    $t->post_ok("/api/v1/jobs/$newerjob/set_done?result=passed")->status_is(200);
+    ok($plugin_mock_callcount == 2, 'mock was called');
+    my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.productmd-compose.test.complete'}};
+    $expected_run = {
+        url => "https://openqa.stg.fedoraproject.org/tests/$newerjob",
+        log => "https://openqa.stg.fedoraproject.org/tests/$newerjob/file/autoinst-log.txt",
+        id  => $newerjob,
+    };
+    $expected_test->{'result'} = 'passed';
+    $expected_error = '';
+    my $expected = get_expected;
+    is_deeply($body, $expected, 'job done (passed) triggers standardized amqp');
+};
+
+subtest 'create update job' => sub {
+    $plugin_mock_callcount = 0;
+    diag("Count: $plugin_mock_callcount");
+    $settings->{BUILD} = 'Update-FEDORA-2019-6bda4c81f4';
+    # this is intentionally not in alphabetical order
+    $settings->{ADVISORY_NVRS} = 'kernel-5.2.7-100.fc29 kernel-tools-5.2.7-100.fc29 kernel-headers-5.2.7-100.fc29';
+    # let's test HDD_* impact on system->{os} here too
+    $settings->{HDD_1}    = 'disk_f40_minimal.qcow2';
+    $settings->{BOOTFROM} = 'c';
+    # some update tests don't have SUBVARIANT, in which case system
+    # variant should not be set
+    delete $settings->{SUBVARIANT};
+    $t->post_ok("/api/v1/jobs" => form => $settings)->status_is(200);
+    ok(my $updatejob = $t->tx->res->json->{id}, 'got ID of update job');
+    diag("Count: $plugin_mock_callcount");
+    ok($plugin_mock_callcount == 1, 'mock was called');
+    my ($body, $headerframe) = @{$published{'org.fedoraproject.stg.ci.fedora-update.test.queued'}};
+    $expected_artifact = {
+        alias   => 'FEDORA-2019-6bda4c81f4',
+        builds  => [
+            {
+                nvr => 'kernel-5.2.7-100.fc29'
+            },
+            {
+                nvr => 'kernel-headers-5.2.7-100.fc29'
+            },
+            {
+                nvr => 'kernel-tools-5.2.7-100.fc29'
+            }
+        ],
+        id      => 'sha256:cf6f57a196a213606246070a99b94053160fee43c7987d071f422de30b2f2953',
+        type    => 'fedora-update',
+        release => {
+            version => '42',
+            name    => 'F42'
+        },
+    };
+    $expected_pipeline = {
+        id   => 'openqa.Update-FEDORA-2019-6bda4c81f4.rainbow.RainbowPC.pink.x86_64',
+        name => 'openqa.Update-FEDORA-2019-6bda4c81f4.rainbow.RainbowPC.pink.x86_64',
+    };
+    $expected_run = {
+        url => "https://openqa.stg.fedoraproject.org/tests/$updatejob",
+        log => "https://openqa.stg.fedoraproject.org/tests/$updatejob/file/autoinst-log.txt",
+        id  => $updatejob,
+    };
+    $expected_system->{'os'}      = 'fedora-40';
+    $expected_test->{'namespace'} = 'update';
+    $expected_test->{'lifetime'}  = 240;
+    delete $expected_system->{'variant'};
+    delete $expected_test->{'result'};
+    my $expected = get_expected;
+    # no image for update message
+    delete $expected->{'image'};
+    is_deeply($body, $expected, 'update job create triggers standardized amqp');
+};
+
+done_testing();

SOURCES/FedoraMessaging.pm

@@ -0,0 +1,300 @@
+# Copyright (C) Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+# This is a plugin for publishing messages to fedora-messaging, Fedora's
+# AMQP message broker. It piggybacks on the upstream AMQP plugin but
+# includes headers required by the fedora-messaging spec and publishes
+# messages in the "CI Messages" spec: https://pagure.io/fedora-ci/messages
+# as well as more 'native' style messages.
+
+package OpenQA::WebAPI::Plugin::FedoraMessaging;
+
+use Digest::SHA qw(sha256_hex);
+use POSIX qw(strftime);
+
+use Mojo::Base 'OpenQA::WebAPI::Plugin::AMQP';
+use OpenQA::Jobs::Constants;
+use OpenQA::Log qw(log_debug log_error);
+use OpenQA::Utils;
+use UUID::URandom 'create_uuid_string';
+
+sub _iso8601_now {
+    # we do this twice, so factor it out
+    my $now = strftime("%Y-%m-%dT%H:%M:%S", gmtime()) . 'Z';
+    return $now;
+}
+
+sub publish_amqp {
+    my ($self, $topic, $event_data, $headerframe) = @_;
+    my $sentat = _iso8601_now;
+    my $messageid = create_uuid_string;
+    # default fedora-messaging compliant header frame. Ridiculous
+    # naming note: AMQP wire format has a header frame and then the
+    # body. Inside the header frame *is a field called headers*. Yo
+    # dawg, I heard you liked headers...fedora-messaging has specific
+    # expectations for some other fields in the header frame, and for
+    # some of the fields in the 'headers' field in the header frame.
+    # Mojo::RabbitMQ::Client tends to call things that represent the
+    # header frame "%headers" and "$headers", so that's fun.
+    my %fullheaderframe = (
+        headers => {
+            fedora_messaging_severity   => 20,
+            fedora_messaging_schema     => 'base.message',
+            "sent-at"                   => $sentat,
+        },
+        content_encoding => 'utf-8',
+        delivery_mode => 2,
+        message_id => $messageid,
+    );
+    # merge in the passed header frame values to allow overriding
+    $headerframe //= {};
+    %fullheaderframe = (%fullheaderframe, %$headerframe);
+    # call parent method
+    $self->SUPER::publish_amqp($topic, $event_data, \%fullheaderframe);
+}
+
+sub log_event_fedora_ci_messages {
+    # this is for publishing messages in the "CI Messages" format:
+    # https://pagure.io/fedora-ci/messages
+    # This is a Fedora/Red Hat-ish thing in a way, but in theory
+    # anyone could adopt it
+    my ($self, $event, $job, $baseurl) = @_;
+    my $stdevent;
+    my $clone_of;
+    my $job_id;
+    # first, get the standard 'state' (from 'queued', 'running',
+    # 'complete', 'error'; we cannot do 'running' at present
+    if ($event eq 'openqa_job_create') {
+        $stdevent = 'queued';
+        $job_id   = $job->id;
+    }
+    elsif ($event eq 'openqa_job_restart' || $event eq 'openqa_job_duplicate') {
+        $stdevent = 'queued';
+        $clone_of = $job->id;
+        $job_id   = $job->clone_id;
+    }
+    elsif ($event eq 'openqa_job_cancel') {
+        $stdevent = 'error';
+        $job_id   = $job->id;
+    }
+    elsif ($event eq 'openqa_job_done') {
+        $job_id = $job->id;
+        # lifecycle note: any job cancelled directly via the web API will
+        # see both job_cancel and job_done with result USER_CANCELLED, so
+        # we emit duplicate standardized fedmsgs in this case. This is
+        # kinda unavoidable, though, as it's possible for a job to wind up
+        # USER_CANCELLED *without* an openqa_job_cancel event happening,
+        # so we can't just throw away all openqa_job_done USER_CANCELLED
+        # events...
+        $stdevent = (grep { $job->result eq $_ } COMPLETE_RESULTS) ? 'complete' : 'error';
+    }
+    else {
+        return undef;
+    }
+
+    # we need this for the system dict; it should be the release of
+    # the system-under-test (the VM in which the test runs) at the
+    # *start* of the test, I think. We're trying to capture info about
+    # the environment in which the test runs
+    my $sysrelease = $job->VERSION;
+    my $hdd1;
+    my $bootfrom;
+    $hdd1     = $job->settings_hash->{HDD_1}    if ($job->settings_hash->{HDD_1});
+    $bootfrom = $job->settings_hash->{BOOTFROM} if ($job->settings_hash->{BOOTFROM});
+    if ($hdd1 && $bootfrom) {
+        $sysrelease = $1 if ($hdd1 =~ /disk_f(\d+)/ && $bootfrom eq 'c');
+    }
+
+    # next, get the 'artifact' (type of thing we tested)
+    my $artifact;
+    my $artifact_alias;
+    my $artifact_builds;
+    my $artifact_id;
+    my $artifact_release;
+    my $compose_type;
+    my $test_namespace;
+    # current date/time in ISO 8601 format
+    my $generated_at = _iso8601_now;
+
+    # this is used as a 'pipeline ID', see
+    # https://pagure.io/fedora-ci/messages/blob/master/f/schemas/pipeline.yaml
+    my $pipeid = join('.', "openqa", $job->BUILD, $job->TEST, $job->MACHINE, $job->FLAVOR, $job->ARCH);
+
+    my $build = $job->BUILD;
+    if ($build =~ /^Fedora/) {
+        $artifact       = 'productmd-compose';
+        $artifact_id    = $build;
+        $compose_type   = 'production';
+        $compose_type   = 'nightly' if ($build =~ /\.n\./);
+        $compose_type   = 'test' if ($build =~ /\.t\./);
+        $test_namespace = 'compose';
+    }
+    elsif ($build =~ /^Update-FEDORA/) {
+        $artifact    = 'fedora-update';
+        $artifact_alias = $build;
+        $artifact_alias =~ s/^Update-//;
+        $artifact_release = {
+            version => $job->VERSION,
+            name => "F" . $job->VERSION
+        };
+        my @nvrs = split(/ /, $job->settings_hash->{ADVISORY_NVRS} || '');
+        unless (@nvrs) {
+            log_error "ADVISORY_NVRS not found for update test $job_id! Cannot publish!";
+            return;
+        }
+        @nvrs = sort(@nvrs);
+        my @builds;
+        my $id = '';
+        foreach my $nvr (@nvrs) {
+            push @builds, {'nvr' => $nvr};
+            $id .= $nvr;
+        }
+        $artifact_builds = \@builds;
+        $artifact_id = 'sha256:' . sha256_hex($id);
+        $test_namespace   = 'update';
+    }
+    else {
+        # unhandled artifact type
+        return undef;
+    }
+
+    # finally, construct the message content
+    my %msg_data = (
+        contact => {
+            name  => 'Fedora openQA',
+            team  => 'Fedora QA',
+            url   => $baseurl,
+            docs  => 'https://fedoraproject.org/wiki/OpenQA',
+            irc   => '#fedora-qa',
+            email => 'qa-devel@lists.fedoraproject.org',
+        },
+        run => {
+            url => "$baseurl/tests/$job_id",
+            log => "$baseurl/tests/$job_id/file/autoinst-log.txt",
+            id  => $job_id,
+        },
+        artifact => {
+            type => $artifact,
+            id   => $artifact_id,
+        },
+        pipeline => {
+            # per https://pagure.io/fedora-ci/messages/issue/61 this
+            # is meant to be unique per test scenario *and* artifact,
+            # so we construct it out of BUILD and the scenario keys.
+            # 'name' is supposed to be a 'human readable name', well,
+            # this is human readable, so we'll just use it twice
+            id   => $pipeid,
+            name => $pipeid,
+        },
+        test => {
+            # openQA tests are pretty much always validation
+            category => 'validation',
+            # test identifier: test name plus scenario keys
+            type      => join(' ', $job->TEST, $job->MACHINE, $job->FLAVOR, $job->ARCH),
+            namespace => $test_namespace,
+        },
+        system => [
+            {
+                # it's interesting whether we should record info on the
+                # *worker host itself* or the *SUT* (the VM run on top of
+                # the worker host environment) here...on the whole I think
+                # SUT is more in line with expectations, so let's do that
+                os => "fedora-${sysrelease}",
+                # openqa provisions itself...we *could* I guess set this
+                # to 'createhdds' if we booted a disk image, but ehhhh
+                provider     => 'openqa',
+                architecture => $job->ARCH,
+            },
+        ],
+        generated_at => $generated_at,
+        version      => "0.2.1",
+    );
+
+    # add keys that don't exist in all cases to the message
+    if ($stdevent eq 'complete') {
+        $msg_data{test}{result} = $job->result;
+        $msg_data{test}{result} = 'info' if $job->result eq 'softfailed';
+    }
+    elsif ($stdevent eq 'error') {
+        $msg_data{error} = {};
+        $msg_data{error}{reason} = $job->result;
+    }
+    elsif ($stdevent eq 'queued') {
+        # this is a hint to consumers that the job probably went away
+        # if they don't get a 'complete' or 'error' in 4 hours
+        # FIXME: we should set this as 2 hours on 'running', but we
+        # can't emit running because there is no internal event for
+        # it, there is no job_running event or anything like it -
+        # this is part of https://progress.opensuse.org/issues/31069
+        $msg_data{test}{lifetime} = 240;
+    }
+    $msg_data{run}{clone_of} = $clone_of if ($clone_of);
+
+    $msg_data{artifact}{release} = $artifact_release if ($artifact_release);
+    $msg_data{artifact}{builds} = $artifact_builds if ($artifact_builds);
+    $msg_data{artifact}{alias} = $artifact_alias if ($artifact_alias);
+
+    $msg_data{artifact}{compose_type} = $compose_type if ($compose_type);
+
+    my $subvariant = $job->settings_hash->{SUBVARIANT} || '';
+    $msg_data{system}[0]{variant} = $subvariant if ($subvariant);
+
+    # record info about the image tested, for compose tests. In theory
+    # we might test more than one image in a job, which would break
+    # the schema. But we don't do that yet fortunately
+    if ($artifact eq 'productmd-compose') {
+        # this is a handy variable which indicates what the 'thing'
+        # the test really tests is (also used for resultsdb)
+        my $target = $job->settings_hash->{TEST_TARGET} || '';
+        my $imgname = $job->settings_hash->{"$target"} || '';
+        if ($imgname) {
+            $msg_data{image} = {
+                id   => $imgname,
+                name => $imgname,
+                type => $target
+            };
+        }
+    }
+
+    # create the topic
+    my $topic = "ci.$artifact.test.$stdevent";
+
+    # prepend the prefix (kinda duplicated with parent log_event)
+    my $prefix = $self->{config}->{amqp}{topic_prefix};
+    $topic     = $prefix . '.' . $topic if ($prefix);
+
+    # finally, send the message
+    log_debug("Sending CI Messages AMQP message for $event");
+    # FIXME: we should set fedora_messaging_schema header here, but the
+    # ci-messages schemas are not currently provided as fedora-messaging
+    # Python classes anywhere, so we kinda can't. See:
+    # https://pagure.io/fedora-ci/messages/issue/33
+    $self->publish_amqp($topic, \%msg_data);
+}
+
+sub on_job_event {
+    # do just enough work to send the 'CI messaging' spec message
+    # (unfortunately a bit of duplication is inevitable)
+    my ($self, $args) = @_;
+    my ($user_id, $connection_id, $event, $event_data) = @$args;
+    my $jobs = $self->{app}->schema->resultset('Jobs');
+    my $job  = $jobs->find({id => $event_data->{id}});
+    my $baseurl = $self->{config}->{global}->{base_url} || "http://UNKNOWN";
+    $self->log_event_fedora_ci_messages($event, $job, $baseurl);
+    # call parent method to send 'native' message
+    $self->SUPER::on_job_event($args);
+}
+
+1;

SOURCES/FedoraUpdateRestart.pm

@@ -0,0 +1,64 @@
+# Copyright (C) Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package OpenQA::WebAPI::Plugin::FedoraUpdateRestart;
+
+use strict;
+use warnings;
+
+use parent 'Mojolicious::Plugin';
+use Mojo::IOLoop;
+use OpenQA::Events;
+use OpenQA::Schema::Result::Jobs;
+
+sub register {
+    my ($self, $app) = @_;
+    OpenQA::Events->singleton->on("openqa_job_done" => sub { shift; $self->on_job_done($app, @_) });
+}
+
+# when a job completes, if it's an update test, it failed, and it's
+# not already a clone, restart it.
+# Also restart all aarch64 install jobs that fail in the first test
+# module, due to https://bugzilla.redhat.com/show_bug.cgi?id=1689037,
+# which seems very common on aarch64
+# We could use the `dup_type_auto` behaviour, but that allows 3
+# restarts, which is kinda a lot.
+sub on_job_done {
+    my ($self, $app, $args) = @_;
+    my ($user_id, $connection_id, $event, $event_data) = @$args;
+    my $schema = $app->schema;
+    my $jobid = $event_data->{id};
+    OpenQA::Utils::log_debug("Update restarter considering job $jobid");
+    my $job = $schema->resultset('Jobs')->find({id => $jobid});
+    my @jobmodules = $job->modules_with_job_prefetched->all;
+    my $firstmodule = $jobmodules[0];
+    my $clone_of = $schema->resultset("Jobs")->find({clone_id => $jobid});
+    my $restart = "";
+    $restart = "update" if (
+            $job->result eq OpenQA::Schema::Result::Jobs::FAILED && !$clone_of &&
+            $job->FLAVOR =~ /^updates-/
+    );
+    $restart = "aarch64 install" if (
+            $job->result eq OpenQA::Schema::Result::Jobs::FAILED && !$clone_of &&
+            $job->TEST =~/^install_/ &&
+            $firstmodule->result eq OpenQA::Schema::Result::Jobs::FAILED
+    );
+    if ($restart) {
+        OpenQA::Utils::log_info("Update restarter restarting job $jobid: $restart");
+        $job->auto_duplicate;
+    }
+}
+
+1;

SOURCES/geekotest.conf

@@ -0,0 +1,3 @@
+# Type Name       ID     GECOS           [HOME]    Shell
+u   geekotest   -     "openQA user" /var/lib/openqa  /bin/bash
+m   geekotest  nogroup

SOURCES/openQA-worker.conf

@@ -0,0 +1,4 @@
+# Type Name            ID     GECOS           [HOME]    Shell
+u   _openqa-worker   -     "openQA worker" /var/lib/empty  /bin/bash
+m   _openqa-worker  nogroup
+m   _openqa-worker  kvm