diff --git a/cockpit.spec b/cockpit.spec
index 4cd0025..08ebcaf 100644
--- a/cockpit.spec
+++ b/cockpit.spec
@@ -815,9 +815,10 @@ via PackageKit.
 
 # The changelog is automatically generated and merged
 %changelog
-* Sat Jul 28 2018 Martin Pitt <martin@piware.de> - UNRELEASED
+* Sat Jul 28 2018 Martin Pitt <martin@piware.de> - 173-2
 - Drop firewalld service (moved to firewalld), add corresponding conflict
   rhbz#1609393
+- Fix CI pipeline tests
 
 * Wed Jul 25 2018 Martin Pitt <martin@piware.de> - 173-1
 
diff --git a/tests/test-canary b/tests/test-canary
deleted file mode 100755
index 376a6e2..0000000
--- a/tests/test-canary
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh -e
-echo "This is a test failure to see if CI is working"
-exit 1
diff --git a/tests/tests.yml b/tests/tests.yml
index e42d276..5c8638c 100644
--- a/tests/tests.yml
+++ b/tests/tests.yml
@@ -5,17 +5,13 @@
     tags:
     - always
 
-  - role: standard-test-overlay
-    tags:
-    - atomic
-    packages:
-    - cockpit-ws
-    - cockpit-dashboard
-
-  - role: standard-test-scripts
+  - role: standard-test-basic
     tags:
     - atomic
     - classic
+    required_packages:
+    - cockpit
     tests:
-    - ./source/tools/debian/tests/smoke
-    - ./test-canary
+    - smoke:
+        dir: ./source/tools/debian/tests/
+        run: ./smoke