diff --git a/cockpit.spec b/cockpit.spec
index 6b201e1..ee9a731 100644
--- a/cockpit.spec
+++ b/cockpit.spec
@@ -1,6 +1,6 @@
 # This spec file has been automatically updated
 Version:	151
-Release: 1%{?dist}
+Release: 2%{?dist}
 #
 # This file is maintained at the following location:
 # https://github.com/cockpit-project/cockpit/blob/master/tools/cockpit.spec
@@ -659,6 +659,9 @@ The Cockpit component for installing package updates, via PackageKit.
 
 # The changelog is automatically generated and merged
 %changelog
+* Mon Sep 25 2017 Stef Walter <stefw@redhat.com> - 151-2
+- Add simulated test failure
+
 * Thu Sep 21 2017 Martin Pitt <martin@piware.de> - 151-1
 
 - Support loading SSH keys from arbitrary paths
diff --git a/tests/test-canary b/tests/test-canary
new file mode 100755
index 0000000..376a6e2
--- /dev/null
+++ b/tests/test-canary
@@ -0,0 +1,3 @@
+#!/bin/sh -e
+echo "This is a test failure to see if CI is working"
+exit 1
diff --git a/tests/tests.yml b/tests/tests.yml
index 62e8575..e42d276 100644
--- a/tests/tests.yml
+++ b/tests/tests.yml
@@ -18,3 +18,4 @@
     - classic
     tests:
     - ./source/tools/debian/tests/smoke
+    - ./test-canary