diff --git a/.gitignore b/.gitignore
index 2b5e04d..7d7f336 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1 @@
-SOURCES/jolokia-1.6.2-source.tar.gz
 /jolokia-1.6.2-source.tar.gz
diff --git a/tests/smoke_test.sh b/tests/smoke_test.sh
new file mode 100755
index 0000000..31adae1
--- /dev/null
+++ b/tests/smoke_test.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+set -e
+
+JAVA=$1
+JAVA_VER=$2
+AGENT_VERSION="1.6.2"
+dir=$(mktemp -d)
+pushd $dir
+cat > jolokia_config.properties <<EOF
+port=7777
+host=localhost
+user=foo
+password=bar
+EOF
+cat > HelloWait.java <<EOF
+public class HelloWait {
+
+    public static void main(String[] args) {
+        System.out.println("Hello World");
+        try {
+            while (true) {
+                Thread.sleep(200);
+            }
+        } catch (InterruptedException e) {
+            // ignore
+        }
+    }
+
+}
+EOF
+javac HelloWait.java
+$JAVA/bin/java -javaagent:/usr/share/java/jolokia-jvm-agent/jolokia-jvm.jar=config=jolokia_config.properties -cp . HelloWait > hello_wait.out 2>&1 &
+num_tries=0
+while [ $num_tries -lt 10 ] && ! grep -q 'Hello World' hello_wait.out; do
+  sleep 1
+  num_tries=$(($num_tries + 1))
+done
+echo 
+if [ $num_tries -eq 10 ]; then
+  echo "Time out reached waiting for HelloWait to come up"
+  exit 1
+fi
+curl -s --user foo:bar http://127.0.0.1:7777/jolokia/ 2>&1 | tee jolokia.out
+echo 
+echo 
+grep -q "\"agent\":\"$AGENT_VERSION\"" jolokia.out
+grep -q '"status":200' jolokia.out
+popd
+rm -rf $dir
+
+# Cleanup HelloWait process
+kill $(jps | grep HelloWait | awk '{ print $1 }')
+
+echo "JDK $JAVA_VER: Tests PASSED!"
diff --git a/tests/tests.yml b/tests/tests.yml
new file mode 100644
index 0000000..083c7e1
--- /dev/null
+++ b/tests/tests.yml
@@ -0,0 +1,16 @@
+---
+- hosts: localhost
+  roles:
+    - role: standard-test-basic
+      tags:
+        - classic
+      required_packages:
+        - jolokia-jvm-agent
+        - curl
+        - java-1.8.0-openjdk-devel
+        - java-11-openjdk-devel
+ 
+      tests:
+        - agent_smoke_test:
+            dir: .
+            run: "./verify.sh"
diff --git a/tests/verify.sh b/tests/verify.sh
new file mode 100755
index 0000000..eed88d4
--- /dev/null
+++ b/tests/verify.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+#
+# Verify Jolokia JVM agent on JDK 8 and JDK 11
+set -e
+
+./smoke_test.sh /usr/lib/jvm/java-1.8.0-openjdk 8
+./smoke_test.sh /usr/lib/jvm/java-11-openjdk 11