diff --git a/tests/roles/standard-test-scripts/README.md b/tests/roles/standard-test-scripts/README.md
index c10c34d..d407bcc 100644
--- a/tests/roles/standard-test-scripts/README.md
+++ b/tests/roles/standard-test-scripts/README.md
@@ -8,7 +8,7 @@ the role will fail.
 
 In the case of test subjects such a host or container, these
 test scripts and related files will be copied to the target into
-/usr/local/bin before execution.
+/var/tmp/tests before execution.
 
 You should define the following variables:
 
diff --git a/tests/roles/standard-test-scripts/tasks/main.yml b/tests/roles/standard-test-scripts/tasks/main.yml
index cb25ba5..bddd620 100644
--- a/tests/roles/standard-test-scripts/tasks/main.yml
+++ b/tests/roles/standard-test-scripts/tasks/main.yml
@@ -1,8 +1,14 @@
 ---
+- name: Install testing requirements
+  package: name={{ item }} state=present
+  with_items:
+   - rsync
+  when: ansible_pkg_mgr != 'unknown'
+
 - name: Copy test scripts to target
   synchronize:
     src: "{{ item }}"
-    dest: /usr/local/bin/
+    dest: /var/tmp/tests/
     ssh_args: "-o UserKnownHostsFile=/dev/null"
   with_items:
   - "{{ tests }}"
@@ -11,16 +17,17 @@
 - block:
   - name: Execute test scripts
     shell: |
-      mkdir -p /tmp/artifacts
-      logfile=/tmp/artifacts/test.$(echo {{ item }} | sed -e 's/\//-/g').log
+      set -e
+      mkdir -p /var/tmp/artifacts
+      logfile=/var/tmp/artifacts/test.$(echo {{ item }} | sed -e 's/\//-/g').log
       exec 2>>$logfile 1>>$logfile
-      cd /usr/local/bin
+      cd /var/tmp/tests
       if [ -x {{ item }} ]; then
         ./$(basename {{ item }}) && result="PASS" || result="FAIL"
       else
         /bin/sh -e ./$(basename {{ item }}) && result="PASS" || result="FAIL"
       fi
-      echo "$result {{ item }}" >> /tmp/artifacts/test.log
+      echo "$result {{ item }}" >> /var/tmp/artifacts/test.log
     with_items:
     - "{{ tests }}"
 
@@ -28,6 +35,14 @@
   - name: Pull out the logs
     synchronize:
       dest: "{{ artifacts }}"
-      src: "/tmp/artifacts/./"
+      src: "/var/tmp/artifacts/./"
       mode: pull
+      ssh_args: "-o UserKnownHostsFile=/dev/null"
     when: artifacts|default("") != ""
+
+  # Can't go in block. See
+  # https://github.com/ansible/ansible/issues/20736
+  - name: Check the results
+    shell: grep "^FAIL" /var/tmp/artifacts/test.log
+    register: test_fails
+    failed_when: test_fails.stdout or test_fails.stderr