diff --git a/booth.spec b/booth.spec
index ffa2d54..f5e16f6 100644
--- a/booth.spec
+++ b/booth.spec
@@ -57,7 +57,7 @@
 
 Name:           booth
 Version:        %{boothver}
-Release:        %{boothrel}%{?dist}
+Release:        %{boothrel}%{?dist}.1
 Summary:        Ticket Manager for Multi-site Clusters
 License:        GPLv2+
 Url:            https://github.com/%{github_owner}/%{name}
@@ -90,6 +90,8 @@ BuildRequires:  pkgconfig(libsystemd)
 # check scriptlet (for hostname and killall respectively)
 BuildRequires:  hostname psmisc
 BuildRequires:  python3-devel
+# For generating tests
+BuildRequires:  sed
 # spec file specifics
 ## for _unitdir, systemd_requires and specific scriptlet macros
 BuildRequires:  systemd
@@ -192,8 +194,6 @@ Automated tests for running Booth, ticket manager for multi-site clusters.
         %{!?with_glue:--without-glue} \
         PYTHON=%{__python3}
 %{make_build}
-# Build testsuite files
-%{make_build} test/runtests.py
 
 %install
 %{make_install}
@@ -211,6 +211,7 @@ rm -rf %{buildroot}/%{_pkgdocdir}/README.upgrade-from-v0.1
 rm -rf %{buildroot}/%{_pkgdocdir}/COPYING
 # tests
 mkdir -p %{buildroot}/%{test_path}
+# Copy tests from tarball
 cp -a -t %{buildroot}/%{test_path} \
         -- conf test unit-tests script/unit-test.py
 chmod +x %{buildroot}/%{test_path}/test/booth_path
@@ -218,6 +219,18 @@ chmod +x %{buildroot}/%{test_path}/test/live_test.sh
 mkdir -p %{buildroot}/%{test_path}/src
 ln -s -t %{buildroot}/%{test_path}/src \
         -- %{_sbindir}/boothd
+# Generate runtests.py and boothtestenv.py
+sed -e 's#PYTHON_SHEBANG#%{__python3}#g' \
+    -e 's#TEST_SRC_DIR#%{test_path}/test#g' \
+    -e 's#TEST_BUILD_DIR#%{test_path}/test#g' \
+    %{buildroot}/%{test_path}/test/runtests.py.in > %{buildroot}/%{test_path}/test/runtests.py
+
+chmod +x %{buildroot}/%{test_path}/test/runtests.py
+
+sed -e 's#PYTHON_SHEBANG#%{__python3}#g' \
+    -e 's#TEST_SRC_DIR#%{test_path}/test#g' \
+    -e 's#TEST_BUILD_DIR#%{test_path}/test#g' \
+    %{buildroot}/%{test_path}/test/boothtestenv.py.in > %{buildroot}/%{test_path}/test/boothtestenv.py
 
 # https://fedoraproject.org/wiki/Packaging:Python_Appendix#Manual_byte_compilation
 %py_byte_compile %{__python3} %{buildroot}/%{test_path}
@@ -267,6 +280,9 @@ VERBOSE=1 make check
 /usr/lib/ocf/resource.d/booth/sharedrsc
 
 %changelog
+* Thu May 28 2020 Jan Friesse <jfriesse@redhat.com> - 1.0-5.385cc25.git.1
+- Fix test subpackage generating
+
 * Wed May 27 2020 Jan Friesse <jfriesse@redhat.com> - 1.0-5.385cc25.git
 - Update to current snapshot (commit 385cc25) to fix build warnings