diff --git a/evolution.spec b/evolution.spec
index 0ffcc55..d5c58db 100644
--- a/evolution.spec
+++ b/evolution.spec
@@ -1,5 +1,7 @@
 %global _changelog_trimtime %(date +%s -d "1 year ago")
 %global _python_bytecompile_extra 0
+
+# correct Obsoletes for evolution-tests when this is changed
 %global enable_installed_tests 0
 
 %define glib2_version 2.46.0
@@ -30,7 +32,7 @@
 
 Name: evolution
 Version: 3.31.1
-Release: 1%{?dist}
+Release: 2%{?dist}
 Group: Applications/Productivity
 Summary: Mail and calendar client for GNOME
 License: GPLv2+ and GFDL
@@ -44,10 +46,15 @@ Obsoletes: libgal2 <= %{last_libgal2_version}
 Obsoletes: evolution-NetworkManager < %{last_evo_nm_version}
 Obsoletes: evolution-perl < %{last_evo_perl_version}
 
+%if !%{enable_installed_tests}
+Obsoletes: evolution-tests <= 3.31.1
+%endif
+
 %global eds_version %{version}
 
 ## Dependencies ###
 
+Requires: %{_bindir}/killall
 Requires: evolution-data-server >= %{eds_version}
 Requires: gvfs
 Requires: gtkspell3
@@ -56,6 +63,7 @@ Requires: %{name}-langpacks = %{version}-%{release}
 
 ### Build Dependencies ###
 
+BuildRequires: %{_bindir}/killall
 BuildRequires: cmake
 BuildRequires: gcc
 BuildRequires: gettext
@@ -511,6 +519,10 @@ grep -v "/usr/share/locale" evolution.lang > help.lang
 %endif
 
 %changelog
+* Fri Oct 26 2018 Milan Crha <mcrha@redhat.com> - 3.31.1-2
+- Obsolete evolution-tests subpackage
+- Add BuildRequires/Requires for 'killall' binary
+
 * Mon Oct 08 2018 Milan Crha <mcrha@redhat.com> - 3.31.1-1
 - Update to 3.31.1