diff --git a/scipy.spec b/scipy.spec
index bc2b6d0..292864d 100644
--- a/scipy.spec
+++ b/scipy.spec
@@ -6,8 +6,10 @@
 # but it is usually not available soon enough for new major Python versions.
 %if 0%{?rhel}
 %bcond_with pythran
+%bcond_with pooch
 %else
 %bcond_without pythran
+%bcond_without pooch
 %endif
 
 # The code is not safe to build with LTO
@@ -34,7 +36,7 @@
 Summary:    Scientific Tools for Python
 Name:       scipy
 Version:    1.11.3
-Release:    8%{?dist}
+Release:    9%{?dist}
 
 # BSD-3-Clause -- whole package except:
 # BSD-2-Clause -- scipy/_lib/_pep440.py
@@ -80,7 +82,9 @@ BuildRequires:  python3-pytest-timeout
 %if ! 0%{?rhel}
 BuildRequires:  python3-pytest-xdist
 %endif
+%if %{with pooch}
 BuildRequires:  python3-pooch
+%endif
 
 %if %{with doc}
 BuildRequires:  python3-sphinx
@@ -142,6 +146,9 @@ Scipy test files
 %if %{without pythran}
 sed -i '/pythran/d' pyproject.toml
 %endif
+%if %{without pooch}
+sed -i '/pooch/d' pyproject.toml
+%endif
 
 cat >> pyproject.toml << EOF
 
@@ -276,7 +283,9 @@ not test_svdp'"
 %endif
 
 pushd %{buildroot}/%{python3_sitearch}
-%{pytest} --timeout=${TIMEOUT} scipy %{?!rhel:--numprocesses=auto}
+# Ignoring the datasets tests as we don't have the optional pooch
+# dependency on RHEL.
+%{pytest} %{!?with_pooch:--ignore=scipy/datasets/tests/test_data.py} --timeout=${TIMEOUT} scipy %{?!rhel:--numprocesses=auto}
 # Remove test remnants
 rm -rf gram{A,B}
 rm -rf .pytest_cache
@@ -302,6 +311,10 @@ popd
 %endif
 
 %changelog
+* Tue May 28 2024 Pavel Simovec <psimovec@redhat.com> - 1.11.3-9
+- Remove python-pooch optional dependency from RHEL
+- Related: RHELMISC-5321
+
 * Thu May 02 2024 Pavel Simovec <psimovec@redhat.com> - 1.11.3-8
 - Build without pythran in RHEL by default
 - Resolves: RHEL-33955