diff --git a/pungi/phases/pkgset/__init__.py b/pungi/phases/pkgset/__init__.py
index 36f9d1ff..66fa4952 100644
--- a/pungi/phases/pkgset/__init__.py
+++ b/pungi/phases/pkgset/__init__.py
@@ -22,8 +22,9 @@ class PkgsetPhase(PhaseBase):
 
     name = "pkgset"
 
-    def __init__(self, *args, **kwargs):
-        super(PkgsetPhase, self).__init__(*args, **kwargs)
+    def __init__(self, compose, *args, **kwargs):
+        super(PkgsetPhase, self).__init__(compose, *args, **kwargs)
+        self.compose = compose
         self.package_sets = []
         self.path_prefix = None
 
@@ -36,3 +37,12 @@ class PkgsetPhase(PhaseBase):
         container = PkgsetSourceContainer()
         SourceClass = container[pkgset_source]
         self.package_sets, self.path_prefix = SourceClass(self.compose)()
+
+    def validate(self):
+        extra_tasks = self.compose.conf.get("pkgset_koji_scratch_tasks", None)
+        sigkeys = tuple(self.compose.conf["sigkeys"] or [None])
+        if extra_tasks is not None and None not in sigkeys and "" not in sigkeys:
+            raise ValueError(
+                "Unsigned packages must be allowed to use the "
+                '"pkgset_koji_scratch_tasks" option'
+            )
diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
index 91a3b0db..67188b3f 100644
--- a/pungi/phases/pkgset/pkgsets.py
+++ b/pungi/phases/pkgset/pkgsets.py
@@ -497,15 +497,9 @@ class KojiPackageSet(PackageSetBase):
         rpm_info, build_info = queue_item
 
         # Check if this RPM is comming from scratch task. In this case, we already
-        # know the path and we must ensure unsigned packages are allowed.
+        # know the path.
         if "path_from_task" in rpm_info:
-            if None in self.sigkey_ordering or "" in self.sigkey_ordering:
-                return rpm_info["path_from_task"]
-            else:
-                self.log_error(
-                    "Scratch RPM %s cannot be used in signed compose." % (rpm_info)
-                )
-                return None
+            return rpm_info["path_from_task"]
 
         pathinfo = self.koji_wrapper.koji_module.pathinfo
         paths = []
diff --git a/tests/test_pkgset_phase.py b/tests/test_pkgset_phase.py
new file mode 100644
index 00000000..60c501cb
--- /dev/null
+++ b/tests/test_pkgset_phase.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from pungi.phases import pkgset
+from tests import helpers
+
+
+class TestPkgsetPhase(helpers.PungiTestCase):
+    def test_validates_pkgset_koji_scratch_tasks_only_signed(self):
+        cfg = {"pkgset_koji_scratch_tasks": ["123"], "sigkeys": ["sigkey"]}
+        compose = helpers.DummyCompose(self.topdir, cfg)
+        phase = pkgset.PkgsetPhase(compose)
+
+        with self.assertRaises(ValueError) as ctx:
+            phase.validate()
+        self.assertIn("Unsigned packages must be allowed", str(ctx.exception))
+
+    def test_validates_pkgset_koji_scratch_tasks_unsigned(self):
+        for unsigned_obj in ["", None]:
+            cfg = {
+                "pkgset_koji_scratch_tasks": ["123"],
+                "sigkeys": ["sigkey", unsigned_obj],
+            }
+            compose = helpers.DummyCompose(self.topdir, cfg)
+            phase = pkgset.PkgsetPhase(compose)
+            phase.validate()