diff --git a/doc/configuration.rst b/doc/configuration.rst
index 35f28504..f4f9a4da 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -471,6 +471,12 @@ Options
     (*dict*) -- A mapping of architectures to repositories with RPMs: ``{arch:
     [repo]}``. Only use when ``pkgset_source = "repos"``.
 
+**pkgset_exclusive_arch_considers_noarch** = True
+    (*bool*) -- If a package includes ``noarch`` in its ``ExclusiveArch`` tag,
+    it will be included in all architectures since ``noarch`` is compatible
+    with everything. Set this option to ``False`` to ignore ``noarch`` in
+    ``ExclusiveArch`` and always consider only binary architectures.
+
 
 Example
 -------
diff --git a/pungi/checks.py b/pungi/checks.py
index 6a45133d..2ddd7164 100644
--- a/pungi/checks.py
+++ b/pungi/checks.py
@@ -757,6 +757,10 @@ def make_schema():
                 "type": "boolean",
                 "default": False
             },
+            "pkgset_exclusive_arch_considers_noarch": {
+                "type": "boolean",
+                "default": True,
+            },
 
             "disc_types": {
                 "type": "object",
diff --git a/pungi/phases/pkgset/common.py b/pungi/phases/pkgset/common.py
index bff1f392..ccb2b421 100644
--- a/pungi/phases/pkgset/common.py
+++ b/pungi/phases/pkgset/common.py
@@ -27,12 +27,13 @@ from pungi.util import is_arch_multilib, find_old_compose
 # TODO: per arch?
 def populate_arch_pkgsets(compose, path_prefix, global_pkgset):
     result = {}
+    exclusive_noarch = compose.conf['pkgset_exclusive_arch_considers_noarch']
     for arch in compose.get_arches():
         compose.log_info("Populating package set for arch: %s" % arch)
         is_multilib = is_arch_multilib(compose.conf, arch)
         arches = get_valid_arches(arch, is_multilib, add_src=True)
         pkgset = pungi.phases.pkgset.pkgsets.PackageSetBase(compose.conf["sigkeys"], logger=compose._logger, arches=arches)
-        pkgset.merge(global_pkgset, arch, arches)
+        pkgset.merge(global_pkgset, arch, arches, exclusive_noarch=exclusive_noarch)
         pkgset.save_file_list(compose.paths.work.package_list(arch=arch), remove_path_prefix=path_prefix)
         result[arch] = pkgset
     return result
diff --git a/pungi/phases/pkgset/pkgsets.py b/pungi/phases/pkgset/pkgsets.py
index bc011bfb..92db809d 100644
--- a/pungi/phases/pkgset/pkgsets.py
+++ b/pungi/phases/pkgset/pkgsets.py
@@ -161,7 +161,7 @@ class PackageSetBase(kobo.log.LoggingBase):
 
         return self.rpms_by_arch
 
-    def merge(self, other, primary_arch, arch_list):
+    def merge(self, other, primary_arch, arch_list, exclusive_noarch=True):
         """
         Merge ``other`` package set into this instance.
         """
@@ -184,6 +184,13 @@ class PackageSetBase(kobo.log.LoggingBase):
         if primary_arch:
             exclusivearch_list = get_valid_arches(
                 primary_arch, multilib=False, add_noarch=False, add_src=False)
+            # We don't want to consider noarch: if a package is true noarch
+            # build (not just a subpackage), it has to have noarch in
+            # ExclusiveArch otherwise rpm will refuse to build it.
+            # This should eventually become a default, but it could have a big
+            # impact and thus it's hidden behind an option.
+            if not exclusive_noarch and 'noarch' in exclusivearch_list:
+                exclusivearch_list.remove('noarch')
         else:
             exclusivearch_list = None
         for arch in arch_list: