diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py
index 829e6e77..497ae749 100644
--- a/pungi/phases/gather/sources/source_module.py
+++ b/pungi/phases/gather/sources/source_module.py
@@ -141,16 +141,10 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
         # list is later used in createrepo phase to generated modules.yaml.
         for nsvc, rpm_nevras in added_rpms.items():
             arch_mmd = variant.arch_mmds[arch][nsvc]
-            artifacts = arch_mmd.get_rpm_artifacts()
-
-            # Modules without artifacts are also valid.
-            if not artifacts or artifacts.size() == 0:
-                continue
 
             added_artifacts = Modulemd.SimpleSet()
             for rpm_nevra in rpm_nevras:
-                if artifacts.contains(rpm_nevra):
-                    added_artifacts.add(rpm_nevra)
+                added_artifacts.add(rpm_nevra)
             arch_mmd.set_rpm_artifacts(added_artifacts)
 
         return packages, groups
@@ -159,28 +153,17 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
 def should_include(rpm_obj, arch, arch_mmd, multilib_arches):
     srpm = kobo.rpmlib.parse_nvr(rpm_obj.sourcerpm)["name"]
 
-    filtered = False
     buildopts = arch_mmd.get_buildopts()
     if buildopts:
         whitelist = buildopts.get_rpm_whitelist()
         if whitelist:
             # We have whitelist, no filtering against components.
-            filtered = True
             if srpm not in whitelist:
                 # Package is not on the list, skip it.
                 return False
 
-    if not filtered:
-        # Skip this mmd if this RPM does not belong to it.
-        if (srpm not in arch_mmd.get_rpm_components().keys() or
-                rpm_obj.nevra not in arch_mmd.get_rpm_artifacts().get()):
-            return False
-
     # Filter out the RPM from artifacts if its filtered in MMD.
     if rpm_obj.name in arch_mmd.get_rpm_filter().get():
-        # No need to check if the rpm_obj is in rpm artifacts,
-        # the .remove() method does that anyway.
-        arch_mmd.get_rpm_artifacts().remove(str(rpm_obj.nevra))
         return False
 
     # Skip the rpm_obj if it's built for multilib arch, but multilib is not
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 5652fabd..ad021457 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -254,14 +254,6 @@ def _add_module_to_variant(variant, mmd, rpms, add_to_variant_modules=False):
             "but according to modulemd, there should be some."
             % nsvc)
 
-    # Add RPMs from build systemto modulemd, so we can track
-    # what RPM is in which module later in gather phase.
-    rpm_artifacts = mmd.get_rpm_artifacts()
-    for rpm_nevra in rpms:
-        if rpm_nevra.endswith(".rpm"):
-            rpm_nevra = rpm_nevra[:-len(".rpm")]
-        rpm_artifacts.add(str(rpm_nevra))
-    mmd.set_rpm_artifacts(rpm_artifacts)
     variant.mmds.append(mmd)
 
     if add_to_variant_modules:
diff --git a/tests/helpers.py b/tests/helpers.py
index 88bc7da5..27ca7bbd 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -59,7 +59,7 @@ class MockVariant(mock.Mock):
     def get_modular_koji_tags(self, arch=None, types=None):
         return []
 
-    def add_fake_module(self, nsvc, rpm_nvrs=None):
+    def add_fake_module(self, nsvc, rpm_nvrs=None, with_artifacts=False):
         if not Modulemd:
             # No support for modules
             return
@@ -85,7 +85,8 @@ class MockVariant(mock.Mock):
                 component.set_name(rpm_name)
                 component.set_rationale("Needed for test")
                 mmd.add_rpm_component(component)
-            mmd.set_rpm_artifacts(artifacts)
+            if with_artifacts:
+                mmd.set_rpm_artifacts(artifacts)
 
         if self.modules is None:
             self.modules = []
diff --git a/tests/test_createrepophase.py b/tests/test_createrepophase.py
index c9739c3c..83d6d525 100644
--- a/tests/test_createrepophase.py
+++ b/tests/test_createrepophase.py
@@ -784,10 +784,14 @@ class TestCreateVariantRepo(PungiTestCase):
         variant = compose.variants['Server']
         variant.arch_mmds["x86_64"] = {
             "test:f27:2018:cafe": variant.add_fake_module(
-                "test:f27:1:2017", rpm_nvrs=["bash-0:4.3.30-2.fc21.x86_64"]
+                "test:f27:1:2017",
+                rpm_nvrs=["bash-0:4.3.30-2.fc21.x86_64"],
+                with_artifacts=True,
             ),
             "test:f28:2018:beef": variant.add_fake_module(
-                "test:f28:1:2017", rpm_nvrs=["pkg-0:2.0.0-1.x86_64"]
+                "test:f28:1:2017",
+                rpm_nvrs=["pkg-0:2.0.0-1.x86_64"],
+                with_artifacts=True,
             ),
         }
         variant.module_uid_to_koji_tag = {