diff --git a/pungi/phases/gather/sources/source_module.py b/pungi/phases/gather/sources/source_module.py
index 3fcb7dd2..e38bb391 100644
--- a/pungi/phases/gather/sources/source_module.py
+++ b/pungi/phases/gather/sources/source_module.py
@@ -38,26 +38,43 @@ class GatherSourceModule(pungi.phases.gather.source.GatherSourceBase):
         if variant is not None and variant.modules:
             variant.arch_mmds.setdefault(arch, {})
 
+            # Contains per-module RPMs added to variant.
+            added_rpms = {}
+
             rpms = sum([
                 variant.pkgset.rpms_by_arch.get(a, [])
                 for a in compatible_arches
             ], [])
             for rpm_obj in rpms:
                 for mmd in variant.mmds:
+                    mmd_id = "%s-%s" % (mmd.name, mmd.stream)
                     # Generate architecture specific modulemd metadata
                     # with list of artifacts only for this architecture.
-                    if mmd.name not in variant.arch_mmds[arch]:
+                    if mmd_id not in variant.arch_mmds[arch]:
                         arch_mmd = yaml.safe_load(mmd.dumps())
-                        arch_mmd["data"]["artifacts"] = {"rpms": []}
-                        variant.arch_mmds[arch][mmd.name] = arch_mmd
+                        variant.arch_mmds[arch][mmd_id] = arch_mmd
                     else:
-                        arch_mmd = variant.arch_mmds[arch][mmd.name]
+                        arch_mmd = variant.arch_mmds[arch][mmd_id]
 
                     srpm = kobo.rpmlib.parse_nvr(rpm_obj.sourcerpm)["name"]
                     if (srpm in mmd.components.rpms.keys() and
                             rpm_obj.name not in mmd.filter.rpms):
                         packages.add((rpm_obj.name, None))
-                        arch_mmd["data"]["artifacts"]["rpms"].append(
-                            str(rpm_obj.nevra))
+                        added_rpms.setdefault(mmd_id, [])
+                        added_rpms[mmd_id].append(str(rpm_obj.nevra))
+
+            # GatherSource returns all the packages in variant and does not
+            # care which package is in which module, but for modular metadata
+            # in the resulting compose repository, we have to know which RPM
+            # is part of which module.
+            # We therefore iterate over all the added packages grouped by
+            # particular module and use them to filter out the packages which
+            # have not been added to variant from the `arch_mmd`. This package
+            # list is later used in createrepo phase to generated modules.yaml.
+            for mmd_id, rpm_nevras in added_rpms.items():
+                arch_mmd = variant.arch_mmds[arch][mmd_id]
+                arch_mmd["data"]["artifacts"]["rpms"] = [
+                    rpm_nevra for rpm_nevra in rpm_nevras
+                    if rpm_nevra in arch_mmd["data"]["artifacts"]["rpms"]]
 
         return packages, groups
diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 08059022..9ae367e3 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -162,6 +162,14 @@ def populate_global_pkgset(compose, koji_wrapper, path_prefix, event_id):
                 pdc_module = get_module(session, module["name"])
                 mmd = modulemd.ModuleMetadata()
                 mmd.loads(pdc_module["modulemd"])
+
+                # Add RPMs from PDC response to modulemd, so we can track
+                # what RPM is in which module later in gather phase.
+                for rpm_nevra in pdc_module["rpms"]:
+                    if rpm_nevra.endswith(".rpm"):
+                        rpm_nevra = rpm_nevra[:-len(".rpm")]
+                    mmd.artifacts.add_rpm(str(rpm_nevra))
+
                 tag = pdc_module["koji_tag"]
                 variant.mmds.append(mmd)
                 variant_tags[variant].append(tag)