diff --git a/pungi/phases/pkgset/sources/source_koji.py b/pungi/phases/pkgset/sources/source_koji.py
index 6f9a6432..a96c9730 100644
--- a/pungi/phases/pkgset/sources/source_koji.py
+++ b/pungi/phases/pkgset/sources/source_koji.py
@@ -324,28 +324,41 @@ def filter_by_whitelist(compose, module_builds, input_modules):
     input_modules. Order may not be preserved.
     """
     specs = set()
-    nvr_prefixes = set()
+    nvr_patterns = set()
     for spec in input_modules:
         # Do not do any filtering in case variant wants all the modules.
         if spec["name"] == "*":
             return module_builds
 
         info = variant_dict_from_str(compose, spec["name"])
-        prefix = ("%s-%s-%s.%s" % (
+        pattern = (
             info["name"],
             info["stream"].replace("-", "_"),
-            info.get("version", ""),
-            info.get("context", ""),
-        )).rstrip("-.")
-        nvr_prefixes.add((prefix, spec["name"]))
+            info.get("version"),
+            info.get("context"),
+        )
+        nvr_patterns.add((pattern, spec["name"]))
         specs.add(spec["name"])
 
     modules_to_keep = []
     used = set()
 
     for mb in module_builds:
-        for (prefix, spec) in nvr_prefixes:
-            if mb["nvr"].startswith(prefix):
+        # Split release from the build into version and context
+        ver, ctx = mb["release"].split(".")
+        # Values in `mb` are from Koji build. There's nvr and name, version and
+        # release. The input pattern specifies modular name, stream, version
+        # and context.
+        for (n, s, v, c), spec in nvr_patterns:
+            if (
+                # We always have a name and stream...
+                mb["name"] == n
+                and mb["version"] == s
+                # ...but version and context can be missing, in which case we
+                # don't want to check them.
+                and (not v or ver == v)
+                and (not c or ctx == c)
+            ):
                 modules_to_keep.append(mb)
                 used.add(spec)
                 break
diff --git a/tests/test_pkgset_source_koji.py b/tests/test_pkgset_source_koji.py
index 6d328a9e..5c0a6509 100644
--- a/tests/test_pkgset_source_koji.py
+++ b/tests/test_pkgset_source_koji.py
@@ -585,6 +585,15 @@ class TestFilterInherited(unittest.TestCase):
 
 
 class TestFilterByWhitelist(unittest.TestCase):
+    def _build(self, n, s, v, c):
+        s = s.replace("-", "_")
+        return {
+            "nvr": "%s-%s-%s.%s" % (n, s, v, c),
+            "name": n,
+            "version": s,
+            "release": "%s.%s" % (v, c),
+        }
+
     def test_no_modules(self):
         compose = mock.Mock()
         module_builds = []
@@ -598,72 +607,55 @@ class TestFilterByWhitelist(unittest.TestCase):
     def test_filter_by_NS(self):
         compose = mock.Mock()
         module_builds = [
-            {"nvr": "foo-1-201809031048.cafebabe"},
-            {"nvr": "foo-1-201809031047.deadbeef"},
-            {"nvr": "foo-2-201809031047.deadbeef"},
+            self._build("foo", "1", "201809031048", "cafebabe"),
+            self._build("foo", "1", "201809031047", "deadbeef"),
+            self._build("foo", "2", "201809031047", "deadbeef"),
         ]
         input_modules = [{"name": "foo:1"}]
 
         result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
 
-        self.assertItemsEqual(
-            result,
-            [
-                {"nvr": "foo-1-201809031048.cafebabe"},
-                {"nvr": "foo-1-201809031047.deadbeef"},
-            ],
-        )
+        self.assertItemsEqual(result, [module_builds[0], module_builds[1]])
 
     def test_filter_by_NSV(self):
         compose = mock.Mock()
         module_builds = [
-            {"nvr": "foo-1-201809031048.cafebabe"},
-            {"nvr": "foo-1-201809031047.deadbeef"},
-            {"nvr": "foo-2-201809031047.deadbeef"},
+            self._build("foo", "1", "201809031048", "cafebabe"),
+            self._build("foo", "1", "201809031047", "deadbeef"),
+            self._build("foo", "2", "201809031047", "deadbeef"),
         ]
         input_modules = [{"name": "foo:1:201809031047"}]
 
         result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
 
-        self.assertItemsEqual(
-            result, [{"nvr": "foo-1-201809031047.deadbeef"}]
-        )
+        self.assertItemsEqual(result, [module_builds[1]])
 
     def test_filter_by_NSVC(self):
         compose = mock.Mock()
         module_builds = [
-            {"nvr": "foo-1-201809031048.cafebabe"},
-            {"nvr": "foo-1-201809031047.deadbeef"},
-            {"nvr": "foo-1-201809031047.cafebabe"},
-            {"nvr": "foo-2-201809031047.deadbeef"},
+            self._build("foo", "1", "201809031048", "cafebabe"),
+            self._build("foo", "1", "201809031047", "deadbeef"),
+            self._build("foo", "1", "201809031047", "cafebabe"),
+            self._build("foo", "2", "201809031047", "deadbeef"),
         ]
         input_modules = [{"name": "foo:1:201809031047:deadbeef"}]
 
         result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
 
-        self.assertItemsEqual(
-            result, [{"nvr": "foo-1-201809031047.deadbeef"}]
-        )
+        self.assertItemsEqual(result, [module_builds[1]])
 
     def test_filter_by_wildcard(self):
         compose = mock.Mock()
         module_builds = [
-            {"nvr": "foo-1-201809031048.cafebabe"},
-            {"nvr": "foo-1-201809031047.deadbeef"},
-            {"nvr": "foo-2-201809031047.deadbeef"},
+            self._build("foo", "1", "201809031048", "cafebabe"),
+            self._build("foo", "1", "201809031047", "deadbeef"),
+            self._build("foo", "2", "201809031047", "deadbeef"),
         ]
         input_modules = [{"name": "*"}]
 
         result = source_koji.filter_by_whitelist(compose, module_builds, input_modules)
 
-        self.assertItemsEqual(
-            result,
-            [
-                {"nvr": "foo-1-201809031048.cafebabe"},
-                {"nvr": "foo-1-201809031047.deadbeef"},
-                {"nvr": "foo-2-201809031047.deadbeef"},
-            ],
-        )
+        self.assertItemsEqual(result, module_builds)
 
 
 class MockModule(object):