diff --git a/pungi/phases/gather/sources/source_comps.py b/pungi/phases/gather/sources/source_comps.py
index 6722af22..b9992080 100644
--- a/pungi/phases/gather/sources/source_comps.py
+++ b/pungi/phases/gather/sources/source_comps.py
@@ -36,9 +36,13 @@ class GatherSourceComps(pungi.phases.gather.source.GatherSourceBase):
         groups = set()
         comps = CompsWrapper(self.compose.paths.work.comps(arch=arch))
 
-        if variant is not None and variant.groups:
-            # Get packages for a particular variant. If the variant has no
-            # groups listed, all groups will be used.
+        if variant is not None and (variant.groups or variant.type != 'variant'):
+            # Get packages for a particular variant. We want to skip the
+            # filtering if the variant is top-level and has no groups (to use
+            # all of them).
+            # For optional we always want to filter (and parent groups will be
+            # added later), for addons and layered products it makes no sense
+            # for the groups to be empty in the first place.
             comps.filter_groups(variant.groups)
 
         for i in comps.get_comps_groups():