diff --git a/pyproject-rpm-macros.spec b/pyproject-rpm-macros.spec
index 4920149..f8364b5 100644
--- a/pyproject-rpm-macros.spec
+++ b/pyproject-rpm-macros.spec
@@ -172,6 +172,7 @@ export HOSTNAME="rpmbuild"  # to speedup tox in network-less mock, see rhbz#1856
 - Fixes: rhbz#2391290
 - On RPM 4.20+ don't put pyproject-macros-specific files in %%buildsubdir
 - Works around https://github.com/rpm-software-management/rpm/issues/3890
+- Speed %%pyproject_save_files up significantly
 
 * Fri Jul 25 2025 Fedora Release Engineering <releng@fedoraproject.org> - 1.18.3-2
 - Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild
diff --git a/pyproject_save_files.py b/pyproject_save_files.py
index 0a831e8..7ae4193 100644
--- a/pyproject_save_files.py
+++ b/pyproject_save_files.py
@@ -137,12 +137,11 @@ def add_file_to_module(paths, module_name, module_type, files_dirs, *files):
     """
     for module in paths["modules"][module_name]:
         if module["type"] == module_type:
-            if files[0] not in module[files_dirs]:
-                module[files_dirs].extend(files)
+            module[files_dirs].update(files)
             break
     else:
         paths["modules"][module_name].append(
-            {"type": module_type, "files": [], "dirs": [], files_dirs: list(files)}
+            {"type": module_type, "files": set(), "dirs": set(), files_dirs: set(files)}
         )