diff --git a/pungi/scripts/create_extra_repo.py b/pungi/scripts/create_extra_repo.py index 34f36f6f..e7133fe7 100644 --- a/pungi/scripts/create_extra_repo.py +++ b/pungi/scripts/create_extra_repo.py @@ -216,7 +216,7 @@ class CreateExtraRepo(PackagesGenerator): ): """ Download all defined packages from a remote repo - :param packages: information about all of packages (including + :param packages: information about all packages (including modularity) in a remote repo :param repo_info: information about a remote repo """ @@ -289,6 +289,18 @@ class CreateExtraRepo(PackagesGenerator): repomd_records = self._get_repomd_records( repo_info=repo_info, ) + repomd_records_dict = {} # type: Dict[str, str] + self._download_repomd_records( + repo_info=repo_info, + repomd_records=repomd_records, + repomd_records_dict=repomd_records_dict, + ) + packages_iterator = cr.PackageIterator( + primary_path=repomd_records_dict['primary'], + filelists_path=repomd_records_dict['filelists'], + other_path=repomd_records_dict['other'], + warningcb=self._warning_callback, + ) # parse the repodata (including modules.yaml.gz) modules_data = self._parse_module_repomd_record( repo_info=repo_info, @@ -300,7 +312,7 @@ class CreateExtraRepo(PackagesGenerator): packages = { f'{package.name}-{package.epoch}:{package.version}-' f'{package.release}.{package.arch}': - package for package in packages.values() + package for package in packages_iterator } self._download_modules( modules_data=modules_data, diff --git a/pungi/scripts/create_packages_json.py b/pungi/scripts/create_packages_json.py index 6fb3e3ff..8aad90c6 100644 --- a/pungi/scripts/create_packages_json.py +++ b/pungi/scripts/create_packages_json.py @@ -325,9 +325,11 @@ class PackagesGenerator: repo_info=repo_info, ) repomd_records_dict = {} # type: Dict[str, str] - self._download_repomd_records(repo_info=repo_info, - repomd_records=repomd_records, - repomd_records_dict=repomd_records_dict) + self._download_repomd_records( + repo_info=repo_info, + repomd_records=repomd_records, + repomd_records_dict=repomd_records_dict, + ) packages_iterator = PackageIterator( primary_path=repomd_records_dict['primary'], filelists_path=repomd_records_dict['filelists'], diff --git a/tests/data/test_create_packages_json/test_repo/repodata/01b82e9eb7ee9151f283c6e761ae450de18ed2d64b5e32de88689eaf95216a80-other.sqlite.bz2 b/tests/data/test_create_packages_json/test_repo/repodata/01b82e9eb7ee9151f283c6e761ae450de18ed2d64b5e32de88689eaf95216a80-other.sqlite.bz2 new file mode 100644 index 00000000..cdb1b024 Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo/repodata/01b82e9eb7ee9151f283c6e761ae450de18ed2d64b5e32de88689eaf95216a80-other.sqlite.bz2 differ diff --git a/tests/data/test_create_packages_json/test_repo/repodata/89cb9cc1181635c9147864a7076d91fb81072641d481cd202832a2d257453576-primary.xml.gz b/tests/data/test_create_packages_json/test_repo/repodata/89cb9cc1181635c9147864a7076d91fb81072641d481cd202832a2d257453576-primary.xml.gz new file mode 100644 index 00000000..2f2521c2 Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo/repodata/89cb9cc1181635c9147864a7076d91fb81072641d481cd202832a2d257453576-primary.xml.gz differ diff --git a/tests/data/test_create_packages_json/test_repo/repodata/8bcf6d40db4e922934ac47e8ac7fb8d15bdacf579af8c819d2134ed54d30550b-filelists.sqlite.bz2 b/tests/data/test_create_packages_json/test_repo/repodata/8bcf6d40db4e922934ac47e8ac7fb8d15bdacf579af8c819d2134ed54d30550b-filelists.sqlite.bz2 new file mode 100644 index 00000000..35141bcc Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo/repodata/8bcf6d40db4e922934ac47e8ac7fb8d15bdacf579af8c819d2134ed54d30550b-filelists.sqlite.bz2 differ diff --git a/tests/data/test_create_packages_json/test_repo/repodata/ab8df35061dfa0285069b843f24a7076e31266d9a8abe8282340bcb936aa61d7-primary.sqlite.bz2 b/tests/data/test_create_packages_json/test_repo/repodata/ab8df35061dfa0285069b843f24a7076e31266d9a8abe8282340bcb936aa61d7-primary.sqlite.bz2 new file mode 100644 index 00000000..1e31fb33 Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo/repodata/ab8df35061dfa0285069b843f24a7076e31266d9a8abe8282340bcb936aa61d7-primary.sqlite.bz2 differ diff --git a/tests/data/test_create_packages_json/test_repo/repodata/b3827bd6c9ea67ffa3912002515c64e4d9fe5c4dacbf7c46b0d8768b7abbb84f-other.xml.gz b/tests/data/test_create_packages_json/test_repo/repodata/b3827bd6c9ea67ffa3912002515c64e4d9fe5c4dacbf7c46b0d8768b7abbb84f-other.xml.gz new file mode 100644 index 00000000..447f87e8 Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo/repodata/b3827bd6c9ea67ffa3912002515c64e4d9fe5c4dacbf7c46b0d8768b7abbb84f-other.xml.gz differ diff --git a/tests/data/test_create_packages_json/test_repo/repodata/f69ca03957574729fd5150335b0d87afddcfb37a97aed5b06272212854f1773d-filelists.xml.gz b/tests/data/test_create_packages_json/test_repo/repodata/f69ca03957574729fd5150335b0d87afddcfb37a97aed5b06272212854f1773d-filelists.xml.gz new file mode 100644 index 00000000..f74b009b Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo/repodata/f69ca03957574729fd5150335b0d87afddcfb37a97aed5b06272212854f1773d-filelists.xml.gz differ diff --git a/tests/data/test_create_packages_json/test_repo/repodata/filelists.xml.gz b/tests/data/test_create_packages_json/test_repo/repodata/filelists.xml.gz deleted file mode 100755 index 920e8153..00000000 Binary files a/tests/data/test_create_packages_json/test_repo/repodata/filelists.xml.gz and /dev/null differ diff --git a/tests/data/test_create_packages_json/test_repo/repodata/other.xml.gz b/tests/data/test_create_packages_json/test_repo/repodata/other.xml.gz deleted file mode 100755 index 18085e5c..00000000 Binary files a/tests/data/test_create_packages_json/test_repo/repodata/other.xml.gz and /dev/null differ diff --git a/tests/data/test_create_packages_json/test_repo/repodata/primary.xml b/tests/data/test_create_packages_json/test_repo/repodata/primary.xml deleted file mode 100644 index f95638eb..00000000 --- a/tests/data/test_create_packages_json/test_repo/repodata/primary.xml +++ /dev/null @@ -1,332 +0,0 @@ - - - - 389-ds-base - x86_64 - - 12505e39ee5a8ce6e954a3cabd4a92165711b4961bfc74a50c938250eeb56356 - 389 Directory Server (base) - 389 Directory Server is an LDAPv3 compliant server. The base package includes -the LDAP server and command line utilities for server administration. - Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla> - https://www.port389.org - - - zziplib-utils - x86_64 - - 26d838b257f0890c3abf408bc200ebcf53ff1e40f16654cad9b1cb4c059f0d42 - Utilities for the zziplib library - The zziplib library is intentionally lightweight, it offers the ability to -easily extract data from files archived in a single zip file. Applications -can bundle files into a single zip archive and access them. The implementation -is based only on the (free) subset of compression with the zlib algorithm -which is actually used by the zip/unzip tools. - -This packages contains all the utilities that come with the zziplib library. - Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla> - http://zziplib.sourceforge.net/ - - - zziplib - i686 - - 207d3ba8e031d11bd13923e6027e17e5db12d76e915a28e9cd1589ef796a6e6b - Lightweight library to easily extract data from zip files - The zziplib library is intentionally lightweight, it offers the ability to -easily extract data from files archived in a single zip file. Applications -can bundle files into a single zip archive and access them. The implementation -is based only on the (free) subset of compression with the zlib algorithm -which is actually used by the zip/unzip tools. - Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla> - http://zziplib.sourceforge.net/ - - - zziplib - x86_64 - - 30cee6c1adcaee7b24c1079072299ee55a9c3bbdcb91697544242c7567d18621 - Lightweight library to easily extract data from zip files - The zziplib library is intentionally lightweight, it offers the ability to -easily extract data from files archived in a single zip file. Applications -can bundle files into a single zip archive and access them. The implementation -is based only on the (free) subset of compression with the zlib algorithm -which is actually used by the zip/unzip tools. - Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla> - http://zziplib.sourceforge.net/ - - diff --git a/tests/data/test_create_packages_json/test_repo/repodata/repomd.xml b/tests/data/test_create_packages_json/test_repo/repodata/repomd.xml index faa24076..dadc686b 100644 --- a/tests/data/test_create_packages_json/test_repo/repodata/repomd.xml +++ b/tests/data/test_create_packages_json/test_repo/repodata/repomd.xml @@ -1,28 +1,55 @@ - 1610968727 - - 2826d3f5dd3b03cfb5d2c079123f7add3a7d068e8dfd210873eb27eb32586a8e - 78efcf6b74f4c56aaab183336eab44fcbcc9cb6c25045fe5980ab83a85e48db7 - - 1610968715 - 3094 - 16878 - - - e41805c927fc4ad1b9bde52509afb37e47acc153283b23da17560d4e250b3a3e - 5f659e8c05b7d056748bf809bec8aa9fa5f791c2b0546d6c49b02a7ebfb26ce2 - - 1610968715 - 3970 - 19897 - - - db6d0d88abcaf06dc8ef09207fdbb9ba5e3ffb505a7dd2bf94fdbc953a6de11e - 3ae1b186b4c3037805e2cf28a78b2204c37b4dc04acbd8bef98a7b24ab5b52a8 - - 1610968715 - 2191 - 8337 - + 1666177486 + + 89cb9cc1181635c9147864a7076d91fb81072641d481cd202832a2d257453576 + 07255d9856f7531b52a6459f6fc7701c6d93c6d6c29d1382d83afcc53f13494a + + 1666177486 + 1387 + 6528 + + + f69ca03957574729fd5150335b0d87afddcfb37a97aed5b06272212854f1773d + c2e1e674d7d48bccaa16cae0a5f70cb55ef4cd7352b4d9d4fdaa619075d07dbc + + 1666177486 + 1252 + 5594 + + + b3827bd6c9ea67ffa3912002515c64e4d9fe5c4dacbf7c46b0d8768b7abbb84f + 9ce24c526239e349d023c577b2ae3872c8b0f1888aed1fb24b9b9aa12063fdf3 + + 1666177486 + 999 + 6320 + + + ab8df35061dfa0285069b843f24a7076e31266d9a8abe8282340bcb936aa61d7 + 2bce9554ce4496cef34b5cd69f186f7f3143c7cabae8fa384fc5c9eeab326f7f + + 1666177486 + 3558 + 106496 + 10 + + + 8bcf6d40db4e922934ac47e8ac7fb8d15bdacf579af8c819d2134ed54d30550b + f7001d1df7f5f7e4898919b15710bea8ed9711ce42faf68e22b757e63169b1fb + + 1666177486 + 2360 + 28672 + 10 + + + 01b82e9eb7ee9151f283c6e761ae450de18ed2d64b5e32de88689eaf95216a80 + 07f5b9750af1e440d37ca216e719dd288149e79e9132f2fdccb6f73b2e5dd541 + + 1666177486 + 2196 + 32768 + 10 + diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/35eff699131e0976429144c6f4514d21568177dc64bb4091c3ff62f76b293725-other.sqlite.bz2 b/tests/data/test_create_packages_json/test_repo_2/repodata/35eff699131e0976429144c6f4514d21568177dc64bb4091c3ff62f76b293725-other.sqlite.bz2 new file mode 100644 index 00000000..0346c4a1 Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo_2/repodata/35eff699131e0976429144c6f4514d21568177dc64bb4091c3ff62f76b293725-other.sqlite.bz2 differ diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/5a60d79d8bce6a805f4fdb22fd891524359dce8ccc665c0b54e7299e79debe84-other.xml.gz b/tests/data/test_create_packages_json/test_repo_2/repodata/5a60d79d8bce6a805f4fdb22fd891524359dce8ccc665c0b54e7299e79debe84-other.xml.gz new file mode 100644 index 00000000..af1858f7 Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo_2/repodata/5a60d79d8bce6a805f4fdb22fd891524359dce8ccc665c0b54e7299e79debe84-other.xml.gz differ diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/6778922d5853d20f213ae7702699a76f1e87e55d6bfb5e4ac6a117d904d47b3c-filelists.xml.gz b/tests/data/test_create_packages_json/test_repo_2/repodata/6778922d5853d20f213ae7702699a76f1e87e55d6bfb5e4ac6a117d904d47b3c-filelists.xml.gz new file mode 100644 index 00000000..d4a1ebf3 Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo_2/repodata/6778922d5853d20f213ae7702699a76f1e87e55d6bfb5e4ac6a117d904d47b3c-filelists.xml.gz differ diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/a1d342aa7cef3a2034fc3f9d6ee02d63572780bc76e61749a57e50b6b3ca9869-primary.xml.gz b/tests/data/test_create_packages_json/test_repo_2/repodata/a1d342aa7cef3a2034fc3f9d6ee02d63572780bc76e61749a57e50b6b3ca9869-primary.xml.gz new file mode 100644 index 00000000..92c90f5f Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo_2/repodata/a1d342aa7cef3a2034fc3f9d6ee02d63572780bc76e61749a57e50b6b3ca9869-primary.xml.gz differ diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/c27bc2ce947173aba305041552c3c6d8db71442c1a2e5dcaf35ff750fe0469fc-primary.sqlite.bz2 b/tests/data/test_create_packages_json/test_repo_2/repodata/c27bc2ce947173aba305041552c3c6d8db71442c1a2e5dcaf35ff750fe0469fc-primary.sqlite.bz2 new file mode 100644 index 00000000..4fe3194d Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo_2/repodata/c27bc2ce947173aba305041552c3c6d8db71442c1a2e5dcaf35ff750fe0469fc-primary.sqlite.bz2 differ diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/ed350865982e7a1e45b144839b56eac888e5d8f680571dd2cd06b37dc83e0fd8-filelists.sqlite.bz2 b/tests/data/test_create_packages_json/test_repo_2/repodata/ed350865982e7a1e45b144839b56eac888e5d8f680571dd2cd06b37dc83e0fd8-filelists.sqlite.bz2 new file mode 100644 index 00000000..ab1e4de2 Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo_2/repodata/ed350865982e7a1e45b144839b56eac888e5d8f680571dd2cd06b37dc83e0fd8-filelists.sqlite.bz2 differ diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/filelists.xml.gz b/tests/data/test_create_packages_json/test_repo_2/repodata/filelists.xml.gz deleted file mode 100755 index 920e8153..00000000 Binary files a/tests/data/test_create_packages_json/test_repo_2/repodata/filelists.xml.gz and /dev/null differ diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/other.xml.gz b/tests/data/test_create_packages_json/test_repo_2/repodata/other.xml.gz deleted file mode 100755 index 18085e5c..00000000 Binary files a/tests/data/test_create_packages_json/test_repo_2/repodata/other.xml.gz and /dev/null differ diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/primary.xml b/tests/data/test_create_packages_json/test_repo_2/repodata/primary.xml deleted file mode 100644 index 9edee311..00000000 --- a/tests/data/test_create_packages_json/test_repo_2/repodata/primary.xml +++ /dev/null @@ -1,332 +0,0 @@ - - - - 389-ds-base-new - x86_64 - - 12505e39ee5a8ce6e954a3cabd4a92165711b4961bfc74a50c938250eeb56356 - 389 Directory Server (base) - 389 Directory Server is an LDAPv3 compliant server. The base package includes -the LDAP server and command line utilities for server administration. - Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla> - https://www.port389.org - - - zziplib-utils - x86_64 - - 26d838b257f0890c3abf408bc200ebcf53ff1e40f16654cad9b1cb4c059f0d42 - Utilities for the zziplib library - The zziplib library is intentionally lightweight, it offers the ability to -easily extract data from files archived in a single zip file. Applications -can bundle files into a single zip archive and access them. The implementation -is based only on the (free) subset of compression with the zlib algorithm -which is actually used by the zip/unzip tools. - -This packages contains all the utilities that come with the zziplib library. - Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla> - http://zziplib.sourceforge.net/ - - - zziplib - i686 - - 207d3ba8e031d11bd13923e6027e17e5db12d76e915a28e9cd1589ef796a6e6b - Lightweight library to easily extract data from zip files - The zziplib library is intentionally lightweight, it offers the ability to -easily extract data from files archived in a single zip file. Applications -can bundle files into a single zip archive and access them. The implementation -is based only on the (free) subset of compression with the zlib algorithm -which is actually used by the zip/unzip tools. - Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla> - http://zziplib.sourceforge.net/ - - - zziplib - x86_64 - - 30cee6c1adcaee7b24c1079072299ee55a9c3bbdcb91697544242c7567d18621 - Lightweight library to easily extract data from zip files - The zziplib library is intentionally lightweight, it offers the ability to -easily extract data from files archived in a single zip file. Applications -can bundle files into a single zip archive and access them. The implementation -is based only on the (free) subset of compression with the zlib algorithm -which is actually used by the zip/unzip tools. - Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla> - http://zziplib.sourceforge.net/ - - diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/repomd.xml b/tests/data/test_create_packages_json/test_repo_2/repodata/repomd.xml index 2d14f784..15d22eac 100644 --- a/tests/data/test_create_packages_json/test_repo_2/repodata/repomd.xml +++ b/tests/data/test_create_packages_json/test_repo_2/repodata/repomd.xml @@ -1,28 +1,55 @@ - 1610968727 - - dcfe621a2c74f0f58fce6da4a4fbbc3670d889ec207350a90ec1d082b206f98a - 14cb7ce7edc80fa750d5139170112073325af0badbf8bd9c294e033bf4518b83 - - 1610968715 - 3193 - 20480 - - - e41805c927fc4ad1b9bde52509afb37e47acc153283b23da17560d4e250b3a3e - 5f659e8c05b7d056748bf809bec8aa9fa5f791c2b0546d6c49b02a7ebfb26ce2 - - 1610968715 - 3970 - 19897 - - - db6d0d88abcaf06dc8ef09207fdbb9ba5e3ffb505a7dd2bf94fdbc953a6de11e - 3ae1b186b4c3037805e2cf28a78b2204c37b4dc04acbd8bef98a7b24ab5b52a8 - - 1610968715 - 2191 - 8337 - + 1666177500 + + a1d342aa7cef3a2034fc3f9d6ee02d63572780bc76e61749a57e50b6b3ca9869 + a9e3eae447dd44282d7d96db5f15f049b757925397adb752f4df982176bab7e0 + + 1666177500 + 3501 + 37296 + + + 6778922d5853d20f213ae7702699a76f1e87e55d6bfb5e4ac6a117d904d47b3c + e30b666d9d88a70de69a08f45e6696bcd600c45485d856bd0213395d7da7bd49 + + 1666177500 + 27624 + 318187 + + + 5a60d79d8bce6a805f4fdb22fd891524359dce8ccc665c0b54e7299e79debe84 + b18138f4a3de45714e578fb1f30b7ec54fdcdaf1a22585891625b6af0894388e + + 1666177500 + 1876 + 28701 + + + c27bc2ce947173aba305041552c3c6d8db71442c1a2e5dcaf35ff750fe0469fc + 586e1af8934229925adb9e746ae5ced119859dfd97f4e3237399bb36a7d7f071 + + 1666177500 + 11528 + 126976 + 10 + + + ed350865982e7a1e45b144839b56eac888e5d8f680571dd2cd06b37dc83e0fd8 + 697903989d0f77de2d44a2b603e75c9b4ca23b3795eb136d175caf5666ce6459 + + 1666177500 + 20440 + 163840 + 10 + + + 35eff699131e0976429144c6f4514d21568177dc64bb4091c3ff62f76b293725 + 3bd999a1bdf300df836a4607b7b75f845d8e1432e3e4e1ab6f0c7cc8a853db39 + + 1666177500 + 4471 + 49152 + 10 + diff --git a/tests/test_create_packages_json.py b/tests/test_create_packages_json.py index df109e71..4855a8a4 100644 --- a/tests/test_create_packages_json.py +++ b/tests/test_create_packages_json.py @@ -65,7 +65,7 @@ class TestPackagesJson(TestCase): test_repo_info_2, ], excluded_packages=['zziplib-utils'], - included_packages=['389-ds-base-new*'], + included_packages=['vim-file*'], ) test_packages = defaultdict( lambda: defaultdict( @@ -74,18 +74,17 @@ class TestPackagesJson(TestCase): ) ) ) - test_packages['TestRepo2']['x86_64']['zziplib'] = \ + test_packages['TestRepo']['x86_64']['zziplib'] = \ [ 'zziplib.i686', 'zziplib.x86_64', ] - test_packages['TestRepo2']['x86_64']['389-ds-base-new'] = \ + test_packages['TestRepo2']['x86_64']['vim'] = \ [ - '389-ds-base-new.x86_64', - ] - test_packages['TestRepo']['x86_64']['zziplib'] = \ - [ - 'zziplib.x86_64', + 'vim-X11.i686', + 'vim-common.i686', + 'vim-enhanced.i686', + 'vim-filesystem.noarch', ] result = pg.generate_packages_json() self.assertEqual(