diff --git a/pungi/scripts/create_extra_repo.py b/pungi/scripts/create_extra_repo.py
index fd5898ae..6cf9c186 100644
--- a/pungi/scripts/create_extra_repo.py
+++ b/pungi/scripts/create_extra_repo.py
@@ -33,7 +33,7 @@ class CreateExtraRepo(PackagesGenerator):
clear_target_repo: bool = True,
):
self.repos = [] # type: List[ExtraRepoInfo]
- super().__init__(repos, [])
+ super().__init__(repos, [], [])
self.auth_headers = {
'Authorization': f'Bearer {bs_auth_token}',
}
diff --git a/pungi/scripts/create_packages_json.py b/pungi/scripts/create_packages_json.py
index 2a875640..72996c03 100644
--- a/pungi/scripts/create_packages_json.py
+++ b/pungi/scripts/create_packages_json.py
@@ -40,6 +40,11 @@ class RepoInfo:
arch: AnyStr
# Is a repo remote or local
is_remote: bool
+ # Is an reference repository (usually it's a RHEL repo)
+ # Layout of packages from such repository will be taken as example
+ # Only layout of specific package (which don't exist
+ # in an reference repository) will be taken as example
+ is_reference: bool = False
class PackagesGenerator:
@@ -47,9 +52,11 @@ class PackagesGenerator:
self,
repos: List[RepoInfo],
excluded_packages: List[AnyStr],
+ included_packages: List[AnyStr],
):
self.repos = repos
self.excluded_packages = excluded_packages
+ self.included_packages = included_packages
@staticmethod
def _warning_callback(warning_type, message):
@@ -277,12 +284,19 @@ class PackagesGenerator:
)
for package in packages.values():
package_key = f'{package.name}.{package.arch}'
- if 'module' in package.release:
+ if 'module' in package.release and not any(
+ re.search(included_package, package.name)
+ for included_package in self.included_packages
+ ):
+ # Even a module package will be added to packages.json if
+ # it presents in the list of included packages
continue
if package_key not in all_packages:
all_packages[package_key]['variant'] = repo_info.name
all_packages[package_key]['arch'] = repo_info.arch
all_packages[package_key]['package'] = package
+ elif not repo_info.is_reference:
+ continue
elif self.compare_pkgs_version(
package,
all_packages[package_key]['package']
@@ -297,8 +311,6 @@ class PackagesGenerator:
package = package_dict['package']
package_name = package.name
package_arch = package.arch
- if 'module' in package.release:
- continue
if any(re.search(excluded_package, package_name)
for excluded_package in self.excluded_packages):
continue
@@ -358,6 +370,14 @@ def create_parser():
choices=['yes', 'no'],
required=True,
)
+ parser.add_argument(
+ '--is-reference',
+ action='append',
+ type=str,
+ help='A repository is used as reference for packages layout',
+ choices=['yes', 'no'],
+ required=True,
+ )
parser.add_argument(
'--excluded-packages',
nargs='+',
@@ -367,6 +387,15 @@ def create_parser():
'All of list elements should be separated by space',
required=False,
)
+ parser.add_argument(
+ '--included-packages',
+ nargs='+',
+ type=str,
+ default=[],
+ help='A list of globally included packages from generated json.'
+ 'All of list elements should be separated by space',
+ required=False,
+ )
parser.add_argument(
'--json-output-path',
type=str,
@@ -380,23 +409,27 @@ def create_parser():
def cli_main():
args = create_parser().parse_args()
repos = []
- for repo_path, repo_folder, repo_name, repo_arch, is_remote in zip(
- args.repo_path,
- args.repo_folder,
- args.repo_name,
- args.repo_arch,
- args.is_remote,
- ):
+ for repo_path, repo_folder, repo_name, \
+ repo_arch, is_remote, is_reference in zip(
+ args.repo_path,
+ args.repo_folder,
+ args.repo_name,
+ args.repo_arch,
+ args.is_remote,
+ args.is_reference,
+ ):
repos.append(RepoInfo(
path=repo_path,
folder=repo_folder,
name=repo_name,
arch=repo_arch,
is_remote=True if is_remote == 'yes' else False,
+ is_reference=True if is_reference == 'yes' else False
))
pg = PackagesGenerator(
repos=repos,
excluded_packages=args.excluded_packages,
+ included_packages=args.included_packages,
)
result = pg.generate_packages_json()
with open(args.json_output_path, 'w') as packages_file:
diff --git a/tests/data/test_create_packages_json/test_repo/repodata/primary.xml b/tests/data/test_create_packages_json/test_repo/repodata/primary.xml
new file mode 100644
index 00000000..f95638eb
--- /dev/null
+++ b/tests/data/test_create_packages_json/test_repo/repodata/primary.xml
@@ -0,0 +1,332 @@
+
+
+
+ 389-ds-base
+ x86_64
+
+ 12505e39ee5a8ce6e954a3cabd4a92165711b4961bfc74a50c938250eeb56356
+ 389 Directory Server (base)
+ 389 Directory Server is an LDAPv3 compliant server. The base package includes
+the LDAP server and command line utilities for server administration.
+ Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
+ https://www.port389.org
+
+
+
+
+ GPLv3+
+ Red Hat, Inc.
+ System Environment/Daemons
+ x86-vm-06.build.eng.bos.redhat.com
+ 389-ds-base-1.4.0.20-10.module+el8.0.0+3096+101825d5.src.rpm
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /etc/dirsrv
+ /etc/dirsrv/config
+ /etc/dirsrv/config/certmap.conf
+ /etc/dirsrv/config/slapd-collations.conf
+ /etc/dirsrv/config/template-initconfig
+ /etc/dirsrv/schema
+ /etc/dirsrv/schema/99user.ldif
+ /etc/sysconfig/dirsrv
+ /etc/sysconfig/dirsrv.systemd
+ /etc/systemd/system/dirsrv.target.wants
+ /usr/bin/dbscan
+ /usr/bin/ds-logpipe.py
+ /usr/bin/ds-replcheck
+ /usr/bin/ldclt
+ /usr/bin/logconv.pl
+ /usr/bin/pwdhash
+ /usr/bin/readnsstate
+ /usr/lib64/dirsrv/bin/jeprof
+ /usr/sbin/bak2db
+ /usr/sbin/db2bak
+ /usr/sbin/db2index
+ /usr/sbin/db2ldif
+ /usr/sbin/dbverify
+ /usr/sbin/ldif2db
+ /usr/sbin/ldif2ldap
+ /usr/sbin/ns-slapd
+ /usr/sbin/restart-dirsrv
+ /usr/sbin/start-dirsrv
+ /usr/sbin/status-dirsrv
+ /usr/sbin/stop-dirsrv
+ /usr/sbin/upgradedb
+ /usr/sbin/vlvindex
+ /usr/share/gdb/auto-load/usr/sbin/ns-slapd-gdb.py
+
+
+
+ zziplib-utils
+ x86_64
+
+ 26d838b257f0890c3abf408bc200ebcf53ff1e40f16654cad9b1cb4c059f0d42
+ Utilities for the zziplib library
+ The zziplib library is intentionally lightweight, it offers the ability to
+easily extract data from files archived in a single zip file. Applications
+can bundle files into a single zip archive and access them. The implementation
+is based only on the (free) subset of compression with the zlib algorithm
+which is actually used by the zip/unzip tools.
+
+This packages contains all the utilities that come with the zziplib library.
+ Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
+ http://zziplib.sourceforge.net/
+
+
+
+
+ LGPLv2+ or MPLv1.1
+ Red Hat, Inc.
+ Applications/Archiving
+ x86-vm-09.build.eng.bos.redhat.com
+ zziplib-0.13.68-8.el8.src.rpm
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /usr/bin/unzip-mem
+ /usr/bin/unzzip
+ /usr/bin/unzzip-big
+ /usr/bin/unzzip-mem
+ /usr/bin/unzzip-mix
+ /usr/bin/zzcat
+ /usr/bin/zzdir
+ /usr/bin/zzxorcat
+ /usr/bin/zzxorcopy
+ /usr/bin/zzxordir
+
+
+
+ zziplib
+ i686
+
+ 207d3ba8e031d11bd13923e6027e17e5db12d76e915a28e9cd1589ef796a6e6b
+ Lightweight library to easily extract data from zip files
+ The zziplib library is intentionally lightweight, it offers the ability to
+easily extract data from files archived in a single zip file. Applications
+can bundle files into a single zip archive and access them. The implementation
+is based only on the (free) subset of compression with the zlib algorithm
+which is actually used by the zip/unzip tools.
+ Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
+ http://zziplib.sourceforge.net/
+
+
+
+
+ LGPLv2+ or MPLv1.1
+ Red Hat, Inc.
+ Applications/Archiving
+ x86-vm-09.build.eng.bos.redhat.com
+ zziplib-0.13.68-8.el8.src.rpm
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ zziplib
+ x86_64
+
+ 30cee6c1adcaee7b24c1079072299ee55a9c3bbdcb91697544242c7567d18621
+ Lightweight library to easily extract data from zip files
+ The zziplib library is intentionally lightweight, it offers the ability to
+easily extract data from files archived in a single zip file. Applications
+can bundle files into a single zip archive and access them. The implementation
+is based only on the (free) subset of compression with the zlib algorithm
+which is actually used by the zip/unzip tools.
+ Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
+ http://zziplib.sourceforge.net/
+
+
+
+
+ LGPLv2+ or MPLv1.1
+ Red Hat, Inc.
+ Applications/Archiving
+ x86-vm-09.build.eng.bos.redhat.com
+ zziplib-0.13.68-8.el8.src.rpm
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/data/test_create_packages_json/test_repo/repodata/primary.xml.gz b/tests/data/test_create_packages_json/test_repo/repodata/primary.xml.gz
deleted file mode 100755
index eb2f101e..00000000
Binary files a/tests/data/test_create_packages_json/test_repo/repodata/primary.xml.gz and /dev/null differ
diff --git a/tests/data/test_create_packages_json/test_repo/repodata/repomd.xml b/tests/data/test_create_packages_json/test_repo/repodata/repomd.xml
index eafb0d2b..faa24076 100644
--- a/tests/data/test_create_packages_json/test_repo/repodata/repomd.xml
+++ b/tests/data/test_create_packages_json/test_repo/repodata/repomd.xml
@@ -4,7 +4,7 @@
2826d3f5dd3b03cfb5d2c079123f7add3a7d068e8dfd210873eb27eb32586a8e
78efcf6b74f4c56aaab183336eab44fcbcc9cb6c25045fe5980ab83a85e48db7
-
+
1610968715
3094
16878
diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/filelists.xml.gz b/tests/data/test_create_packages_json/test_repo_2/repodata/filelists.xml.gz
new file mode 100755
index 00000000..920e8153
Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo_2/repodata/filelists.xml.gz differ
diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/other.xml.gz b/tests/data/test_create_packages_json/test_repo_2/repodata/other.xml.gz
new file mode 100755
index 00000000..18085e5c
Binary files /dev/null and b/tests/data/test_create_packages_json/test_repo_2/repodata/other.xml.gz differ
diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/primary.xml b/tests/data/test_create_packages_json/test_repo_2/repodata/primary.xml
new file mode 100644
index 00000000..9edee311
--- /dev/null
+++ b/tests/data/test_create_packages_json/test_repo_2/repodata/primary.xml
@@ -0,0 +1,332 @@
+
+
+
+ 389-ds-base-new
+ x86_64
+
+ 12505e39ee5a8ce6e954a3cabd4a92165711b4961bfc74a50c938250eeb56356
+ 389 Directory Server (base)
+ 389 Directory Server is an LDAPv3 compliant server. The base package includes
+the LDAP server and command line utilities for server administration.
+ Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
+ https://www.port389.org
+
+
+
+
+ GPLv3+
+ Red Hat, Inc.
+ System Environment/Daemons
+ x86-vm-06.build.eng.bos.redhat.com
+ 389-ds-base-new-1.4.0.20-10.module+el8.0.0+3096+101825d5.src.rpm
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /etc/dirsrv
+ /etc/dirsrv/config
+ /etc/dirsrv/config/certmap.conf
+ /etc/dirsrv/config/slapd-collations.conf
+ /etc/dirsrv/config/template-initconfig
+ /etc/dirsrv/schema
+ /etc/dirsrv/schema/99user.ldif
+ /etc/sysconfig/dirsrv
+ /etc/sysconfig/dirsrv.systemd
+ /etc/systemd/system/dirsrv.target.wants
+ /usr/bin/dbscan
+ /usr/bin/ds-logpipe.py
+ /usr/bin/ds-replcheck
+ /usr/bin/ldclt
+ /usr/bin/logconv.pl
+ /usr/bin/pwdhash
+ /usr/bin/readnsstate
+ /usr/lib64/dirsrv/bin/jeprof
+ /usr/sbin/bak2db
+ /usr/sbin/db2bak
+ /usr/sbin/db2index
+ /usr/sbin/db2ldif
+ /usr/sbin/dbverify
+ /usr/sbin/ldif2db
+ /usr/sbin/ldif2ldap
+ /usr/sbin/ns-slapd
+ /usr/sbin/restart-dirsrv
+ /usr/sbin/start-dirsrv
+ /usr/sbin/status-dirsrv
+ /usr/sbin/stop-dirsrv
+ /usr/sbin/upgradedb
+ /usr/sbin/vlvindex
+ /usr/share/gdb/auto-load/usr/sbin/ns-slapd-gdb.py
+
+
+
+ zziplib-utils
+ x86_64
+
+ 26d838b257f0890c3abf408bc200ebcf53ff1e40f16654cad9b1cb4c059f0d42
+ Utilities for the zziplib library
+ The zziplib library is intentionally lightweight, it offers the ability to
+easily extract data from files archived in a single zip file. Applications
+can bundle files into a single zip archive and access them. The implementation
+is based only on the (free) subset of compression with the zlib algorithm
+which is actually used by the zip/unzip tools.
+
+This packages contains all the utilities that come with the zziplib library.
+ Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
+ http://zziplib.sourceforge.net/
+
+
+
+
+ LGPLv2+ or MPLv1.1
+ Red Hat, Inc.
+ Applications/Archiving
+ x86-vm-09.build.eng.bos.redhat.com
+ zziplib-0.13.68-8.el8.src.rpm
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /usr/bin/unzip-mem
+ /usr/bin/unzzip
+ /usr/bin/unzzip-big
+ /usr/bin/unzzip-mem
+ /usr/bin/unzzip-mix
+ /usr/bin/zzcat
+ /usr/bin/zzdir
+ /usr/bin/zzxorcat
+ /usr/bin/zzxorcopy
+ /usr/bin/zzxordir
+
+
+
+ zziplib
+ i686
+
+ 207d3ba8e031d11bd13923e6027e17e5db12d76e915a28e9cd1589ef796a6e6b
+ Lightweight library to easily extract data from zip files
+ The zziplib library is intentionally lightweight, it offers the ability to
+easily extract data from files archived in a single zip file. Applications
+can bundle files into a single zip archive and access them. The implementation
+is based only on the (free) subset of compression with the zlib algorithm
+which is actually used by the zip/unzip tools.
+ Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
+ http://zziplib.sourceforge.net/
+
+
+
+
+ LGPLv2+ or MPLv1.1
+ Red Hat, Inc.
+ Applications/Archiving
+ x86-vm-09.build.eng.bos.redhat.com
+ zziplib-0.13.68-8.el8.src.rpm
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ zziplib
+ x86_64
+
+ 30cee6c1adcaee7b24c1079072299ee55a9c3bbdcb91697544242c7567d18621
+ Lightweight library to easily extract data from zip files
+ The zziplib library is intentionally lightweight, it offers the ability to
+easily extract data from files archived in a single zip file. Applications
+can bundle files into a single zip archive and access them. The implementation
+is based only on the (free) subset of compression with the zlib algorithm
+which is actually used by the zip/unzip tools.
+ Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>
+ http://zziplib.sourceforge.net/
+
+
+
+
+ LGPLv2+ or MPLv1.1
+ Red Hat, Inc.
+ Applications/Archiving
+ x86-vm-09.build.eng.bos.redhat.com
+ zziplib-0.13.68-8.el8.src.rpm
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/data/test_create_packages_json/test_repo_2/repodata/repomd.xml b/tests/data/test_create_packages_json/test_repo_2/repodata/repomd.xml
new file mode 100644
index 00000000..2d14f784
--- /dev/null
+++ b/tests/data/test_create_packages_json/test_repo_2/repodata/repomd.xml
@@ -0,0 +1,28 @@
+
+
+ 1610968727
+
+ dcfe621a2c74f0f58fce6da4a4fbbc3670d889ec207350a90ec1d082b206f98a
+ 14cb7ce7edc80fa750d5139170112073325af0badbf8bd9c294e033bf4518b83
+
+ 1610968715
+ 3193
+ 20480
+
+
+ e41805c927fc4ad1b9bde52509afb37e47acc153283b23da17560d4e250b3a3e
+ 5f659e8c05b7d056748bf809bec8aa9fa5f791c2b0546d6c49b02a7ebfb26ce2
+
+ 1610968715
+ 3970
+ 19897
+
+
+ db6d0d88abcaf06dc8ef09207fdbb9ba5e3ffb505a7dd2bf94fdbc953a6de11e
+ 3ae1b186b4c3037805e2cf28a78b2204c37b4dc04acbd8bef98a7b24ab5b52a8
+
+ 1610968715
+ 2191
+ 8337
+
+
diff --git a/tests/test_create_packages_json.py b/tests/test_create_packages_json.py
index f5ffbcf4..122fd870 100644
--- a/tests/test_create_packages_json.py
+++ b/tests/test_create_packages_json.py
@@ -19,6 +19,7 @@ test_repo_info = RepoInfo(
name='TestRepo',
arch='x86_64',
is_remote=False,
+ is_reference=True,
)
test_repo_info_2 = RepoInfo(
@@ -27,6 +28,7 @@ test_repo_info_2 = RepoInfo(
name='TestRepo2',
arch='x86_64',
is_remote=False,
+ is_reference=True,
)
@@ -64,6 +66,7 @@ class TestPackagesJson(TestCase):
test_repo_info_2,
],
excluded_packages=['zziplib-utils'],
+ included_packages=['389-ds-base-new*'],
)
test_packages = defaultdict(
lambda: defaultdict(
@@ -80,6 +83,10 @@ class TestPackagesJson(TestCase):
[
'zziplib.i686',
]
+ test_packages['TestRepo2']['x86_64']['389-ds-base-new'] = \
+ [
+ '389-ds-base-new.x86_64',
+ ]
result = pg.generate_packages_json()
self.assertEqual(
test_packages,