Compare commits
198 Commits
centos_mas
...
master
Author | SHA1 | Date |
---|---|---|
Stepan Oksanichenko | bc8c776872 | |
Stepan Oksanichenko | 91d282708e | |
Stepan Oksanichenko | ccaf31bc87 | |
Stepan Oksanichenko | 5fe0504265 | |
Stepan Oksanichenko | d79f163685 | |
Stepan Oksanichenko | 793fb23958 | |
Stepan Oksanichenko | 65d0c09e97 | |
Stepan Oksanichenko | 0a9e5df66c | |
Stepan Oksanichenko | ae527a2e01 | |
Aditya Bisoi | 4991144a01 | |
Lubomír Sedlář | 68d94ff488 | |
Ozan Unsal | ce45fdc39a | |
Lubomír Sedlář | b625ccea06 | |
Lubomír Sedlář | 8eccfc5a03 | |
Lubomír Sedlář | f5a0e06af5 | |
Lubomír Sedlář | f6f54b56ca | |
Aditya Bisoi | fcee346c7c | |
Lubomír Sedlář | 82ec38ad60 | |
Lubomír Sedlář | c9cbd80569 | |
Aditya Bisoi | 035fca1e6d | |
Lubomír Sedlář | 0f8cae69b7 | |
Lubomír Sedlář | f17628dd5f | |
Lubomír Sedlář | f3485410ad | |
Haibo Lin | cccfaea14e | |
Lubomír Sedlář | e2057b75c5 | |
Lubomír Sedlář | 44ea4d4419 | |
Lubomír Sedlář | d4425f7935 | |
Lubomír Sedlář | c8118527ea | |
Lubomír Sedlář | a8ea322907 | |
Lubomír Sedlář | c4995c8f4b | |
Lubomír Sedlář | 997e372f25 | |
Lubomír Sedlář | 42f1c62528 | |
Lubomír Sedlář | 3fd29d0ee0 | |
Lubomír Sedlář | c1f2fa5035 | |
Aurélien Bompard | 85c9e9e776 | |
Lubomír Sedlář | 33012ab31e | |
Lubomír Sedlář | 72ddf65e62 | |
Haibo Lin | c402ff3d60 | |
Haibo Lin | 8dd344f9ee | |
Lubomír Sedlář | d07f517a90 | |
Lubomír Sedlář | 48366177cc | |
Lubomír Sedlář | 4cb8671fe4 | |
Lubomír Sedlář | 135bbbfe7e | |
Lubomír Sedlář | 5624829564 | |
Haibo Lin | 5fb4f86312 | |
Lubomír Sedlář | e891fe7b09 | |
Haibo Lin | 4cd7d39914 | |
Lubomír Sedlář | 5de829d05b | |
Lubomír Sedlář | 2930a1cc54 | |
Lubomír Sedlář | 9c4d3d496d | |
Haibo Lin | 4637fd6697 | |
Lubomír Sedlář | 2ff8132eaf | |
Lubomír Sedlář | f9190d1fd1 | |
Lubomír Sedlář | 80ad0448ec | |
Lubomír Sedlář | 027380f969 | |
Lubomír Sedlář | 41048f60b7 | |
Ondrej Nosek | 9f8f6a7956 | |
Lubomír Sedlář | 3d3e4bafdf | |
Lubomír Sedlář | 8fe0257e93 | |
Fedora Release Engineering | d7b5fd2278 | |
Lubomír Sedlář | 8b49d4ad61 | |
Lubomír Sedlář | 57443cd0aa | |
Python Maint | 1d146bb8d5 | |
Lubomír Sedlář | 790091b7d7 | |
Lubomír Sedlář | 28aad3ea40 | |
Pierre-Yves Chibon | 7373b4dbbf | |
Lubomír Sedlář | 218b11f1b7 | |
Haibo Lin | bfbe9095d2 | |
Lubomír Sedlář | eb17182c04 | |
Stepan Oksanichenko | f91f90cf64 | |
Stepan Oksanichenko | 49931082b2 | |
Stepan Oksanichenko | 8ba8609bda | |
Stepan Oksanichenko | 6f495a8133 | |
Stepan Oksanichenko | 2b4bddbfe0 | |
Stepan Oksanichenko | 032cf725de | |
Stepan Oksanichenko | 8b11bb81af | |
soksanichenko | 114a73f100 | |
soksanichenko | 1c3e5dce5e | |
soksanichenko | e55abb17f1 | |
soksanichenko | e81d78a1d1 | |
soksanichenko | 68915d04f8 | |
soksanichenko | a25bf72fb8 | |
Stepan Oksanichenko | 68aee1fa2d | |
soksanichenko | 6592735aec | |
soksanichenko | 943fd8e77d | |
soksanichenko | 004fc4382f | |
soksanichenko | 596c5c0b7f | |
soksanichenko | 141d00e941 | |
soksanichenko | 4b64d20826 | |
soksanichenko | 0747e967b0 | |
soksanichenko | 6d58bc2ed8 | |
Stepan Oksanichenko | 60a347a4a2 | |
soksanichenko | 53ed7386f3 | |
soksanichenko | ed43f0038e | |
soksanichenko | fcc9b4f1ca | |
soksanichenko | d32c293bca | |
soksanichenko | f0bd1af999 | |
soksanichenko | 1b4747b915 | |
Lubomír Sedlář | 6aabfc9285 | |
Tomáš Hozza | 9e014fed6a | |
Tomáš Hozza | 7ccb1d4849 | |
Tomáš Hozza | abec28256d | |
Lubomír Sedlář | 46216b4f17 | |
Lubomír Sedlář | 02b3adbaeb | |
Lubomír Sedlář | d17e578645 | |
Lubomír Sedlář | 6c1c9d9efd | |
Stepan Oksanichenko | 8dd7d8326f | |
soksanichenko | d7b173cae5 | |
soksanichenko | fa4640f03e | |
Stepan Oksanichenko | d66eb0dea8 | |
soksanichenko | d56227ab4a | |
soksanichenko | 12433157dd | |
soksanichenko | 623955cb1f | |
soksanichenko | 4e0d2d14c9 | |
soksanichenko | b61e59d676 | |
soksanichenko | eb35d7baac | |
soksanichenko | 54209f3643 | |
soksanichenko | 80c4536eaa | |
soksanichenko | 9bb5550d36 | |
soksanichenko | 364ed6c3af | |
soksanichenko | 0b965096ee | |
soksanichenko | d914626d92 | |
soksanichenko | 32215d955a | |
soksanichenko | d711f8a2d6 | |
soksanichenko | bd9d800b52 | |
soksanichenko | e03648589d | |
soksanichenko | b5fe2e8129 | |
soksanichenko | b14e85324c | |
soksanichenko | 5a19ad2258 | |
soksanichenko | 9ae49dae5b | |
soksanichenko | c82cbfdc32 | |
soksanichenko | ee9c9a74e6 | |
soksanichenko | ea0f933315 | |
soksanichenko | 323d31df2b | |
soksanichenko | 9acd7f5fa4 | |
soksanichenko | a2b16eb44f | |
soksanichenko | ff946d3f7b | |
soksanichenko | ede91bcd03 | |
soksanichenko | 0fa459eb9e | |
soksanichenko | b49ffee06d | |
soksanichenko | fce5493f09 | |
soksanichenko | 750499eda1 | |
soksanichenko | d999960235 | |
soksanichenko | 6edece449d | |
Stepan Oksanichenko | dd22d94a9e | |
soksanichenko | b157a1825a | |
soksanichenko | fd298d4f17 | |
soksanichenko | f21ed6f607 | |
Stepan Oksanichenko | cfe6ec3f4e | |
soksanichenko | e6c6f74176 | |
soksanichenko | 8676941655 | |
soksanichenko | 5f74175c33 | |
soksanichenko | 1e18e8995d | |
soksanichenko | 38ea822260 | |
soksanichenko | 34eb45c7ec | |
soksanichenko | 7422d1e045 | |
soksanichenko | 97801e772e | |
soksanichenko | dff346eedb | |
soksanichenko | de53dd0bbd | |
soksanichenko | 88121619bc | |
soksanichenko | 0484426e0c | |
soksanichenko | b9d86b90e1 | |
soksanichenko | 58a16e5688 | |
soksanichenko | f2ed64d952 | |
stepan_oksanichenko | b2c49dcaf6 | |
stepan_oksanichenko | 14dd6a195f | |
stepan_oksanichenko | 084321dd97 | |
stepan_oksanichenko | 941d6b064a | |
stepan_oksanichenko | aaeee7132d | |
stepan_oksanichenko | cc4d99441c | |
stepan_oksanichenko | a435eeed06 | |
stepan_oksanichenko | b9f554bf39 | |
stepan_oksanichenko | ebf028ca3b | |
stepan_oksanichenko | 305103a38e | |
stepan_oksanichenko | 01bce26275 | |
soksanichenko | 4d763514c1 | |
Danylo Kuropiatnyk | 41381df6a5 | |
soksanichenko | 02686d7bdf | |
soksanichenko | 2e48c9a56f | |
soksanichenko | b3a8c3f28a | |
soksanichenko | 5434d24027 | |
soksanichenko | 3b5501b4bf | |
soksanichenko | cea8d92906 | |
soksanichenko | 1a29de435e | |
soksanichenko | 69ed7699e8 | |
Stepan Oksanichenko | 103c3dc608 | |
Stepan Oksanichenko | 94ad7603b8 | |
oshyshatskyi | 903db91c0f | |
oshyshatskyi | 552343fffe | |
oshyshatskyi | 5806217041 | |
Andrew Lukoshko | 67eacf8483 | |
Ken Dreyer | 38789d07ee | |
Lubomír Sedlář | 3735aaa443 | |
Haibo Lin | 2c1603c414 | |
Haibo Lin | f2fd10b0ab | |
Sergey Fokin | ac601ab8ea | |
oshyshatskyi | 757a6ed653 | |
Oleksandr Shyshatskyi | b2e439e561 |
1
TODO
1
TODO
|
@ -47,6 +47,7 @@ Split Pungi into smaller well-defined tools
|
||||||
|
|
||||||
* create install images
|
* create install images
|
||||||
* lorax
|
* lorax
|
||||||
|
* buildinstall
|
||||||
|
|
||||||
* create isos
|
* create isos
|
||||||
* isos
|
* isos
|
||||||
|
|
|
@ -53,7 +53,7 @@ copyright = "2016, Red Hat, Inc."
|
||||||
# The short X.Y version.
|
# The short X.Y version.
|
||||||
version = "4.5"
|
version = "4.5"
|
||||||
# The full version, including alpha/beta/rc tags.
|
# The full version, including alpha/beta/rc tags.
|
||||||
release = "4.5.1"
|
release = "4.5.0"
|
||||||
|
|
||||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||||
# for a list of supported languages.
|
# for a list of supported languages.
|
||||||
|
|
|
@ -642,7 +642,7 @@ Options
|
||||||
-------
|
-------
|
||||||
|
|
||||||
**buildinstall_method**
|
**buildinstall_method**
|
||||||
(*str*) -- "lorax" (f16+, rhel7+)
|
(*str*) -- "lorax" (f16+, rhel7+) or "buildinstall" (older releases)
|
||||||
**lorax_options**
|
**lorax_options**
|
||||||
(*list*) -- special options passed on to *lorax*.
|
(*list*) -- special options passed on to *lorax*.
|
||||||
|
|
||||||
|
@ -1652,8 +1652,6 @@ OSBuild Composer for building images
|
||||||
* ``arches`` -- list of architectures for which to build the image. By
|
* ``arches`` -- list of architectures for which to build the image. By
|
||||||
default, the variant arches are used. This option can only restrict it,
|
default, the variant arches are used. This option can only restrict it,
|
||||||
not add a new one.
|
not add a new one.
|
||||||
* ``manifest_type`` -- the image type that is put into the manifest by
|
|
||||||
pungi. If not supplied then it is autodetected from the Koji output.
|
|
||||||
* ``ostree_url`` -- URL of the repository that's used to fetch the parent
|
* ``ostree_url`` -- URL of the repository that's used to fetch the parent
|
||||||
commit from.
|
commit from.
|
||||||
* ``ostree_ref`` -- name of the ostree branch
|
* ``ostree_ref`` -- name of the ostree branch
|
||||||
|
|
|
@ -30,14 +30,17 @@ packages to architectures.
|
||||||
Buildinstall
|
Buildinstall
|
||||||
------------
|
------------
|
||||||
|
|
||||||
Spawns a bunch of threads, each of which runs the ``lorax`` command. The
|
Spawns a bunch of threads, each of which runs either ``lorax`` or
|
||||||
|
``buildinstall`` command (the latter coming from ``anaconda`` package). The
|
||||||
commands create ``boot.iso`` and other boot configuration files. The image is
|
commands create ``boot.iso`` and other boot configuration files. The image is
|
||||||
finally linked into the ``compose/`` directory as netinstall media.
|
finally linked into the ``compose/`` directory as netinstall media.
|
||||||
|
|
||||||
The created images are also needed for creating live media or other images in
|
The created images are also needed for creating live media or other images in
|
||||||
later phases.
|
later phases.
|
||||||
|
|
||||||
With ``lorax`` this phase runs one task per variant.arch combination.
|
With ``lorax`` this phase runs one task per variant.arch combination. For
|
||||||
|
``buildinstall`` command there is only one task per architecture and
|
||||||
|
``product.img`` should be used to customize the results.
|
||||||
|
|
||||||
Gather
|
Gather
|
||||||
------
|
------
|
||||||
|
|
937
pungi.spec
937
pungi.spec
File diff suppressed because it is too large
Load Diff
|
@ -572,6 +572,7 @@ def make_schema():
|
||||||
},
|
},
|
||||||
"required": ["kickstart"],
|
"required": ["kickstart"],
|
||||||
"additionalProperties": False,
|
"additionalProperties": False,
|
||||||
|
"type": "object",
|
||||||
},
|
},
|
||||||
"osbs_config": {
|
"osbs_config": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
@ -608,6 +609,7 @@ def make_schema():
|
||||||
"release_discinfo_description": {"type": "string"},
|
"release_discinfo_description": {"type": "string"},
|
||||||
"treeinfo_version": {"type": "string"},
|
"treeinfo_version": {"type": "string"},
|
||||||
"compose_type": {"type": "string", "enum": COMPOSE_TYPES},
|
"compose_type": {"type": "string", "enum": COMPOSE_TYPES},
|
||||||
|
"label": {"type": "string"},
|
||||||
"base_product_name": {"type": "string"},
|
"base_product_name": {"type": "string"},
|
||||||
"base_product_short": {"type": "string"},
|
"base_product_short": {"type": "string"},
|
||||||
"base_product_version": {"type": "string"},
|
"base_product_version": {"type": "string"},
|
||||||
|
@ -685,7 +687,11 @@ def make_schema():
|
||||||
"pkgset_allow_reuse": {"type": "boolean", "default": True},
|
"pkgset_allow_reuse": {"type": "boolean", "default": True},
|
||||||
"createiso_allow_reuse": {"type": "boolean", "default": True},
|
"createiso_allow_reuse": {"type": "boolean", "default": True},
|
||||||
"extraiso_allow_reuse": {"type": "boolean", "default": True},
|
"extraiso_allow_reuse": {"type": "boolean", "default": True},
|
||||||
"pkgset_source": {"type": "string", "enum": ["koji", "repos"]},
|
"pkgset_source": {"type": "string", "enum": [
|
||||||
|
"koji",
|
||||||
|
"repos",
|
||||||
|
"kojimock",
|
||||||
|
]},
|
||||||
"createrepo_c": {"type": "boolean", "default": True},
|
"createrepo_c": {"type": "boolean", "default": True},
|
||||||
"createrepo_checksum": {
|
"createrepo_checksum": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
@ -812,7 +818,15 @@ def make_schema():
|
||||||
"buildinstall_allow_reuse": {"type": "boolean", "default": False},
|
"buildinstall_allow_reuse": {"type": "boolean", "default": False},
|
||||||
"buildinstall_method": {
|
"buildinstall_method": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"enum": ["lorax"],
|
"enum": ["lorax", "buildinstall"],
|
||||||
|
},
|
||||||
|
# In phase `buildinstall` we should add to compose only the
|
||||||
|
# images that will be used only as netinstall
|
||||||
|
"netinstall_variants": {
|
||||||
|
"$ref": "#/definitions/list_of_strings",
|
||||||
|
"default": [
|
||||||
|
"BaseOS",
|
||||||
|
],
|
||||||
},
|
},
|
||||||
"buildinstall_topdir": {"type": "string"},
|
"buildinstall_topdir": {"type": "string"},
|
||||||
"buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"},
|
"buildinstall_kickstart": {"$ref": "#/definitions/str_or_scm_dict"},
|
||||||
|
@ -1243,7 +1257,6 @@ def make_schema():
|
||||||
"ostree_url": {"type": "string"},
|
"ostree_url": {"type": "string"},
|
||||||
"ostree_ref": {"type": "string"},
|
"ostree_ref": {"type": "string"},
|
||||||
"ostree_parent": {"type": "string"},
|
"ostree_parent": {"type": "string"},
|
||||||
"manifest_type": {"type": "string"},
|
|
||||||
"upload_options": {
|
"upload_options": {
|
||||||
# this should be really 'oneOf', but the minimal
|
# this should be really 'oneOf', but the minimal
|
||||||
# required properties in AWSEC2 and GCP options
|
# required properties in AWSEC2 and GCP options
|
||||||
|
@ -1497,6 +1510,7 @@ def get_num_cpus():
|
||||||
CONFIG_DEPS = {
|
CONFIG_DEPS = {
|
||||||
"buildinstall_method": {
|
"buildinstall_method": {
|
||||||
"conflicts": (
|
"conflicts": (
|
||||||
|
(lambda val: val == "buildinstall", ["lorax_options"]),
|
||||||
(lambda val: not val, ["lorax_options", "buildinstall_kickstart"]),
|
(lambda val: not val, ["lorax_options", "buildinstall_kickstart"]),
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
|
|
@ -67,6 +67,10 @@ def make_image(f, opts):
|
||||||
os.path.join("$TEMPLATE", "config_files/ppc"),
|
os.path.join("$TEMPLATE", "config_files/ppc"),
|
||||||
hfs_compat=opts.hfs_compat,
|
hfs_compat=opts.hfs_compat,
|
||||||
)
|
)
|
||||||
|
elif opts.buildinstall_method == "buildinstall":
|
||||||
|
mkisofs_kwargs["boot_args"] = iso.get_boot_options(
|
||||||
|
opts.arch, "/usr/lib/anaconda-runtime/boot"
|
||||||
|
)
|
||||||
|
|
||||||
# ppc(64) doesn't seem to support utf-8
|
# ppc(64) doesn't seem to support utf-8
|
||||||
if opts.arch in ("ppc", "ppc64", "ppc64le"):
|
if opts.arch in ("ppc", "ppc64", "ppc64le"):
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from functools import cmp_to_key
|
from functools import cmp_to_key
|
||||||
from itertools import count, groupby
|
from itertools import count, groupby
|
||||||
import errno
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
@ -1068,11 +1067,8 @@ class Gather(GatherBase):
|
||||||
# Link downloaded package in (or link package from file repo)
|
# Link downloaded package in (or link package from file repo)
|
||||||
try:
|
try:
|
||||||
linker.link(pkg.localPkg(), target)
|
linker.link(pkg.localPkg(), target)
|
||||||
except Exception as ex:
|
except Exception:
|
||||||
if ex.errno == errno.EEXIST:
|
self.logger.error("Unable to link %s from the yum cache." % pkg.name)
|
||||||
self.logger.warning("Downloaded package exists in %s", target)
|
|
||||||
else:
|
|
||||||
self.logger.error("Unable to link %s from the yum cache.", pkg.name)
|
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def log_count(self, msg, method, *args):
|
def log_count(self, msg, method, *args):
|
||||||
|
|
|
@ -219,6 +219,10 @@ class BuildinstallPhase(PhaseBase):
|
||||||
return repos
|
return repos
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
lorax = LoraxWrapper()
|
||||||
|
product = self.compose.conf["release_name"]
|
||||||
|
version = self.compose.conf["release_version"]
|
||||||
|
release = self.compose.conf["release_version"]
|
||||||
disc_type = self.compose.conf["disc_types"].get("dvd", "dvd")
|
disc_type = self.compose.conf["disc_types"].get("dvd", "dvd")
|
||||||
|
|
||||||
# Prepare kickstart file for final images.
|
# Prepare kickstart file for final images.
|
||||||
|
@ -271,6 +275,23 @@ class BuildinstallPhase(PhaseBase):
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
elif self.buildinstall_method == "buildinstall":
|
||||||
|
volid = get_volid(self.compose, arch, disc_type=disc_type)
|
||||||
|
commands.append(
|
||||||
|
(
|
||||||
|
None,
|
||||||
|
lorax.get_buildinstall_cmd(
|
||||||
|
product,
|
||||||
|
version,
|
||||||
|
release,
|
||||||
|
repo_baseurls,
|
||||||
|
output_dir,
|
||||||
|
is_final=self.compose.supported,
|
||||||
|
buildarch=arch,
|
||||||
|
volid=volid,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Unsupported buildinstall method: %s" % self.buildinstall_method
|
"Unsupported buildinstall method: %s" % self.buildinstall_method
|
||||||
|
@ -525,6 +546,13 @@ def link_boot_iso(compose, arch, variant, can_fail):
|
||||||
img.volume_id = iso.get_volume_id(new_boot_iso_path)
|
img.volume_id = iso.get_volume_id(new_boot_iso_path)
|
||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
pass
|
pass
|
||||||
|
# In this phase we should add to compose only the images that
|
||||||
|
# will be used only as netinstall.
|
||||||
|
# On this step lorax generates environment
|
||||||
|
# for creating isos and create them.
|
||||||
|
# On step `extra_isos` we overwrite the not needed iso `boot Minimal` by
|
||||||
|
# new iso. It already contains necessary packages from incldued variants.
|
||||||
|
if variant.uid in compose.conf['netinstall_variants']:
|
||||||
compose.im.add(variant.uid, arch, img)
|
compose.im.add(variant.uid, arch, img)
|
||||||
compose.log_info("[DONE ] %s" % msg)
|
compose.log_info("[DONE ] %s" % msg)
|
||||||
|
|
||||||
|
@ -806,6 +834,8 @@ class BuildinstallThread(WorkerThread):
|
||||||
if buildinstall_method == "lorax":
|
if buildinstall_method == "lorax":
|
||||||
packages += ["lorax"]
|
packages += ["lorax"]
|
||||||
chown_paths.append(_get_log_dir(compose, variant, arch))
|
chown_paths.append(_get_log_dir(compose, variant, arch))
|
||||||
|
elif buildinstall_method == "buildinstall":
|
||||||
|
packages += ["anaconda"]
|
||||||
packages += get_arch_variant_data(
|
packages += get_arch_variant_data(
|
||||||
compose.conf, "buildinstall_packages", arch, variant
|
compose.conf, "buildinstall_packages", arch, variant
|
||||||
)
|
)
|
||||||
|
|
|
@ -154,13 +154,6 @@ class CreateisoPhase(PhaseLoggerMixin, PhaseBase):
|
||||||
disc_num=cmd["disc_num"],
|
disc_num=cmd["disc_num"],
|
||||||
disc_count=cmd["disc_count"],
|
disc_count=cmd["disc_count"],
|
||||||
)
|
)
|
||||||
if self.compose.notifier:
|
|
||||||
self.compose.notifier.send(
|
|
||||||
"createiso-imagedone",
|
|
||||||
file=cmd["iso_path"],
|
|
||||||
arch=arch,
|
|
||||||
variant=str(variant),
|
|
||||||
)
|
|
||||||
|
|
||||||
def try_reuse(self, cmd, variant, arch, opts):
|
def try_reuse(self, cmd, variant, arch, opts):
|
||||||
"""Try to reuse image from previous compose.
|
"""Try to reuse image from previous compose.
|
||||||
|
@ -546,6 +539,7 @@ def run_createiso_command(num, compose, bootable, arch, cmd, mounts, log_file):
|
||||||
if bootable:
|
if bootable:
|
||||||
extra_packages = {
|
extra_packages = {
|
||||||
"lorax": ["lorax", "which"],
|
"lorax": ["lorax", "which"],
|
||||||
|
"buildinstall": ["anaconda"],
|
||||||
}
|
}
|
||||||
packages.extend(extra_packages[compose.conf["buildinstall_method"]])
|
packages.extend(extra_packages[compose.conf["buildinstall_method"]])
|
||||||
|
|
||||||
|
|
|
@ -420,6 +420,12 @@ def get_iso_contents(
|
||||||
original_treeinfo,
|
original_treeinfo,
|
||||||
os.path.join(extra_files_dir, ".treeinfo"),
|
os.path.join(extra_files_dir, ".treeinfo"),
|
||||||
)
|
)
|
||||||
|
tweak_repo_treeinfo(
|
||||||
|
compose,
|
||||||
|
include_variants,
|
||||||
|
original_treeinfo,
|
||||||
|
original_treeinfo,
|
||||||
|
)
|
||||||
|
|
||||||
# Add extra files specific for the ISO
|
# Add extra files specific for the ISO
|
||||||
files.update(
|
files.update(
|
||||||
|
@ -431,6 +437,45 @@ def get_iso_contents(
|
||||||
return gp
|
return gp
|
||||||
|
|
||||||
|
|
||||||
|
def tweak_repo_treeinfo(compose, include_variants, source_file, dest_file):
|
||||||
|
"""
|
||||||
|
The method includes the variants to file .treeinfo of a variant. It takes
|
||||||
|
the variants which are described
|
||||||
|
by options `extra_isos -> include_variants`.
|
||||||
|
"""
|
||||||
|
ti = productmd.treeinfo.TreeInfo()
|
||||||
|
ti.load(source_file)
|
||||||
|
main_variant = next(iter(ti.variants))
|
||||||
|
for variant_uid in include_variants:
|
||||||
|
variant = compose.all_variants[variant_uid]
|
||||||
|
var = productmd.treeinfo.Variant(ti)
|
||||||
|
var.id = variant.id
|
||||||
|
var.uid = variant.uid
|
||||||
|
var.name = variant.name
|
||||||
|
var.type = variant.type
|
||||||
|
ti.variants.add(var)
|
||||||
|
|
||||||
|
for variant_id in ti.variants:
|
||||||
|
var = ti.variants[variant_id]
|
||||||
|
if variant_id == main_variant:
|
||||||
|
var.paths.packages = 'Packages'
|
||||||
|
var.paths.repository = '.'
|
||||||
|
else:
|
||||||
|
var.paths.packages = os.path.join(
|
||||||
|
'../../..',
|
||||||
|
var.uid,
|
||||||
|
var.arch,
|
||||||
|
'os/Packages',
|
||||||
|
)
|
||||||
|
var.paths.repository = os.path.join(
|
||||||
|
'../../..',
|
||||||
|
var.uid,
|
||||||
|
var.arch,
|
||||||
|
'os',
|
||||||
|
)
|
||||||
|
ti.dump(dest_file, main_variant=main_variant)
|
||||||
|
|
||||||
|
|
||||||
def tweak_treeinfo(compose, include_variants, source_file, dest_file):
|
def tweak_treeinfo(compose, include_variants, source_file, dest_file):
|
||||||
ti = load_and_tweak_treeinfo(source_file)
|
ti = load_and_tweak_treeinfo(source_file)
|
||||||
for variant_uid in include_variants:
|
for variant_uid in include_variants:
|
||||||
|
@ -446,7 +491,6 @@ def tweak_treeinfo(compose, include_variants, source_file, dest_file):
|
||||||
var = ti.variants[variant_id]
|
var = ti.variants[variant_id]
|
||||||
var.paths.packages = os.path.join(var.uid, "Packages")
|
var.paths.packages = os.path.join(var.uid, "Packages")
|
||||||
var.paths.repository = var.uid
|
var.paths.repository = var.uid
|
||||||
|
|
||||||
ti.dump(dest_file)
|
ti.dump(dest_file)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -23,6 +23,7 @@ import threading
|
||||||
from kobo.rpmlib import parse_nvra
|
from kobo.rpmlib import parse_nvra
|
||||||
from kobo.shortcuts import run
|
from kobo.shortcuts import run
|
||||||
from productmd.rpms import Rpms
|
from productmd.rpms import Rpms
|
||||||
|
from pungi.phases.pkgset.common import get_all_arches
|
||||||
from six.moves import cPickle as pickle
|
from six.moves import cPickle as pickle
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -649,6 +650,11 @@ def _make_lookaside_repo(compose, variant, arch, pkg_map, package_sets=None):
|
||||||
pungi.wrappers.kojiwrapper.KojiWrapper(compose).koji_module.config.topdir,
|
pungi.wrappers.kojiwrapper.KojiWrapper(compose).koji_module.config.topdir,
|
||||||
).rstrip("/")
|
).rstrip("/")
|
||||||
+ "/",
|
+ "/",
|
||||||
|
"kojimock": lambda: pungi.wrappers.kojiwrapper.KojiMockWrapper(
|
||||||
|
compose,
|
||||||
|
get_all_arches(compose),
|
||||||
|
).koji_module.config.topdir.rstrip("/")
|
||||||
|
+ "/",
|
||||||
}
|
}
|
||||||
path_prefix = prefixes[compose.conf["pkgset_source"]]()
|
path_prefix = prefixes[compose.conf["pkgset_source"]]()
|
||||||
package_list = set()
|
package_list = set()
|
||||||
|
|
|
@ -249,24 +249,7 @@ class RunOSBuildThread(WorkerThread):
|
||||||
|
|
||||||
# Update image manifest
|
# Update image manifest
|
||||||
img = Image(compose.im)
|
img = Image(compose.im)
|
||||||
|
img.type = archive["type_name"] if archive["type_name"] != "iso" else "dvd"
|
||||||
# Get the manifest type from the config if supplied, otherwise we
|
|
||||||
# determine the manifest type based on the koji output
|
|
||||||
img.type = config.get("manifest_type")
|
|
||||||
if not img.type:
|
|
||||||
if archive["type_name"] != "iso":
|
|
||||||
img.type = archive["type_name"]
|
|
||||||
else:
|
|
||||||
fn = archive["filename"].lower()
|
|
||||||
if "ostree" in fn:
|
|
||||||
img.type = "dvd-ostree-osbuild"
|
|
||||||
elif "live" in fn:
|
|
||||||
img.type = "live-osbuild"
|
|
||||||
elif "netinst" in fn or "boot" in fn:
|
|
||||||
img.type = "boot"
|
|
||||||
else:
|
|
||||||
img.type = "dvd"
|
|
||||||
|
|
||||||
img.format = suffix
|
img.format = suffix
|
||||||
img.path = os.path.join(rel_image_dir, archive["filename"])
|
img.path = os.path.join(rel_image_dir, archive["filename"])
|
||||||
img.mtime = util.get_mtime(image_dest)
|
img.mtime = util.get_mtime(image_dest)
|
||||||
|
|
|
@ -23,6 +23,8 @@ import itertools
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
|
import pgpy
|
||||||
|
import rpm
|
||||||
from six.moves import cPickle as pickle
|
from six.moves import cPickle as pickle
|
||||||
from functools import partial
|
from functools import partial
|
||||||
|
|
||||||
|
@ -152,9 +154,15 @@ class PackageSetBase(kobo.log.LoggingBase):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def nvr_formatter(package_info):
|
def nvr_formatter(package_info):
|
||||||
# joins NVR parts of the package with '-' character.
|
epoch_suffix = ''
|
||||||
return "-".join(
|
if package_info['epoch'] is not None:
|
||||||
(package_info["name"], package_info["version"], package_info["release"])
|
epoch_suffix = ':' + package_info['epoch']
|
||||||
|
return (
|
||||||
|
f"{package_info['name']}"
|
||||||
|
f"{epoch_suffix}-"
|
||||||
|
f"{package_info['version']}-"
|
||||||
|
f"{package_info['release']}."
|
||||||
|
f"{package_info['arch']}"
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_error(sigkeys, infos):
|
def get_error(sigkeys, infos):
|
||||||
|
@ -503,7 +511,8 @@ class KojiPackageSet(PackageSetBase):
|
||||||
|
|
||||||
response = None
|
response = None
|
||||||
if self.cache_region:
|
if self.cache_region:
|
||||||
cache_key = "KojiPackageSet.get_latest_rpms_%s_%s_%s" % (
|
cache_key = "%s.get_latest_rpms_%s_%s_%s" % (
|
||||||
|
str(self.__class__.__name__),
|
||||||
str(tag),
|
str(tag),
|
||||||
str(event),
|
str(event),
|
||||||
str(inherit),
|
str(inherit),
|
||||||
|
@ -525,6 +534,8 @@ class KojiPackageSet(PackageSetBase):
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def get_package_path(self, queue_item):
|
def get_package_path(self, queue_item):
|
||||||
rpm_info, build_info = queue_item
|
rpm_info, build_info = queue_item
|
||||||
|
|
||||||
|
@ -614,7 +625,7 @@ class KojiPackageSet(PackageSetBase):
|
||||||
result_srpms = []
|
result_srpms = []
|
||||||
include_packages = set(include_packages or [])
|
include_packages = set(include_packages or [])
|
||||||
|
|
||||||
if isinstance(event, dict):
|
if type(event) is dict:
|
||||||
event = event["id"]
|
event = event["id"]
|
||||||
|
|
||||||
msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (
|
msg = "Getting latest RPMs (tag: %s, event: %s, inherit: %s)" % (
|
||||||
|
@ -882,6 +893,67 @@ class KojiPackageSet(PackageSetBase):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class KojiMockPackageSet(KojiPackageSet):
|
||||||
|
|
||||||
|
def _is_rpm_signed(self, rpm_path) -> bool:
|
||||||
|
ts = rpm.TransactionSet()
|
||||||
|
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES)
|
||||||
|
sigkeys = [
|
||||||
|
sigkey.lower() for sigkey in self.sigkey_ordering
|
||||||
|
if sigkey is not None
|
||||||
|
]
|
||||||
|
if not sigkeys:
|
||||||
|
return True
|
||||||
|
with open(rpm_path, 'rb') as fd:
|
||||||
|
header = ts.hdrFromFdno(fd)
|
||||||
|
signature = header[rpm.RPMTAG_SIGGPG] or header[rpm.RPMTAG_SIGPGP]
|
||||||
|
if signature is None:
|
||||||
|
return False
|
||||||
|
pgp_msg = pgpy.PGPMessage.from_blob(signature)
|
||||||
|
return any(
|
||||||
|
signature.signer.lower() in sigkeys
|
||||||
|
for signature in pgp_msg.signatures
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_package_path(self, queue_item):
|
||||||
|
rpm_info, build_info = queue_item
|
||||||
|
|
||||||
|
# Check if this RPM is coming from scratch task.
|
||||||
|
# In this case, we already know the path.
|
||||||
|
if "path_from_task" in rpm_info:
|
||||||
|
return rpm_info["path_from_task"]
|
||||||
|
|
||||||
|
# we replaced this part because pungi uses way
|
||||||
|
# of guessing path of package on koji based on sigkey
|
||||||
|
# we don't need that because all our packages will
|
||||||
|
# be ready for release
|
||||||
|
# signature verification is still done during deps resolution
|
||||||
|
pathinfo = self.koji_wrapper.koji_module.pathinfo
|
||||||
|
|
||||||
|
rpm_path = os.path.join(pathinfo.topdir, pathinfo.rpm(rpm_info))
|
||||||
|
if os.path.isfile(rpm_path):
|
||||||
|
if not self._is_rpm_signed(rpm_path):
|
||||||
|
self._invalid_sigkey_rpms.append(rpm_info)
|
||||||
|
self.log_error(
|
||||||
|
'RPM "%s" not found for sigs: "%s". Path checked: "%s"',
|
||||||
|
rpm_info, self.sigkey_ordering, rpm_path
|
||||||
|
)
|
||||||
|
return
|
||||||
|
return rpm_path
|
||||||
|
else:
|
||||||
|
self.log_warning("RPM %s not found" % rpm_path)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def populate(self, tag, event=None, inherit=True, include_packages=None):
|
||||||
|
result = super().populate(
|
||||||
|
tag=tag,
|
||||||
|
event=event,
|
||||||
|
inherit=inherit,
|
||||||
|
include_packages=include_packages,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _is_src(rpm_info):
|
def _is_src(rpm_info):
|
||||||
"""Check if rpm info object returned by Koji refers to source packages."""
|
"""Check if rpm info object returned by Koji refers to source packages."""
|
||||||
return rpm_info["arch"] in ("src", "nosrc")
|
return rpm_info["arch"] in ("src", "nosrc")
|
||||||
|
|
|
@ -15,8 +15,10 @@
|
||||||
|
|
||||||
from .source_koji import PkgsetSourceKoji
|
from .source_koji import PkgsetSourceKoji
|
||||||
from .source_repos import PkgsetSourceRepos
|
from .source_repos import PkgsetSourceRepos
|
||||||
|
from .source_kojimock import PkgsetSourceKojiMock
|
||||||
|
|
||||||
ALL_SOURCES = {
|
ALL_SOURCES = {
|
||||||
"koji": PkgsetSourceKoji,
|
"koji": PkgsetSourceKoji,
|
||||||
"repos": PkgsetSourceRepos,
|
"repos": PkgsetSourceRepos,
|
||||||
|
"kojimock": PkgsetSourceKojiMock,
|
||||||
}
|
}
|
||||||
|
|
|
@ -487,16 +487,7 @@ def filter_inherited(koji_proxy, event, module_builds, top_tag):
|
||||||
# And keep only builds from that topmost tag
|
# And keep only builds from that topmost tag
|
||||||
result.extend(build for build in builds if build["tag_name"] == tag)
|
result.extend(build for build in builds if build["tag_name"] == tag)
|
||||||
|
|
||||||
# If the same module was inherited multiple times, it will be in result
|
return result
|
||||||
# multiple times. We need to deduplicate.
|
|
||||||
deduplicated_result = []
|
|
||||||
included_nvrs = set()
|
|
||||||
for build in result:
|
|
||||||
if build["nvr"] not in included_nvrs:
|
|
||||||
deduplicated_result.append(build)
|
|
||||||
included_nvrs.add(build["nvr"])
|
|
||||||
|
|
||||||
return deduplicated_result
|
|
||||||
|
|
||||||
|
|
||||||
def filter_by_whitelist(compose, module_builds, input_modules, expected_modules):
|
def filter_by_whitelist(compose, module_builds, input_modules, expected_modules):
|
||||||
|
@ -901,13 +892,7 @@ def populate_global_pkgset(compose, koji_wrapper, event):
|
||||||
if pkgset.reuse is None:
|
if pkgset.reuse is None:
|
||||||
pkgset.populate(
|
pkgset.populate(
|
||||||
compose_tag,
|
compose_tag,
|
||||||
# We care about packages as they existed on the specified
|
event,
|
||||||
# event. However, modular content tags are not expected to
|
|
||||||
# change, so the event doesn't matter there. If an exact NSVC
|
|
||||||
# of a module is specified, the code above would happily find
|
|
||||||
# its content tag, but fail here if the content tag doesn't
|
|
||||||
# exist at the given event.
|
|
||||||
event=event if is_traditional else None,
|
|
||||||
inherit=should_inherit,
|
inherit=should_inherit,
|
||||||
include_packages=modular_packages,
|
include_packages=modular_packages,
|
||||||
)
|
)
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -94,7 +94,7 @@ class Runroot(kobo.log.LoggingBase):
|
||||||
log_file = os.path.join(log_dir, "program.log")
|
log_file = os.path.join(log_dir, "program.log")
|
||||||
try:
|
try:
|
||||||
with open(log_file) as f:
|
with open(log_file) as f:
|
||||||
for line in f:
|
for line in f.readlines():
|
||||||
if "losetup: cannot find an unused loop device" in line:
|
if "losetup: cannot find an unused loop device" in line:
|
||||||
return True
|
return True
|
||||||
if re.match("losetup: .* failed to set up loop device", line):
|
if re.match("losetup: .* failed to set up loop device", line):
|
||||||
|
|
|
@ -0,0 +1,441 @@
|
||||||
|
# coding=utf-8
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from shutil import rmtree
|
||||||
|
from typing import (
|
||||||
|
AnyStr,
|
||||||
|
List,
|
||||||
|
Dict,
|
||||||
|
Optional,
|
||||||
|
)
|
||||||
|
|
||||||
|
import createrepo_c as cr
|
||||||
|
import requests
|
||||||
|
import yaml
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
from .create_packages_json import (
|
||||||
|
PackagesGenerator,
|
||||||
|
RepoInfo,
|
||||||
|
VariantInfo,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ExtraVariantInfo(VariantInfo):
|
||||||
|
|
||||||
|
modules: List[AnyStr] = field(default_factory=list)
|
||||||
|
packages: List[AnyStr] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class CreateExtraRepo(PackagesGenerator):
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
variants: List[ExtraVariantInfo],
|
||||||
|
bs_auth_token: AnyStr,
|
||||||
|
local_repository_path: AnyStr,
|
||||||
|
clear_target_repo: bool = True,
|
||||||
|
):
|
||||||
|
self.variants = [] # type: List[ExtraVariantInfo]
|
||||||
|
super().__init__(variants, [], [])
|
||||||
|
self.auth_headers = {
|
||||||
|
'Authorization': f'Bearer {bs_auth_token}',
|
||||||
|
}
|
||||||
|
# modules data of modules.yaml.gz from an existing local repo
|
||||||
|
self.local_modules_data = []
|
||||||
|
self.local_repository_path = local_repository_path
|
||||||
|
# path to modules.yaml, which generated by the class
|
||||||
|
self.default_modules_yaml_path = os.path.join(
|
||||||
|
local_repository_path,
|
||||||
|
'modules.yaml',
|
||||||
|
)
|
||||||
|
if clear_target_repo:
|
||||||
|
if os.path.exists(self.local_repository_path):
|
||||||
|
rmtree(self.local_repository_path)
|
||||||
|
os.makedirs(self.local_repository_path, exist_ok=True)
|
||||||
|
else:
|
||||||
|
self._read_local_modules_yaml()
|
||||||
|
|
||||||
|
def _read_local_modules_yaml(self):
|
||||||
|
"""
|
||||||
|
Read modules data from an existin local repo
|
||||||
|
"""
|
||||||
|
repomd_file_path = os.path.join(
|
||||||
|
self.local_repository_path,
|
||||||
|
'repodata',
|
||||||
|
'repomd.xml',
|
||||||
|
)
|
||||||
|
repomd_object = self._parse_repomd(repomd_file_path)
|
||||||
|
for repomd_record in repomd_object.records:
|
||||||
|
if repomd_record.type != 'modules':
|
||||||
|
continue
|
||||||
|
modules_yaml_path = os.path.join(
|
||||||
|
self.local_repository_path,
|
||||||
|
repomd_record.location_href,
|
||||||
|
)
|
||||||
|
self.local_modules_data = list(self._parse_modules_file(
|
||||||
|
modules_yaml_path,
|
||||||
|
))
|
||||||
|
break
|
||||||
|
|
||||||
|
def _dump_local_modules_yaml(self):
|
||||||
|
"""
|
||||||
|
Dump merged modules data to an local repo
|
||||||
|
"""
|
||||||
|
if self.local_modules_data:
|
||||||
|
with open(self.default_modules_yaml_path, 'w') as yaml_file:
|
||||||
|
yaml.dump_all(
|
||||||
|
self.local_modules_data,
|
||||||
|
yaml_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_repo_info_from_bs_repo(
|
||||||
|
auth_token: AnyStr,
|
||||||
|
build_id: AnyStr,
|
||||||
|
arch: AnyStr,
|
||||||
|
packages: Optional[List[AnyStr]] = None,
|
||||||
|
modules: Optional[List[AnyStr]] = None,
|
||||||
|
) -> List[ExtraVariantInfo]:
|
||||||
|
"""
|
||||||
|
Get info about a BS repo and save it to
|
||||||
|
an object of class ExtraRepoInfo
|
||||||
|
:param auth_token: Auth token to Build System
|
||||||
|
:param build_id: ID of a build from BS
|
||||||
|
:param arch: an architecture of repo which will be used
|
||||||
|
:param packages: list of names of packages which will be put to an
|
||||||
|
local repo from a BS repo
|
||||||
|
:param modules: list of names of modules which will be put to an
|
||||||
|
local repo from a BS repo
|
||||||
|
:return: list of ExtraRepoInfo with info about the BS repos
|
||||||
|
"""
|
||||||
|
|
||||||
|
bs_url = 'https://build.cloudlinux.com'
|
||||||
|
api_uri = 'api/v1'
|
||||||
|
bs_repo_suffix = 'build_repos'
|
||||||
|
|
||||||
|
variants_info = []
|
||||||
|
|
||||||
|
# get the full info about a BS repo
|
||||||
|
repo_request = requests.get(
|
||||||
|
url=os.path.join(
|
||||||
|
bs_url,
|
||||||
|
api_uri,
|
||||||
|
'builds',
|
||||||
|
build_id,
|
||||||
|
),
|
||||||
|
headers={
|
||||||
|
'Authorization': f'Bearer {auth_token}',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
repo_request.raise_for_status()
|
||||||
|
result = repo_request.json()
|
||||||
|
for build_platform in result['build_platforms']:
|
||||||
|
platform_name = build_platform['name']
|
||||||
|
for architecture in build_platform['architectures']:
|
||||||
|
# skip repo with unsuitable architecture
|
||||||
|
if architecture != arch:
|
||||||
|
continue
|
||||||
|
variant_info = ExtraVariantInfo(
|
||||||
|
name=f'{build_id}-{platform_name}-{architecture}',
|
||||||
|
arch=architecture,
|
||||||
|
packages=packages,
|
||||||
|
modules=modules,
|
||||||
|
repos=[
|
||||||
|
RepoInfo(
|
||||||
|
path=os.path.join(
|
||||||
|
bs_url,
|
||||||
|
bs_repo_suffix,
|
||||||
|
build_id,
|
||||||
|
platform_name,
|
||||||
|
),
|
||||||
|
folder=architecture,
|
||||||
|
is_remote=True,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
variants_info.append(variant_info)
|
||||||
|
return variants_info
|
||||||
|
|
||||||
|
def _create_local_extra_repo(self):
|
||||||
|
"""
|
||||||
|
Call `createrepo_c <path_to_repo>` for creating a local repo
|
||||||
|
"""
|
||||||
|
subprocess.call(
|
||||||
|
f'createrepo_c {self.local_repository_path}',
|
||||||
|
shell=True,
|
||||||
|
)
|
||||||
|
# remove an unnecessary temporary modules.yaml
|
||||||
|
if os.path.exists(self.default_modules_yaml_path):
|
||||||
|
os.remove(self.default_modules_yaml_path)
|
||||||
|
|
||||||
|
def get_remote_file_content(
|
||||||
|
self,
|
||||||
|
file_url: AnyStr,
|
||||||
|
) -> AnyStr:
|
||||||
|
"""
|
||||||
|
Get content from a remote file and write it to a temp file
|
||||||
|
:param file_url: url of a remote file
|
||||||
|
:return: path to a temp file
|
||||||
|
"""
|
||||||
|
|
||||||
|
file_request = requests.get(
|
||||||
|
url=file_url,
|
||||||
|
# for the case when we get a file from BS
|
||||||
|
headers=self.auth_headers,
|
||||||
|
)
|
||||||
|
file_request.raise_for_status()
|
||||||
|
with tempfile.NamedTemporaryFile(delete=False) as file_stream:
|
||||||
|
file_stream.write(file_request.content)
|
||||||
|
return file_stream.name
|
||||||
|
|
||||||
|
def _download_rpm_to_local_repo(
|
||||||
|
self,
|
||||||
|
package_location: AnyStr,
|
||||||
|
repo_info: RepoInfo,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Download a rpm package from a remote repo and save it to a local repo
|
||||||
|
:param package_location: relative uri of a package in a remote repo
|
||||||
|
:param repo_info: info about a remote repo which contains a specific
|
||||||
|
rpm package
|
||||||
|
"""
|
||||||
|
rpm_package_remote_path = os.path.join(
|
||||||
|
repo_info.path,
|
||||||
|
repo_info.folder,
|
||||||
|
package_location,
|
||||||
|
)
|
||||||
|
rpm_package_local_path = os.path.join(
|
||||||
|
self.local_repository_path,
|
||||||
|
os.path.basename(package_location),
|
||||||
|
)
|
||||||
|
rpm_request = requests.get(
|
||||||
|
url=rpm_package_remote_path,
|
||||||
|
headers=self.auth_headers,
|
||||||
|
)
|
||||||
|
rpm_request.raise_for_status()
|
||||||
|
with open(rpm_package_local_path, 'wb') as rpm_file:
|
||||||
|
rpm_file.write(rpm_request.content)
|
||||||
|
|
||||||
|
def _download_packages(
|
||||||
|
self,
|
||||||
|
packages: Dict[AnyStr, cr.Package],
|
||||||
|
variant_info: ExtraVariantInfo
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Download all defined packages from a remote repo
|
||||||
|
:param packages: information about all packages (including
|
||||||
|
modularity) in a remote repo
|
||||||
|
:param variant_info: information about a remote variant
|
||||||
|
"""
|
||||||
|
for package in packages.values():
|
||||||
|
package_name = package.name
|
||||||
|
# Skip a current package from a remote repo if we defined
|
||||||
|
# the list packages and a current package doesn't belong to it
|
||||||
|
if variant_info.packages and \
|
||||||
|
package_name not in variant_info.packages:
|
||||||
|
continue
|
||||||
|
for repo_info in variant_info.repos:
|
||||||
|
self._download_rpm_to_local_repo(
|
||||||
|
package_location=package.location_href,
|
||||||
|
repo_info=repo_info,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _download_modules(
|
||||||
|
self,
|
||||||
|
modules_data: List[Dict],
|
||||||
|
variant_info: ExtraVariantInfo,
|
||||||
|
packages: Dict[AnyStr, cr.Package]
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Download all defined modularity packages and their data from
|
||||||
|
a remote repo
|
||||||
|
:param modules_data: information about all modules in a remote repo
|
||||||
|
:param variant_info: information about a remote variant
|
||||||
|
:param packages: information about all packages (including
|
||||||
|
modularity) in a remote repo
|
||||||
|
"""
|
||||||
|
for module in modules_data:
|
||||||
|
module_data = module['data']
|
||||||
|
# Skip a current module from a remote repo if we defined
|
||||||
|
# the list modules and a current module doesn't belong to it
|
||||||
|
if variant_info.modules and \
|
||||||
|
module_data['name'] not in variant_info.modules:
|
||||||
|
continue
|
||||||
|
# we should add info about a module if the local repodata
|
||||||
|
# doesn't have it
|
||||||
|
if module not in self.local_modules_data:
|
||||||
|
self.local_modules_data.append(module)
|
||||||
|
# just skip a module's record if it doesn't have rpm artifact
|
||||||
|
if module['document'] != 'modulemd' or \
|
||||||
|
'artifacts' not in module_data or \
|
||||||
|
'rpms' not in module_data['artifacts']:
|
||||||
|
continue
|
||||||
|
for rpm in module['data']['artifacts']['rpms']:
|
||||||
|
# Empty repo_info.packages means that we will download
|
||||||
|
# all packages from repo including
|
||||||
|
# the modularity packages
|
||||||
|
if not variant_info.packages:
|
||||||
|
break
|
||||||
|
# skip a rpm if it doesn't belong to a processed repo
|
||||||
|
if rpm not in packages:
|
||||||
|
continue
|
||||||
|
for repo_info in variant_info.repos:
|
||||||
|
self._download_rpm_to_local_repo(
|
||||||
|
package_location=packages[rpm].location_href,
|
||||||
|
repo_info=repo_info,
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_extra_repo(self):
|
||||||
|
"""
|
||||||
|
1. Get from the remote repos the specific (or all) packages/modules
|
||||||
|
2. Save them to a local repo
|
||||||
|
3. Save info about the modules to a local repo
|
||||||
|
3. Call `createrepo_c` which creates a local repo
|
||||||
|
with the right repodata
|
||||||
|
"""
|
||||||
|
for variant_info in self.variants:
|
||||||
|
for repo_info in variant_info.repos:
|
||||||
|
repomd_records = self._get_repomd_records(
|
||||||
|
repo_info=repo_info,
|
||||||
|
)
|
||||||
|
packages_iterator = self.get_packages_iterator(repo_info)
|
||||||
|
# parse the repodata (including modules.yaml.gz)
|
||||||
|
modules_data = self._parse_module_repomd_record(
|
||||||
|
repo_info=repo_info,
|
||||||
|
repomd_records=repomd_records,
|
||||||
|
)
|
||||||
|
# convert the packages dict to more usable form
|
||||||
|
# for future checking that a rpm from the module's artifacts
|
||||||
|
# belongs to a processed repository
|
||||||
|
packages = {
|
||||||
|
f'{package.name}-{package.epoch}:{package.version}-'
|
||||||
|
f'{package.release}.{package.arch}':
|
||||||
|
package for package in packages_iterator
|
||||||
|
}
|
||||||
|
self._download_modules(
|
||||||
|
modules_data=modules_data,
|
||||||
|
variant_info=variant_info,
|
||||||
|
packages=packages,
|
||||||
|
)
|
||||||
|
self._download_packages(
|
||||||
|
packages=packages,
|
||||||
|
variant_info=variant_info,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._dump_local_modules_yaml()
|
||||||
|
self._create_local_extra_repo()
|
||||||
|
|
||||||
|
|
||||||
|
def create_parser():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
'--bs-auth-token',
|
||||||
|
help='Auth token for Build System',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--local-repo-path',
|
||||||
|
help='Path to a local repo. E.g. /var/repo/test_repo',
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--clear-local-repo',
|
||||||
|
help='Clear a local repo before creating a new',
|
||||||
|
action='store_true',
|
||||||
|
default=False,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--repo',
|
||||||
|
action='append',
|
||||||
|
help='Path to a folder with repofolders or build id. E.g. '
|
||||||
|
'"http://koji.cloudlinux.com/mirrors/rhel_mirror" or '
|
||||||
|
'"601809b3c2f5b0e458b14cd3"',
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--repo-folder',
|
||||||
|
action='append',
|
||||||
|
help='A folder which contains folder repodata . E.g. "baseos-stream"',
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--repo-arch',
|
||||||
|
action='append',
|
||||||
|
help='What architecture packages a repository contains. E.g. "x86_64"',
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--packages',
|
||||||
|
action='append',
|
||||||
|
type=str,
|
||||||
|
default=[],
|
||||||
|
help='A list of packages names which we want to download to local '
|
||||||
|
'extra repo. We will download all of packages if param is empty',
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--modules',
|
||||||
|
action='append',
|
||||||
|
type=str,
|
||||||
|
default=[],
|
||||||
|
help='A list of modules names which we want to download to local '
|
||||||
|
'extra repo. We will download all of modules if param is empty',
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def cli_main():
|
||||||
|
args = create_parser().parse_args()
|
||||||
|
repos_info = []
|
||||||
|
for repo, repo_folder, repo_arch, packages, modules in zip(
|
||||||
|
args.repo,
|
||||||
|
args.repo_folder,
|
||||||
|
args.repo_arch,
|
||||||
|
args.packages,
|
||||||
|
args.modules,
|
||||||
|
):
|
||||||
|
modules = modules.split()
|
||||||
|
packages = packages.split()
|
||||||
|
if repo.startswith('http://'):
|
||||||
|
repos_info.append(
|
||||||
|
ExtraVariantInfo(
|
||||||
|
name=repo_folder,
|
||||||
|
arch=repo_arch,
|
||||||
|
repos=[
|
||||||
|
RepoInfo(
|
||||||
|
path=repo,
|
||||||
|
folder=repo_folder,
|
||||||
|
is_remote=True,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
modules=modules,
|
||||||
|
packages=packages,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
repos_info.extend(
|
||||||
|
CreateExtraRepo.get_repo_info_from_bs_repo(
|
||||||
|
auth_token=args.bs_auth_token,
|
||||||
|
build_id=repo,
|
||||||
|
arch=repo_arch,
|
||||||
|
modules=modules,
|
||||||
|
packages=packages,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cer = CreateExtraRepo(
|
||||||
|
variants=repos_info,
|
||||||
|
bs_auth_token=args.bs_auth_token,
|
||||||
|
local_repository_path=args.local_repo_path,
|
||||||
|
clear_target_repo=args.clear_local_repo,
|
||||||
|
)
|
||||||
|
cer.create_extra_repo()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
cli_main()
|
|
@ -0,0 +1,514 @@
|
||||||
|
# coding=utf-8
|
||||||
|
"""
|
||||||
|
The tool allow to generate package.json. This file is used by pungi
|
||||||
|
# as parameter `gather_prepopulate`
|
||||||
|
Sample of using repodata files taken from
|
||||||
|
https://github.com/rpm-software-management/createrepo_c/blob/master/examples/python/repodata_parsing.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import lzma
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
from collections import defaultdict
|
||||||
|
from itertools import tee
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import (
|
||||||
|
AnyStr,
|
||||||
|
Dict,
|
||||||
|
List,
|
||||||
|
Any,
|
||||||
|
Iterator,
|
||||||
|
Optional,
|
||||||
|
Tuple,
|
||||||
|
Union,
|
||||||
|
)
|
||||||
|
|
||||||
|
import binascii
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import rpm
|
||||||
|
import yaml
|
||||||
|
from createrepo_c import (
|
||||||
|
Package,
|
||||||
|
PackageIterator,
|
||||||
|
Repomd,
|
||||||
|
RepomdRecord,
|
||||||
|
)
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from kobo.rpmlib import parse_nvra
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_compressed_file(first_two_bytes: bytes, initial_bytes: bytes):
|
||||||
|
return binascii.hexlify(first_two_bytes) == initial_bytes
|
||||||
|
|
||||||
|
|
||||||
|
def is_gzip_file(first_two_bytes):
|
||||||
|
return _is_compressed_file(
|
||||||
|
first_two_bytes=first_two_bytes,
|
||||||
|
initial_bytes=b'1f8b',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def is_xz_file(first_two_bytes):
|
||||||
|
return _is_compressed_file(
|
||||||
|
first_two_bytes=first_two_bytes,
|
||||||
|
initial_bytes=b'fd37',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class RepoInfo:
|
||||||
|
# path to a directory with repo directories. E.g. '/var/repos' contains
|
||||||
|
# 'appstream', 'baseos', etc.
|
||||||
|
# Or 'http://koji.cloudlinux.com/mirrors/rhel_mirror' if you are
|
||||||
|
# using remote repo
|
||||||
|
path: str
|
||||||
|
# name of folder with a repodata folder. E.g. 'baseos', 'appstream', etc
|
||||||
|
folder: str
|
||||||
|
# Is a repo remote or local
|
||||||
|
is_remote: bool
|
||||||
|
# Is a reference repository (usually it's a RHEL repo)
|
||||||
|
# Layout of packages from such repository will be taken as example
|
||||||
|
# Only layout of specific package (which doesn't exist
|
||||||
|
# in a reference repository) will be taken as example
|
||||||
|
is_reference: bool = False
|
||||||
|
# The packages from 'present' repo will be added to a variant.
|
||||||
|
# The packages from 'absent' repo will be removed from a variant.
|
||||||
|
repo_type: str = 'present'
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class VariantInfo:
|
||||||
|
# name of variant. E.g. 'BaseOS', 'AppStream', etc
|
||||||
|
name: AnyStr
|
||||||
|
# architecture of variant. E.g. 'x86_64', 'i686', etc
|
||||||
|
arch: AnyStr
|
||||||
|
# The packages which will be not added to a variant
|
||||||
|
excluded_packages: List[str] = field(default_factory=list)
|
||||||
|
# Repos of a variant
|
||||||
|
repos: List[RepoInfo] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class PackagesGenerator:
|
||||||
|
|
||||||
|
repo_arches = defaultdict(lambda: list(('noarch',)))
|
||||||
|
addon_repos = {
|
||||||
|
'x86_64': ['i686'],
|
||||||
|
'ppc64le': [],
|
||||||
|
'aarch64': [],
|
||||||
|
's390x': [],
|
||||||
|
'i686': [],
|
||||||
|
}
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
variants: List[VariantInfo],
|
||||||
|
excluded_packages: List[AnyStr],
|
||||||
|
included_packages: List[AnyStr],
|
||||||
|
):
|
||||||
|
self.variants = variants
|
||||||
|
self.pkgs = dict()
|
||||||
|
self.excluded_packages = excluded_packages
|
||||||
|
self.included_packages = included_packages
|
||||||
|
self.tmp_files = [] # type: list[Path]
|
||||||
|
for arch, arch_list in self.addon_repos.items():
|
||||||
|
self.repo_arches[arch].extend(arch_list)
|
||||||
|
self.repo_arches[arch].append(arch)
|
||||||
|
|
||||||
|
def __del__(self):
|
||||||
|
for tmp_file in self.tmp_files:
|
||||||
|
if tmp_file.exists():
|
||||||
|
tmp_file.unlink()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_full_repo_path(repo_info: RepoInfo):
|
||||||
|
result = os.path.join(
|
||||||
|
repo_info.path,
|
||||||
|
repo_info.folder
|
||||||
|
)
|
||||||
|
if repo_info.is_remote:
|
||||||
|
result = urljoin(
|
||||||
|
repo_info.path + '/',
|
||||||
|
repo_info.folder,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _warning_callback(warning_type, message):
|
||||||
|
"""
|
||||||
|
Warning callback for createrepo_c parsing functions
|
||||||
|
"""
|
||||||
|
print(f'Warning message: "{message}"; warning type: "{warning_type}"')
|
||||||
|
return True
|
||||||
|
|
||||||
|
def get_remote_file_content(self, file_url: AnyStr) -> AnyStr:
|
||||||
|
"""
|
||||||
|
Get content from a remote file and write it to a temp file
|
||||||
|
:param file_url: url of a remote file
|
||||||
|
:return: path to a temp file
|
||||||
|
"""
|
||||||
|
|
||||||
|
file_request = requests.get(
|
||||||
|
url=file_url,
|
||||||
|
)
|
||||||
|
file_request.raise_for_status()
|
||||||
|
with tempfile.NamedTemporaryFile(delete=False) as file_stream:
|
||||||
|
file_stream.write(file_request.content)
|
||||||
|
self.tmp_files.append(Path(file_stream.name))
|
||||||
|
return file_stream.name
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_repomd(repomd_file_path: AnyStr) -> Repomd:
|
||||||
|
"""
|
||||||
|
Parse file repomd.xml and create object Repomd
|
||||||
|
:param repomd_file_path: path to local repomd.xml
|
||||||
|
"""
|
||||||
|
return Repomd(repomd_file_path)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _parse_modules_file(
|
||||||
|
cls,
|
||||||
|
modules_file_path: AnyStr,
|
||||||
|
|
||||||
|
) -> Iterator[Any]:
|
||||||
|
"""
|
||||||
|
Parse modules.yaml.gz and returns parsed data
|
||||||
|
:param modules_file_path: path to local modules.yaml.gz
|
||||||
|
:return: List of dict for each module in a repo
|
||||||
|
"""
|
||||||
|
|
||||||
|
with open(modules_file_path, 'rb') as modules_file:
|
||||||
|
data = modules_file.read()
|
||||||
|
if is_gzip_file(data[:2]):
|
||||||
|
data = gzip.decompress(data)
|
||||||
|
elif is_xz_file(data[:2]):
|
||||||
|
data = lzma.decompress(data)
|
||||||
|
return yaml.load_all(
|
||||||
|
data,
|
||||||
|
Loader=yaml.BaseLoader,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_repomd_records(
|
||||||
|
self,
|
||||||
|
repo_info: RepoInfo,
|
||||||
|
) -> List[RepomdRecord]:
|
||||||
|
"""
|
||||||
|
Get, parse file repomd.xml and extract from it repomd records
|
||||||
|
:param repo_info: structure which contains info about a current repo
|
||||||
|
:return: list with repomd records
|
||||||
|
"""
|
||||||
|
repomd_file_path = os.path.join(
|
||||||
|
repo_info.path,
|
||||||
|
repo_info.folder,
|
||||||
|
'repodata',
|
||||||
|
'repomd.xml',
|
||||||
|
)
|
||||||
|
if repo_info.is_remote:
|
||||||
|
repomd_file_path = urljoin(
|
||||||
|
urljoin(
|
||||||
|
repo_info.path + '/',
|
||||||
|
repo_info.folder
|
||||||
|
) + '/',
|
||||||
|
'repodata/repomd.xml'
|
||||||
|
)
|
||||||
|
repomd_file_path = self.get_remote_file_content(repomd_file_path)
|
||||||
|
|
||||||
|
repomd_object = self._parse_repomd(repomd_file_path)
|
||||||
|
if repo_info.is_remote:
|
||||||
|
os.remove(repomd_file_path)
|
||||||
|
return repomd_object.records
|
||||||
|
|
||||||
|
def _download_repomd_records(
|
||||||
|
self,
|
||||||
|
repo_info: RepoInfo,
|
||||||
|
repomd_records: List[RepomdRecord],
|
||||||
|
repomd_records_dict: Dict[str, str],
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Download repomd records
|
||||||
|
:param repo_info: structure which contains info about a current repo
|
||||||
|
:param repomd_records: list with repomd records
|
||||||
|
:param repomd_records_dict: dict with paths to repodata files
|
||||||
|
"""
|
||||||
|
for repomd_record in repomd_records:
|
||||||
|
if repomd_record.type not in (
|
||||||
|
'primary',
|
||||||
|
'filelists',
|
||||||
|
'other',
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
repomd_record_file_path = os.path.join(
|
||||||
|
repo_info.path,
|
||||||
|
repo_info.folder,
|
||||||
|
repomd_record.location_href,
|
||||||
|
)
|
||||||
|
if repo_info.is_remote:
|
||||||
|
repomd_record_file_path = self.get_remote_file_content(
|
||||||
|
repomd_record_file_path)
|
||||||
|
repomd_records_dict[repomd_record.type] = repomd_record_file_path
|
||||||
|
|
||||||
|
def _parse_module_repomd_record(
|
||||||
|
self,
|
||||||
|
repo_info: RepoInfo,
|
||||||
|
repomd_records: List[RepomdRecord],
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""
|
||||||
|
Download repomd records
|
||||||
|
:param repo_info: structure which contains info about a current repo
|
||||||
|
:param repomd_records: list with repomd records
|
||||||
|
"""
|
||||||
|
for repomd_record in repomd_records:
|
||||||
|
if repomd_record.type != 'modules':
|
||||||
|
continue
|
||||||
|
repomd_record_file_path = os.path.join(
|
||||||
|
repo_info.path,
|
||||||
|
repo_info.folder,
|
||||||
|
repomd_record.location_href,
|
||||||
|
)
|
||||||
|
if repo_info.is_remote:
|
||||||
|
repomd_record_file_path = self.get_remote_file_content(
|
||||||
|
repomd_record_file_path)
|
||||||
|
return list(self._parse_modules_file(
|
||||||
|
repomd_record_file_path,
|
||||||
|
))
|
||||||
|
return []
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def compare_pkgs_version(package_1: Package, package_2: Package) -> int:
|
||||||
|
version_tuple_1 = (
|
||||||
|
package_1.epoch,
|
||||||
|
package_1.version,
|
||||||
|
package_1.release,
|
||||||
|
)
|
||||||
|
version_tuple_2 = (
|
||||||
|
package_2.epoch,
|
||||||
|
package_2.version,
|
||||||
|
package_2.release,
|
||||||
|
)
|
||||||
|
return rpm.labelCompare(version_tuple_1, version_tuple_2)
|
||||||
|
|
||||||
|
def get_packages_iterator(
|
||||||
|
self,
|
||||||
|
repo_info: RepoInfo,
|
||||||
|
) -> Union[PackageIterator, Iterator]:
|
||||||
|
full_repo_path = self._get_full_repo_path(repo_info)
|
||||||
|
pkgs_iterator = self.pkgs.get(full_repo_path)
|
||||||
|
if pkgs_iterator is None:
|
||||||
|
repomd_records = self._get_repomd_records(
|
||||||
|
repo_info=repo_info,
|
||||||
|
)
|
||||||
|
repomd_records_dict = {} # type: Dict[str, str]
|
||||||
|
self._download_repomd_records(
|
||||||
|
repo_info=repo_info,
|
||||||
|
repomd_records=repomd_records,
|
||||||
|
repomd_records_dict=repomd_records_dict,
|
||||||
|
)
|
||||||
|
pkgs_iterator = PackageIterator(
|
||||||
|
primary_path=repomd_records_dict['primary'],
|
||||||
|
filelists_path=repomd_records_dict['filelists'],
|
||||||
|
other_path=repomd_records_dict['other'],
|
||||||
|
warningcb=self._warning_callback,
|
||||||
|
)
|
||||||
|
pkgs_iterator, self.pkgs[full_repo_path] = tee(pkgs_iterator)
|
||||||
|
return pkgs_iterator
|
||||||
|
|
||||||
|
def get_package_arch(
|
||||||
|
self,
|
||||||
|
package: Package,
|
||||||
|
variant_arch: str,
|
||||||
|
) -> str:
|
||||||
|
result = variant_arch
|
||||||
|
if package.arch in self.repo_arches[variant_arch]:
|
||||||
|
result = package.arch
|
||||||
|
return result
|
||||||
|
|
||||||
|
def is_skipped_module_package(
|
||||||
|
self,
|
||||||
|
package: Package,
|
||||||
|
variant_arch: str,
|
||||||
|
) -> bool:
|
||||||
|
package_key = self.get_package_key(package, variant_arch)
|
||||||
|
# Even a module package will be added to packages.json if
|
||||||
|
# it presents in the list of included packages
|
||||||
|
return 'module' in package.release and not any(
|
||||||
|
re.search(
|
||||||
|
f'^{included_pkg}$',
|
||||||
|
package_key,
|
||||||
|
) or included_pkg in (package.name, package_key)
|
||||||
|
for included_pkg in self.included_packages
|
||||||
|
)
|
||||||
|
|
||||||
|
def is_excluded_package(
|
||||||
|
self,
|
||||||
|
package: Package,
|
||||||
|
variant_arch: str,
|
||||||
|
excluded_packages: List[str],
|
||||||
|
) -> bool:
|
||||||
|
package_key = self.get_package_key(package, variant_arch)
|
||||||
|
return any(
|
||||||
|
re.search(
|
||||||
|
f'^{excluded_pkg}$',
|
||||||
|
package_key,
|
||||||
|
) or excluded_pkg in (package.name, package_key)
|
||||||
|
for excluded_pkg in excluded_packages
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_source_rpm_name(package: Package) -> str:
|
||||||
|
source_rpm_nvra = parse_nvra(package.rpm_sourcerpm)
|
||||||
|
return source_rpm_nvra['name']
|
||||||
|
|
||||||
|
def get_package_key(self, package: Package, variant_arch: str) -> str:
|
||||||
|
return (
|
||||||
|
f'{package.name}.'
|
||||||
|
f'{self.get_package_arch(package, variant_arch)}'
|
||||||
|
)
|
||||||
|
|
||||||
|
def generate_packages_json(
|
||||||
|
self
|
||||||
|
) -> Dict[AnyStr, Dict[AnyStr, Dict[AnyStr, List[AnyStr]]]]:
|
||||||
|
"""
|
||||||
|
Generate packages.json
|
||||||
|
"""
|
||||||
|
packages = defaultdict(lambda: defaultdict(lambda: {
|
||||||
|
'variants': list(),
|
||||||
|
}))
|
||||||
|
for variant_info in self.variants:
|
||||||
|
for repo_info in variant_info.repos:
|
||||||
|
is_reference = repo_info.is_reference
|
||||||
|
for package in self.get_packages_iterator(repo_info=repo_info):
|
||||||
|
if self.is_skipped_module_package(
|
||||||
|
package=package,
|
||||||
|
variant_arch=variant_info.arch,
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
if self.is_excluded_package(
|
||||||
|
package=package,
|
||||||
|
variant_arch=variant_info.arch,
|
||||||
|
excluded_packages=self.excluded_packages,
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
if self.is_excluded_package(
|
||||||
|
package=package,
|
||||||
|
variant_arch=variant_info.arch,
|
||||||
|
excluded_packages=variant_info.excluded_packages,
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
package_key = self.get_package_key(
|
||||||
|
package,
|
||||||
|
variant_info.arch,
|
||||||
|
)
|
||||||
|
source_rpm_name = self.get_source_rpm_name(package)
|
||||||
|
package_info = packages[source_rpm_name][package_key]
|
||||||
|
if 'is_reference' not in package_info:
|
||||||
|
package_info['variants'].append(variant_info.name)
|
||||||
|
package_info['is_reference'] = is_reference
|
||||||
|
package_info['package'] = package
|
||||||
|
elif not package_info['is_reference'] or \
|
||||||
|
package_info['is_reference'] == is_reference and \
|
||||||
|
self.compare_pkgs_version(
|
||||||
|
package_1=package,
|
||||||
|
package_2=package_info['package'],
|
||||||
|
) > 0:
|
||||||
|
package_info['variants'] = [variant_info.name]
|
||||||
|
package_info['is_reference'] = is_reference
|
||||||
|
package_info['package'] = package
|
||||||
|
elif self.compare_pkgs_version(
|
||||||
|
package_1=package,
|
||||||
|
package_2=package_info['package'],
|
||||||
|
) == 0 and repo_info.repo_type != 'absent':
|
||||||
|
package_info['variants'].append(variant_info.name)
|
||||||
|
result = defaultdict(lambda: defaultdict(
|
||||||
|
lambda: defaultdict(list),
|
||||||
|
))
|
||||||
|
for variant_info in self.variants:
|
||||||
|
for source_rpm_name, packages_info in packages.items():
|
||||||
|
for package_key, package_info in packages_info.items():
|
||||||
|
variant_pkgs = result[variant_info.name][variant_info.arch]
|
||||||
|
if variant_info.name not in package_info['variants']:
|
||||||
|
continue
|
||||||
|
variant_pkgs[source_rpm_name].append(package_key)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def create_parser():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
'-c',
|
||||||
|
'--config',
|
||||||
|
type=Path,
|
||||||
|
default=Path('config.yaml'),
|
||||||
|
required=False,
|
||||||
|
help='Path to a config',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'-o',
|
||||||
|
'--json-output-path',
|
||||||
|
type=str,
|
||||||
|
help='Full path to output json file',
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def read_config(config_path: Path) -> Optional[Dict]:
|
||||||
|
if not config_path.exists():
|
||||||
|
logging.error('A config by path "%s" does not exist', config_path)
|
||||||
|
exit(1)
|
||||||
|
with config_path.open('r') as config_fd:
|
||||||
|
return yaml.safe_load(config_fd)
|
||||||
|
|
||||||
|
|
||||||
|
def process_config(config_data: Dict) -> Tuple[
|
||||||
|
List[VariantInfo],
|
||||||
|
List[str],
|
||||||
|
List[str],
|
||||||
|
]:
|
||||||
|
excluded_packages = config_data.get('excluded_packages', [])
|
||||||
|
included_packages = config_data.get('included_packages', [])
|
||||||
|
variants = [VariantInfo(
|
||||||
|
name=variant_name,
|
||||||
|
arch=variant_info['arch'],
|
||||||
|
excluded_packages=variant_info.get('excluded_packages', []),
|
||||||
|
repos=[RepoInfo(
|
||||||
|
path=variant_repo['path'],
|
||||||
|
folder=variant_repo['folder'],
|
||||||
|
is_remote=variant_repo['remote'],
|
||||||
|
is_reference=variant_repo['reference'],
|
||||||
|
repo_type=variant_repo.get('repo_type', 'present'),
|
||||||
|
) for variant_repo in variant_info['repos']]
|
||||||
|
) for variant_name, variant_info in config_data['variants'].items()]
|
||||||
|
return variants, excluded_packages, included_packages
|
||||||
|
|
||||||
|
|
||||||
|
def cli_main():
|
||||||
|
args = create_parser().parse_args()
|
||||||
|
variants, excluded_packages, included_packages = process_config(
|
||||||
|
config_data=read_config(args.config)
|
||||||
|
)
|
||||||
|
pg = PackagesGenerator(
|
||||||
|
variants=variants,
|
||||||
|
excluded_packages=excluded_packages,
|
||||||
|
included_packages=included_packages,
|
||||||
|
)
|
||||||
|
result = pg.generate_packages_json()
|
||||||
|
with open(args.json_output_path, 'w') as packages_file:
|
||||||
|
json.dump(
|
||||||
|
result,
|
||||||
|
packages_file,
|
||||||
|
indent=4,
|
||||||
|
sort_keys=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
cli_main()
|
|
@ -0,0 +1,255 @@
|
||||||
|
import gzip
|
||||||
|
import lzma
|
||||||
|
import os
|
||||||
|
from argparse import ArgumentParser, FileType
|
||||||
|
from glob import iglob
|
||||||
|
from io import BytesIO
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, AnyStr, Iterable, Union, Optional
|
||||||
|
import logging
|
||||||
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
import createrepo_c as cr
|
||||||
|
from typing.io import BinaryIO
|
||||||
|
|
||||||
|
from .create_packages_json import PackagesGenerator, is_gzip_file, is_xz_file
|
||||||
|
|
||||||
|
EMPTY_FILE = '.empty'
|
||||||
|
|
||||||
|
|
||||||
|
def read_modules_yaml(modules_yaml_path: Union[str, Path]) -> BytesIO:
|
||||||
|
with open(modules_yaml_path, 'rb') as fp:
|
||||||
|
return BytesIO(fp.read())
|
||||||
|
|
||||||
|
|
||||||
|
def grep_list_of_modules_yaml(repos_path: AnyStr) -> Iterable[BytesIO]:
|
||||||
|
"""
|
||||||
|
Find all of valid *modules.yaml.gz in repos
|
||||||
|
:param repos_path: path to a directory which contains repo dirs
|
||||||
|
:return: iterable object of content from *modules.yaml.*
|
||||||
|
"""
|
||||||
|
|
||||||
|
return (
|
||||||
|
read_modules_yaml_from_specific_repo(repo_path=Path(path).parent)
|
||||||
|
for path in iglob(
|
||||||
|
str(Path(repos_path).joinpath('**/repodata')),
|
||||||
|
recursive=True
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_remote(path: str):
|
||||||
|
return any(str(path).startswith(protocol)
|
||||||
|
for protocol in ('http', 'https'))
|
||||||
|
|
||||||
|
|
||||||
|
def read_modules_yaml_from_specific_repo(
|
||||||
|
repo_path: Union[str, Path]
|
||||||
|
) -> Optional[BytesIO]:
|
||||||
|
"""
|
||||||
|
Read modules_yaml from a specific repo (remote or local)
|
||||||
|
:param repo_path: path/url to a specific repo
|
||||||
|
(final dir should contain dir `repodata`)
|
||||||
|
:return: iterable object of content from *modules.yaml.*
|
||||||
|
"""
|
||||||
|
|
||||||
|
if _is_remote(repo_path):
|
||||||
|
repomd_url = urljoin(
|
||||||
|
repo_path + '/',
|
||||||
|
'repodata/repomd.xml',
|
||||||
|
)
|
||||||
|
packages_generator = PackagesGenerator(
|
||||||
|
variants=[],
|
||||||
|
excluded_packages=[],
|
||||||
|
included_packages=[],
|
||||||
|
)
|
||||||
|
repomd_file_path = packages_generator.get_remote_file_content(
|
||||||
|
file_url=repomd_url
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
repomd_file_path = os.path.join(
|
||||||
|
repo_path,
|
||||||
|
'repodata/repomd.xml',
|
||||||
|
)
|
||||||
|
repomd_obj = cr.Repomd(str(repomd_file_path))
|
||||||
|
for record in repomd_obj.records:
|
||||||
|
if record.type != 'modules':
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
if _is_remote(repo_path):
|
||||||
|
modules_yaml_url = urljoin(
|
||||||
|
repo_path + '/',
|
||||||
|
record.location_href,
|
||||||
|
)
|
||||||
|
packages_generator = PackagesGenerator(
|
||||||
|
variants=[],
|
||||||
|
excluded_packages=[],
|
||||||
|
included_packages=[],
|
||||||
|
)
|
||||||
|
modules_yaml_path = packages_generator.get_remote_file_content(
|
||||||
|
file_url=modules_yaml_url
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
modules_yaml_path = os.path.join(
|
||||||
|
repo_path,
|
||||||
|
record.location_href,
|
||||||
|
)
|
||||||
|
return read_modules_yaml(modules_yaml_path=modules_yaml_path)
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _should_grep_defaults(
|
||||||
|
document_type: str,
|
||||||
|
grep_only_modules_data: bool = False,
|
||||||
|
grep_only_modules_defaults_data: bool = False,
|
||||||
|
) -> bool:
|
||||||
|
xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
|
||||||
|
if document_type == 'modulemd' and (xor_flag or grep_only_modules_data):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _should_grep_modules(
|
||||||
|
document_type: str,
|
||||||
|
grep_only_modules_data: bool = False,
|
||||||
|
grep_only_modules_defaults_data: bool = False,
|
||||||
|
) -> bool:
|
||||||
|
xor_flag = grep_only_modules_data == grep_only_modules_defaults_data
|
||||||
|
if document_type == 'modulemd-defaults' and \
|
||||||
|
(xor_flag or grep_only_modules_defaults_data):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def collect_modules(
|
||||||
|
modules_paths: List[BinaryIO],
|
||||||
|
target_dir: str,
|
||||||
|
grep_only_modules_data: bool = False,
|
||||||
|
grep_only_modules_defaults_data: bool = False,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Read given modules.yaml.gz files and export modules
|
||||||
|
and modulemd files from it.
|
||||||
|
Returns:
|
||||||
|
object:
|
||||||
|
"""
|
||||||
|
xor_flag = grep_only_modules_defaults_data is grep_only_modules_data
|
||||||
|
modules_path = os.path.join(target_dir, 'modules')
|
||||||
|
module_defaults_path = os.path.join(target_dir, 'module_defaults')
|
||||||
|
if grep_only_modules_data or xor_flag:
|
||||||
|
os.makedirs(modules_path, exist_ok=True)
|
||||||
|
if grep_only_modules_defaults_data or xor_flag:
|
||||||
|
os.makedirs(module_defaults_path, exist_ok=True)
|
||||||
|
# Defaults modules can be empty, but pungi detects
|
||||||
|
# empty folder while copying and raises the exception in this case
|
||||||
|
Path(os.path.join(module_defaults_path, EMPTY_FILE)).touch()
|
||||||
|
|
||||||
|
for module_file in modules_paths:
|
||||||
|
data = module_file.read()
|
||||||
|
if is_gzip_file(data[:2]):
|
||||||
|
data = gzip.decompress(data)
|
||||||
|
elif is_xz_file(data[:2]):
|
||||||
|
data = lzma.decompress(data)
|
||||||
|
documents = yaml.load_all(data, Loader=yaml.BaseLoader)
|
||||||
|
for doc in documents:
|
||||||
|
path = None
|
||||||
|
if _should_grep_modules(
|
||||||
|
doc['document'],
|
||||||
|
grep_only_modules_data,
|
||||||
|
grep_only_modules_defaults_data,
|
||||||
|
):
|
||||||
|
name = f"{doc['data']['module']}.yaml"
|
||||||
|
path = os.path.join(module_defaults_path, name)
|
||||||
|
logging.info('Found %s module defaults', name)
|
||||||
|
elif _should_grep_defaults(
|
||||||
|
doc['document'],
|
||||||
|
grep_only_modules_data,
|
||||||
|
grep_only_modules_defaults_data,
|
||||||
|
):
|
||||||
|
# pungi.phases.pkgset.sources.source_koji.get_koji_modules
|
||||||
|
stream = doc['data']['stream'].replace('-', '_')
|
||||||
|
doc_data = doc['data']
|
||||||
|
name = f"{doc_data['name']}-{stream}-" \
|
||||||
|
f"{doc_data['version']}.{doc_data['context']}"
|
||||||
|
arch_dir = os.path.join(
|
||||||
|
modules_path,
|
||||||
|
doc_data['arch']
|
||||||
|
)
|
||||||
|
os.makedirs(arch_dir, exist_ok=True)
|
||||||
|
path = os.path.join(
|
||||||
|
arch_dir,
|
||||||
|
name,
|
||||||
|
)
|
||||||
|
logging.info('Found module %s', name)
|
||||||
|
|
||||||
|
if 'artifacts' not in doc['data']:
|
||||||
|
logging.warning(
|
||||||
|
'RPM %s does not have explicit list of artifacts',
|
||||||
|
name
|
||||||
|
)
|
||||||
|
if path is not None:
|
||||||
|
with open(path, 'w') as f:
|
||||||
|
yaml.dump(doc, f, default_flow_style=False)
|
||||||
|
|
||||||
|
|
||||||
|
def cli_main():
|
||||||
|
parser = ArgumentParser()
|
||||||
|
content_type_group = parser.add_mutually_exclusive_group(required=False)
|
||||||
|
content_type_group.add_argument(
|
||||||
|
'--get-only-modules-data',
|
||||||
|
action='store_true',
|
||||||
|
help='Parse and get only modules data',
|
||||||
|
)
|
||||||
|
content_type_group.add_argument(
|
||||||
|
'--get-only-modules-defaults-data',
|
||||||
|
action='store_true',
|
||||||
|
help='Parse and get only modules_defaults data',
|
||||||
|
)
|
||||||
|
path_group = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
path_group.add_argument(
|
||||||
|
'-p', '--path',
|
||||||
|
type=FileType('rb'), nargs='+',
|
||||||
|
help='Path to modules.yaml.gz file. '
|
||||||
|
'You may pass multiple files by passing -p path1 path2'
|
||||||
|
)
|
||||||
|
path_group.add_argument(
|
||||||
|
'-rp', '--repo-path',
|
||||||
|
required=False,
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help='Path to a directory which contains repodirs. E.g. /var/repos'
|
||||||
|
)
|
||||||
|
path_group.add_argument(
|
||||||
|
'-rd', '--repodata-paths',
|
||||||
|
required=False,
|
||||||
|
type=str,
|
||||||
|
nargs='+',
|
||||||
|
default=[],
|
||||||
|
help='Paths/urls to the directories with directory `repodata`',
|
||||||
|
)
|
||||||
|
parser.add_argument('-t', '--target', required=True)
|
||||||
|
|
||||||
|
namespace = parser.parse_args()
|
||||||
|
if namespace.repodata_paths:
|
||||||
|
modules = []
|
||||||
|
for repodata_path in namespace.repodata_paths:
|
||||||
|
modules.append(read_modules_yaml_from_specific_repo(
|
||||||
|
repodata_path,
|
||||||
|
))
|
||||||
|
elif namespace.path is not None:
|
||||||
|
modules = namespace.path
|
||||||
|
else:
|
||||||
|
modules = grep_list_of_modules_yaml(namespace.repo_path)
|
||||||
|
modules = list(filter(lambda i: i is not None, modules))
|
||||||
|
collect_modules(
|
||||||
|
modules,
|
||||||
|
namespace.target,
|
||||||
|
namespace.get_only_modules_data,
|
||||||
|
namespace.get_only_modules_defaults_data,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
cli_main()
|
|
@ -0,0 +1,96 @@
|
||||||
|
import re
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
|
||||||
|
import os
|
||||||
|
from glob import iglob
|
||||||
|
from typing import List
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from productmd.common import parse_nvra
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Package:
|
||||||
|
nvra: dict
|
||||||
|
path: Path
|
||||||
|
|
||||||
|
|
||||||
|
def search_rpms(top_dir: Path) -> List[Package]:
|
||||||
|
"""
|
||||||
|
Search for all *.rpm files recursively
|
||||||
|
in given top directory
|
||||||
|
Returns:
|
||||||
|
list: list of paths
|
||||||
|
"""
|
||||||
|
return [Package(
|
||||||
|
nvra=parse_nvra(Path(path).stem),
|
||||||
|
path=Path(path),
|
||||||
|
) for path in iglob(str(top_dir.joinpath('**/*.rpm')), recursive=True)]
|
||||||
|
|
||||||
|
|
||||||
|
def is_excluded_package(
|
||||||
|
package: Package,
|
||||||
|
excluded_packages: List[str],
|
||||||
|
) -> bool:
|
||||||
|
package_key = f'{package.nvra["name"]}.{package.nvra["arch"]}'
|
||||||
|
return any(
|
||||||
|
re.search(
|
||||||
|
f'^{excluded_pkg}$',
|
||||||
|
package_key,
|
||||||
|
) or excluded_pkg in (package.nvra['name'], package_key)
|
||||||
|
for excluded_pkg in excluded_packages
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def copy_rpms(
|
||||||
|
packages: List[Package],
|
||||||
|
target_top_dir: Path,
|
||||||
|
excluded_packages: List[str],
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Search synced repos for rpms and prepare
|
||||||
|
koji-like structure for pungi
|
||||||
|
|
||||||
|
Instead of repos, use following structure:
|
||||||
|
# ls /mnt/koji/
|
||||||
|
i686/ noarch/ x86_64/
|
||||||
|
Returns:
|
||||||
|
Nothing:
|
||||||
|
"""
|
||||||
|
for package in packages:
|
||||||
|
if is_excluded_package(package, excluded_packages):
|
||||||
|
continue
|
||||||
|
target_arch_dir = target_top_dir.joinpath(package.nvra['arch'])
|
||||||
|
target_file = target_arch_dir.joinpath(package.path.name)
|
||||||
|
os.makedirs(target_arch_dir, exist_ok=True)
|
||||||
|
|
||||||
|
if not target_file.exists():
|
||||||
|
try:
|
||||||
|
os.link(package.path, target_file)
|
||||||
|
except OSError:
|
||||||
|
# hardlink failed, try symlinking
|
||||||
|
package.path.symlink_to(target_file)
|
||||||
|
|
||||||
|
|
||||||
|
def cli_main():
|
||||||
|
parser = ArgumentParser()
|
||||||
|
parser.add_argument('-p', '--path', required=True, type=Path)
|
||||||
|
parser.add_argument('-t', '--target', required=True, type=Path)
|
||||||
|
parser.add_argument(
|
||||||
|
'-e',
|
||||||
|
'--excluded-packages',
|
||||||
|
required=False,
|
||||||
|
nargs='+',
|
||||||
|
type=str,
|
||||||
|
default=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
namespace = parser.parse_args()
|
||||||
|
|
||||||
|
rpms = search_rpms(namespace.path)
|
||||||
|
copy_rpms(rpms, namespace.target, namespace.excluded_packages)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
cli_main()
|
|
@ -252,9 +252,15 @@ def main():
|
||||||
kobo.log.add_stderr_logger(logger)
|
kobo.log.add_stderr_logger(logger)
|
||||||
|
|
||||||
conf = util.load_config(opts.config)
|
conf = util.load_config(opts.config)
|
||||||
|
|
||||||
compose_type = opts.compose_type or conf.get("compose_type", "production")
|
compose_type = opts.compose_type or conf.get("compose_type", "production")
|
||||||
if compose_type == "production" and not opts.label and not opts.no_label:
|
label = opts.label or conf.get("label")
|
||||||
|
if label:
|
||||||
|
try:
|
||||||
|
productmd.composeinfo.verify_label(label)
|
||||||
|
except ValueError as ex:
|
||||||
|
abort(str(ex))
|
||||||
|
|
||||||
|
if compose_type == "production" and not label and not opts.no_label:
|
||||||
abort("must specify label for a production compose")
|
abort("must specify label for a production compose")
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
@ -304,7 +310,7 @@ def main():
|
||||||
opts.target_dir,
|
opts.target_dir,
|
||||||
conf,
|
conf,
|
||||||
compose_type=compose_type,
|
compose_type=compose_type,
|
||||||
compose_label=opts.label,
|
compose_label=label,
|
||||||
parent_compose_ids=opts.parent_compose_id,
|
parent_compose_ids=opts.parent_compose_id,
|
||||||
respin_of=opts.respin_of,
|
respin_of=opts.respin_of,
|
||||||
)
|
)
|
||||||
|
@ -315,7 +321,7 @@ def main():
|
||||||
ci = Compose.get_compose_info(
|
ci = Compose.get_compose_info(
|
||||||
conf,
|
conf,
|
||||||
compose_type=compose_type,
|
compose_type=compose_type,
|
||||||
compose_label=opts.label,
|
compose_label=label,
|
||||||
parent_compose_ids=opts.parent_compose_id,
|
parent_compose_ids=opts.parent_compose_id,
|
||||||
respin_of=opts.respin_of,
|
respin_of=opts.respin_of,
|
||||||
)
|
)
|
||||||
|
|
|
@ -306,6 +306,8 @@ class CompsWrapper(object):
|
||||||
append_common_info(doc, group_node, group, force_description=True)
|
append_common_info(doc, group_node, group, force_description=True)
|
||||||
append_bool(doc, group_node, "default", group.default)
|
append_bool(doc, group_node, "default", group.default)
|
||||||
append_bool(doc, group_node, "uservisible", group.uservisible)
|
append_bool(doc, group_node, "uservisible", group.uservisible)
|
||||||
|
if group.display_order is not None:
|
||||||
|
append(doc, group_node, "display_order", str(group.display_order))
|
||||||
|
|
||||||
if group.lang_only:
|
if group.lang_only:
|
||||||
append(doc, group_node, "langonly", group.lang_only)
|
append(doc, group_node, "langonly", group.lang_only)
|
||||||
|
|
|
@ -88,5 +88,12 @@ def parse_output(output):
|
||||||
packages.add((name, arch, frozenset(flags)))
|
packages.add((name, arch, frozenset(flags)))
|
||||||
else:
|
else:
|
||||||
name, arch = nevra.rsplit(".", 1)
|
name, arch = nevra.rsplit(".", 1)
|
||||||
modules.add(name.split(":", 1)[1])
|
# replace dash by underscore in stream of module's nerva
|
||||||
|
# source of name looks like
|
||||||
|
# module:llvm-toolset:rhel8:8040020210411062713:9f9e2e7e.x86_64
|
||||||
|
name = ':'.join(
|
||||||
|
item.replace('-', '_') if i == 1 else item for
|
||||||
|
i, item in enumerate(name.split(':')[1:])
|
||||||
|
)
|
||||||
|
modules.add(name)
|
||||||
return packages, modules
|
return packages, modules
|
||||||
|
|
|
@ -0,0 +1,299 @@
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from attr import dataclass
|
||||||
|
from kobo.rpmlib import parse_nvra
|
||||||
|
|
||||||
|
from pungi.module_util import Modulemd
|
||||||
|
|
||||||
|
# just a random value which we don't
|
||||||
|
# use in mock currently
|
||||||
|
# originally builds are filtered by this value
|
||||||
|
# to get consistent snapshot of tags and packages
|
||||||
|
from pungi.scripts.gather_rpms import search_rpms
|
||||||
|
|
||||||
|
LAST_EVENT_ID = 999999
|
||||||
|
# last event time is not important but build
|
||||||
|
# time should be less then it
|
||||||
|
LAST_EVENT_TIME = time.time()
|
||||||
|
BUILD_TIME = 0
|
||||||
|
# virtual build that collects all
|
||||||
|
# packages built for some arch
|
||||||
|
RELEASE_BUILD_ID = 15270
|
||||||
|
# tag that should have all packages available
|
||||||
|
ALL_PACKAGES_TAG = 'dist-c8-compose'
|
||||||
|
# tag that should have all modules available
|
||||||
|
ALL_MODULES_TAG = 'dist-c8-module-compose'
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Module:
|
||||||
|
build_id: int
|
||||||
|
name: str
|
||||||
|
nvr: str
|
||||||
|
stream: str
|
||||||
|
version: str
|
||||||
|
context: str
|
||||||
|
arch: str
|
||||||
|
|
||||||
|
|
||||||
|
class KojiMock:
|
||||||
|
"""
|
||||||
|
Class that acts like real koji (for some needed methods)
|
||||||
|
but uses local storage as data source
|
||||||
|
"""
|
||||||
|
def __init__(self, packages_dir, modules_dir, all_arches):
|
||||||
|
self._modules = self._gather_modules(modules_dir)
|
||||||
|
self._modules_dir = modules_dir
|
||||||
|
self._packages_dir = packages_dir
|
||||||
|
self._all_arches = all_arches
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _gather_modules(modules_dir):
|
||||||
|
modules = {}
|
||||||
|
for index, (f, arch) in enumerate(
|
||||||
|
(sub_path.name, sub_path.parent.name)
|
||||||
|
for path in Path(modules_dir).glob('*')
|
||||||
|
for sub_path in path.iterdir()
|
||||||
|
):
|
||||||
|
parsed = parse_nvra(f)
|
||||||
|
modules[index] = Module(
|
||||||
|
name=parsed['name'],
|
||||||
|
nvr=f,
|
||||||
|
version=parsed['release'],
|
||||||
|
context=parsed['arch'],
|
||||||
|
stream=parsed['version'],
|
||||||
|
build_id=index,
|
||||||
|
arch=arch,
|
||||||
|
)
|
||||||
|
return modules
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def getLastEvent(*args, **kwargs):
|
||||||
|
return {'id': LAST_EVENT_ID, 'ts': LAST_EVENT_TIME}
|
||||||
|
|
||||||
|
def listTagged(self, tag_name, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Returns list of virtual 'builds' that contain packages by given tag
|
||||||
|
There are two kinds of tags: modular and distributive.
|
||||||
|
For now, only one kind, distributive one, is needed.
|
||||||
|
"""
|
||||||
|
if tag_name != ALL_MODULES_TAG:
|
||||||
|
raise ValueError("I don't know what tag is %s" % tag_name)
|
||||||
|
|
||||||
|
builds = []
|
||||||
|
for module in self._modules.values():
|
||||||
|
builds.append({
|
||||||
|
'build_id': module.build_id,
|
||||||
|
'owner_name': 'centos',
|
||||||
|
'package_name': module.name,
|
||||||
|
'nvr': module.nvr,
|
||||||
|
'version': module.stream,
|
||||||
|
'release': '%s.%s' % (module.version, module.context),
|
||||||
|
'name': module.name,
|
||||||
|
'id': module.build_id,
|
||||||
|
'tag_name': tag_name,
|
||||||
|
'arch': module.arch,
|
||||||
|
# Following fields are currently not
|
||||||
|
# used but returned by real koji
|
||||||
|
# left them here just for reference
|
||||||
|
#
|
||||||
|
# 'task_id': None,
|
||||||
|
# 'state': 1,
|
||||||
|
# 'start_time': '2020-12-23 16:43:59',
|
||||||
|
# 'creation_event_id': 309485,
|
||||||
|
# 'creation_time': '2020-12-23 17:05:33.553748',
|
||||||
|
# 'epoch': None, 'tag_id': 533,
|
||||||
|
# 'completion_time': '2020-12-23 17:05:23',
|
||||||
|
# 'volume_id': 0,
|
||||||
|
# 'package_id': 3221,
|
||||||
|
# 'owner_id': 11,
|
||||||
|
# 'volume_name': 'DEFAULT',
|
||||||
|
})
|
||||||
|
|
||||||
|
return builds
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def getFullInheritance(*args, **kwargs):
|
||||||
|
"""
|
||||||
|
Unneeded because we use local storage.
|
||||||
|
"""
|
||||||
|
return []
|
||||||
|
|
||||||
|
def getBuild(self, build_id, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Used to get information about build
|
||||||
|
(used in pungi only for modules currently)
|
||||||
|
"""
|
||||||
|
module = self._modules[build_id]
|
||||||
|
|
||||||
|
result = {
|
||||||
|
'id': build_id,
|
||||||
|
'name': module.name,
|
||||||
|
'version': module.stream,
|
||||||
|
'release': '%s.%s' % (module.version, module.context),
|
||||||
|
'completion_ts': BUILD_TIME,
|
||||||
|
'state': 'COMPLETE',
|
||||||
|
'arch': module.arch,
|
||||||
|
'extra': {
|
||||||
|
'typeinfo': {
|
||||||
|
'module': {
|
||||||
|
'stream': module.stream,
|
||||||
|
'version': module.version,
|
||||||
|
'name': module.name,
|
||||||
|
'context': module.context,
|
||||||
|
'content_koji_tag': '-'.join([
|
||||||
|
module.name,
|
||||||
|
module.stream,
|
||||||
|
module.version
|
||||||
|
]) + '.' + module.context
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
|
||||||
|
def listArchives(self, build_id, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Originally lists artifacts for build, but in pungi used
|
||||||
|
only to get list of modulemd files for some module
|
||||||
|
"""
|
||||||
|
module = self._modules[build_id]
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
'build_id': module.build_id,
|
||||||
|
'filename': f'modulemd.{module.arch}.txt',
|
||||||
|
'btype': 'module'
|
||||||
|
},
|
||||||
|
# noone ever uses this file
|
||||||
|
# but it should be because pungi ignores builds
|
||||||
|
# with len(files) <= 1
|
||||||
|
{
|
||||||
|
'build_id': module.build_id,
|
||||||
|
'filename': 'modulemd.txt',
|
||||||
|
'btype': 'module'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
def listTaggedRPMS(self, tag_name, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
Get information about packages that are tagged by tag.
|
||||||
|
There are two kings of tags: per-module and per-distr.
|
||||||
|
"""
|
||||||
|
if tag_name == ALL_PACKAGES_TAG:
|
||||||
|
builds, packages = self._get_release_packages()
|
||||||
|
else:
|
||||||
|
builds, packages = self._get_module_packages(tag_name)
|
||||||
|
return [
|
||||||
|
packages,
|
||||||
|
builds
|
||||||
|
]
|
||||||
|
|
||||||
|
def _get_release_packages(self):
|
||||||
|
"""
|
||||||
|
Search packages dir and keep only
|
||||||
|
packages that are non-modular.
|
||||||
|
|
||||||
|
This is quite the way how real koji works:
|
||||||
|
- modular packages are tagged by module-* tag
|
||||||
|
- all other packages are tagged with dist* tag
|
||||||
|
"""
|
||||||
|
packages = []
|
||||||
|
|
||||||
|
# get all rpms in folder
|
||||||
|
rpms = search_rpms(Path(self._packages_dir))
|
||||||
|
|
||||||
|
for rpm in rpms:
|
||||||
|
info = parse_nvra(rpm.path.stem)
|
||||||
|
if 'module' in info['release']:
|
||||||
|
continue
|
||||||
|
packages.append({
|
||||||
|
"build_id": RELEASE_BUILD_ID,
|
||||||
|
"name": info['name'],
|
||||||
|
"extra": None,
|
||||||
|
"arch": info['arch'],
|
||||||
|
"epoch": info['epoch'] or None,
|
||||||
|
"version": info['version'],
|
||||||
|
"metadata_only": False,
|
||||||
|
"release": info['release'],
|
||||||
|
# not used currently
|
||||||
|
# "id": 262555,
|
||||||
|
# "size": 0
|
||||||
|
})
|
||||||
|
builds = []
|
||||||
|
return builds, packages
|
||||||
|
|
||||||
|
def _get_module_packages(self, tag_name):
|
||||||
|
"""
|
||||||
|
Get list of builds for module and given module tag name.
|
||||||
|
"""
|
||||||
|
builds = []
|
||||||
|
packages = []
|
||||||
|
modules = self._get_modules_by_name(tag_name)
|
||||||
|
for module in modules:
|
||||||
|
if module is None:
|
||||||
|
raise ValueError('Module %s is not found' % tag_name)
|
||||||
|
path = os.path.join(
|
||||||
|
self._modules_dir,
|
||||||
|
module.arch,
|
||||||
|
tag_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
builds.append({
|
||||||
|
"build_id": module.build_id,
|
||||||
|
"package_name": module.name,
|
||||||
|
"nvr": module.nvr,
|
||||||
|
"tag_name": module.nvr,
|
||||||
|
"version": module.stream,
|
||||||
|
"release": module.version,
|
||||||
|
"id": module.build_id,
|
||||||
|
"name": module.name,
|
||||||
|
"volume_name": "DEFAULT",
|
||||||
|
# Following fields are currently not
|
||||||
|
# used but returned by real koji
|
||||||
|
# left them here just for reference
|
||||||
|
#
|
||||||
|
# "owner_name": "mbox-mbs-backend",
|
||||||
|
# "task_id": 195937,
|
||||||
|
# "state": 1,
|
||||||
|
# "start_time": "2020-12-22 19:20:12.504578",
|
||||||
|
# "creation_event_id": 306731,
|
||||||
|
# "creation_time": "2020-12-22 19:20:12.504578",
|
||||||
|
# "epoch": None,
|
||||||
|
# "tag_id": 1192,
|
||||||
|
# "completion_time": "2020-12-22 19:34:34.716615",
|
||||||
|
# "volume_id": 0,
|
||||||
|
# "package_id": 104,
|
||||||
|
# "owner_id": 6,
|
||||||
|
})
|
||||||
|
|
||||||
|
if os.path.exists(path):
|
||||||
|
info = Modulemd.ModuleStream.read_string(open(path).read(), strict=True)
|
||||||
|
for art in info.get_rpm_artifacts():
|
||||||
|
data = parse_nvra(art)
|
||||||
|
packages.append({
|
||||||
|
"build_id": module.build_id,
|
||||||
|
"name": data['name'],
|
||||||
|
"extra": None,
|
||||||
|
"arch": data['arch'],
|
||||||
|
"epoch": data['epoch'] or None,
|
||||||
|
"version": data['version'],
|
||||||
|
"metadata_only": False,
|
||||||
|
"release": data['release'],
|
||||||
|
"id": 262555,
|
||||||
|
"size": 0
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
raise RuntimeError('Unable to find module %s' % path)
|
||||||
|
return builds, packages
|
||||||
|
|
||||||
|
def _get_modules_by_name(self, tag_name):
|
||||||
|
modules = []
|
||||||
|
for arch in self._all_arches:
|
||||||
|
for module in self._modules.values():
|
||||||
|
if module.nvr != tag_name or module.arch != arch:
|
||||||
|
continue
|
||||||
|
modules.append(module)
|
||||||
|
return modules
|
|
@ -32,6 +32,7 @@ import six.moves.xmlrpc_client as xmlrpclib
|
||||||
from flufl.lock import Lock
|
from flufl.lock import Lock
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
|
from .kojimock import KojiMock
|
||||||
from .. import util
|
from .. import util
|
||||||
from ..arch_utils import getBaseArch
|
from ..arch_utils import getBaseArch
|
||||||
|
|
||||||
|
@ -869,6 +870,45 @@ class KojiWrapper(object):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class KojiMockWrapper(object):
|
||||||
|
lock = threading.Lock()
|
||||||
|
|
||||||
|
def __init__(self, compose, all_arches):
|
||||||
|
self.all_arches = all_arches
|
||||||
|
self.compose = compose
|
||||||
|
try:
|
||||||
|
self.profile = self.compose.conf["koji_profile"]
|
||||||
|
except KeyError:
|
||||||
|
raise RuntimeError("Koji profile must be configured")
|
||||||
|
with self.lock:
|
||||||
|
self.koji_module = koji.get_profile_module(self.profile)
|
||||||
|
session_opts = {}
|
||||||
|
for key in (
|
||||||
|
"timeout",
|
||||||
|
"keepalive",
|
||||||
|
"max_retries",
|
||||||
|
"retry_interval",
|
||||||
|
"anon_retry",
|
||||||
|
"offline_retry",
|
||||||
|
"offline_retry_interval",
|
||||||
|
"debug",
|
||||||
|
"debug_xmlrpc",
|
||||||
|
"serverca",
|
||||||
|
"use_fast_upload",
|
||||||
|
):
|
||||||
|
value = getattr(self.koji_module.config, key, None)
|
||||||
|
if value is not None:
|
||||||
|
session_opts[key] = value
|
||||||
|
self.koji_proxy = KojiMock(
|
||||||
|
packages_dir=self.koji_module.config.topdir,
|
||||||
|
modules_dir=os.path.join(
|
||||||
|
self.koji_module.config.topdir,
|
||||||
|
'modules',
|
||||||
|
),
|
||||||
|
all_arches=self.all_arches,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_buildroot_rpms(compose, task_id):
|
def get_buildroot_rpms(compose, task_id):
|
||||||
"""Get build root RPMs - either from runroot or local"""
|
"""Get build root RPMs - either from runroot or local"""
|
||||||
result = []
|
result = []
|
||||||
|
|
|
@ -109,3 +109,55 @@ class LoraxWrapper(object):
|
||||||
# TODO: workdir
|
# TODO: workdir
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
|
|
||||||
|
def get_buildinstall_cmd(
|
||||||
|
self,
|
||||||
|
product,
|
||||||
|
version,
|
||||||
|
release,
|
||||||
|
repo_baseurl,
|
||||||
|
output_dir,
|
||||||
|
variant=None,
|
||||||
|
bugurl=None,
|
||||||
|
nomacboot=False,
|
||||||
|
noupgrade=False,
|
||||||
|
is_final=False,
|
||||||
|
buildarch=None,
|
||||||
|
volid=None,
|
||||||
|
brand=None,
|
||||||
|
):
|
||||||
|
# RHEL 6 compatibility
|
||||||
|
# Usage: buildinstall [--debug] --version <version> --brand <brand> --product <product> --release <comment> --final [--output outputdir] [--discs <discstring>] <root> # noqa: E501
|
||||||
|
|
||||||
|
brand = brand or "redhat"
|
||||||
|
# HACK: ignore provided release
|
||||||
|
release = "%s %s" % (brand, version)
|
||||||
|
bugurl = bugurl or "https://bugzilla.redhat.com"
|
||||||
|
|
||||||
|
cmd = ["/usr/lib/anaconda-runtime/buildinstall"]
|
||||||
|
|
||||||
|
cmd.append("--debug")
|
||||||
|
|
||||||
|
cmd.extend(["--version", version])
|
||||||
|
cmd.extend(["--brand", brand])
|
||||||
|
cmd.extend(["--product", product])
|
||||||
|
cmd.extend(["--release", release])
|
||||||
|
|
||||||
|
if is_final:
|
||||||
|
cmd.append("--final")
|
||||||
|
|
||||||
|
if buildarch:
|
||||||
|
cmd.extend(["--buildarch", buildarch])
|
||||||
|
|
||||||
|
if bugurl:
|
||||||
|
cmd.extend(["--bugurl", bugurl])
|
||||||
|
|
||||||
|
output_dir = os.path.abspath(output_dir)
|
||||||
|
cmd.extend(["--output", output_dir])
|
||||||
|
|
||||||
|
for i in force_list(repo_baseurl):
|
||||||
|
if "://" not in i:
|
||||||
|
i = "file://%s" % os.path.abspath(i)
|
||||||
|
cmd.append(i)
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
|
8
setup.py
8
setup.py
|
@ -20,7 +20,7 @@ packages = sorted(packages)
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="pungi",
|
name="pungi",
|
||||||
version="4.5.1",
|
version="4.5.0",
|
||||||
description="Distribution compose tool",
|
description="Distribution compose tool",
|
||||||
url="https://pagure.io/pungi",
|
url="https://pagure.io/pungi",
|
||||||
author="Dennis Gilmore",
|
author="Dennis Gilmore",
|
||||||
|
@ -42,6 +42,10 @@ setup(
|
||||||
"pungi-config-dump = pungi.scripts.config_dump:cli_main",
|
"pungi-config-dump = pungi.scripts.config_dump:cli_main",
|
||||||
"pungi-config-validate = pungi.scripts.config_validate:cli_main",
|
"pungi-config-validate = pungi.scripts.config_validate:cli_main",
|
||||||
"pungi-cache-cleanup = pungi.scripts.cache_cleanup:main",
|
"pungi-cache-cleanup = pungi.scripts.cache_cleanup:main",
|
||||||
|
"pungi-gather-modules = pungi.scripts.gather_modules:cli_main",
|
||||||
|
"pungi-gather-rpms = pungi.scripts.gather_rpms:cli_main",
|
||||||
|
"pungi-generate-packages-json = pungi.scripts.create_packages_json:cli_main", # noqa: E501
|
||||||
|
"pungi-create-extra-repo = pungi.scripts.create_extra_repo:cli_main"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"],
|
scripts=["contrib/yum-dnf-compare/pungi-compare-depsolving"],
|
||||||
|
@ -62,5 +66,5 @@ setup(
|
||||||
"dogpile.cache",
|
"dogpile.cache",
|
||||||
],
|
],
|
||||||
extras_require={':python_version=="2.7"': ["enum34", "lockfile"]},
|
extras_require={':python_version=="2.7"': ["enum34", "lockfile"]},
|
||||||
tests_require=["mock", "pytest", "pytest-cov"],
|
tests_require=["mock", "pytest", "pytest-cov", "pyfakefs"],
|
||||||
)
|
)
|
||||||
|
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,36 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
|
||||||
|
<revision>1612479076</revision>
|
||||||
|
<data type="primary">
|
||||||
|
<checksum type="sha256">08941fae6bdb14f3b22bfad38b9d7dcb685a9df58fe8f515a3a0b2fe1af903bb</checksum>
|
||||||
|
<open-checksum type="sha256">2a15e618f049a883d360ccbf3e764b30640255f47dc526c633b1722fe23cbcbc</open-checksum>
|
||||||
|
<location href="repodata/08941fae6bdb14f3b22bfad38b9d7dcb685a9df58fe8f515a3a0b2fe1af903bb-primary.xml.gz"/>
|
||||||
|
<timestamp>1612479075</timestamp>
|
||||||
|
<size>1240</size>
|
||||||
|
<open-size>3888</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="filelists">
|
||||||
|
<checksum type="sha256">e37a0b4a63b2b245dca1727195300cd3961f80aebc82ae7b9849dbf7482f5d0f</checksum>
|
||||||
|
<open-checksum type="sha256">b1782bc4207a5b7c3e64115d5a1d001802e8d363f022ea165df7cdab6f14651c</open-checksum>
|
||||||
|
<location href="repodata/e37a0b4a63b2b245dca1727195300cd3961f80aebc82ae7b9849dbf7482f5d0f-filelists.xml.gz"/>
|
||||||
|
<timestamp>1612479075</timestamp>
|
||||||
|
<size>439</size>
|
||||||
|
<open-size>1295</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="other">
|
||||||
|
<checksum type="sha256">92992176bce71dcde9e4b6ad1442e7b5c7f3de9b7f019a2cd27d042ab38ea2b1</checksum>
|
||||||
|
<open-checksum type="sha256">3b847919691ad32279b13463de6c08f1f8b32f51e87b7d8d7e95a3ec2f46ef51</open-checksum>
|
||||||
|
<location href="repodata/92992176bce71dcde9e4b6ad1442e7b5c7f3de9b7f019a2cd27d042ab38ea2b1-other.xml.gz"/>
|
||||||
|
<timestamp>1612479075</timestamp>
|
||||||
|
<size>630</size>
|
||||||
|
<open-size>1911</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="modules">
|
||||||
|
<checksum type="sha256">e7a671401f8e207e4cd3b90b4ac92d621f84a34dc9026f57c3f427fbed444c57</checksum>
|
||||||
|
<open-checksum type="sha256">d59fee86c18018cc18bb7325aa74aa0abf923c64d29a4ec45e08dcd01a0c3966</open-checksum>
|
||||||
|
<location href="repodata/e7a671401f8e207e4cd3b90b4ac92d621f84a34dc9026f57c3f427fbed444c57-modules.yaml.gz"/>
|
||||||
|
<timestamp>1612479075</timestamp>
|
||||||
|
<size>920</size>
|
||||||
|
<open-size>3308</open-size>
|
||||||
|
</data>
|
||||||
|
</repomd>
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,55 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
|
||||||
|
<revision>1666177486</revision>
|
||||||
|
<data type="primary">
|
||||||
|
<checksum type="sha256">89cb9cc1181635c9147864a7076d91fb81072641d481cd202832a2d257453576</checksum>
|
||||||
|
<open-checksum type="sha256">07255d9856f7531b52a6459f6fc7701c6d93c6d6c29d1382d83afcc53f13494a</open-checksum>
|
||||||
|
<location href="repodata/89cb9cc1181635c9147864a7076d91fb81072641d481cd202832a2d257453576-primary.xml.gz"/>
|
||||||
|
<timestamp>1666177486</timestamp>
|
||||||
|
<size>1387</size>
|
||||||
|
<open-size>6528</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="filelists">
|
||||||
|
<checksum type="sha256">f69ca03957574729fd5150335b0d87afddcfb37a97aed5b06272212854f1773d</checksum>
|
||||||
|
<open-checksum type="sha256">c2e1e674d7d48bccaa16cae0a5f70cb55ef4cd7352b4d9d4fdaa619075d07dbc</open-checksum>
|
||||||
|
<location href="repodata/f69ca03957574729fd5150335b0d87afddcfb37a97aed5b06272212854f1773d-filelists.xml.gz"/>
|
||||||
|
<timestamp>1666177486</timestamp>
|
||||||
|
<size>1252</size>
|
||||||
|
<open-size>5594</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="other">
|
||||||
|
<checksum type="sha256">b3827bd6c9ea67ffa3912002515c64e4d9fe5c4dacbf7c46b0d8768b7abbb84f</checksum>
|
||||||
|
<open-checksum type="sha256">9ce24c526239e349d023c577b2ae3872c8b0f1888aed1fb24b9b9aa12063fdf3</open-checksum>
|
||||||
|
<location href="repodata/b3827bd6c9ea67ffa3912002515c64e4d9fe5c4dacbf7c46b0d8768b7abbb84f-other.xml.gz"/>
|
||||||
|
<timestamp>1666177486</timestamp>
|
||||||
|
<size>999</size>
|
||||||
|
<open-size>6320</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="primary_db">
|
||||||
|
<checksum type="sha256">ab8df35061dfa0285069b843f24a7076e31266d9a8abe8282340bcb936aa61d7</checksum>
|
||||||
|
<open-checksum type="sha256">2bce9554ce4496cef34b5cd69f186f7f3143c7cabae8fa384fc5c9eeab326f7f</open-checksum>
|
||||||
|
<location href="repodata/ab8df35061dfa0285069b843f24a7076e31266d9a8abe8282340bcb936aa61d7-primary.sqlite.bz2"/>
|
||||||
|
<timestamp>1666177486</timestamp>
|
||||||
|
<size>3558</size>
|
||||||
|
<open-size>106496</open-size>
|
||||||
|
<database_version>10</database_version>
|
||||||
|
</data>
|
||||||
|
<data type="filelists_db">
|
||||||
|
<checksum type="sha256">8bcf6d40db4e922934ac47e8ac7fb8d15bdacf579af8c819d2134ed54d30550b</checksum>
|
||||||
|
<open-checksum type="sha256">f7001d1df7f5f7e4898919b15710bea8ed9711ce42faf68e22b757e63169b1fb</open-checksum>
|
||||||
|
<location href="repodata/8bcf6d40db4e922934ac47e8ac7fb8d15bdacf579af8c819d2134ed54d30550b-filelists.sqlite.bz2"/>
|
||||||
|
<timestamp>1666177486</timestamp>
|
||||||
|
<size>2360</size>
|
||||||
|
<open-size>28672</open-size>
|
||||||
|
<database_version>10</database_version>
|
||||||
|
</data>
|
||||||
|
<data type="other_db">
|
||||||
|
<checksum type="sha256">01b82e9eb7ee9151f283c6e761ae450de18ed2d64b5e32de88689eaf95216a80</checksum>
|
||||||
|
<open-checksum type="sha256">07f5b9750af1e440d37ca216e719dd288149e79e9132f2fdccb6f73b2e5dd541</open-checksum>
|
||||||
|
<location href="repodata/01b82e9eb7ee9151f283c6e761ae450de18ed2d64b5e32de88689eaf95216a80-other.sqlite.bz2"/>
|
||||||
|
<timestamp>1666177486</timestamp>
|
||||||
|
<size>2196</size>
|
||||||
|
<open-size>32768</open-size>
|
||||||
|
<database_version>10</database_version>
|
||||||
|
</data>
|
||||||
|
</repomd>
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,55 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">
|
||||||
|
<revision>1666177500</revision>
|
||||||
|
<data type="primary">
|
||||||
|
<checksum type="sha256">a1d342aa7cef3a2034fc3f9d6ee02d63572780bc76e61749a57e50b6b3ca9869</checksum>
|
||||||
|
<open-checksum type="sha256">a9e3eae447dd44282d7d96db5f15f049b757925397adb752f4df982176bab7e0</open-checksum>
|
||||||
|
<location href="repodata/a1d342aa7cef3a2034fc3f9d6ee02d63572780bc76e61749a57e50b6b3ca9869-primary.xml.gz"/>
|
||||||
|
<timestamp>1666177500</timestamp>
|
||||||
|
<size>3501</size>
|
||||||
|
<open-size>37296</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="filelists">
|
||||||
|
<checksum type="sha256">6778922d5853d20f213ae7702699a76f1e87e55d6bfb5e4ac6a117d904d47b3c</checksum>
|
||||||
|
<open-checksum type="sha256">e30b666d9d88a70de69a08f45e6696bcd600c45485d856bd0213395d7da7bd49</open-checksum>
|
||||||
|
<location href="repodata/6778922d5853d20f213ae7702699a76f1e87e55d6bfb5e4ac6a117d904d47b3c-filelists.xml.gz"/>
|
||||||
|
<timestamp>1666177500</timestamp>
|
||||||
|
<size>27624</size>
|
||||||
|
<open-size>318187</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="other">
|
||||||
|
<checksum type="sha256">5a60d79d8bce6a805f4fdb22fd891524359dce8ccc665c0b54e7299e79debe84</checksum>
|
||||||
|
<open-checksum type="sha256">b18138f4a3de45714e578fb1f30b7ec54fdcdaf1a22585891625b6af0894388e</open-checksum>
|
||||||
|
<location href="repodata/5a60d79d8bce6a805f4fdb22fd891524359dce8ccc665c0b54e7299e79debe84-other.xml.gz"/>
|
||||||
|
<timestamp>1666177500</timestamp>
|
||||||
|
<size>1876</size>
|
||||||
|
<open-size>28701</open-size>
|
||||||
|
</data>
|
||||||
|
<data type="primary_db">
|
||||||
|
<checksum type="sha256">c27bc2ce947173aba305041552c3c6d8db71442c1a2e5dcaf35ff750fe0469fc</checksum>
|
||||||
|
<open-checksum type="sha256">586e1af8934229925adb9e746ae5ced119859dfd97f4e3237399bb36a7d7f071</open-checksum>
|
||||||
|
<location href="repodata/c27bc2ce947173aba305041552c3c6d8db71442c1a2e5dcaf35ff750fe0469fc-primary.sqlite.bz2"/>
|
||||||
|
<timestamp>1666177500</timestamp>
|
||||||
|
<size>11528</size>
|
||||||
|
<open-size>126976</open-size>
|
||||||
|
<database_version>10</database_version>
|
||||||
|
</data>
|
||||||
|
<data type="filelists_db">
|
||||||
|
<checksum type="sha256">ed350865982e7a1e45b144839b56eac888e5d8f680571dd2cd06b37dc83e0fd8</checksum>
|
||||||
|
<open-checksum type="sha256">697903989d0f77de2d44a2b603e75c9b4ca23b3795eb136d175caf5666ce6459</open-checksum>
|
||||||
|
<location href="repodata/ed350865982e7a1e45b144839b56eac888e5d8f680571dd2cd06b37dc83e0fd8-filelists.sqlite.bz2"/>
|
||||||
|
<timestamp>1666177500</timestamp>
|
||||||
|
<size>20440</size>
|
||||||
|
<open-size>163840</open-size>
|
||||||
|
<database_version>10</database_version>
|
||||||
|
</data>
|
||||||
|
<data type="other_db">
|
||||||
|
<checksum type="sha256">35eff699131e0976429144c6f4514d21568177dc64bb4091c3ff62f76b293725</checksum>
|
||||||
|
<open-checksum type="sha256">3bd999a1bdf300df836a4607b7b75f845d8e1432e3e4e1ab6f0c7cc8a853db39</open-checksum>
|
||||||
|
<location href="repodata/35eff699131e0976429144c6f4514d21568177dc64bb4091c3ff62f76b293725-other.sqlite.bz2"/>
|
||||||
|
<timestamp>1666177500</timestamp>
|
||||||
|
<size>4471</size>
|
||||||
|
<open-size>49152</open-size>
|
||||||
|
<database_version>10</database_version>
|
||||||
|
</data>
|
||||||
|
</repomd>
|
|
@ -0,0 +1,58 @@
|
||||||
|
[checksums]
|
||||||
|
images/boot.iso = sha256:fc8a4be604b6425746f12fa706116eb940f93358f036b8fbbe518b516cb6870c
|
||||||
|
|
||||||
|
[general]
|
||||||
|
; WARNING.0 = This section provides compatibility with pre-productmd treeinfos.
|
||||||
|
; WARNING.1 = Read productmd documentation for details about new format.
|
||||||
|
arch = x86_64
|
||||||
|
family = Test
|
||||||
|
name = Test 1.0
|
||||||
|
packagedir = Packages
|
||||||
|
platforms = x86_64,xen
|
||||||
|
repository = .
|
||||||
|
timestamp = 1531881582
|
||||||
|
variant = Server
|
||||||
|
variants = Client,Server
|
||||||
|
version = 1.0
|
||||||
|
|
||||||
|
[header]
|
||||||
|
type = productmd.treeinfo
|
||||||
|
version = 1.2
|
||||||
|
|
||||||
|
[images-x86_64]
|
||||||
|
boot.iso = images/boot.iso
|
||||||
|
|
||||||
|
[images-xen]
|
||||||
|
initrd = images/pxeboot/initrd.img
|
||||||
|
kernel = images/pxeboot/vmlinuz
|
||||||
|
|
||||||
|
[release]
|
||||||
|
name = Test
|
||||||
|
short = T
|
||||||
|
version = 1.0
|
||||||
|
|
||||||
|
[stage2]
|
||||||
|
mainimage = images/install.img
|
||||||
|
|
||||||
|
[tree]
|
||||||
|
arch = x86_64
|
||||||
|
build_timestamp = 1531881582
|
||||||
|
platforms = x86_64,xen
|
||||||
|
variants = Client,Server
|
||||||
|
|
||||||
|
[variant-Client]
|
||||||
|
id = Client
|
||||||
|
name = Client
|
||||||
|
packages = ../../../Client/x86_64/os/Packages
|
||||||
|
repository = ../../../Client/x86_64/os
|
||||||
|
type = variant
|
||||||
|
uid = Client
|
||||||
|
|
||||||
|
[variant-Server]
|
||||||
|
id = Server
|
||||||
|
name = Server
|
||||||
|
packages = Packages
|
||||||
|
repository = .
|
||||||
|
type = variant
|
||||||
|
uid = Server
|
||||||
|
|
|
@ -0,0 +1,20 @@
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: module
|
||||||
|
stream: master
|
||||||
|
version: 20190318
|
||||||
|
context: abcdef
|
||||||
|
arch: x86_64
|
||||||
|
summary: Dummy module
|
||||||
|
description: Dummy module
|
||||||
|
license:
|
||||||
|
module:
|
||||||
|
- Beerware
|
||||||
|
content:
|
||||||
|
- Beerware
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- foobar-0:1.0-1.noarch
|
||||||
|
...
|
|
@ -0,0 +1,20 @@
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: module
|
||||||
|
stream: master
|
||||||
|
version: 20190318
|
||||||
|
context: abcdef
|
||||||
|
arch: x86_64
|
||||||
|
summary: Dummy module
|
||||||
|
description: Dummy module
|
||||||
|
license:
|
||||||
|
module:
|
||||||
|
- Beerware
|
||||||
|
content:
|
||||||
|
- Beerware
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- foobar-0:1.0-1.noarch
|
||||||
|
...
|
|
@ -0,0 +1,20 @@
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: scratch-module
|
||||||
|
stream: master
|
||||||
|
version: 20200710
|
||||||
|
context: abcdef
|
||||||
|
arch: x86_64
|
||||||
|
summary: Dummy module
|
||||||
|
description: Dummy module
|
||||||
|
license:
|
||||||
|
module:
|
||||||
|
- Beerware
|
||||||
|
content:
|
||||||
|
- Beerware
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- foobar-0:1.0-1.noarch
|
||||||
|
...
|
|
@ -0,0 +1,20 @@
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: scratch-module
|
||||||
|
stream: master
|
||||||
|
version: 20200710
|
||||||
|
context: abcdef
|
||||||
|
arch: x86_64
|
||||||
|
summary: Dummy module
|
||||||
|
description: Dummy module
|
||||||
|
license:
|
||||||
|
module:
|
||||||
|
- Beerware
|
||||||
|
content:
|
||||||
|
- Beerware
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- foobar-0:1.0-1.noarch
|
||||||
|
...
|
|
@ -7,7 +7,7 @@ import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import six
|
import six
|
||||||
from kobo.rpmlib import parse_nvr
|
from kobo.rpmlib import parse_nvr
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from pungi.arch import (
|
from pungi.arch import (
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
|
|
|
@ -1,15 +1,15 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import six
|
import six
|
||||||
from copy import copy
|
from copy import copy
|
||||||
from six.moves import StringIO
|
from six.moves import StringIO
|
||||||
|
from ddt import ddt, data
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
@ -473,6 +473,70 @@ class TestBuildinstallPhase(PungiTestCase):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@mock.patch("pungi.phases.buildinstall.ThreadPool")
|
||||||
|
@mock.patch("pungi.phases.buildinstall.LoraxWrapper")
|
||||||
|
@mock.patch("pungi.phases.buildinstall.get_volid")
|
||||||
|
def test_starts_threads_for_each_cmd_with_buildinstall(
|
||||||
|
self, get_volid, loraxCls, poolCls
|
||||||
|
):
|
||||||
|
compose = BuildInstallCompose(
|
||||||
|
self.topdir,
|
||||||
|
{
|
||||||
|
"bootable": True,
|
||||||
|
"release_name": "Test",
|
||||||
|
"release_short": "t",
|
||||||
|
"release_version": "1",
|
||||||
|
"buildinstall_method": "buildinstall",
|
||||||
|
"disc_types": {"dvd": "DVD"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
get_volid.return_value = "vol_id"
|
||||||
|
|
||||||
|
phase = BuildinstallPhase(compose, self._make_pkgset_phase(["p1"]))
|
||||||
|
|
||||||
|
phase.run()
|
||||||
|
|
||||||
|
# Two items added for processing in total.
|
||||||
|
pool = poolCls.return_value
|
||||||
|
self.assertEqual(2, len(pool.queue_put.mock_calls))
|
||||||
|
|
||||||
|
# Obtained correct lorax commands.
|
||||||
|
six.assertCountEqual(
|
||||||
|
self,
|
||||||
|
loraxCls.return_value.get_buildinstall_cmd.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
"Test",
|
||||||
|
"1",
|
||||||
|
"1",
|
||||||
|
[self.topdir + "/work/x86_64/repo/p1"],
|
||||||
|
self.topdir + "/work/x86_64/buildinstall",
|
||||||
|
buildarch="x86_64",
|
||||||
|
is_final=True,
|
||||||
|
volid="vol_id",
|
||||||
|
),
|
||||||
|
mock.call(
|
||||||
|
"Test",
|
||||||
|
"1",
|
||||||
|
"1",
|
||||||
|
[self.topdir + "/work/amd64/repo/p1"],
|
||||||
|
self.topdir + "/work/amd64/buildinstall",
|
||||||
|
buildarch="amd64",
|
||||||
|
is_final=True,
|
||||||
|
volid="vol_id",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
six.assertCountEqual(
|
||||||
|
self,
|
||||||
|
get_volid.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call(compose, "x86_64", disc_type="DVD"),
|
||||||
|
mock.call(compose, "amd64", disc_type="DVD"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
@mock.patch("pungi.phases.buildinstall.get_file")
|
@mock.patch("pungi.phases.buildinstall.get_file")
|
||||||
@mock.patch("pungi.phases.buildinstall.ThreadPool")
|
@mock.patch("pungi.phases.buildinstall.ThreadPool")
|
||||||
@mock.patch("pungi.phases.buildinstall.LoraxWrapper")
|
@mock.patch("pungi.phases.buildinstall.LoraxWrapper")
|
||||||
|
@ -1336,6 +1400,142 @@ class BuildinstallThreadTestCase(PungiTestCase):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@mock.patch("pungi.phases.buildinstall.link_boot_iso")
|
||||||
|
@mock.patch("pungi.phases.buildinstall.tweak_buildinstall")
|
||||||
|
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
|
||||||
|
@mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
|
||||||
|
@mock.patch("pungi.phases.buildinstall.run")
|
||||||
|
def test_buildinstall_thread_with_buildinstall_in_runroot(
|
||||||
|
self, run, get_buildroot_rpms, KojiWrapperMock, mock_tweak, mock_link
|
||||||
|
):
|
||||||
|
compose = BuildInstallCompose(
|
||||||
|
self.topdir,
|
||||||
|
{
|
||||||
|
"buildinstall_method": "buildinstall",
|
||||||
|
"runroot_tag": "rrt",
|
||||||
|
"koji_profile": "koji",
|
||||||
|
"koji_cache": "/tmp",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
get_buildroot_rpms.return_value = ["bash", "zsh"]
|
||||||
|
|
||||||
|
get_runroot_cmd = KojiWrapperMock.return_value.get_runroot_cmd
|
||||||
|
|
||||||
|
run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd
|
||||||
|
run_runroot_cmd.return_value = {
|
||||||
|
"output": "Foo bar baz",
|
||||||
|
"retcode": 0,
|
||||||
|
"task_id": 1234,
|
||||||
|
}
|
||||||
|
|
||||||
|
t = BuildinstallThread(self.pool)
|
||||||
|
|
||||||
|
with mock.patch("time.sleep"):
|
||||||
|
pkgset_phase = self._make_pkgset_phase(["p1"])
|
||||||
|
t.process((compose, "amd64", None, self.cmd, pkgset_phase), 0)
|
||||||
|
|
||||||
|
destdir = os.path.join(self.topdir, "work/amd64/buildinstall")
|
||||||
|
self.assertEqual(
|
||||||
|
get_runroot_cmd.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
"rrt",
|
||||||
|
"amd64",
|
||||||
|
self.cmd,
|
||||||
|
channel=None,
|
||||||
|
use_shell=True,
|
||||||
|
packages=["anaconda"],
|
||||||
|
mounts=[self.topdir],
|
||||||
|
weight=None,
|
||||||
|
chown_paths=[destdir],
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
run_runroot_cmd.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
get_runroot_cmd.return_value,
|
||||||
|
log_file=self.topdir + "/logs/amd64/buildinstall.amd64.log",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
with open(self.topdir + "/logs/amd64/buildinstall-RPMs.amd64.log") as f:
|
||||||
|
rpms = f.read().strip().split("\n")
|
||||||
|
six.assertCountEqual(self, rpms, ["bash", "zsh"])
|
||||||
|
six.assertCountEqual(self, self.pool.finished_tasks, [(None, "amd64")])
|
||||||
|
six.assertCountEqual(
|
||||||
|
self,
|
||||||
|
mock_tweak.call_args_list,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
compose,
|
||||||
|
destdir,
|
||||||
|
os.path.join(self.topdir, "compose", var, "amd64/os"),
|
||||||
|
"amd64",
|
||||||
|
var,
|
||||||
|
"",
|
||||||
|
"dummy-volid",
|
||||||
|
self.pool.kickstart_file,
|
||||||
|
)
|
||||||
|
for var in ["Client", "Server"]
|
||||||
|
],
|
||||||
|
)
|
||||||
|
six.assertCountEqual(
|
||||||
|
self,
|
||||||
|
mock_link.call_args_list,
|
||||||
|
[
|
||||||
|
mock.call(compose, "amd64", compose.variants["Client"], False),
|
||||||
|
mock.call(compose, "amd64", compose.variants["Server"], False),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
|
||||||
|
@mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
|
||||||
|
@mock.patch("pungi.phases.buildinstall.run")
|
||||||
|
def test_buildinstall_fail_exit_code(
|
||||||
|
self, run, get_buildroot_rpms, KojiWrapperMock
|
||||||
|
):
|
||||||
|
compose = BuildInstallCompose(
|
||||||
|
self.topdir,
|
||||||
|
{
|
||||||
|
"buildinstall_method": "buildinstall",
|
||||||
|
"runroot_tag": "rrt",
|
||||||
|
"koji_profile": "koji",
|
||||||
|
"koji_cache": "/tmp",
|
||||||
|
"failable_deliverables": [("^.+$", {"*": ["buildinstall"]})],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
get_buildroot_rpms.return_value = ["bash", "zsh"]
|
||||||
|
|
||||||
|
run_runroot_cmd = KojiWrapperMock.return_value.run_runroot_cmd
|
||||||
|
run_runroot_cmd.return_value = {
|
||||||
|
"output": "Foo bar baz",
|
||||||
|
"retcode": 1,
|
||||||
|
"task_id": 1234,
|
||||||
|
}
|
||||||
|
|
||||||
|
t = BuildinstallThread(self.pool)
|
||||||
|
|
||||||
|
with mock.patch("time.sleep"):
|
||||||
|
pkgset_phase = self._make_pkgset_phase(["p1"])
|
||||||
|
t.process((compose, "x86_64", None, self.cmd, pkgset_phase), 0)
|
||||||
|
|
||||||
|
compose._logger.error.assert_has_calls(
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
"[FAIL] Buildinstall (variant None, arch x86_64) failed, but going on anyway." # noqa: E501
|
||||||
|
),
|
||||||
|
mock.call(
|
||||||
|
"Runroot task failed: 1234. See %s/logs/x86_64/buildinstall.x86_64.log for more details." # noqa: E501
|
||||||
|
% self.topdir
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
self.assertEqual(self.pool.finished_tasks, set())
|
||||||
|
|
||||||
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
|
@mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
|
||||||
@mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
|
@mock.patch("pungi.wrappers.kojiwrapper.get_buildroot_rpms")
|
||||||
@mock.patch("pungi.phases.buildinstall.run")
|
@mock.patch("pungi.phases.buildinstall.run")
|
||||||
|
@ -1814,6 +2014,7 @@ class BuildinstallThreadTestCase(PungiTestCase):
|
||||||
self.assertEqual(ret, None)
|
self.assertEqual(ret, None)
|
||||||
|
|
||||||
|
|
||||||
|
@ddt
|
||||||
class TestSymlinkIso(PungiTestCase):
|
class TestSymlinkIso(PungiTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestSymlinkIso, self).setUp()
|
super(TestSymlinkIso, self).setUp()
|
||||||
|
@ -1829,8 +2030,13 @@ class TestSymlinkIso(PungiTestCase):
|
||||||
@mock.patch("pungi.phases.buildinstall.get_file_size")
|
@mock.patch("pungi.phases.buildinstall.get_file_size")
|
||||||
@mock.patch("pungi.phases.buildinstall.iso")
|
@mock.patch("pungi.phases.buildinstall.iso")
|
||||||
@mock.patch("pungi.phases.buildinstall.run")
|
@mock.patch("pungi.phases.buildinstall.run")
|
||||||
def test_hardlink(self, run, iso, get_file_size, get_mtime, ImageCls):
|
@data(['Server'], ['BaseOS'])
|
||||||
self.compose.conf = {"buildinstall_symlink": False, "disc_types": {}}
|
def test_hardlink(self, netinstall_variants, run, iso, get_file_size, get_mtime, ImageCls):
|
||||||
|
self.compose.conf = {
|
||||||
|
"buildinstall_symlink": False,
|
||||||
|
"disc_types": {},
|
||||||
|
"netinstall_variants": netinstall_variants,
|
||||||
|
}
|
||||||
get_file_size.return_value = 1024
|
get_file_size.return_value = 1024
|
||||||
get_mtime.return_value = 13579
|
get_mtime.return_value = 13579
|
||||||
|
|
||||||
|
@ -1880,9 +2086,14 @@ class TestSymlinkIso(PungiTestCase):
|
||||||
self.assertEqual(image.bootable, True)
|
self.assertEqual(image.bootable, True)
|
||||||
self.assertEqual(image.implant_md5, iso.get_implanted_md5.return_value)
|
self.assertEqual(image.implant_md5, iso.get_implanted_md5.return_value)
|
||||||
self.assertEqual(image.can_fail, False)
|
self.assertEqual(image.can_fail, False)
|
||||||
|
if 'Server' in netinstall_variants:
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.compose.im.add.mock_calls, [mock.call("Server", "x86_64", image)]
|
self.compose.im.add.mock_calls, [mock.call("Server", "x86_64", image)]
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
self.assertEqual(
|
||||||
|
self.compose.im.add.mock_calls, []
|
||||||
|
)
|
||||||
|
|
||||||
@mock.patch("pungi.phases.buildinstall.Image")
|
@mock.patch("pungi.phases.buildinstall.Image")
|
||||||
@mock.patch("pungi.phases.buildinstall.get_mtime")
|
@mock.patch("pungi.phases.buildinstall.get_mtime")
|
||||||
|
@ -1895,6 +2106,7 @@ class TestSymlinkIso(PungiTestCase):
|
||||||
self.compose.conf = {
|
self.compose.conf = {
|
||||||
"buildinstall_symlink": False,
|
"buildinstall_symlink": False,
|
||||||
"disc_types": {"boot": "netinst"},
|
"disc_types": {"boot": "netinst"},
|
||||||
|
"netinstall_variants": ['Server'],
|
||||||
}
|
}
|
||||||
get_file_size.return_value = 1024
|
get_file_size.return_value = 1024
|
||||||
get_mtime.return_value = 13579
|
get_mtime.return_value = 13579
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
|
|
|
@ -7,7 +7,7 @@ except ImportError:
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
import six
|
import six
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
from pungi import checks
|
from pungi import checks
|
||||||
from tests.helpers import load_config, PKGSET_REPOS
|
from tests.helpers import load_config, PKGSET_REPOS
|
||||||
|
@ -223,6 +223,22 @@ class BuildinstallConfigTestCase(ConfigTestCase):
|
||||||
|
|
||||||
self.assertValidation(cfg, [])
|
self.assertValidation(cfg, [])
|
||||||
|
|
||||||
|
def test_buildinstall_with_lorax_options(self):
|
||||||
|
cfg = load_config(
|
||||||
|
PKGSET_REPOS,
|
||||||
|
buildinstall_method="buildinstall",
|
||||||
|
lorax_options=[("^Server$", {})],
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertValidation(
|
||||||
|
cfg,
|
||||||
|
[
|
||||||
|
checks.CONFLICTS.format(
|
||||||
|
"buildinstall_method", "buildinstall", "lorax_options"
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
def test_lorax_with_lorax_options(self):
|
def test_lorax_with_lorax_options(self):
|
||||||
cfg = load_config(PKGSET_REPOS, buildinstall_method="lorax", lorax_options=[])
|
cfg = load_config(PKGSET_REPOS, buildinstall_method="lorax", lorax_options=[])
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import os
|
import os
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,228 @@
|
||||||
|
# coding=utf-8
|
||||||
|
|
||||||
|
import os
|
||||||
|
from unittest import TestCase, mock, main
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from pungi.scripts.create_extra_repo import CreateExtraRepo, ExtraVariantInfo, RepoInfo
|
||||||
|
|
||||||
|
FOLDER_WITH_TEST_DATA = os.path.join(
|
||||||
|
os.path.dirname(
|
||||||
|
os.path.abspath(__file__)
|
||||||
|
),
|
||||||
|
'data/test_create_extra_repo/',
|
||||||
|
)
|
||||||
|
|
||||||
|
TEST_MODULE_INFO = yaml.load("""
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: perl-App-cpanminus
|
||||||
|
stream: 1.7044
|
||||||
|
version: 8030020210126085450
|
||||||
|
context: 3a33b840
|
||||||
|
arch: x86_64
|
||||||
|
summary: Get, unpack, build and install CPAN modules
|
||||||
|
description: >
|
||||||
|
This is a CPAN client that requires zero configuration, and stands alone but it's
|
||||||
|
maintainable and extensible with plug-ins and friendly to shell scripting.
|
||||||
|
license:
|
||||||
|
module:
|
||||||
|
- MIT
|
||||||
|
content:
|
||||||
|
- (GPL+ or Artistic) and GPLv2+
|
||||||
|
- ASL 2.0
|
||||||
|
- GPL+ or Artistic
|
||||||
|
dependencies:
|
||||||
|
- buildrequires:
|
||||||
|
perl: [5.30]
|
||||||
|
platform: [el8.3.0]
|
||||||
|
requires:
|
||||||
|
perl: [5.30]
|
||||||
|
perl-YAML: []
|
||||||
|
platform: [el8]
|
||||||
|
references:
|
||||||
|
community: https://metacpan.org/release/App-cpanminus
|
||||||
|
profiles:
|
||||||
|
common:
|
||||||
|
description: App-cpanminus distribution
|
||||||
|
rpms:
|
||||||
|
- perl-App-cpanminus
|
||||||
|
api:
|
||||||
|
rpms:
|
||||||
|
- perl-App-cpanminus
|
||||||
|
filter:
|
||||||
|
rpms:
|
||||||
|
- perl-CPAN-DistnameInfo-dummy
|
||||||
|
- perl-Test-Deep
|
||||||
|
buildopts:
|
||||||
|
rpms:
|
||||||
|
macros: >
|
||||||
|
%_without_perl_CPAN_Meta_Check_enables_extra_test 1
|
||||||
|
components:
|
||||||
|
rpms:
|
||||||
|
perl-App-cpanminus:
|
||||||
|
rationale: The API.
|
||||||
|
ref: perl-App-cpanminus-1.7044-5.module+el8.2.0+4278+abcfa81a.src.rpm
|
||||||
|
buildorder: 1
|
||||||
|
arches: [i686, x86_64]
|
||||||
|
perl-CPAN-DistnameInfo:
|
||||||
|
rationale: Run-time dependency.
|
||||||
|
ref: stream-0.12-rhel-8.3.0
|
||||||
|
arches: [i686, x86_64]
|
||||||
|
perl-CPAN-Meta-Check:
|
||||||
|
rationale: Run-time dependency.
|
||||||
|
ref: perl-CPAN-Meta-Check-0.014-6.module+el8.2.0+4278+abcfa81a.src.rpm
|
||||||
|
buildorder: 1
|
||||||
|
arches: [i686, x86_64]
|
||||||
|
perl-File-pushd:
|
||||||
|
rationale: Run-time dependency.
|
||||||
|
ref: perl-File-pushd-1.014-6.module+el8.2.0+4278+abcfa81a.src.rpm
|
||||||
|
arches: [i686, x86_64]
|
||||||
|
perl-Module-CPANfile:
|
||||||
|
rationale: Run-time dependency.
|
||||||
|
ref: perl-Module-CPANfile-1.1002-7.module+el8.2.0+4278+abcfa81a.src.rpm
|
||||||
|
arches: [i686, x86_64]
|
||||||
|
perl-Parse-PMFile:
|
||||||
|
rationale: Run-time dependency.
|
||||||
|
ref: perl-Parse-PMFile-0.41-7.module+el8.2.0+4278+abcfa81a.src.rpm
|
||||||
|
arches: [i686, x86_64]
|
||||||
|
perl-String-ShellQuote:
|
||||||
|
rationale: Run-time dependency.
|
||||||
|
ref: perl-String-ShellQuote-1.04-24.module+el8.2.0+4278+abcfa81a.src.rpm
|
||||||
|
arches: [i686, x86_64]
|
||||||
|
perl-Test-Deep:
|
||||||
|
rationale: Build-time dependency.
|
||||||
|
ref: stream-1.127-rhel-8.3.0
|
||||||
|
arches: [i686, x86_64]
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- perl-App-cpanminus-0:1.7044-5.module_el8.3.0+2027+c8990d1d.noarch
|
||||||
|
- perl-App-cpanminus-0:1.7044-5.module_el8.3.0+2027+c8990d1d.src
|
||||||
|
- perl-CPAN-Meta-Check-0:0.014-6.module_el8.3.0+2027+c8990d1d.noarch
|
||||||
|
- perl-CPAN-Meta-Check-0:0.014-6.module_el8.3.0+2027+c8990d1d.src
|
||||||
|
- perl-File-pushd-0:1.014-6.module_el8.3.0+2027+c8990d1d.noarch
|
||||||
|
- perl-File-pushd-0:1.014-6.module_el8.3.0+2027+c8990d1d.src
|
||||||
|
- perl-Module-CPANfile-0:1.1002-7.module_el8.3.0+2027+c8990d1d.noarch
|
||||||
|
- perl-Module-CPANfile-0:1.1002-7.module_el8.3.0+2027+c8990d1d.src
|
||||||
|
- perl-Parse-PMFile-0:0.41-7.module_el8.3.0+2027+c8990d1d.noarch
|
||||||
|
- perl-Parse-PMFile-0:0.41-7.module_el8.3.0+2027+c8990d1d.src
|
||||||
|
- perl-String-ShellQuote-0:1.04-24.module_el8.3.0+2027+c8990d1d.noarch
|
||||||
|
- perl-String-ShellQuote-0:1.04-24.module_el8.3.0+2027+c8990d1d.src
|
||||||
|
...
|
||||||
|
""", Loader=yaml.BaseLoader)
|
||||||
|
|
||||||
|
TEST_REPO_INFO = RepoInfo(
|
||||||
|
path=FOLDER_WITH_TEST_DATA,
|
||||||
|
folder='test_repo',
|
||||||
|
is_remote=False,
|
||||||
|
)
|
||||||
|
TEST_VARIANT_INFO = ExtraVariantInfo(
|
||||||
|
name='TestRepo',
|
||||||
|
arch='x86_64',
|
||||||
|
packages=[],
|
||||||
|
modules=[],
|
||||||
|
repos=[TEST_REPO_INFO]
|
||||||
|
)
|
||||||
|
|
||||||
|
BS_BUILD_INFO = {
|
||||||
|
'build_platforms': [
|
||||||
|
{
|
||||||
|
'architectures': ['non_fake_arch', 'fake_arch'],
|
||||||
|
'name': 'fake_platform'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestCreteExtraRepo(TestCase):
|
||||||
|
|
||||||
|
maxDiff = None
|
||||||
|
|
||||||
|
def test_01_get_repo_info_from_bs_repo(self):
|
||||||
|
auth_token = 'fake_auth_token'
|
||||||
|
build_id = 'fake_build_id'
|
||||||
|
arch = 'fake_arch'
|
||||||
|
packages = ['fake_package1', 'fake_package2']
|
||||||
|
modules = ['fake_module1', 'fake_module2']
|
||||||
|
|
||||||
|
request_object = mock.Mock()
|
||||||
|
request_object.raise_for_status = lambda: True
|
||||||
|
request_object.json = lambda: BS_BUILD_INFO
|
||||||
|
with mock.patch(
|
||||||
|
'pungi.scripts.create_extra_repo.requests.get',
|
||||||
|
return_value=request_object,
|
||||||
|
) as mock_request_get:
|
||||||
|
repos_info = CreateExtraRepo.get_repo_info_from_bs_repo(
|
||||||
|
auth_token=auth_token,
|
||||||
|
build_id=build_id,
|
||||||
|
arch=arch,
|
||||||
|
packages=packages,
|
||||||
|
modules=modules,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
[
|
||||||
|
ExtraVariantInfo(
|
||||||
|
name=f'{build_id}-fake_platform-{arch}',
|
||||||
|
arch=arch,
|
||||||
|
packages=packages,
|
||||||
|
modules=modules,
|
||||||
|
repos=[
|
||||||
|
RepoInfo(
|
||||||
|
path='https://build.cloudlinux.com/'
|
||||||
|
f'build_repos/{build_id}/fake_platform',
|
||||||
|
folder=arch,
|
||||||
|
is_remote=True,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
],
|
||||||
|
repos_info,
|
||||||
|
)
|
||||||
|
mock_request_get.assert_called_once_with(
|
||||||
|
url=f'https://build.cloudlinux.com/api/v1/builds/{build_id}',
|
||||||
|
headers={
|
||||||
|
'Authorization': f'Bearer {auth_token}',
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_02_create_extra_repo(self):
|
||||||
|
with mock.patch(
|
||||||
|
'pungi.scripts.create_extra_repo.'
|
||||||
|
'CreateExtraRepo._read_local_modules_yaml',
|
||||||
|
return_value=[],
|
||||||
|
) as mock__read_local_modules_yaml, mock.patch(
|
||||||
|
'pungi.scripts.create_extra_repo.'
|
||||||
|
'CreateExtraRepo._download_rpm_to_local_repo',
|
||||||
|
) as mock__download_rpm_to_local_repo, mock.patch(
|
||||||
|
'pungi.scripts.create_extra_repo.'
|
||||||
|
'CreateExtraRepo._dump_local_modules_yaml'
|
||||||
|
) as mock__dump_local_modules_yaml, mock.patch(
|
||||||
|
'pungi.scripts.create_extra_repo.'
|
||||||
|
'CreateExtraRepo._create_local_extra_repo'
|
||||||
|
) as mock__create_local_extra_repo:
|
||||||
|
cer = CreateExtraRepo(
|
||||||
|
variants=[TEST_VARIANT_INFO],
|
||||||
|
bs_auth_token='fake_auth_token',
|
||||||
|
local_repository_path='/path/to/local/repo',
|
||||||
|
clear_target_repo=False,
|
||||||
|
)
|
||||||
|
mock__read_local_modules_yaml.assert_called_once_with()
|
||||||
|
cer.create_extra_repo()
|
||||||
|
mock__download_rpm_to_local_repo.assert_called_once_with(
|
||||||
|
package_location='perl-App-cpanminus-1.7044-5.'
|
||||||
|
'module_el8.3.0+2027+c8990d1d.noarch.rpm',
|
||||||
|
repo_info=TEST_REPO_INFO,
|
||||||
|
)
|
||||||
|
mock__dump_local_modules_yaml.assert_called_once_with()
|
||||||
|
mock__create_local_extra_repo.assert_called_once_with()
|
||||||
|
self.assertEqual(
|
||||||
|
[TEST_MODULE_INFO],
|
||||||
|
cer.local_modules_data,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -0,0 +1,112 @@
|
||||||
|
# coding=utf-8
|
||||||
|
|
||||||
|
import os
|
||||||
|
from collections import defaultdict
|
||||||
|
from unittest import TestCase, mock, main
|
||||||
|
|
||||||
|
from pungi.scripts.create_packages_json import (
|
||||||
|
PackagesGenerator,
|
||||||
|
RepoInfo,
|
||||||
|
VariantInfo,
|
||||||
|
)
|
||||||
|
|
||||||
|
FOLDER_WITH_TEST_DATA = os.path.join(
|
||||||
|
os.path.dirname(
|
||||||
|
os.path.abspath(__file__)
|
||||||
|
),
|
||||||
|
'data/test_create_packages_json/',
|
||||||
|
)
|
||||||
|
|
||||||
|
test_repo_info = RepoInfo(
|
||||||
|
path=FOLDER_WITH_TEST_DATA,
|
||||||
|
folder='test_repo',
|
||||||
|
is_remote=False,
|
||||||
|
is_reference=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
test_repo_info_2 = RepoInfo(
|
||||||
|
path=FOLDER_WITH_TEST_DATA,
|
||||||
|
folder='test_repo_2',
|
||||||
|
is_remote=False,
|
||||||
|
is_reference=True,
|
||||||
|
)
|
||||||
|
variant_info_1 = VariantInfo(
|
||||||
|
name='TestRepo',
|
||||||
|
arch='x86_64',
|
||||||
|
repos=[test_repo_info]
|
||||||
|
)
|
||||||
|
variant_info_2 = VariantInfo(
|
||||||
|
name='TestRepo2',
|
||||||
|
arch='x86_64',
|
||||||
|
repos=[test_repo_info_2]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestPackagesJson(TestCase):
|
||||||
|
def test_01_get_remote_file_content(self):
|
||||||
|
"""
|
||||||
|
Test the getting of content from a remote file
|
||||||
|
"""
|
||||||
|
request_object = mock.Mock()
|
||||||
|
request_object.raise_for_status = lambda: True
|
||||||
|
request_object.content = b'TestContent'
|
||||||
|
with mock.patch(
|
||||||
|
'pungi.scripts.create_packages_json.requests.get',
|
||||||
|
return_value=request_object,
|
||||||
|
) as mock_requests_get, mock.patch(
|
||||||
|
'pungi.scripts.create_packages_json.tempfile.NamedTemporaryFile',
|
||||||
|
) as mock_tempfile:
|
||||||
|
mock_tempfile.return_value.__enter__.return_value.name = 'tmpfile'
|
||||||
|
packages_generator = PackagesGenerator(
|
||||||
|
variants=[],
|
||||||
|
excluded_packages=[],
|
||||||
|
included_packages=[],
|
||||||
|
)
|
||||||
|
file_name = packages_generator.get_remote_file_content(
|
||||||
|
file_url='fakeurl')
|
||||||
|
mock_requests_get.assert_called_once_with(url='fakeurl')
|
||||||
|
mock_tempfile.assert_called_once_with(delete=False)
|
||||||
|
mock_tempfile.return_value.__enter__().\
|
||||||
|
write.assert_called_once_with(b'TestContent')
|
||||||
|
self.assertEqual(
|
||||||
|
file_name,
|
||||||
|
'tmpfile',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_02_generate_additional_packages(self):
|
||||||
|
pg = PackagesGenerator(
|
||||||
|
variants=[
|
||||||
|
variant_info_1,
|
||||||
|
variant_info_2,
|
||||||
|
],
|
||||||
|
excluded_packages=['zziplib-utils'],
|
||||||
|
included_packages=['vim-file*'],
|
||||||
|
)
|
||||||
|
test_packages = defaultdict(
|
||||||
|
lambda: defaultdict(
|
||||||
|
lambda: defaultdict(
|
||||||
|
list,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
test_packages['TestRepo']['x86_64']['zziplib'] = \
|
||||||
|
[
|
||||||
|
'zziplib.i686',
|
||||||
|
'zziplib.x86_64',
|
||||||
|
]
|
||||||
|
test_packages['TestRepo2']['x86_64']['vim'] = \
|
||||||
|
[
|
||||||
|
'vim-X11.i686',
|
||||||
|
'vim-common.i686',
|
||||||
|
'vim-enhanced.i686',
|
||||||
|
'vim-filesystem.noarch',
|
||||||
|
]
|
||||||
|
result = pg.generate_packages_json()
|
||||||
|
self.assertEqual(
|
||||||
|
test_packages,
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import mock
|
from unittest import mock
|
||||||
import six
|
import six
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
from parameterized import parameterized
|
from parameterized import parameterized
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
@ -267,6 +267,58 @@ class CreateIsoScriptTest(helpers.PungiTestCase):
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_bootable_run_buildinstall(self):
|
||||||
|
createiso.write_script(
|
||||||
|
createiso.CreateIsoOpts(
|
||||||
|
output_dir=self.outdir,
|
||||||
|
iso_name="DP-1.0-20160405.t.3-ppc64.iso",
|
||||||
|
volid="DP-1.0-20160405.t.3",
|
||||||
|
graft_points="graft-list",
|
||||||
|
arch="ppc64",
|
||||||
|
buildinstall_method="buildinstall",
|
||||||
|
),
|
||||||
|
self.out,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertScript(
|
||||||
|
[
|
||||||
|
" ".join(
|
||||||
|
[
|
||||||
|
"/usr/bin/genisoimage",
|
||||||
|
"-untranslated-filenames",
|
||||||
|
"-volid",
|
||||||
|
"DP-1.0-20160405.t.3",
|
||||||
|
"-J",
|
||||||
|
"-joliet-long",
|
||||||
|
"-rational-rock",
|
||||||
|
"-translation-table",
|
||||||
|
"-x",
|
||||||
|
"./lost+found",
|
||||||
|
"-part",
|
||||||
|
"-hfs",
|
||||||
|
"-r",
|
||||||
|
"-l",
|
||||||
|
"-sysid",
|
||||||
|
"PPC",
|
||||||
|
"-no-desktop",
|
||||||
|
"-allow-multidot",
|
||||||
|
"-chrp-boot",
|
||||||
|
"-map",
|
||||||
|
"/usr/lib/anaconda-runtime/boot/mapping",
|
||||||
|
"-hfs-bless",
|
||||||
|
"/ppc/mac",
|
||||||
|
"-o",
|
||||||
|
"DP-1.0-20160405.t.3-ppc64.iso",
|
||||||
|
"-graft-points",
|
||||||
|
"-path-list",
|
||||||
|
"graft-list",
|
||||||
|
]
|
||||||
|
),
|
||||||
|
" ".join(["/usr/bin/implantisomd5", "DP-1.0-20160405.t.3-ppc64.iso"]),
|
||||||
|
"isoinfo -R -f -i DP-1.0-20160405.t.3-ppc64.iso | grep -v '/TRANS.TBL$' | sort >> DP-1.0-20160405.t.3-ppc64.iso.manifest", # noqa: E501
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
@mock.patch("sys.stderr")
|
@mock.patch("sys.stderr")
|
||||||
@mock.patch("kobo.shortcuts.run")
|
@mock.patch("kobo.shortcuts.run")
|
||||||
def test_run_with_jigdo_bad_args(self, run, stderr):
|
def test_run_with_jigdo_bad_args(self, run, stderr):
|
||||||
|
|
|
@ -8,7 +8,7 @@ except ImportError:
|
||||||
import glob
|
import glob
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from pungi.module_util import Modulemd
|
from pungi.module_util import Modulemd
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from productmd.extra_files import ExtraFiles
|
from productmd.extra_files import ExtraFiles
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
from typing import AnyStr, List
|
||||||
import logging
|
from unittest import mock
|
||||||
import mock
|
|
||||||
import six
|
import six
|
||||||
|
import logging
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
@ -614,6 +614,7 @@ class GetExtraFilesTest(helpers.PungiTestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@mock.patch("pungi.phases.extra_isos.tweak_repo_treeinfo")
|
||||||
@mock.patch("pungi.phases.extra_isos.tweak_treeinfo")
|
@mock.patch("pungi.phases.extra_isos.tweak_treeinfo")
|
||||||
@mock.patch("pungi.wrappers.iso.write_graft_points")
|
@mock.patch("pungi.wrappers.iso.write_graft_points")
|
||||||
@mock.patch("pungi.wrappers.iso.get_graft_points")
|
@mock.patch("pungi.wrappers.iso.get_graft_points")
|
||||||
|
@ -623,7 +624,7 @@ class GetIsoContentsTest(helpers.PungiTestCase):
|
||||||
self.compose = helpers.DummyCompose(self.topdir, {})
|
self.compose = helpers.DummyCompose(self.topdir, {})
|
||||||
self.variant = self.compose.variants["Server"]
|
self.variant = self.compose.variants["Server"]
|
||||||
|
|
||||||
def test_non_bootable_binary(self, ggp, wgp, tt):
|
def test_non_bootable_binary(self, ggp, wgp, tt, trt):
|
||||||
gp = {
|
gp = {
|
||||||
"compose/Client/x86_64/os/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
|
"compose/Client/x86_64/os/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
|
||||||
"compose/Client/x86_64/os/repodata": {
|
"compose/Client/x86_64/os/repodata": {
|
||||||
|
@ -693,7 +694,15 @@ class GetIsoContentsTest(helpers.PungiTestCase):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_inherit_extra_files(self, ggp, wgp, tt):
|
# Check correct call to tweak_repo_treeinfo
|
||||||
|
self._tweak_repo_treeinfo_call_list_checker(
|
||||||
|
trt_mock=trt,
|
||||||
|
main_variant='Server',
|
||||||
|
addon_variants=['Client'],
|
||||||
|
sub_path='x86_64/os',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_inherit_extra_files(self, ggp, wgp, tt, trt):
|
||||||
gp = {
|
gp = {
|
||||||
"compose/Client/x86_64/os/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
|
"compose/Client/x86_64/os/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
|
||||||
"compose/Client/x86_64/os/repodata": {
|
"compose/Client/x86_64/os/repodata": {
|
||||||
|
@ -767,7 +776,15 @@ class GetIsoContentsTest(helpers.PungiTestCase):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_source(self, ggp, wgp, tt):
|
# Check correct call to tweak_repo_treeinfo
|
||||||
|
self._tweak_repo_treeinfo_call_list_checker(
|
||||||
|
trt_mock=trt,
|
||||||
|
main_variant='Server',
|
||||||
|
addon_variants=['Client'],
|
||||||
|
sub_path='x86_64/os',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_source(self, ggp, wgp, tt, trt):
|
||||||
gp = {
|
gp = {
|
||||||
"compose/Client/source/tree/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
|
"compose/Client/source/tree/Packages": {"f/foo.rpm": "/mnt/f/foo.rpm"},
|
||||||
"compose/Client/source/tree/repodata": {
|
"compose/Client/source/tree/repodata": {
|
||||||
|
@ -837,7 +854,15 @@ class GetIsoContentsTest(helpers.PungiTestCase):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_bootable(self, ggp, wgp, tt):
|
# Check correct call to tweak_repo_treeinfo
|
||||||
|
self._tweak_repo_treeinfo_call_list_checker(
|
||||||
|
trt_mock=trt,
|
||||||
|
main_variant='Server',
|
||||||
|
addon_variants=['Client'],
|
||||||
|
sub_path='source/tree',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_bootable(self, ggp, wgp, tt, trt):
|
||||||
self.compose.conf["buildinstall_method"] = "lorax"
|
self.compose.conf["buildinstall_method"] = "lorax"
|
||||||
|
|
||||||
bi_dir = os.path.join(self.topdir, "work/x86_64/buildinstall/Server")
|
bi_dir = os.path.join(self.topdir, "work/x86_64/buildinstall/Server")
|
||||||
|
@ -939,6 +964,42 @@ class GetIsoContentsTest(helpers.PungiTestCase):
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Check correct call to tweak_repo_treeinfo
|
||||||
|
self._tweak_repo_treeinfo_call_list_checker(
|
||||||
|
trt_mock=trt,
|
||||||
|
main_variant='Server',
|
||||||
|
addon_variants=['Client'],
|
||||||
|
sub_path='x86_64/os',
|
||||||
|
)
|
||||||
|
|
||||||
|
def _tweak_repo_treeinfo_call_list_checker(
|
||||||
|
self,
|
||||||
|
trt_mock: mock.Mock,
|
||||||
|
main_variant: AnyStr,
|
||||||
|
addon_variants: List[AnyStr],
|
||||||
|
sub_path: AnyStr) -> None:
|
||||||
|
"""
|
||||||
|
Check correct call to tweak_repo_treeinfo
|
||||||
|
"""
|
||||||
|
path_to_treeinfo = os.path.join(
|
||||||
|
self.topdir,
|
||||||
|
'compose',
|
||||||
|
main_variant,
|
||||||
|
sub_path,
|
||||||
|
'.treeinfo',
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
trt_mock.call_args_list,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
self.compose,
|
||||||
|
addon_variants,
|
||||||
|
path_to_treeinfo,
|
||||||
|
path_to_treeinfo,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class GetFilenameTest(helpers.PungiTestCase):
|
class GetFilenameTest(helpers.PungiTestCase):
|
||||||
def test_use_original_name(self):
|
def test_use_original_name(self):
|
||||||
|
@ -1016,6 +1077,15 @@ class TweakTreeinfoTest(helpers.PungiTestCase):
|
||||||
|
|
||||||
self.assertFilesEqual(output, expected)
|
self.assertFilesEqual(output, expected)
|
||||||
|
|
||||||
|
def test_repo_tweak(self):
|
||||||
|
compose = helpers.DummyCompose(self.topdir, {})
|
||||||
|
input = os.path.join(helpers.FIXTURE_DIR, "extraiso.treeinfo")
|
||||||
|
output = os.path.join(self.topdir, "actual-treeinfo")
|
||||||
|
expected = os.path.join(helpers.FIXTURE_DIR, "extraiso-tweaked-expected.treeinfo")
|
||||||
|
extra_isos.tweak_repo_treeinfo(compose, ["Client"], input, output)
|
||||||
|
|
||||||
|
self.assertFilesEqual(output, expected)
|
||||||
|
|
||||||
|
|
||||||
class PrepareMetadataTest(helpers.PungiTestCase):
|
class PrepareMetadataTest(helpers.PungiTestCase):
|
||||||
@mock.patch("pungi.metadata.create_media_repo")
|
@mock.patch("pungi.metadata.create_media_repo")
|
||||||
|
|
|
@ -153,7 +153,10 @@ class TestParseOutput(unittest.TestCase):
|
||||||
self.assertEqual(modules, set())
|
self.assertEqual(modules, set())
|
||||||
|
|
||||||
def test_extracts_modules(self):
|
def test_extracts_modules(self):
|
||||||
touch(self.file, "module:mod:master:20181003:cafebeef.x86_64@repo-0\n")
|
touch(
|
||||||
|
self.file,
|
||||||
|
"module:mod:master-1:20181003:cafebeef.x86_64@repo-0\n"
|
||||||
|
)
|
||||||
packages, modules = fus.parse_output(self.file)
|
packages, modules = fus.parse_output(self.file)
|
||||||
self.assertEqual(packages, set())
|
self.assertEqual(packages, set())
|
||||||
self.assertEqual(modules, set(["mod:master:20181003:cafebeef"]))
|
self.assertEqual(modules, set(["mod:master_1:20181003:cafebeef"]))
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
from pungi.phases.gather.methods import method_deps as deps
|
from pungi.phases.gather.methods import method_deps as deps
|
||||||
from tests import helpers
|
from tests import helpers
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
import copy
|
import copy
|
||||||
import mock
|
from unittest import mock
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import six
|
import six
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import six
|
import six
|
||||||
|
|
|
@ -0,0 +1,124 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import gzip
|
||||||
|
import os
|
||||||
|
from io import StringIO
|
||||||
|
import yaml
|
||||||
|
from pungi.scripts.gather_modules import collect_modules, EMPTY_FILE
|
||||||
|
import unittest
|
||||||
|
from pyfakefs.fake_filesystem_unittest import TestCase
|
||||||
|
|
||||||
|
MARIADB_MODULE = yaml.load("""
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: mariadb-devel
|
||||||
|
stream: 10.3-1
|
||||||
|
version: 8010020200108182321
|
||||||
|
context: cdc1202b
|
||||||
|
arch: x86_64
|
||||||
|
summary: MariaDB Module
|
||||||
|
description: >-
|
||||||
|
MariaDB is a community developed branch of MySQL.
|
||||||
|
components:
|
||||||
|
rpms:
|
||||||
|
Judy:
|
||||||
|
rationale: MariaDB dependency for OQgraph computation engine
|
||||||
|
ref: a3583b33f939e74a530f2a1dff0552dff2c8ea73
|
||||||
|
buildorder: 4
|
||||||
|
arches: [aarch64, i686, ppc64le, x86_64]
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- Judy-0:1.0.5-18.module_el8.1.0+217+4d875839.i686
|
||||||
|
- Judy-debuginfo-0:1.0.5-18.module_el8.1.0+217+4d875839.i686
|
||||||
|
""", Loader=yaml.BaseLoader)
|
||||||
|
|
||||||
|
JAVAPACKAGES_TOOLS_MODULE = yaml.load("""
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: javapackages-tools
|
||||||
|
stream: 201801
|
||||||
|
version: 8000020190628172923
|
||||||
|
context: b07bea58
|
||||||
|
arch: x86_64
|
||||||
|
summary: Tools and macros for Java packaging support
|
||||||
|
description: >-
|
||||||
|
Java Packages Tools is a collection of tools that make it easier to build RPM
|
||||||
|
packages containing software running on Java platform.
|
||||||
|
components:
|
||||||
|
rpms:
|
||||||
|
ant:
|
||||||
|
rationale: "Runtime dependency of ant-contrib"
|
||||||
|
ref: 2eaf095676540e2805ee7e8c7f6f78285c428fdc
|
||||||
|
arches: [aarch64, i686, ppc64le, x86_64]
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.noarch
|
||||||
|
- ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.src
|
||||||
|
""", Loader=yaml.BaseLoader)
|
||||||
|
|
||||||
|
ANT_DEFAULTS = yaml.load("""
|
||||||
|
data:
|
||||||
|
module: ant
|
||||||
|
profiles:
|
||||||
|
'1.10':
|
||||||
|
- common
|
||||||
|
stream: '1.10'
|
||||||
|
document: modulemd-defaults
|
||||||
|
version: '1'
|
||||||
|
""", Loader=yaml.BaseLoader)
|
||||||
|
|
||||||
|
|
||||||
|
PATH_TO_KOJI = '/path/to/koji'
|
||||||
|
MODULES_YAML_GZ = 'modules.yaml.gz'
|
||||||
|
|
||||||
|
|
||||||
|
class TestModulesYamlParser(TestCase):
|
||||||
|
|
||||||
|
maxDiff = None
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.setUpPyfakefs()
|
||||||
|
|
||||||
|
def _prepare_test_data(self):
|
||||||
|
"""
|
||||||
|
Create modules.yaml.gz with some test data
|
||||||
|
"""
|
||||||
|
os.makedirs(PATH_TO_KOJI)
|
||||||
|
modules_gz_path = os.path.join(PATH_TO_KOJI, MODULES_YAML_GZ)
|
||||||
|
# dump modules into compressed file as in generic repos for rpm
|
||||||
|
io = StringIO()
|
||||||
|
yaml.dump_all([MARIADB_MODULE, JAVAPACKAGES_TOOLS_MODULE, ANT_DEFAULTS], io)
|
||||||
|
with open(os.path.join(PATH_TO_KOJI, MODULES_YAML_GZ), 'wb') as f:
|
||||||
|
f.write(gzip.compress(io.getvalue().encode()))
|
||||||
|
return modules_gz_path
|
||||||
|
|
||||||
|
def test_export_modules(self):
|
||||||
|
modules_gz_path = self._prepare_test_data()
|
||||||
|
|
||||||
|
paths = [open(modules_gz_path, 'rb')]
|
||||||
|
collect_modules(paths, PATH_TO_KOJI)
|
||||||
|
|
||||||
|
# check directory structure matches expected
|
||||||
|
self.assertEqual([MODULES_YAML_GZ, 'modules', 'module_defaults'], os.listdir(PATH_TO_KOJI))
|
||||||
|
self.assertEqual(['mariadb-devel-10.3_1-8010020200108182321.cdc1202b',
|
||||||
|
'javapackages-tools-201801-8000020190628172923.b07bea58'],
|
||||||
|
os.listdir(os.path.join(PATH_TO_KOJI, 'modules/x86_64')))
|
||||||
|
self.assertEqual([EMPTY_FILE, 'ant.yaml'],
|
||||||
|
os.listdir(os.path.join(PATH_TO_KOJI, 'module_defaults')))
|
||||||
|
|
||||||
|
# check that modules were exported
|
||||||
|
self.assertEqual(MARIADB_MODULE, yaml.safe_load(
|
||||||
|
open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'mariadb-devel-10.3_1-8010020200108182321.cdc1202b'))))
|
||||||
|
self.assertEqual(JAVAPACKAGES_TOOLS_MODULE, yaml.safe_load(
|
||||||
|
open(os.path.join(PATH_TO_KOJI, 'modules/x86_64', 'javapackages-tools-201801-8000020190628172923.b07bea58'))))
|
||||||
|
|
||||||
|
# check that defaults were copied
|
||||||
|
self.assertEqual(ANT_DEFAULTS, yaml.safe_load(
|
||||||
|
open(os.path.join(PATH_TO_KOJI, 'module_defaults', 'ant.yaml'))))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -4,7 +4,7 @@ import copy
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
|
|
|
@ -0,0 +1,151 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from pyfakefs.fake_filesystem_unittest import TestCase
|
||||||
|
|
||||||
|
from pungi.scripts.gather_rpms import search_rpms, copy_rpms, Package
|
||||||
|
from productmd.common import parse_nvra
|
||||||
|
|
||||||
|
PATH_TO_REPOS = '/path/to/repos'
|
||||||
|
MODULES_YAML_GZ = 'modules.yaml.gz'
|
||||||
|
|
||||||
|
|
||||||
|
class TestGatherRpms(TestCase):
|
||||||
|
maxDiff = None
|
||||||
|
|
||||||
|
FILES_TO_CREATE = [
|
||||||
|
'powertools/Packages/libvirt-6.0.0-28.module_el'
|
||||||
|
'8.3.0+555+a55c8938.i686.rpm',
|
||||||
|
'powertools/Packages/libgit2-devel-0.26.8-2.el8.x86_64.rpm',
|
||||||
|
'powertools/Packages/xalan-j2-2.7.1-38.module_el'
|
||||||
|
'8.0.0+30+832da3a1.noarch.rpm',
|
||||||
|
'appstream/Packages/bnd-maven-plugin-3.5.0-4.module_el'
|
||||||
|
'8.0.0+30+832da3a1.noarch.rpm',
|
||||||
|
'appstream/Packages/OpenEXR-devel-2.2.0-11.el8.i686.rpm',
|
||||||
|
'appstream/Packages/mingw-binutils-generic-2.30-1.el8.x86_64.rpm',
|
||||||
|
'appstream/Packages/somenonrpm',
|
||||||
|
]
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.setUpPyfakefs()
|
||||||
|
|
||||||
|
os.makedirs(PATH_TO_REPOS)
|
||||||
|
|
||||||
|
for filepath in self.FILES_TO_CREATE:
|
||||||
|
os.makedirs(
|
||||||
|
os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)),
|
||||||
|
exist_ok=True,
|
||||||
|
)
|
||||||
|
open(os.path.join(PATH_TO_REPOS, filepath), 'w').close()
|
||||||
|
|
||||||
|
def test_gather_rpms(self):
|
||||||
|
self.assertEqual(
|
||||||
|
[Package(nvra=parse_nvra('libvirt-6.0.0-28.module_'
|
||||||
|
'el8.3.0+555+a55c8938.i686'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/powertools/Packages/'
|
||||||
|
f'libvirt-6.0.0-28.module_el'
|
||||||
|
f'8.3.0+555+a55c8938.i686.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('libgit2-devel-0.26.8-2.el8.x86_64'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/powertools/Packages/'
|
||||||
|
f'libgit2-devel-0.26.8-2.el8.x86_64.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('xalan-j2-2.7.1-38.module_el'
|
||||||
|
'8.0.0+30+832da3a1.noarch'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/powertools/Packages/'
|
||||||
|
f'xalan-j2-2.7.1-38.module_el'
|
||||||
|
f'8.0.0+30+832da3a1.noarch.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('bnd-maven-plugin-3.5.0-4.module_el'
|
||||||
|
'8.0.0+30+832da3a1.noarch'),
|
||||||
|
path=Path(
|
||||||
|
'/path/to/repos/appstream/Packages/'
|
||||||
|
'bnd-maven-plugin-3.5.0-4.module_el'
|
||||||
|
'8.0.0+30+832da3a1.noarch.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('OpenEXR-devel-2.2.0-11.el8.i686'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/appstream/Packages/'
|
||||||
|
f'OpenEXR-devel-2.2.0-11.el8.i686.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('mingw-binutils-generic-'
|
||||||
|
'2.30-1.el8.x86_64'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/appstream/Packages/'
|
||||||
|
f'mingw-binutils-generic-2.30-1.el8.x86_64.rpm'
|
||||||
|
))
|
||||||
|
],
|
||||||
|
search_rpms(Path(PATH_TO_REPOS))
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_copy_rpms(self):
|
||||||
|
target_path = Path('/mnt/koji')
|
||||||
|
packages = [
|
||||||
|
|
||||||
|
Package(nvra=parse_nvra('libvirt-6.0.0-28.module_'
|
||||||
|
'el8.3.0+555+a55c8938.i686'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/powertools/Packages/'
|
||||||
|
f'libvirt-6.0.0-28.module_el'
|
||||||
|
f'8.3.0+555+a55c8938.i686.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('libgit2-devel-0.26.8-2.el8.x86_64'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/powertools/Packages/'
|
||||||
|
f'libgit2-devel-0.26.8-2.el8.x86_64.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('xalan-j2-2.7.1-38.module_'
|
||||||
|
'el8.0.0+30+832da3a1.noarch'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/powertools/Packages/'
|
||||||
|
f'xalan-j2-2.7.1-38.module_el'
|
||||||
|
f'8.0.0+30+832da3a1.noarch.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('bnd-maven-plugin-3.5.0-4.module_el'
|
||||||
|
'8.0.0+30+832da3a1.noarch'),
|
||||||
|
path=Path(
|
||||||
|
'/path/to/repos/appstream/Packages/'
|
||||||
|
'bnd-maven-plugin-3.5.0-4.module_el'
|
||||||
|
'8.0.0+30+832da3a1.noarch.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('OpenEXR-devel-2.2.0-11.el8.i686'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/appstream/Packages/'
|
||||||
|
f'OpenEXR-devel-2.2.0-11.el8.i686.rpm'
|
||||||
|
)),
|
||||||
|
Package(nvra=parse_nvra('mingw-binutils-generic-'
|
||||||
|
'2.30-1.el8.x86_64'),
|
||||||
|
path=Path(
|
||||||
|
f'{PATH_TO_REPOS}/appstream/Packages/'
|
||||||
|
f'mingw-binutils-generic-2.30-1.el8.x86_64.rpm'
|
||||||
|
))
|
||||||
|
]
|
||||||
|
copy_rpms(packages, target_path, [])
|
||||||
|
|
||||||
|
self.assertCountEqual([
|
||||||
|
'xalan-j2-2.7.1-38.module_el8.0.0+30+832da3a1.noarch.rpm',
|
||||||
|
'bnd-maven-plugin-3.5.0-4.module_el8.0.0+30+832da3a1.noarch.rpm'
|
||||||
|
], os.listdir(target_path / 'noarch'))
|
||||||
|
|
||||||
|
self.assertCountEqual([
|
||||||
|
'libgit2-devel-0.26.8-2.el8.x86_64.rpm',
|
||||||
|
'mingw-binutils-generic-2.30-1.el8.x86_64.rpm'
|
||||||
|
], os.listdir(target_path / 'x86_64'))
|
||||||
|
|
||||||
|
self.assertCountEqual([
|
||||||
|
'libvirt-6.0.0-28.module_el8.3.0+555+a55c8938.i686.rpm',
|
||||||
|
'OpenEXR-devel-2.2.0-11.el8.i686.rpm'
|
||||||
|
], os.listdir(target_path / 'i686'))
|
||||||
|
|
||||||
|
self.assertCountEqual([
|
||||||
|
'i686', 'x86_64', 'noarch'
|
||||||
|
], os.listdir(target_path))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -5,7 +5,7 @@ try:
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from pungi.phases.gather.sources.source_module import GatherSourceModule
|
from pungi.phases.gather.sources.source_module import GatherSourceModule
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import unittest
|
import unittest
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
|
@ -5,7 +5,7 @@ try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import unittest
|
import unittest
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import itertools
|
import itertools
|
||||||
import mock
|
from unittest import mock
|
||||||
import os
|
import os
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,364 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import os
|
||||||
|
import ddt
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from pyfakefs.fake_filesystem_unittest import TestCase
|
||||||
|
|
||||||
|
from pungi.wrappers.kojimock import KojiMock, RELEASE_BUILD_ID
|
||||||
|
|
||||||
|
PATH_TO_REPOS = '/path/to/repos'
|
||||||
|
MODULES_YAML_GZ = 'modules.yaml.gz'
|
||||||
|
|
||||||
|
|
||||||
|
@ddt.ddt
|
||||||
|
class TestLocalKojiMock(TestCase):
|
||||||
|
maxDiff = None
|
||||||
|
|
||||||
|
FILES_TO_CREATE = [
|
||||||
|
# modular package that should be excluded from global list
|
||||||
|
'powertools/Packages/ant-1.10.5-1.module_el8.0.0+30+832da3a1.noarch.rpm',
|
||||||
|
# packages that should be gathered
|
||||||
|
'powertools/Packages/libgit2-devel-0.26.8-2.el8.x86_64.rpm',
|
||||||
|
'appstream/Packages/OpenEXR-devel-2.2.0-11.el8.i686.rpm',
|
||||||
|
'appstream/Packages/mingw-binutils-generic-2.30-1.el8.x86_64.rpm',
|
||||||
|
# non-rpm
|
||||||
|
'appstream/Packages/somenonrpm',
|
||||||
|
]
|
||||||
|
|
||||||
|
MARIADB_MODULE = """
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: mariadb-devel
|
||||||
|
stream: 10.3
|
||||||
|
version: 8010020200108182321
|
||||||
|
context: cdc1202b
|
||||||
|
arch: x86_64
|
||||||
|
summary: MariaDB Module
|
||||||
|
license:
|
||||||
|
content:
|
||||||
|
- (CDDL or GPLv2 with exceptions) and ASL 2.0
|
||||||
|
module:
|
||||||
|
- MIT
|
||||||
|
description: >-
|
||||||
|
MariaDB is a community developed branch of MySQL.
|
||||||
|
components:
|
||||||
|
rpms:
|
||||||
|
Judy:
|
||||||
|
rationale: MariaDB dependency for OQgraph computation engine
|
||||||
|
ref: a3583b33f939e74a530f2a1dff0552dff2c8ea73
|
||||||
|
buildorder: 4
|
||||||
|
arches: [aarch64, i686, ppc64le, x86_64]
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- Judy-0:1.0.5-18.module_el8.1.0+217+4d875839.i686
|
||||||
|
- Judy-debuginfo-0:1.0.5-18.module_el8.1.0+217+4d875839.i686
|
||||||
|
"""
|
||||||
|
|
||||||
|
JAVAPACKAGES_TOOLS_MODULE = """
|
||||||
|
---
|
||||||
|
document: modulemd
|
||||||
|
version: 2
|
||||||
|
data:
|
||||||
|
name: javapackages-tools
|
||||||
|
stream: 201801
|
||||||
|
version: 8000020190628172923
|
||||||
|
context: b07bea58
|
||||||
|
arch: x86_64
|
||||||
|
summary: Tools and macros for Java packaging support
|
||||||
|
license:
|
||||||
|
content:
|
||||||
|
- (CDDL or GPLv2 with exceptions) and ASL 2.0
|
||||||
|
module:
|
||||||
|
- MIT
|
||||||
|
description: >-
|
||||||
|
Java Packages Tools is a collection of tools that make it easier to build RPM
|
||||||
|
packages containing software running on Java platform.
|
||||||
|
components:
|
||||||
|
rpms:
|
||||||
|
ant:
|
||||||
|
rationale: "Runtime dependency of ant-contrib"
|
||||||
|
ref: 2eaf095676540e2805ee7e8c7f6f78285c428fdc
|
||||||
|
arches: [aarch64, i686, ppc64le, x86_64]
|
||||||
|
artifacts:
|
||||||
|
rpms:
|
||||||
|
- ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.noarch
|
||||||
|
- ant-0:1.10.5-1.module_el8.0.0+30+832da3a1.src
|
||||||
|
"""
|
||||||
|
|
||||||
|
ANT_DEFAULTS = """
|
||||||
|
data:
|
||||||
|
module: ant
|
||||||
|
profiles:
|
||||||
|
'1.10':
|
||||||
|
- common
|
||||||
|
stream: '1.10'
|
||||||
|
document: modulemd-defaults
|
||||||
|
version: '1'
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.setUpPyfakefs()
|
||||||
|
|
||||||
|
os.makedirs(PATH_TO_REPOS)
|
||||||
|
os.makedirs(os.path.join(PATH_TO_REPOS, 'modules/x86_64'))
|
||||||
|
|
||||||
|
with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
|
||||||
|
'javapackages-tools-201801-8000020190628172923.b07bea58'), 'w') as f:
|
||||||
|
f.write(self.JAVAPACKAGES_TOOLS_MODULE)
|
||||||
|
|
||||||
|
with open(os.path.join(PATH_TO_REPOS, 'modules/x86_64',
|
||||||
|
'mariadb-devel-10.3-8010020200108182321.cdc1202b'), 'w') as f:
|
||||||
|
f.write(self.MARIADB_MODULE)
|
||||||
|
|
||||||
|
for filepath in self.FILES_TO_CREATE:
|
||||||
|
os.makedirs(os.path.join(PATH_TO_REPOS, os.path.dirname(filepath)), exist_ok=True)
|
||||||
|
open(os.path.join(PATH_TO_REPOS, filepath), 'w').close()
|
||||||
|
|
||||||
|
self._koji = KojiMock(
|
||||||
|
PATH_TO_REPOS,
|
||||||
|
os.path.join(PATH_TO_REPOS, 'modules'),
|
||||||
|
['x86_64', 'noarch', 'i686'],
|
||||||
|
)
|
||||||
|
|
||||||
|
@ddt.data(
|
||||||
|
[0, {
|
||||||
|
'completion_ts': 0,
|
||||||
|
'arch': 'x86_64',
|
||||||
|
'extra': {
|
||||||
|
'typeinfo': {
|
||||||
|
'module': {
|
||||||
|
'content_koji_tag': 'javapackages-tools-201801-8000020190628172923.b07bea58',
|
||||||
|
'context': 'b07bea58',
|
||||||
|
'name': 'javapackages-tools',
|
||||||
|
'stream': '201801',
|
||||||
|
'version': '8000020190628172923'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'id': 0,
|
||||||
|
'name': 'javapackages-tools',
|
||||||
|
'release': '8000020190628172923.b07bea58',
|
||||||
|
'state': 'COMPLETE',
|
||||||
|
'version': '201801'
|
||||||
|
}],
|
||||||
|
[1, {
|
||||||
|
'completion_ts': 0,
|
||||||
|
'arch': 'x86_64',
|
||||||
|
'extra': {
|
||||||
|
'typeinfo': {
|
||||||
|
'module': {
|
||||||
|
'content_koji_tag': 'mariadb-devel-10.3-8010020200108182321.cdc1202b',
|
||||||
|
'context': 'cdc1202b',
|
||||||
|
'name': 'mariadb-devel',
|
||||||
|
'stream': '10.3',
|
||||||
|
'version': '8010020200108182321'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'id': 1,
|
||||||
|
'name': 'mariadb-devel',
|
||||||
|
'release': '8010020200108182321.cdc1202b',
|
||||||
|
'state': 'COMPLETE',
|
||||||
|
'version': '10.3'
|
||||||
|
}]
|
||||||
|
)
|
||||||
|
@ddt.unpack
|
||||||
|
def test_get_build_info(self, build_id, result):
|
||||||
|
"""
|
||||||
|
Check that we are able to get build information from getBuild method
|
||||||
|
"""
|
||||||
|
build_info = self._koji.getBuild(build_id)
|
||||||
|
|
||||||
|
self.assertEqual(result, build_info)
|
||||||
|
|
||||||
|
@ddt.data(
|
||||||
|
[0, [{'btype': 'module', 'build_id': 0, 'filename': 'modulemd.x86_64.txt'},
|
||||||
|
{'btype': 'module', 'build_id': 0, 'filename': 'modulemd.txt'}]],
|
||||||
|
[1, [{'btype': 'module', 'build_id': 1, 'filename': 'modulemd.x86_64.txt'},
|
||||||
|
{'btype': 'module', 'build_id': 1, 'filename': 'modulemd.txt'}]]
|
||||||
|
)
|
||||||
|
@ddt.unpack
|
||||||
|
def test_list_archives(self, build_id, result):
|
||||||
|
"""
|
||||||
|
Provides list of archives of module descriptions.
|
||||||
|
Always should contain at least two files, so
|
||||||
|
I did a little hack and added modulemd.txt (it is on real koji)
|
||||||
|
but it is not used later by pungi
|
||||||
|
"""
|
||||||
|
build_info = self._koji.listArchives(build_id)
|
||||||
|
|
||||||
|
self.assertEqual(result, build_info)
|
||||||
|
|
||||||
|
@ddt.data(
|
||||||
|
[
|
||||||
|
'javapackages-tools-201801-8000020190628172923.b07bea58',
|
||||||
|
[
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'arch': 'noarch',
|
||||||
|
'build_id': 0,
|
||||||
|
'epoch': '0',
|
||||||
|
'extra': None,
|
||||||
|
'id': 262555,
|
||||||
|
'metadata_only': False,
|
||||||
|
'name': 'ant',
|
||||||
|
'release': '1.module_el8.0.0+30+832da3a1',
|
||||||
|
'size': 0,
|
||||||
|
'version': '1.10.5'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'arch': 'src',
|
||||||
|
'build_id': 0,
|
||||||
|
'epoch': '0',
|
||||||
|
'extra': None,
|
||||||
|
'id': 262555,
|
||||||
|
'metadata_only': False,
|
||||||
|
'name': 'ant',
|
||||||
|
'release': '1.module_el8.0.0+30+832da3a1',
|
||||||
|
'size': 0,
|
||||||
|
'version': '1.10.5'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'build_id': 0,
|
||||||
|
'id': 0,
|
||||||
|
'name': 'javapackages-tools',
|
||||||
|
'nvr': 'javapackages-tools-201801-8000020190628172923.b07bea58',
|
||||||
|
'package_name': 'javapackages-tools',
|
||||||
|
'release': '8000020190628172923',
|
||||||
|
'tag_name': 'javapackages-tools-201801-8000020190628172923.b07bea58',
|
||||||
|
'version': '201801',
|
||||||
|
'volume_name': 'DEFAULT'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'mariadb-devel-10.3-8010020200108182321.cdc1202b',
|
||||||
|
[
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'arch': 'i686',
|
||||||
|
'build_id': 1,
|
||||||
|
'epoch': '0',
|
||||||
|
'extra': None,
|
||||||
|
'id': 262555,
|
||||||
|
'metadata_only': False,
|
||||||
|
'name': 'Judy',
|
||||||
|
'release': '18.module_el8.1.0+217+4d875839',
|
||||||
|
'size': 0,
|
||||||
|
'version': '1.0.5'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'arch': 'i686',
|
||||||
|
'build_id': 1,
|
||||||
|
'epoch': '0',
|
||||||
|
'extra': None,
|
||||||
|
'id': 262555,
|
||||||
|
'metadata_only': False,
|
||||||
|
'name': 'Judy-debuginfo',
|
||||||
|
'release': '18.module_el8.1.0+217+4d875839',
|
||||||
|
'size': 0,
|
||||||
|
'version': '1.0.5'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
[
|
||||||
|
{'build_id': 1,
|
||||||
|
'id': 1,
|
||||||
|
'name': 'mariadb-devel',
|
||||||
|
'nvr': 'mariadb-devel-10.3-8010020200108182321.cdc1202b',
|
||||||
|
'package_name': 'mariadb-devel',
|
||||||
|
'release': '8010020200108182321',
|
||||||
|
'tag_name': 'mariadb-devel-10.3-8010020200108182321.cdc1202b',
|
||||||
|
'version': '10.3',
|
||||||
|
'volume_name': 'DEFAULT'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'dist-c8-compose',
|
||||||
|
[
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'arch': 'x86_64',
|
||||||
|
'build_id': RELEASE_BUILD_ID,
|
||||||
|
'epoch': None,
|
||||||
|
'extra': None,
|
||||||
|
'metadata_only': False,
|
||||||
|
'name': 'libgit2-devel',
|
||||||
|
'release': '2.el8',
|
||||||
|
'version': '0.26.8'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'arch': 'i686',
|
||||||
|
'build_id': RELEASE_BUILD_ID,
|
||||||
|
'epoch': None,
|
||||||
|
'extra': None,
|
||||||
|
'metadata_only': False,
|
||||||
|
'name': 'OpenEXR-devel',
|
||||||
|
'release': '11.el8',
|
||||||
|
'version': '2.2.0'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'arch': 'x86_64',
|
||||||
|
'build_id': RELEASE_BUILD_ID,
|
||||||
|
'epoch': None,
|
||||||
|
'extra': None,
|
||||||
|
'metadata_only': False,
|
||||||
|
'name': 'mingw-binutils-generic',
|
||||||
|
'release': '1.el8',
|
||||||
|
'version': '2.30'
|
||||||
|
}
|
||||||
|
],
|
||||||
|
# no build needed in this case because pungi does not use them
|
||||||
|
[]
|
||||||
|
]
|
||||||
|
],
|
||||||
|
)
|
||||||
|
@ddt.unpack
|
||||||
|
def test_list_tagged_rpms(self, tag, result):
|
||||||
|
"""
|
||||||
|
This method is used by pungi to get list of rpms:
|
||||||
|
either modular or just prepared for release
|
||||||
|
"""
|
||||||
|
self.assertEqual(result, self._koji.listTaggedRPMS(tag))
|
||||||
|
|
||||||
|
def test_list_tagged(self):
|
||||||
|
"""
|
||||||
|
Used only to get list of modules for some release.
|
||||||
|
"""
|
||||||
|
result = self._koji.listTagged('dist-c8-module-compose')
|
||||||
|
self.assertEqual([
|
||||||
|
{
|
||||||
|
'arch': 'x86_64',
|
||||||
|
'build_id': 0,
|
||||||
|
'id': 0,
|
||||||
|
'name': 'javapackages-tools',
|
||||||
|
'nvr': 'javapackages-tools-201801-8000020190628172923.b07bea58',
|
||||||
|
'owner_name': 'centos',
|
||||||
|
'package_name': 'javapackages-tools',
|
||||||
|
'release': '8000020190628172923.b07bea58',
|
||||||
|
'tag_name': 'dist-c8-module-compose',
|
||||||
|
'version': '201801'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'arch': 'x86_64',
|
||||||
|
'build_id': 1,
|
||||||
|
'id': 1,
|
||||||
|
'name': 'mariadb-devel',
|
||||||
|
'nvr': 'mariadb-devel-10.3-8010020200108182321.cdc1202b',
|
||||||
|
'owner_name': 'centos',
|
||||||
|
'package_name': 'mariadb-devel',
|
||||||
|
'release': '8010020200108182321.cdc1202b',
|
||||||
|
'tag_name': 'dist-c8-module-compose',
|
||||||
|
'version': '10.3'
|
||||||
|
}], result)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
|
@ -1,7 +1,7 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import errno
|
import errno
|
||||||
import os
|
import os
|
||||||
import stat
|
import stat
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@ try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import unittest
|
import unittest
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
from pungi import media_split
|
from pungi import media_split
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import mock
|
from unittest import mock
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import six
|
import six
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import json
|
import json
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import json
|
import json
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import six
|
import six
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import os
|
import os
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import mock
|
from unittest import mock
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from pungi.module_util import Modulemd
|
from pungi.module_util import Modulemd
|
||||||
|
@ -135,16 +135,16 @@ class TestMaterializedPkgsetCreate(helpers.PungiTestCase):
|
||||||
amm.assert_called_once()
|
amm.assert_called_once()
|
||||||
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
amm.mock_calls[0].args[1], os.path.join(self.topdir, "work/x86_64/repo/foo")
|
amm.mock_calls[0][1][1], os.path.join(self.topdir, "work/x86_64/repo/foo")
|
||||||
)
|
)
|
||||||
self.assertIsInstance(amm.mock_calls[0].args[2], Modulemd.ModuleIndex)
|
self.assertIsInstance(amm.mock_calls[0][1][2], Modulemd.ModuleIndex)
|
||||||
self.assertIsNotNone(amm.mock_calls[0].args[2].get_module("mod_name"))
|
self.assertIsNotNone(amm.mock_calls[0][1][2].get_module("mod_name"))
|
||||||
# Check if proper Index is used by add_modular_metadata
|
# Check if proper Index is used by add_modular_metadata
|
||||||
self.assertIsNotNone(
|
self.assertIsNotNone(
|
||||||
amm.mock_calls[0].args[2].get_module("mod_name").get_obsoletes()
|
amm.mock_calls[0][1][2].get_module("mod_name").get_obsoletes()
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
amm.mock_calls[0].args[3],
|
amm.mock_calls[0][1][3],
|
||||||
os.path.join(self.topdir, "logs/x86_64/arch_repo_modulemd.foo.x86_64.log"),
|
os.path.join(self.topdir, "logs/x86_64/arch_repo_modulemd.foo.x86_64.log"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
import ddt
|
||||||
import mock
|
from unittest import mock
|
||||||
import os
|
import os
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
@ -141,9 +141,25 @@ class DummySystem(object):
|
||||||
return self.methods
|
return self.methods
|
||||||
|
|
||||||
|
|
||||||
|
@ddt.ddt
|
||||||
@mock.patch("pungi.phases.pkgset.pkgsets.ReaderPool", new=FakePool)
|
@mock.patch("pungi.phases.pkgset.pkgsets.ReaderPool", new=FakePool)
|
||||||
@mock.patch("kobo.pkgset.FileCache", new=MockFileCache)
|
@mock.patch("kobo.pkgset.FileCache", new=MockFileCache)
|
||||||
class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
|
||||||
|
cls.patcher = mock.patch.object(
|
||||||
|
pkgsets.KojiMockPackageSet,
|
||||||
|
'_is_rpm_signed',
|
||||||
|
return_value=True,
|
||||||
|
)
|
||||||
|
cls.patcher.start()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls) -> None:
|
||||||
|
cls.patcher.stop()
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestKojiPkgset, self).setUp()
|
super(TestKojiPkgset, self).setUp()
|
||||||
with open(os.path.join(helpers.FIXTURE_DIR, "tagged-rpms.json")) as f:
|
with open(os.path.join(helpers.FIXTURE_DIR, "tagged-rpms.json")) as f:
|
||||||
|
@ -167,7 +183,11 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
six.assertCountEqual(self, v1, v2)
|
six.assertCountEqual(self, v1, v2)
|
||||||
self.assertEqual({}, actual, msg="Some architectures were missing")
|
self.assertEqual({}, actual, msg="Some architectures were missing")
|
||||||
|
|
||||||
def test_all_arches(self):
|
@ddt.data(
|
||||||
|
pkgsets.KojiMockPackageSet,
|
||||||
|
pkgsets.KojiPackageSet,
|
||||||
|
)
|
||||||
|
def test_all_arches(self, package_set):
|
||||||
self._touch_files(
|
self._touch_files(
|
||||||
[
|
[
|
||||||
"rpms/pungi@4.1.3@3.fc25@noarch",
|
"rpms/pungi@4.1.3@3.fc25@noarch",
|
||||||
|
@ -180,7 +200,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
pkgset = pkgsets.KojiPackageSet(
|
pkgset = package_set(
|
||||||
"pkgset", self.koji_wrapper, [None], downloader=self.koji_downloader
|
"pkgset", self.koji_wrapper, [None], downloader=self.koji_downloader
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -207,7 +227,11 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_only_one_arch(self):
|
@ddt.data(
|
||||||
|
pkgsets.KojiPackageSet,
|
||||||
|
pkgsets.KojiMockPackageSet,
|
||||||
|
)
|
||||||
|
def test_only_one_arch(self, package_set):
|
||||||
self._touch_files(
|
self._touch_files(
|
||||||
[
|
[
|
||||||
"rpms/bash@4.3.42@4.fc24@x86_64",
|
"rpms/bash@4.3.42@4.fc24@x86_64",
|
||||||
|
@ -215,7 +239,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
pkgset = pkgsets.KojiPackageSet(
|
pkgset = package_set(
|
||||||
"pkgset",
|
"pkgset",
|
||||||
self.koji_wrapper,
|
self.koji_wrapper,
|
||||||
[None],
|
[None],
|
||||||
|
@ -325,7 +349,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
figure = re.compile(
|
figure = re.compile(
|
||||||
r"^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$", # noqa: E501
|
r"^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24\.x86_64.+bash-debuginfo-4\.3\.42-4\.fc24\.x86_64$", # noqa: E501
|
||||||
re.DOTALL,
|
re.DOTALL,
|
||||||
)
|
)
|
||||||
self.assertRegex(str(ctx.exception), figure)
|
self.assertRegex(str(ctx.exception), figure)
|
||||||
|
@ -404,7 +428,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkey_rpms)
|
pkgset.raise_invalid_sigkeys_exception(pkgset.invalid_sigkey_rpms)
|
||||||
|
|
||||||
figure = re.compile(
|
figure = re.compile(
|
||||||
r"^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24.+bash-debuginfo-4\.3\.42-4\.fc24$", # noqa: E501
|
r"^RPM\(s\) not found for sigs: .+Check log for details.+bash-4\.3\.42-4\.fc24\.x86_64.+bash-debuginfo-4\.3\.42-4\.fc24\.x86_64$", # noqa: E501
|
||||||
re.DOTALL,
|
re.DOTALL,
|
||||||
)
|
)
|
||||||
self.assertRegex(str(ctx.exception), figure)
|
self.assertRegex(str(ctx.exception), figure)
|
||||||
|
@ -458,7 +482,11 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
# Two packages making three attempts each, so two waits per package.
|
# Two packages making three attempts each, so two waits per package.
|
||||||
self.assertEqual(time.call_args_list, [mock.call(5)] * 4)
|
self.assertEqual(time.call_args_list, [mock.call(5)] * 4)
|
||||||
|
|
||||||
def test_packages_attribute(self):
|
@ddt.data(
|
||||||
|
pkgsets.KojiPackageSet,
|
||||||
|
pkgsets.KojiMockPackageSet,
|
||||||
|
)
|
||||||
|
def test_packages_attribute(self, package_set):
|
||||||
self._touch_files(
|
self._touch_files(
|
||||||
[
|
[
|
||||||
"rpms/pungi@4.1.3@3.fc25@noarch",
|
"rpms/pungi@4.1.3@3.fc25@noarch",
|
||||||
|
@ -471,7 +499,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
pkgset = pkgsets.KojiPackageSet(
|
pkgset = package_set(
|
||||||
"pkgset",
|
"pkgset",
|
||||||
self.koji_wrapper,
|
self.koji_wrapper,
|
||||||
[None],
|
[None],
|
||||||
|
@ -496,8 +524,12 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_get_extra_rpms_from_tasks(self):
|
@ddt.data(
|
||||||
pkgset = pkgsets.KojiPackageSet(
|
pkgsets.KojiPackageSet,
|
||||||
|
pkgsets.KojiMockPackageSet,
|
||||||
|
)
|
||||||
|
def test_get_extra_rpms_from_tasks(self, package_set):
|
||||||
|
pkgset = package_set(
|
||||||
"pkgset",
|
"pkgset",
|
||||||
self.koji_wrapper,
|
self.koji_wrapper,
|
||||||
[None],
|
[None],
|
||||||
|
@ -563,7 +595,11 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
rpms = pkgset.get_extra_rpms_from_tasks()
|
rpms = pkgset.get_extra_rpms_from_tasks()
|
||||||
self.assertEqual(rpms, expected_rpms)
|
self.assertEqual(rpms, expected_rpms)
|
||||||
|
|
||||||
def test_get_latest_rpms_cache(self):
|
@ddt.data(
|
||||||
|
pkgsets.KojiMockPackageSet,
|
||||||
|
pkgsets.KojiPackageSet,
|
||||||
|
)
|
||||||
|
def test_get_latest_rpms_cache(self, package_set):
|
||||||
self._touch_files(
|
self._touch_files(
|
||||||
[
|
[
|
||||||
"rpms/bash@4.3.42@4.fc24@x86_64",
|
"rpms/bash@4.3.42@4.fc24@x86_64",
|
||||||
|
@ -572,7 +608,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
cache_region = make_region().configure("dogpile.cache.memory")
|
cache_region = make_region().configure("dogpile.cache.memory")
|
||||||
pkgset = pkgsets.KojiPackageSet(
|
pkgset = package_set(
|
||||||
"pkgset",
|
"pkgset",
|
||||||
self.koji_wrapper,
|
self.koji_wrapper,
|
||||||
[None],
|
[None],
|
||||||
|
@ -603,7 +639,11 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_get_latest_rpms_cache_different_id(self):
|
@ddt.data(
|
||||||
|
pkgsets.KojiMockPackageSet,
|
||||||
|
pkgsets.KojiPackageSet,
|
||||||
|
)
|
||||||
|
def test_get_latest_rpms_cache_different_id(self, package_set):
|
||||||
self._touch_files(
|
self._touch_files(
|
||||||
[
|
[
|
||||||
"rpms/bash@4.3.42@4.fc24@x86_64",
|
"rpms/bash@4.3.42@4.fc24@x86_64",
|
||||||
|
@ -612,7 +652,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
cache_region = make_region().configure("dogpile.cache.memory")
|
cache_region = make_region().configure("dogpile.cache.memory")
|
||||||
pkgset = pkgsets.KojiPackageSet(
|
pkgset = package_set(
|
||||||
"pkgset",
|
"pkgset",
|
||||||
self.koji_wrapper,
|
self.koji_wrapper,
|
||||||
[None],
|
[None],
|
||||||
|
@ -640,7 +680,11 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_extra_builds_attribute(self):
|
@ddt.data(
|
||||||
|
pkgsets.KojiMockPackageSet,
|
||||||
|
pkgsets.KojiPackageSet,
|
||||||
|
)
|
||||||
|
def test_extra_builds_attribute(self, package_set):
|
||||||
self._touch_files(
|
self._touch_files(
|
||||||
[
|
[
|
||||||
"rpms/pungi@4.1.3@3.fc25@noarch",
|
"rpms/pungi@4.1.3@3.fc25@noarch",
|
||||||
|
@ -671,7 +715,7 @@ class TestKojiPkgset(PkgsetCompareMixin, helpers.PungiTestCase):
|
||||||
[b for b in self.tagged_rpms[1] if b["package_name"] != "pungi"],
|
[b for b in self.tagged_rpms[1] if b["package_name"] != "pungi"],
|
||||||
]
|
]
|
||||||
|
|
||||||
pkgset = pkgsets.KojiPackageSet(
|
pkgset = package_set(
|
||||||
"pkgset",
|
"pkgset",
|
||||||
self.koji_wrapper,
|
self.koji_wrapper,
|
||||||
[None],
|
[None],
|
||||||
|
@ -747,6 +791,211 @@ class TestReuseKojiPkgset(helpers.PungiTestCase):
|
||||||
)
|
)
|
||||||
self.assert_not_reuse()
|
self.assert_not_reuse()
|
||||||
|
|
||||||
|
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
|
||||||
|
def test_reuse_build_under_tag_changed(self, mock_old_topdir):
|
||||||
|
mock_old_topdir.return_value = self.old_compose_dir
|
||||||
|
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
|
||||||
|
self.koji_wrapper.koji_proxy.queryHistory.return_value = {"tag_listing": [{}]}
|
||||||
|
|
||||||
|
self.pkgset.try_to_reuse(self.compose, self.tag)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
self.pkgset.log_debug.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
"Koji event doesn't match, querying changes between event 1 and 3"
|
||||||
|
),
|
||||||
|
mock.call("Builds under tag %s changed. Can't reuse." % self.tag),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assert_not_reuse()
|
||||||
|
|
||||||
|
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
|
||||||
|
def test_reuse_build_under_inherited_tag_changed(self, mock_old_topdir):
|
||||||
|
mock_old_topdir.return_value = self.old_compose_dir
|
||||||
|
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
|
||||||
|
self.koji_wrapper.koji_proxy.queryHistory.side_effect = [
|
||||||
|
{"tag_listing": [], "tag_inheritance": []},
|
||||||
|
{"tag_listing": [{}], "tag_inheritance": []},
|
||||||
|
]
|
||||||
|
self.koji_wrapper.koji_proxy.getFullInheritance.return_value = [
|
||||||
|
{"name": self.inherited_tag}
|
||||||
|
]
|
||||||
|
|
||||||
|
self.pkgset.try_to_reuse(self.compose, self.tag)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
self.pkgset.log_debug.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
"Koji event doesn't match, querying changes between event 1 and 3"
|
||||||
|
),
|
||||||
|
mock.call(
|
||||||
|
"Builds under inherited tag %s changed. Can't reuse."
|
||||||
|
% self.inherited_tag
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assert_not_reuse()
|
||||||
|
|
||||||
|
@mock.patch("pungi.paths.os.path.exists", return_value=True)
|
||||||
|
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
|
||||||
|
def test_reuse_failed_load_reuse_file(self, mock_old_topdir, mock_exists):
|
||||||
|
mock_old_topdir.return_value = self.old_compose_dir
|
||||||
|
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
|
||||||
|
self.koji_wrapper.koji_proxy.queryHistory.return_value = {
|
||||||
|
"tag_listing": [], "tag_inheritance": []
|
||||||
|
}
|
||||||
|
self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
|
||||||
|
self.pkgset.load_old_file_cache = mock.Mock(
|
||||||
|
side_effect=Exception("unknown error")
|
||||||
|
)
|
||||||
|
|
||||||
|
self.pkgset.try_to_reuse(self.compose, self.tag)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
self.pkgset.log_debug.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
"Koji event doesn't match, querying changes between event 1 and 3"
|
||||||
|
),
|
||||||
|
mock.call(
|
||||||
|
"Loading reuse file: %s"
|
||||||
|
% os.path.join(
|
||||||
|
self.old_compose_dir,
|
||||||
|
"work/global",
|
||||||
|
"pkgset_%s_reuse.pickle" % self.tag,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
mock.call("Failed to load reuse file: unknown error"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assert_not_reuse()
|
||||||
|
|
||||||
|
@mock.patch("pungi.paths.os.path.exists", return_value=True)
|
||||||
|
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
|
||||||
|
def test_reuse_criteria_not_match(self, mock_old_topdir, mock_exists):
|
||||||
|
mock_old_topdir.return_value = self.old_compose_dir
|
||||||
|
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
|
||||||
|
self.koji_wrapper.koji_proxy.queryHistory.return_value = {
|
||||||
|
"tag_listing": [], "tag_inheritance": []
|
||||||
|
}
|
||||||
|
self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
|
||||||
|
self.pkgset.load_old_file_cache = mock.Mock(
|
||||||
|
return_value={"allow_invalid_sigkeys": True}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.pkgset.try_to_reuse(self.compose, self.tag)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
self.pkgset.log_debug.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call(
|
||||||
|
"Koji event doesn't match, querying changes between event 1 and 3"
|
||||||
|
),
|
||||||
|
mock.call(
|
||||||
|
"Loading reuse file: %s"
|
||||||
|
% os.path.join(
|
||||||
|
self.old_compose_dir,
|
||||||
|
"work/global",
|
||||||
|
"pkgset_%s_reuse.pickle" % self.tag,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
self.pkgset.log_info.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call("Trying to reuse pkgset data of old compose"),
|
||||||
|
mock.call("Criteria does not match. Nothing to reuse."),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assert_not_reuse()
|
||||||
|
|
||||||
|
@mock.patch("pungi.phases.pkgset.pkgsets.copy_all")
|
||||||
|
@mock.patch("pungi.paths.os.path.exists", return_value=True)
|
||||||
|
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
|
||||||
|
def test_reuse_pkgset(self, mock_old_topdir, mock_exists, mock_copy_all):
|
||||||
|
mock_old_topdir.return_value = self.old_compose_dir
|
||||||
|
self.pkgset._get_koji_event_from_file = mock.Mock(side_effect=[3, 1])
|
||||||
|
self.koji_wrapper.koji_proxy.queryHistory.return_value = {
|
||||||
|
"tag_listing": [], "tag_inheritance": []
|
||||||
|
}
|
||||||
|
self.koji_wrapper.koji_proxy.getFullInheritance.return_value = []
|
||||||
|
self.pkgset.load_old_file_cache = mock.Mock(
|
||||||
|
return_value={
|
||||||
|
"allow_invalid_sigkeys": self.pkgset._allow_invalid_sigkeys,
|
||||||
|
"packages": self.pkgset.packages,
|
||||||
|
"populate_only_packages": self.pkgset.populate_only_packages,
|
||||||
|
"extra_builds": self.pkgset.extra_builds,
|
||||||
|
"sigkeys": self.pkgset.sigkey_ordering,
|
||||||
|
"include_packages": None,
|
||||||
|
"rpms_by_arch": mock.Mock(),
|
||||||
|
"srpms_by_name": mock.Mock(),
|
||||||
|
"inherit_to_noarch": True,
|
||||||
|
"exclusive_noarch": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.pkgset.old_file_cache = mock.Mock()
|
||||||
|
|
||||||
|
self.pkgset.try_to_reuse(self.compose, self.tag)
|
||||||
|
|
||||||
|
old_repo_dir = os.path.join(self.old_compose_dir, "work/global/repo", self.tag)
|
||||||
|
self.assertEqual(
|
||||||
|
self.pkgset.log_info.mock_calls,
|
||||||
|
[
|
||||||
|
mock.call("Trying to reuse pkgset data of old compose"),
|
||||||
|
mock.call("Copying repo data for reuse: %s" % old_repo_dir),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assertEqual(old_repo_dir, self.pkgset.reuse)
|
||||||
|
self.assertEqual(self.pkgset.file_cache, self.pkgset.old_file_cache)
|
||||||
|
|
||||||
|
|
||||||
|
class TestReuseKojiMockPkgset(helpers.PungiTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(TestReuseKojiMockPkgset, self).setUp()
|
||||||
|
self.old_compose_dir = tempfile.mkdtemp()
|
||||||
|
self.old_compose = helpers.DummyCompose(self.old_compose_dir, {})
|
||||||
|
self.compose = helpers.DummyCompose(
|
||||||
|
self.topdir, {"old_composes": os.path.dirname(self.old_compose_dir)}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.koji_wrapper = mock.Mock()
|
||||||
|
|
||||||
|
self.tag = "test-tag"
|
||||||
|
self.inherited_tag = "inherited-test-tag"
|
||||||
|
self.pkgset = pkgsets.KojiMockPackageSet(
|
||||||
|
self.tag, self.koji_wrapper, [None], arches=["x86_64"]
|
||||||
|
)
|
||||||
|
self.pkgset.log_debug = mock.Mock()
|
||||||
|
self.pkgset.log_info = mock.Mock()
|
||||||
|
|
||||||
|
def assert_not_reuse(self):
|
||||||
|
self.assertIsNone(getattr(self.pkgset, "reuse", None))
|
||||||
|
|
||||||
|
def test_resue_no_old_compose_found(self):
|
||||||
|
self.pkgset.try_to_reuse(self.compose, self.tag)
|
||||||
|
self.pkgset.log_info.assert_called_once_with(
|
||||||
|
"Trying to reuse pkgset data of old compose"
|
||||||
|
)
|
||||||
|
self.pkgset.log_debug.assert_called_once_with(
|
||||||
|
"No old compose found. Nothing to reuse."
|
||||||
|
)
|
||||||
|
self.assert_not_reuse()
|
||||||
|
|
||||||
|
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
|
||||||
|
def test_reuse_read_koji_event_file_failed(self, mock_old_topdir):
|
||||||
|
mock_old_topdir.return_value = self.old_compose_dir
|
||||||
|
self.pkgset._get_koji_event_from_file = mock.Mock(
|
||||||
|
side_effect=Exception("unknown error")
|
||||||
|
)
|
||||||
|
self.pkgset.try_to_reuse(self.compose, self.tag)
|
||||||
|
self.pkgset.log_debug.assert_called_once_with(
|
||||||
|
"Can't read koji event from file: unknown error"
|
||||||
|
)
|
||||||
|
self.assert_not_reuse()
|
||||||
|
|
||||||
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
|
@mock.patch.object(helpers.paths.Paths, "get_old_compose_topdir")
|
||||||
def test_reuse_build_under_tag_changed(self, mock_old_topdir):
|
def test_reuse_build_under_tag_changed(self, mock_old_topdir):
|
||||||
mock_old_topdir.return_value = self.old_compose_dir
|
mock_old_topdir.return_value = self.old_compose_dir
|
||||||
|
|
|
@ -1,17 +1,23 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import mock
|
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import six
|
import six
|
||||||
|
from ddt import ddt, data, unpack
|
||||||
|
from typing import AnyStr, List, Set, Dict, Tuple
|
||||||
|
|
||||||
|
from tests.test_gather_method_hybrid import MockModule
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import unittest2 as unittest
|
import unittest2 as unittest
|
||||||
|
from unittest2 import mock
|
||||||
except ImportError:
|
except ImportError:
|
||||||
import unittest
|
import unittest
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
from pungi.phases.pkgset.sources import source_koji
|
from pungi.phases.pkgset.sources import source_koji, source_kojimock
|
||||||
from tests import helpers
|
from tests import helpers
|
||||||
from pungi.module_util import Modulemd
|
from pungi.module_util import Modulemd
|
||||||
from pungi.util import read_single_module_stream_from_file
|
from pungi.util import read_single_module_stream_from_file
|
||||||
|
@ -31,17 +37,6 @@ TAG_INFO = {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _mk_module_build(r, t):
|
|
||||||
"""Create a dict as returned Koji buildinfo."""
|
|
||||||
return {
|
|
||||||
"name": "foo",
|
|
||||||
"version": "1",
|
|
||||||
"release": r,
|
|
||||||
"nvr": "foo-1-%s" % r,
|
|
||||||
"tag_name": t,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class TestGetKojiEvent(helpers.PungiTestCase):
|
class TestGetKojiEvent(helpers.PungiTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestGetKojiEvent, self).setUp()
|
super(TestGetKojiEvent, self).setUp()
|
||||||
|
@ -110,14 +105,7 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
|
||||||
self.assertEqual(len(pkgsets), 1)
|
self.assertEqual(len(pkgsets), 1)
|
||||||
self.assertIs(pkgsets[0], orig_pkgset)
|
self.assertIs(pkgsets[0], orig_pkgset)
|
||||||
pkgsets[0].assert_has_calls(
|
pkgsets[0].assert_has_calls(
|
||||||
[
|
[mock.call.populate("f25", 123456, inherit=True, include_packages=set())],
|
||||||
mock.call.populate(
|
|
||||||
"f25",
|
|
||||||
event=123456,
|
|
||||||
inherit=True,
|
|
||||||
include_packages=set(),
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def mock_materialize(self, compose, pkgset, prefix, mmd):
|
def mock_materialize(self, compose, pkgset, prefix, mmd):
|
||||||
|
@ -158,16 +146,12 @@ class TestPopulateGlobalPkgset(helpers.PungiTestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
pkgsets[0].assert_has_calls(
|
pkgsets[0].assert_has_calls(
|
||||||
[
|
[mock.call.populate("f25", 123456, inherit=True, include_packages=set())]
|
||||||
mock.call.populate(
|
|
||||||
"f25", event=123456, inherit=True, include_packages=set()
|
|
||||||
),
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
pkgsets[1].assert_has_calls(
|
pkgsets[1].assert_has_calls(
|
||||||
[
|
[
|
||||||
mock.call.populate(
|
mock.call.populate(
|
||||||
"f25-extra", event=123456, inherit=True, include_packages=set()
|
"f25-extra", 123456, inherit=True, include_packages=set()
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
@ -260,10 +244,10 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase):
|
||||||
self.compose, self.koji_wrapper, event, module_info_str
|
self.compose, self.koji_wrapper, event, module_info_str
|
||||||
)
|
)
|
||||||
|
|
||||||
assert isinstance(result, list)
|
assert type(result) is list
|
||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
module = result[0]
|
module = result[0]
|
||||||
assert isinstance(module, dict)
|
assert type(module) is dict
|
||||||
self.assertIn("module_stream", module)
|
self.assertIn("module_stream", module)
|
||||||
self.assertIn("module_version", module)
|
self.assertIn("module_version", module)
|
||||||
self.assertIn("module_context", module)
|
self.assertIn("module_context", module)
|
||||||
|
@ -389,11 +373,11 @@ class TestGetPackageSetFromKoji(helpers.PungiTestCase):
|
||||||
self.compose, self.koji_wrapper, event, module_info_str
|
self.compose, self.koji_wrapper, event, module_info_str
|
||||||
)
|
)
|
||||||
|
|
||||||
assert isinstance(result, list)
|
assert type(result) is list
|
||||||
assert len(result) == 2
|
assert len(result) == 2
|
||||||
module = result[0]
|
module = result[0]
|
||||||
for module in result:
|
for module in result:
|
||||||
assert isinstance(module, dict)
|
assert type(module) is dict
|
||||||
self.assertIn("module_stream", module)
|
self.assertIn("module_stream", module)
|
||||||
self.assertIn("module_version", module)
|
self.assertIn("module_version", module)
|
||||||
self.assertIn("module_context", module)
|
self.assertIn("module_context", module)
|
||||||
|
@ -569,15 +553,19 @@ class TestFilterInherited(unittest.TestCase):
|
||||||
{"name": "middle-tag"},
|
{"name": "middle-tag"},
|
||||||
{"name": "bottom-tag"},
|
{"name": "bottom-tag"},
|
||||||
]
|
]
|
||||||
|
module_builds = [
|
||||||
m1 = _mk_module_build("1", "top-tag")
|
{"name": "foo", "version": "1", "release": "1", "tag_name": "top-tag"},
|
||||||
m2 = _mk_module_build("2", "middle-tag")
|
{"name": "foo", "version": "1", "release": "2", "tag_name": "bottom-tag"},
|
||||||
m3 = _mk_module_build("3", "bottom-tag")
|
{"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"},
|
||||||
module_builds = [m1, m2, m3]
|
]
|
||||||
|
|
||||||
result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)
|
result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)
|
||||||
|
|
||||||
six.assertCountEqual(self, result, [m1])
|
six.assertCountEqual(
|
||||||
|
self,
|
||||||
|
result,
|
||||||
|
[{"name": "foo", "version": "1", "release": "1", "tag_name": "top-tag"}],
|
||||||
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
koji_proxy.mock_calls,
|
koji_proxy.mock_calls,
|
||||||
[mock.call.getFullInheritance("top-tag", event=123456)],
|
[mock.call.getFullInheritance("top-tag", event=123456)],
|
||||||
|
@ -592,33 +580,18 @@ class TestFilterInherited(unittest.TestCase):
|
||||||
{"name": "middle-tag"},
|
{"name": "middle-tag"},
|
||||||
{"name": "bottom-tag"},
|
{"name": "bottom-tag"},
|
||||||
]
|
]
|
||||||
m2 = _mk_module_build("2", "bottom-tag")
|
module_builds = [
|
||||||
m3 = _mk_module_build("3", "middle-tag")
|
{"name": "foo", "version": "1", "release": "2", "tag_name": "bottom-tag"},
|
||||||
module_builds = [m2, m3]
|
{"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"},
|
||||||
|
|
||||||
result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)
|
|
||||||
|
|
||||||
six.assertCountEqual(self, result, [m3])
|
|
||||||
self.assertEqual(
|
|
||||||
koji_proxy.mock_calls,
|
|
||||||
[mock.call.getFullInheritance("top-tag", event=123456)],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_build_in_multiple_tags(self):
|
|
||||||
event = {"id": 123456}
|
|
||||||
koji_proxy = mock.Mock()
|
|
||||||
top_tag = "top-tag"
|
|
||||||
|
|
||||||
koji_proxy.getFullInheritance.return_value = [
|
|
||||||
{"name": "middle-tag"},
|
|
||||||
{"name": "bottom-tag"},
|
|
||||||
]
|
]
|
||||||
m = _mk_module_build("1", "middle-tag")
|
|
||||||
module_builds = [m, m]
|
|
||||||
|
|
||||||
result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)
|
result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)
|
||||||
|
|
||||||
six.assertCountEqual(self, result, [m])
|
six.assertCountEqual(
|
||||||
|
self,
|
||||||
|
result,
|
||||||
|
[{"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"}],
|
||||||
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
koji_proxy.mock_calls,
|
koji_proxy.mock_calls,
|
||||||
[mock.call.getFullInheritance("top-tag", event=123456)],
|
[mock.call.getFullInheritance("top-tag", event=123456)],
|
||||||
|
@ -859,6 +832,153 @@ class TestAddModuleToVariant(helpers.PungiTestCase):
|
||||||
self.assertEqual(variant.modules, [])
|
self.assertEqual(variant.modules, [])
|
||||||
|
|
||||||
|
|
||||||
|
@mock.patch("pungi.module_util.Modulemd.ModuleStream.read_file", new=MockModule)
|
||||||
|
@unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
|
||||||
|
class TestAddModuleToVariantForKojiMock(helpers.PungiTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(TestAddModuleToVariantForKojiMock, self).setUp()
|
||||||
|
self.koji = mock.Mock()
|
||||||
|
self.koji.koji_module.pathinfo.typedir.return_value = "/koji"
|
||||||
|
self.compose = helpers.DummyCompose(self.topdir, {})
|
||||||
|
self.koji.koji_module.pathinfo.topdir = MMDS_DIR
|
||||||
|
files = [
|
||||||
|
"modulemd.x86_64.txt",
|
||||||
|
"scratch-module.x86_64.txt",
|
||||||
|
]
|
||||||
|
self.koji.koji_proxy.listArchives.return_value = [
|
||||||
|
{"btype": "module", "filename": fname} for fname in files
|
||||||
|
]
|
||||||
|
self.buildinfo = {
|
||||||
|
"id": 1234,
|
||||||
|
"arch": "x86_64",
|
||||||
|
"extra": {
|
||||||
|
"typeinfo": {
|
||||||
|
"module": {
|
||||||
|
"name": "module",
|
||||||
|
"stream": "master",
|
||||||
|
"version": "20190318",
|
||||||
|
"context": "abcdef",
|
||||||
|
'content_koji_tag': 'module:master-20190318-abcdef'
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_adding_module(self):
|
||||||
|
variant = mock.Mock(
|
||||||
|
arches=[
|
||||||
|
"x86_64"
|
||||||
|
],
|
||||||
|
arch_mmds={},
|
||||||
|
modules=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
source_kojimock._add_module_to_variant(
|
||||||
|
self.koji,
|
||||||
|
variant,
|
||||||
|
self.buildinfo,
|
||||||
|
)
|
||||||
|
|
||||||
|
mod = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
|
||||||
|
self.assertEqual(mod.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
|
||||||
|
self.assertEqual(len(variant.arch_mmds), 1)
|
||||||
|
self.assertEqual(variant.modules, [])
|
||||||
|
|
||||||
|
def test_adding_module_to_existing(self):
|
||||||
|
variant = mock.Mock(
|
||||||
|
arches=[
|
||||||
|
"x86_64"
|
||||||
|
],
|
||||||
|
arch_mmds={
|
||||||
|
"x86_64": {
|
||||||
|
"m1:latest:20190101:cafe": read_single_module_stream_from_file(
|
||||||
|
os.path.join(MMDS_DIR, "m1.x86_64.txt")
|
||||||
|
)}
|
||||||
|
},
|
||||||
|
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
source_koji._add_module_to_variant(
|
||||||
|
self.koji, variant, self.buildinfo, compose=self.compose
|
||||||
|
)
|
||||||
|
|
||||||
|
mod = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
|
||||||
|
self.assertEqual(mod.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
variant.modules,
|
||||||
|
[{"name": "m1:latest-20190101:cafe", "glob": False}]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_adding_module_with_add_module(self):
|
||||||
|
variant = mock.Mock(arches=[
|
||||||
|
"x86_64"
|
||||||
|
], arch_mmds={}, modules=[])
|
||||||
|
|
||||||
|
source_kojimock._add_module_to_variant(
|
||||||
|
self.koji, variant, self.buildinfo, add_to_variant_modules=True
|
||||||
|
)
|
||||||
|
|
||||||
|
mod = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
|
||||||
|
self.assertEqual(mod.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
variant.modules, [{"name": "module:master:20190318:abcdef", "glob": False}]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_adding_module_to_existing_with_add_module(self):
|
||||||
|
variant = mock.Mock(
|
||||||
|
arches=[
|
||||||
|
"x86_64"
|
||||||
|
],
|
||||||
|
arch_mmds={
|
||||||
|
"x86_64": {"m1:latest:20190101:cafe": read_single_module_stream_from_file(
|
||||||
|
os.path.join(MMDS_DIR, "m1.x86_64.txt")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
|
||||||
|
)
|
||||||
|
|
||||||
|
source_kojimock._add_module_to_variant(
|
||||||
|
self.koji, variant, self.buildinfo, add_to_variant_modules=True
|
||||||
|
)
|
||||||
|
|
||||||
|
mod = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
|
||||||
|
self.assertEqual(mod.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
variant.modules,
|
||||||
|
[
|
||||||
|
{"name": "m1:latest-20190101:cafe", "glob": False},
|
||||||
|
{"name": "module:master:20190318:abcdef", "glob": False},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_adding_module_but_filtered(self):
|
||||||
|
compose = helpers.DummyCompose(
|
||||||
|
self.topdir, {"filter_modules": [(".*", {"*": ["module:*"]})]}
|
||||||
|
)
|
||||||
|
variant = mock.Mock(
|
||||||
|
arches=[
|
||||||
|
"x86_64"
|
||||||
|
], arch_mmds={}, modules=[], uid="Variant"
|
||||||
|
)
|
||||||
|
|
||||||
|
nsvc = source_kojimock._add_module_to_variant(
|
||||||
|
self.koji,
|
||||||
|
variant,
|
||||||
|
self.buildinfo,
|
||||||
|
add_to_variant_modules=True,
|
||||||
|
compose=compose,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsNone(nsvc)
|
||||||
|
self.assertEqual(variant.arch_mmds, {})
|
||||||
|
self.assertEqual(variant.modules, [])
|
||||||
|
|
||||||
|
|
||||||
class TestIsModuleFiltered(helpers.PungiTestCase):
|
class TestIsModuleFiltered(helpers.PungiTestCase):
|
||||||
def assertIsFiltered(self, name, stream):
|
def assertIsFiltered(self, name, stream):
|
||||||
self.assertTrue(
|
self.assertTrue(
|
||||||
|
@ -921,7 +1041,10 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase):
|
||||||
|
|
||||||
def test_adding_scratch_module(self):
|
def test_adding_scratch_module(self):
|
||||||
variant = mock.Mock(
|
variant = mock.Mock(
|
||||||
arches=["armhfp", "x86_64"],
|
arches=[
|
||||||
|
# "armhfp",
|
||||||
|
"x86_64"
|
||||||
|
],
|
||||||
arch_mmds={},
|
arch_mmds={},
|
||||||
modules=[],
|
modules=[],
|
||||||
module_uid_to_koji_tag={},
|
module_uid_to_koji_tag={},
|
||||||
|
@ -958,3 +1081,124 @@ class TestAddScratchModuleToVariant(helpers.PungiTestCase):
|
||||||
self.compose.log_warning.assert_called_once_with(
|
self.compose.log_warning.assert_called_once_with(
|
||||||
"Only test composes could include scratch module builds"
|
"Only test composes could include scratch module builds"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ddt
|
||||||
|
class TestSourceKoji(unittest.TestCase):
|
||||||
|
|
||||||
|
@unpack
|
||||||
|
@data(
|
||||||
|
(
|
||||||
|
'AppStream',
|
||||||
|
[
|
||||||
|
'x86_64',
|
||||||
|
'i386'
|
||||||
|
],
|
||||||
|
{
|
||||||
|
'python39-devel:3.9',
|
||||||
|
'python39:3.9',
|
||||||
|
},
|
||||||
|
[
|
||||||
|
(
|
||||||
|
'^(BaseOS|AppStream|PowerTools)$',
|
||||||
|
{
|
||||||
|
'x86_64': [
|
||||||
|
'python39:3.9',
|
||||||
|
],
|
||||||
|
'aarch64': [
|
||||||
|
'python39-devel:3.9',
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
],
|
||||||
|
{
|
||||||
|
'python39-devel:3.9',
|
||||||
|
}
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'AppStream',
|
||||||
|
[
|
||||||
|
'x86_64',
|
||||||
|
'i386'
|
||||||
|
],
|
||||||
|
{
|
||||||
|
'python39-devel:3.9',
|
||||||
|
'python39:3.9',
|
||||||
|
'python38-devel:3.8',
|
||||||
|
'python38:3.8',
|
||||||
|
},
|
||||||
|
[
|
||||||
|
(
|
||||||
|
'^(BaseOS|AppStream|PowerTools)$',
|
||||||
|
{
|
||||||
|
'x86_64': [
|
||||||
|
'python39:3.9',
|
||||||
|
],
|
||||||
|
'*': [
|
||||||
|
'python38-devel:3.8',
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
],
|
||||||
|
{
|
||||||
|
'python39-devel:3.9',
|
||||||
|
'python38:3.8',
|
||||||
|
}
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'AppStream',
|
||||||
|
[
|
||||||
|
'x86_64',
|
||||||
|
'i386'
|
||||||
|
],
|
||||||
|
{
|
||||||
|
'python39-devel:3.9',
|
||||||
|
'python39:3.9',
|
||||||
|
'python38-devel:3.8',
|
||||||
|
'python38:3.8',
|
||||||
|
},
|
||||||
|
[
|
||||||
|
(
|
||||||
|
'^(BaseOS|AppStream|PowerTools)$',
|
||||||
|
{
|
||||||
|
'x86_64': [
|
||||||
|
'python39:3.9',
|
||||||
|
],
|
||||||
|
'aarch64': [
|
||||||
|
'python38-devel:3.8',
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'*',
|
||||||
|
{
|
||||||
|
'*': [
|
||||||
|
'python38-devel:3.8',
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
],
|
||||||
|
{
|
||||||
|
'python39-devel:3.9',
|
||||||
|
'python38:3.8',
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
def test__filter_expected_modules(
|
||||||
|
self,
|
||||||
|
variant_name: AnyStr,
|
||||||
|
variant_arches: List[AnyStr],
|
||||||
|
expected_modules: Set[AnyStr],
|
||||||
|
filtered_modules: List[Tuple[AnyStr, Dict[AnyStr, List[AnyStr]]]],
|
||||||
|
expected_result: Set[AnyStr],
|
||||||
|
) -> None:
|
||||||
|
real_result = source_kojimock._filter_expected_modules(
|
||||||
|
variant_name=variant_name,
|
||||||
|
variant_arches=variant_arches,
|
||||||
|
expected_modules=expected_modules,
|
||||||
|
filtered_modules=filtered_modules,
|
||||||
|
)
|
||||||
|
self.assertSetEqual(
|
||||||
|
real_result,
|
||||||
|
expected_result,
|
||||||
|
)
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue