leapp-repository/SOURCES/0041-repomap-Process-repositories-based-on-distro.patch
2025-06-24 11:34:57 +00:00

1143 lines
51 KiB
Diff

From ff21cc8f8d81766658bc63f23dcf7914cef978e1 Mon Sep 17 00:00:00 2001
From: Matej Matuska <mmatuska@redhat.com>
Date: Mon, 28 Apr 2025 15:07:22 +0200
Subject: [PATCH 41/43] repomap: Process repositories based on distro
The original solution expect just IPU of RHEL systems and so only
source RHEL DNF repositories have been mapped to the target RHEL
repositories. Making this more distro agnostic for CentOS-like
systems, the mapping needs to be updated to respect distro specific
repositories.
Hence a 'distro' field has been added in the 1.3.0 version of the
repomap.json format. The field is required for all the `repositories`
entries. RepomapDataHandler now takes in distro as a constructor
parameter and only consider entries with the matching distro value.
Reflects also changes in the repomap json schema for version 1.3.0,
which drops enumeration for the `rhui` field and replace it by simple
String type to make the solution more scalable. This also means that
vendor of the repomap data file must validate this field "manually".
All related unit-tests have been updated. As the commit is already
large, keeping the update of data files on a separate commit.
Update the PESIDRepositoryEntry model:
* add the distro field represented by String
* change rhui type from StringEnum to String
Jira: RHEL-80336
WIP setuptarget repos tests for distro
Add repositories mapping unit tests for distro
---
.../libraries/pes_events_scanner.py | 1 +
.../libraries/peseventsscanner_repomap.py | 51 +++-
.../tests/test_pes_event_scanner.py | 4 +-
.../tests/test_repositoriesblacklist.py | 6 +-
.../libraries/repositoriesmapping.py | 5 +-
.../tests/files/repomap_example.json | 43 ++-
.../tests/unit_test_repositoriesmapping.py | 37 ++-
.../common/actors/setuptargetrepos/actor.py | 13 +-
.../libraries/setuptargetrepos_repomap.py | 51 +++-
.../tests/test_repomapping.py | 265 ++++++++++++++----
.../tests/test_setuptargetrepos.py | 18 +-
.../common/models/repositoriesmap.py | 7 +-
12 files changed, 397 insertions(+), 104 deletions(-)
diff --git a/repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_events_scanner.py b/repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_events_scanner.py
index 50336150..e6741293 100644
--- a/repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_events_scanner.py
+++ b/repos/system_upgrade/common/actors/peseventsscanner/libraries/pes_events_scanner.py
@@ -400,6 +400,7 @@ def get_pesid_to_repoid_map(target_pesids):
repo_type='rpm',
channel='ga',
rhui='',
+ distro=api.current_actor().configuration.os_release.release_id,
)
for pesid in target_pesids:
diff --git a/repos/system_upgrade/common/actors/peseventsscanner/libraries/peseventsscanner_repomap.py b/repos/system_upgrade/common/actors/peseventsscanner/libraries/peseventsscanner_repomap.py
index c2976ddf..37be03f1 100644
--- a/repos/system_upgrade/common/actors/peseventsscanner/libraries/peseventsscanner_repomap.py
+++ b/repos/system_upgrade/common/actors/peseventsscanner/libraries/peseventsscanner_repomap.py
@@ -23,7 +23,7 @@ class RepoMapDataHandler(object):
Provide the basic functionality to work with the repository data easily.
"""
- def __init__(self, repo_map, cloud_provider='', default_channels=None):
+ def __init__(self, repo_map, distro='', cloud_provider='', default_channels=None):
"""
Initialize the object based on the given RepositoriesMapping msg.
@@ -32,6 +32,8 @@ class RepoMapDataHandler(object):
:param repo_map: A valid RepositoryMapping message.
:type repo_map: RepositoryMapping
+ :param distro: Which distribution's mappings to use, default to current
+ :type distro: str
:param default_channels: A list of default channels to use when a target repository
equivalent exactly matching a source repository was not found.
:type default_channels: List[str]
@@ -42,6 +44,7 @@ class RepoMapDataHandler(object):
# ideal for work, but there is not any significant impact..
self.repositories = repo_map.repositories
self.mapping = repo_map.mapping
+ self.distro = distro or api.current_actor().configuration.os_release.release_id
# FIXME(pstodulk): what about default_channel -> fallback_channel
# hardcoded always as ga? instead of list of channels..
# it'd be possibly confusing naming now...
@@ -93,6 +96,8 @@ class RepoMapDataHandler(object):
If multiple pesid repo entries with the same repoid were found, the entry with rhui matching the source
system's rhui info will be returned. If no entry with matching rhui exists, the CDN one is returned if any.
+ Note that repositories are automatically filtered based on the specified OS release ID (self.distro).
+
:param repoid: RepoID that should the PESIDRepositoryEntry match.
:type repoid: str
:param major_version: RepoID that should the PESIDRepositoryEntry match.
@@ -103,9 +108,23 @@ class RepoMapDataHandler(object):
"""
matching_pesid_repos = []
for pesid_repo in self.repositories:
- if pesid_repo.repoid == repoid and pesid_repo.major_version == major_version:
+ # FIXME(pstodulk): Why we do not check actually architecture here?
+ # It seems obvious we should check it but the fixme comment below
+ # suggests that it's expected - for not obvious reason.
+ # For the investigation:
+ # # check repoids matching various architectures
+ # # check repoids without $arch in substring on how many architectures they are present
+ # Investigate and: add a comment with an explanation or fix the
+ # condition.
+ if (
+ pesid_repo.repoid == repoid
+ and pesid_repo.major_version == major_version
+ and pesid_repo.distro == self.distro
+ ):
matching_pesid_repos.append(pesid_repo)
+ # FIXME: when a PESID is present for multiple architectures, there
+ # multiple matching repos even though there should really be just one
if len(matching_pesid_repos) == 1:
# Perform no heuristics if only a single pesid repository with matching repoid found
return matching_pesid_repos[0]
@@ -136,27 +155,34 @@ class RepoMapDataHandler(object):
pesids.update(repomap.target)
return sorted(pesids)
- def get_pesid_repos(self, pesid, major_version):
+ def get_pesid_repos(self, pesid, major_version, distro):
"""
- Get the list of PESIDRepositoryEntry with the specified PES ID and OS major version.
+ Get the list of PESIDRepositoryEntry with the specified PES ID,
+ OS major version, and OS release ID.
:param pesid: PES ID of the repositories to be retrieved.
:type pesid: str
- :param major_version: OS major version of the repositories to be retrieved.
+ :param major_version: OS major version of repositories to be retrieved.
:type major_version: str
- :return: A list of PESIDRepositoryEntries that match the provided PES ID and OS major version.
+ :param distro: OS release ID of repositories to be retrieved,
+ :type distro: str
+ :return: A list of PESIDRepositoryEntries that match the provided PES ID, OS major version, and OS release ID.
:rtype: List[PESIDRepositoryEntry]
"""
pesid_repos = []
for pesid_repo in self.repositories:
- if pesid_repo.pesid == pesid and pesid_repo.major_version == major_version:
+ if (
+ pesid_repo.pesid == pesid
+ and pesid_repo.major_version == major_version
+ and pesid_repo.distro == distro
+ ):
pesid_repos.append(pesid_repo)
return pesid_repos
def get_source_pesid_repos(self, pesid):
"""
Return the list of PESIDRepositoryEntry objects for a specified PES ID
- matching the source OS major version.
+ matching the source OS.
:param pesid: The PES ID for which to retrieve PESIDRepositoryEntries.
:type pesid: str
@@ -164,12 +190,12 @@ class RepoMapDataHandler(object):
the OS Major version same as the source OS.
:rtype: List[PESIDRepositoryEntry]
"""
- return self.get_pesid_repos(pesid, get_source_major_version())
+ return self.get_pesid_repos(pesid, get_source_major_version(), self.distro)
def get_target_pesid_repos(self, pesid):
"""
Return the list of PESIDRepositoryEntry objects for a specified PES ID
- matching the target OS major version.
+ matching the target OS.
:param pesid: The PES ID for which to retrieve PESIDRepositoryEntries.
:type pesid: str
@@ -177,7 +203,7 @@ class RepoMapDataHandler(object):
the OS Major version same as the target OS.
:rtype: List[PESIDRepositoryEntry]
"""
- return self.get_pesid_repos(pesid, get_target_major_version())
+ return self.get_pesid_repos(pesid, get_target_major_version(), self.distro)
def _find_repository_target_equivalent(self, src_pesidrepo, target_pesid):
"""
@@ -197,8 +223,9 @@ class RepoMapDataHandler(object):
matches_rhui = candidate.rhui == src_pesidrepo.rhui
matches_repo_type = candidate.repo_type == 'rpm'
matches_arch = candidate.arch == api.current_actor().configuration.architecture
+ matches_distro = candidate.distro == self.distro
- if matches_rhui and matches_arch and matches_repo_type:
+ if matches_rhui and matches_arch and matches_distro and matches_repo_type:
# user can specify in future the specific channel should be
# prioritized always (e.g. want to go to EUS...).
channel = self.prio_channel or src_pesidrepo.channel
diff --git a/repos/system_upgrade/common/actors/peseventsscanner/tests/test_pes_event_scanner.py b/repos/system_upgrade/common/actors/peseventsscanner/tests/test_pes_event_scanner.py
index 9a499baa..09a1e82d 100644
--- a/repos/system_upgrade/common/actors/peseventsscanner/tests/test_pes_event_scanner.py
+++ b/repos/system_upgrade/common/actors/peseventsscanner/tests/test_pes_event_scanner.py
@@ -246,9 +246,9 @@ def test_actor_performs(monkeypatch):
],
repositories=[
PESIDRepositoryEntry(pesid='rhel7-base', major_version='7', repoid='rhel7-repo', arch='x86_64',
- repo_type='rpm', channel='ga', rhui=''),
+ repo_type='rpm', channel='ga', rhui='', distro='rhel'),
PESIDRepositoryEntry(pesid='rhel8-BaseOS', major_version='8', repoid='rhel8-repo', arch='x86_64',
- repo_type='rpm', channel='ga', rhui='')]
+ repo_type='rpm', channel='ga', rhui='', distro='rhel')]
)
enabled_modules = EnabledModules(modules=[])
diff --git a/repos/system_upgrade/common/actors/repositoriesblacklist/tests/test_repositoriesblacklist.py b/repos/system_upgrade/common/actors/repositoriesblacklist/tests/test_repositoriesblacklist.py
index 28da781e..c4f9a36e 100644
--- a/repos/system_upgrade/common/actors/repositoriesblacklist/tests/test_repositoriesblacklist.py
+++ b/repos/system_upgrade/common/actors/repositoriesblacklist/tests/test_repositoriesblacklist.py
@@ -6,7 +6,6 @@ from leapp.libraries.common.testutils import create_report_mocked, CurrentActorM
from leapp.libraries.stdlib import api
from leapp.models import (
CustomTargetRepository,
- EnvVar,
PESIDRepositoryEntry,
RepoMapEntry,
RepositoriesBlacklisted,
@@ -42,6 +41,7 @@ def rhel7_optional_pesidrepo():
arch='x86_64',
channel='ga',
repo_type='rpm',
+ distro='rhel',
)
@@ -54,7 +54,9 @@ def rhel8_crb_pesidrepo():
rhui='',
arch='x86_64',
channel='ga',
- repo_type='rpm')
+ repo_type='rpm',
+ distro='rhel',
+ )
@pytest.fixture
diff --git a/repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py b/repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py
index 58089195..d4a64793 100644
--- a/repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py
+++ b/repos/system_upgrade/common/actors/repositoriesmapping/libraries/repositoriesmapping.py
@@ -17,7 +17,7 @@ REPOMAP_FILE = 'repomap.json'
class RepoMapData(object):
- VERSION_FORMAT = '1.2.1'
+ VERSION_FORMAT = '1.3.0'
def __init__(self):
self.repositories = []
@@ -40,7 +40,8 @@ class RepoMapData(object):
repo_type=data['repo_type'],
arch=data['arch'],
major_version=data['major_version'],
- pesid=pesid
+ pesid=pesid,
+ distro=data['distro'],
))
def get_repositories(self, valid_major_versions):
diff --git a/repos/system_upgrade/common/actors/repositoriesmapping/tests/files/repomap_example.json b/repos/system_upgrade/common/actors/repositoriesmapping/tests/files/repomap_example.json
index a5fc5fe1..1f04d72d 100644
--- a/repos/system_upgrade/common/actors/repositoriesmapping/tests/files/repomap_example.json
+++ b/repos/system_upgrade/common/actors/repositoriesmapping/tests/files/repomap_example.json
@@ -1,6 +1,6 @@
{
"datetime": "202107141655Z",
- "version_format": "1.2.1",
+ "version_format": "1.3.0",
"mapping": [
{
"source_major_version": "7",
@@ -38,7 +38,8 @@
"repoid": "some-rhel-7-repoid",
"arch": "x86_64",
"repo_type": "rpm",
- "channel": "eus"
+ "channel": "eus",
+ "distro": "rhel"
}
]
},
@@ -50,7 +51,8 @@
"repoid": "some-rhel-8-repoid1",
"arch": "x86_64",
"repo_type": "rpm",
- "channel": "eus"
+ "channel": "eus",
+ "distro": "rhel"
}
]
},
@@ -62,7 +64,8 @@
"repoid": "some-rhel-8-repoid2",
"arch": "x86_64",
"repo_type": "rpm",
- "channel": "eus"
+ "channel": "eus",
+ "distro": "rhel"
}
]
},
@@ -74,7 +77,8 @@
"repoid": "some-rhel-9-repo1",
"arch": "x86_64",
"repo_type": "rpm",
- "channel": "eus"
+ "channel": "eus",
+ "distro": "rhel"
}
]
},
@@ -86,7 +90,34 @@
"repoid": "some-rhel-9-repo2",
"arch": "x86_64",
"repo_type": "rpm",
- "channel": "eus"
+ "channel": "eus",
+ "distro": "rhel"
+ }
+ ]
+ },
+ {
+ "pesid": "pesid6",
+ "entries": [
+ {
+ "major_version": "7",
+ "repoid": "some-centos-9-repoid1",
+ "arch": "x86_64",
+ "repo_type": "rpm",
+ "channel": "ga",
+ "distro": "centos"
+ }
+ ]
+ },
+ {
+ "pesid": "pesid7",
+ "entries": [
+ {
+ "major_version": "8",
+ "repoid": "some-centos-10-repoid1",
+ "arch": "x86_64",
+ "repo_type": "rpm",
+ "channel": "ga",
+ "distro": "centos"
}
]
}
diff --git a/repos/system_upgrade/common/actors/repositoriesmapping/tests/unit_test_repositoriesmapping.py b/repos/system_upgrade/common/actors/repositoriesmapping/tests/unit_test_repositoriesmapping.py
index 243b8809..9d781125 100644
--- a/repos/system_upgrade/common/actors/repositoriesmapping/tests/unit_test_repositoriesmapping.py
+++ b/repos/system_upgrade/common/actors/repositoriesmapping/tests/unit_test_repositoriesmapping.py
@@ -7,10 +7,9 @@ import requests
from leapp.exceptions import StopActorExecutionError
from leapp.libraries.actor import repositoriesmapping
from leapp.libraries.common import fetch
-from leapp.libraries.common.config import architecture, version
from leapp.libraries.common.testutils import CurrentActorMocked, produce_mocked
from leapp.libraries.stdlib import api
-from leapp.models import ConsumedDataAsset, PESIDRepositoryEntry, RPM
+from leapp.models import ConsumedDataAsset, PESIDRepositoryEntry
CUR_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -53,7 +52,7 @@ def test_scan_existing_valid_data(monkeypatch, adjust_cwd):
# 2. Verify that only repositories valid for the current IPU are produced
pesid_repos = repo_mapping.repositories
fail_description = 'Actor produced incorrect number of IPU-relevant pesid repos.'
- assert len(pesid_repos) == 3, fail_description
+ assert len(pesid_repos) == 5, fail_description
expected_pesid_repos = [
PESIDRepositoryEntry(
@@ -63,7 +62,8 @@ def test_scan_existing_valid_data(monkeypatch, adjust_cwd):
arch='x86_64',
repo_type='rpm',
channel='eus',
- rhui=''
+ rhui='',
+ distro='rhel',
),
PESIDRepositoryEntry(
pesid='pesid2',
@@ -72,7 +72,8 @@ def test_scan_existing_valid_data(monkeypatch, adjust_cwd):
arch='x86_64',
repo_type='rpm',
channel='eus',
- rhui=''
+ rhui='',
+ distro='rhel',
),
PESIDRepositoryEntry(
pesid='pesid3',
@@ -81,7 +82,28 @@ def test_scan_existing_valid_data(monkeypatch, adjust_cwd):
arch='x86_64',
repo_type='rpm',
channel='eus',
- rhui=''
+ rhui='',
+ distro='rhel',
+ ),
+ PESIDRepositoryEntry(
+ pesid='pesid6',
+ major_version='7',
+ repoid='some-centos-9-repoid1',
+ arch='x86_64',
+ repo_type='rpm',
+ channel='ga',
+ rhui='',
+ distro='centos',
+ ),
+ PESIDRepositoryEntry(
+ pesid='pesid7',
+ major_version='8',
+ repoid='some-centos-10-repoid1',
+ arch='x86_64',
+ repo_type='rpm',
+ channel='ga',
+ rhui='',
+ distro='centos',
),
]
@@ -177,7 +199,8 @@ def test_scan_repositories_with_mapping_to_pesid_without_repos(monkeypatch):
'repoid': 'some-rhel-7-repo',
'arch': 'x86_64',
'repo_type': 'rpm',
- 'channel': 'eus'
+ 'channel': 'eus',
+ 'distro': 'rhel',
}
]
}
diff --git a/repos/system_upgrade/common/actors/setuptargetrepos/actor.py b/repos/system_upgrade/common/actors/setuptargetrepos/actor.py
index 767fa00c..91855818 100644
--- a/repos/system_upgrade/common/actors/setuptargetrepos/actor.py
+++ b/repos/system_upgrade/common/actors/setuptargetrepos/actor.py
@@ -17,11 +17,16 @@ from leapp.tags import FactsPhaseTag, IPUWorkflowTag
class SetupTargetRepos(Actor):
"""
- Produces list of repositories that should be available to be used by Upgrade process.
+ Produces list of repositories that should be available to be used during IPU process.
- Based on current set of Red Hat Enterprise Linux repositories, produces the list of target
- repositories. Additionally process request to use custom repositories during the upgrade
- transaction.
+ The list of expected target repositories is produced based on:
+ * required custom target repositories,
+ * discovered enabled repositories,
+ * repositories from which originates the installed content,
+ * and the system distribution.
+
+ The list of repositories can be additionally affected in case of RHEL by
+ required channel (e.g. eus) or by detected use of RHUI.
"""
name = 'setuptargetrepos'
diff --git a/repos/system_upgrade/common/actors/setuptargetrepos/libraries/setuptargetrepos_repomap.py b/repos/system_upgrade/common/actors/setuptargetrepos/libraries/setuptargetrepos_repomap.py
index c2976ddf..37be03f1 100644
--- a/repos/system_upgrade/common/actors/setuptargetrepos/libraries/setuptargetrepos_repomap.py
+++ b/repos/system_upgrade/common/actors/setuptargetrepos/libraries/setuptargetrepos_repomap.py
@@ -23,7 +23,7 @@ class RepoMapDataHandler(object):
Provide the basic functionality to work with the repository data easily.
"""
- def __init__(self, repo_map, cloud_provider='', default_channels=None):
+ def __init__(self, repo_map, distro='', cloud_provider='', default_channels=None):
"""
Initialize the object based on the given RepositoriesMapping msg.
@@ -32,6 +32,8 @@ class RepoMapDataHandler(object):
:param repo_map: A valid RepositoryMapping message.
:type repo_map: RepositoryMapping
+ :param distro: Which distribution's mappings to use, default to current
+ :type distro: str
:param default_channels: A list of default channels to use when a target repository
equivalent exactly matching a source repository was not found.
:type default_channels: List[str]
@@ -42,6 +44,7 @@ class RepoMapDataHandler(object):
# ideal for work, but there is not any significant impact..
self.repositories = repo_map.repositories
self.mapping = repo_map.mapping
+ self.distro = distro or api.current_actor().configuration.os_release.release_id
# FIXME(pstodulk): what about default_channel -> fallback_channel
# hardcoded always as ga? instead of list of channels..
# it'd be possibly confusing naming now...
@@ -93,6 +96,8 @@ class RepoMapDataHandler(object):
If multiple pesid repo entries with the same repoid were found, the entry with rhui matching the source
system's rhui info will be returned. If no entry with matching rhui exists, the CDN one is returned if any.
+ Note that repositories are automatically filtered based on the specified OS release ID (self.distro).
+
:param repoid: RepoID that should the PESIDRepositoryEntry match.
:type repoid: str
:param major_version: RepoID that should the PESIDRepositoryEntry match.
@@ -103,9 +108,23 @@ class RepoMapDataHandler(object):
"""
matching_pesid_repos = []
for pesid_repo in self.repositories:
- if pesid_repo.repoid == repoid and pesid_repo.major_version == major_version:
+ # FIXME(pstodulk): Why we do not check actually architecture here?
+ # It seems obvious we should check it but the fixme comment below
+ # suggests that it's expected - for not obvious reason.
+ # For the investigation:
+ # # check repoids matching various architectures
+ # # check repoids without $arch in substring on how many architectures they are present
+ # Investigate and: add a comment with an explanation or fix the
+ # condition.
+ if (
+ pesid_repo.repoid == repoid
+ and pesid_repo.major_version == major_version
+ and pesid_repo.distro == self.distro
+ ):
matching_pesid_repos.append(pesid_repo)
+ # FIXME: when a PESID is present for multiple architectures, there
+ # multiple matching repos even though there should really be just one
if len(matching_pesid_repos) == 1:
# Perform no heuristics if only a single pesid repository with matching repoid found
return matching_pesid_repos[0]
@@ -136,27 +155,34 @@ class RepoMapDataHandler(object):
pesids.update(repomap.target)
return sorted(pesids)
- def get_pesid_repos(self, pesid, major_version):
+ def get_pesid_repos(self, pesid, major_version, distro):
"""
- Get the list of PESIDRepositoryEntry with the specified PES ID and OS major version.
+ Get the list of PESIDRepositoryEntry with the specified PES ID,
+ OS major version, and OS release ID.
:param pesid: PES ID of the repositories to be retrieved.
:type pesid: str
- :param major_version: OS major version of the repositories to be retrieved.
+ :param major_version: OS major version of repositories to be retrieved.
:type major_version: str
- :return: A list of PESIDRepositoryEntries that match the provided PES ID and OS major version.
+ :param distro: OS release ID of repositories to be retrieved,
+ :type distro: str
+ :return: A list of PESIDRepositoryEntries that match the provided PES ID, OS major version, and OS release ID.
:rtype: List[PESIDRepositoryEntry]
"""
pesid_repos = []
for pesid_repo in self.repositories:
- if pesid_repo.pesid == pesid and pesid_repo.major_version == major_version:
+ if (
+ pesid_repo.pesid == pesid
+ and pesid_repo.major_version == major_version
+ and pesid_repo.distro == distro
+ ):
pesid_repos.append(pesid_repo)
return pesid_repos
def get_source_pesid_repos(self, pesid):
"""
Return the list of PESIDRepositoryEntry objects for a specified PES ID
- matching the source OS major version.
+ matching the source OS.
:param pesid: The PES ID for which to retrieve PESIDRepositoryEntries.
:type pesid: str
@@ -164,12 +190,12 @@ class RepoMapDataHandler(object):
the OS Major version same as the source OS.
:rtype: List[PESIDRepositoryEntry]
"""
- return self.get_pesid_repos(pesid, get_source_major_version())
+ return self.get_pesid_repos(pesid, get_source_major_version(), self.distro)
def get_target_pesid_repos(self, pesid):
"""
Return the list of PESIDRepositoryEntry objects for a specified PES ID
- matching the target OS major version.
+ matching the target OS.
:param pesid: The PES ID for which to retrieve PESIDRepositoryEntries.
:type pesid: str
@@ -177,7 +203,7 @@ class RepoMapDataHandler(object):
the OS Major version same as the target OS.
:rtype: List[PESIDRepositoryEntry]
"""
- return self.get_pesid_repos(pesid, get_target_major_version())
+ return self.get_pesid_repos(pesid, get_target_major_version(), self.distro)
def _find_repository_target_equivalent(self, src_pesidrepo, target_pesid):
"""
@@ -197,8 +223,9 @@ class RepoMapDataHandler(object):
matches_rhui = candidate.rhui == src_pesidrepo.rhui
matches_repo_type = candidate.repo_type == 'rpm'
matches_arch = candidate.arch == api.current_actor().configuration.architecture
+ matches_distro = candidate.distro == self.distro
- if matches_rhui and matches_arch and matches_repo_type:
+ if matches_rhui and matches_arch and matches_distro and matches_repo_type:
# user can specify in future the specific channel should be
# prioritized always (e.g. want to go to EUS...).
channel = self.prio_channel or src_pesidrepo.channel
diff --git a/repos/system_upgrade/common/actors/setuptargetrepos/tests/test_repomapping.py b/repos/system_upgrade/common/actors/setuptargetrepos/tests/test_repomapping.py
index ba5906f4..af40c443 100644
--- a/repos/system_upgrade/common/actors/setuptargetrepos/tests/test_repomapping.py
+++ b/repos/system_upgrade/common/actors/setuptargetrepos/tests/test_repomapping.py
@@ -5,12 +5,21 @@ import pytest
from leapp.libraries.actor import setuptargetrepos_repomap
from leapp.libraries.actor.setuptargetrepos_repomap import get_default_repository_channels, RepoMapDataHandler
-from leapp.libraries.common.testutils import CurrentActorMocked, produce_mocked
+from leapp.libraries.common.testutils import CurrentActorMocked
from leapp.libraries.stdlib import api
from leapp.models import PESIDRepositoryEntry, RepoMapEntry, RepositoriesMapping
-def make_pesid_repo(pesid, major_version, repoid, arch='x86_64', repo_type='rpm', channel='ga', rhui=''):
+def make_pesid_repo(
+ pesid,
+ major_version,
+ repoid,
+ arch='x86_64',
+ repo_type='rpm',
+ channel='ga',
+ rhui='',
+ distro='rhel',
+):
"""
PESIDRepositoryEntry factory function allowing shorter data description in tests by providing default values.
"""
@@ -21,7 +30,8 @@ def make_pesid_repo(pesid, major_version, repoid, arch='x86_64', repo_type='rpm'
arch=arch,
repo_type=repo_type,
channel=channel,
- rhui=rhui
+ rhui=rhui,
+ distro=distro,
)
@@ -49,6 +59,27 @@ def repomap_data_for_pesid_repo_retrieval():
return repomap_data
+@pytest.fixture
+def repomap_data_multiple_distros():
+ repomap_data = RepositoriesMapping(
+ mapping=[
+ RepoMapEntry(source="pesid1", target=["pesid3", "pesid2"]),
+ ],
+ repositories=[
+ make_pesid_repo("pesid1", "9", "pesid1-repoid"),
+ make_pesid_repo("pesid1", "9", "pesid1-repoid-eus", channel="eus"),
+ make_pesid_repo("pesid1", "9", "pesid1-repoid-centos", distro="centos"),
+ make_pesid_repo("pesid2", "10", "pesid2-repoid"),
+ make_pesid_repo("pesid2", "10", "pesid2-repoid-centos", distro="centos"),
+ make_pesid_repo("pesid3", "10", "pesid3-repoid"),
+ make_pesid_repo("pesid3", "10", "pesid3-repoid-eus", channel="eus"),
+ make_pesid_repo("pesid3", "10", "pesid3-repoid-aws", rhui="aws"),
+ make_pesid_repo("pesid3", "10", "pesid3-repoid-centos", distro="centos"),
+ ],
+ )
+ return repomap_data
+
+
def test_get_pesid_repo_entry(monkeypatch, repomap_data_for_pesid_repo_retrieval):
"""
Test for the RepoMapDataHandler.get_pesid_repo_entry method.
@@ -75,6 +106,40 @@ def test_get_pesid_repo_entry(monkeypatch, repomap_data_for_pesid_repo_retrieval
assert handler.get_pesid_repo_entry('nonexisting-repo', '7') is None, fail_description
+@pytest.mark.parametrize('distro', ('rhel', 'centos'))
+def test_get_pesid_repo_entry_distro(
+ monkeypatch, repomap_data_multiple_distros, distro
+):
+ """
+ Test for the RepoMapDataHandler.get_pesid_repo_entry method.
+
+ Verifies that the method correctly retrieves PESIDRepositoryEntry that are
+ matching the OS major version, repoid and the distro.
+ """
+ monkeypatch.setattr(
+ api,
+ "current_actor",
+ CurrentActorMocked(
+ arch="x86_64", src_ver="9.6", dst_ver="10.2", release_id=distro
+ ),
+ )
+ handler = RepoMapDataHandler(repomap_data_multiple_distros)
+ repositories = [
+ repo
+ for repo in repomap_data_multiple_distros.repositories
+ if repo.distro == distro
+ ]
+
+ fail_description = (
+ "get_pesid_repo_entry method failed to find correct pesid repository that matches given parameters."
+ )
+ for exp_repo in repositories:
+ result_repo = handler.get_pesid_repo_entry(
+ exp_repo.repoid, exp_repo.major_version
+ )
+ assert result_repo == exp_repo, fail_description
+
+
def test_get_target_pesids(monkeypatch, repomap_data_for_pesid_repo_retrieval):
"""
Test for the RepoMapDataHandler.get_target_pesids method.
@@ -98,54 +163,112 @@ def test_get_target_pesids(monkeypatch, repomap_data_for_pesid_repo_retrieval):
assert [] == handler.get_target_pesids('pesid_no_mapping'), fail_description
-def test_get_pesid_repos(monkeypatch, repomap_data_for_pesid_repo_retrieval):
+@pytest.mark.parametrize('distro', ('rhel', 'centos'))
+def test_get_target_pesids_distro(
+ monkeypatch, repomap_data_multiple_distros, distro
+):
+ """
+ Test for the RepoMapDataHandler.get_target_pesids method.
+
+ Verifies that the method correctly tells what target pesids is the given source pesid mapped to.
+ """
+ monkeypatch.setattr(
+ api,
+ "current_actor",
+ CurrentActorMocked(
+ arch="x86_64", src_ver="7.9", dst_ver="8.4", release_id=distro
+ ),
+ )
+ handler = RepoMapDataHandler(repomap_data_multiple_distros)
+
+ expected_target_pesids = ['pesid2', 'pesid3']
+ actual_target_pesids = handler.get_target_pesids('pesid1')
+
+ fail_description = (
+ 'The get_target_pesids method did not correctly identify what is the given source pesid mapped to.')
+ assert expected_target_pesids == actual_target_pesids, fail_description
+
+ fail_description = (
+ 'The get_target_pesids method found target pesids even if the source repository is not mapped.')
+ assert [] == handler.get_target_pesids('pesid2'), fail_description
+ assert [] == handler.get_target_pesids('pesid_no_mapping'), fail_description
+
+
+@pytest.mark.parametrize(
+ 'distro,expect_pesid3,expect_pesid1',
+ [
+ ('rhel', [5, 6, 7], [0, 1]),
+ ('centos', [8], [2]),
+ ]
+)
+def test_get_pesid_repos(
+ monkeypatch, repomap_data_multiple_distros, distro, expect_pesid3, expect_pesid1
+):
"""
Test for the RepoMapDataHandler.get_pesid_repos method.
Verifies that the method is able to collect all PESIDRepositoryEntry present in the repomap data that
- match the given OS major version and the given pesid.
- """
- monkeypatch.setattr(api, 'current_actor',
- CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4'))
- handler = RepoMapDataHandler(repomap_data_for_pesid_repo_retrieval)
- repositories = repomap_data_for_pesid_repo_retrieval.repositories
+ matches
+ * the given OS major version,
+ * the given pesid,
+ * and the given distro.
+ """
+ monkeypatch.setattr(
+ api,
+ "current_actor",
+ CurrentActorMocked(
+ arch="x86_64", src_ver="9.6", dst_ver="10.4", release_id=distro
+ ),
+ )
+ handler = RepoMapDataHandler(repomap_data_multiple_distros)
- actual_pesid_repos = handler.get_pesid_repos('pesid3', '8')
- expected_pesid_repos = [repositories[3], repositories[4], repositories[5]]
+ actual_pesid_repos = handler.get_pesid_repos('pesid3', '10', distro)
+ expected_pesid_repos = [
+ repomap_data_multiple_distros.repositories[repo] for repo in expect_pesid3
+ ]
fail_description = 'The get_pesid_repos failed to find pesid repos matching the given criteria.'
assert len(expected_pesid_repos) == len(actual_pesid_repos), fail_description
for actual_pesid_repo in actual_pesid_repos:
assert actual_pesid_repo in expected_pesid_repos, fail_description
- actual_pesid_repos = handler.get_pesid_repos('pesid1', '7')
- expected_pesid_repos = [repositories[0], repositories[1]]
+ actual_pesid_repos = handler.get_pesid_repos('pesid1', '9', distro)
+ expected_pesid_repos = [
+ repomap_data_multiple_distros.repositories[repo] for repo in expect_pesid1
+ ]
assert len(expected_pesid_repos) == len(actual_pesid_repos), fail_description
for actual_pesid_repo in actual_pesid_repos:
assert actual_pesid_repo in expected_pesid_repos, fail_description
fail_description = (
'The get_pesid_repos found some pesid repositories matching criteria, but there are no such repositories.')
- assert [] == handler.get_pesid_repos('pesid3', '7'), fail_description
- assert [] == handler.get_pesid_repos('pesid1', '8'), fail_description
- assert [] == handler.get_pesid_repos('nonexisting_pesid', '7'), fail_description
+ assert [] == handler.get_pesid_repos('pesid3', '7', 'rhel'), fail_description
+ assert [] == handler.get_pesid_repos('pesid1', '8', 'rhel'), fail_description
+ assert [] == handler.get_pesid_repos('nonexisting_pesid', '7', 'rhel'), fail_description
-def test_get_source_pesid_repos(monkeypatch, repomap_data_for_pesid_repo_retrieval):
+@pytest.mark.parametrize(
+ 'distro,expected_repos_index',
+ [
+ ('rhel', [0, 1]),
+ ('centos', []),
+ ]
+)
+def test_get_source_pesid_repos(monkeypatch, repomap_data_for_pesid_repo_retrieval, distro, expected_repos_index):
"""
Test for the RepoMapDataHandler.get_source_pesid_repos method.
Verifies that the method is able to collect all PESIDRepositoryEntry that match the given PES ID and
- have the major version same as the source system.
+ have the same major version and distro as the source system.
"""
monkeypatch.setattr(api, 'current_actor',
- CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4'))
+ CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4', release_id=distro))
handler = RepoMapDataHandler(repomap_data_for_pesid_repo_retrieval)
repositories = repomap_data_for_pesid_repo_retrieval.repositories
fail_description = (
'The get_source_pesid_repos method failed to retrieve all pesid repos that match given pesid '
- 'and have the major version same as the source system.')
- expected_pesid_repos = [repositories[0], repositories[1]]
+ 'and have the same major version and distro as the source system.')
+ expected_pesid_repos = [repositories[i] for i in expected_repos_index]
actual_pesid_repos = handler.get_source_pesid_repos('pesid1')
assert len(expected_pesid_repos) == len(actual_pesid_repos), fail_description
for actual_pesid_repo in actual_pesid_repos:
@@ -163,22 +286,29 @@ def test_get_source_pesid_repos(monkeypatch, repomap_data_for_pesid_repo_retriev
assert [] == handler.get_source_pesid_repos('nonexisting_pesid'), fail_description
-def test_get_target_pesid_repos(monkeypatch, repomap_data_for_pesid_repo_retrieval):
+@pytest.mark.parametrize(
+ 'distro,expected_repos_index',
+ [
+ ('rhel', [3, 4, 5]),
+ ('centos', []),
+ ]
+)
+def test_get_target_pesid_repos(monkeypatch, repomap_data_for_pesid_repo_retrieval, distro, expected_repos_index):
"""
Test for the RepoMapDataHandler.get_target_pesid_repos method.
Verifies that the method is able to collect all PESIDRepositoryEntry that match the given PES ID and
- have the major version same as the source system.
+ have the same major version and distro as the source system.
"""
monkeypatch.setattr(api, 'current_actor',
- CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4'))
+ CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4', release_id=distro))
handler = RepoMapDataHandler(repomap_data_for_pesid_repo_retrieval)
repositories = repomap_data_for_pesid_repo_retrieval.repositories
fail_description = (
'The get_target_pesid_repos method failed to retrieve all pesid repos that match given pesid '
- 'and have the major version same as the target system.')
- expected_pesid_repos = [repositories[3], repositories[4], repositories[5]]
+ 'and have the same major version and distro as the target system.')
+ expected_pesid_repos = [repositories[i] for i in expected_repos_index]
actual_pesid_repos = handler.get_target_pesid_repos('pesid3')
assert len(expected_pesid_repos) == len(actual_pesid_repos), fail_description
for actual_pesid_repo in actual_pesid_repos:
@@ -188,7 +318,7 @@ def test_get_target_pesid_repos(monkeypatch, repomap_data_for_pesid_repo_retriev
'The get_target_pesid_repos method doesn\'t take into account the target system version correctly.'
)
monkeypatch.setattr(api, 'current_actor',
- CurrentActorMocked(arch='x86_64', src_ver='9.4', dst_ver='10.0'))
+ CurrentActorMocked(arch='x86_64', src_ver='9.4', dst_ver='10.0', release_id=distro))
# Repeat the same test as above to make sure it respects the target OS major version
assert [] == handler.get_target_pesid_repos('pesid3'), fail_description
@@ -202,6 +332,7 @@ def mapping_data_for_find_repository_equiv():
repositories = [
make_pesid_repo('pesid1', '7', 'pesid1-repoid'),
make_pesid_repo('pesid1', '7', 'pesid1-repoid', channel='e4s', rhui='aws'),
+ make_pesid_repo('pesid2', '8', 'pesid2-repoid1-centos', distro='centos'),
make_pesid_repo('pesid2', '8', 'pesid2-repoid1'),
make_pesid_repo('pesid2', '8', 'pesid2-repoid2-s390x', arch='s390x'),
# This repository is a better candidate than the full match equivalent, but _find_repository_target_equivalent
@@ -218,18 +349,28 @@ def mapping_data_for_find_repository_equiv():
)
-def test_find_repository_target_equivalent_fullmatch(monkeypatch, mapping_data_for_find_repository_equiv):
+@pytest.mark.parametrize(
+ 'distro,expected_repo_index',
+ [
+ ('rhel', 3),
+ ('centos', 2),
+ ]
+)
+def test_find_repository_target_equivalent_fullmatch(
+ monkeypatch, mapping_data_for_find_repository_equiv, distro, expected_repo_index
+):
"""
Test for the RepoMapDataHandler._find_repository_target_equivalent method.
Verifies that the method can find the target equivalent for a repository that matches the source
pesid repo parameters exactly when such repository is available in the repository mapping data.
"""
- monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4'))
+ monkeypatch.setattr(api, 'current_actor',
+ CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4', release_id=distro))
handler = RepoMapDataHandler(mapping_data_for_find_repository_equiv)
- expected_target_equivalent = mapping_data_for_find_repository_equiv.repositories[2]
+ expected_target_equivalent = mapping_data_for_find_repository_equiv.repositories[expected_repo_index]
actual_target_equivalent = handler._find_repository_target_equivalent(
mapping_data_for_find_repository_equiv.repositories[0], 'pesid2')
@@ -239,6 +380,11 @@ def test_find_repository_target_equivalent_fullmatch(monkeypatch, mapping_data_f
assert expected_target_equivalent == actual_target_equivalent, fail_description
+# #############################################################################
+# Note that tests below cover usually only distro 'rhel' as the tested
+# functionality is not relevant for other distributions (RHUI and different
+# channels are relevant only for RHEL).
+
def test_find_repository_target_equivalent_fallback_to_default(monkeypatch,
mapping_data_for_find_repository_equiv):
"""
@@ -255,13 +401,13 @@ def test_find_repository_target_equivalent_fallback_to_default(monkeypatch,
fail_description = (
'The _find_repository_target_equivalent failed to find repository with some of the fallback channels.')
- expected_target_equivalent = repositories[6]
+ expected_target_equivalent = repositories[7]
actual_target_equivalent = handler._find_repository_target_equivalent(repositories[1], 'pesid2')
assert expected_target_equivalent == actual_target_equivalent, fail_description
handler.set_default_channels(['eus', 'ga'])
- expected_target_equivalent = repositories[7]
+ expected_target_equivalent = repositories[8]
actual_target_equivalent = handler._find_repository_target_equivalent(repositories[1], 'pesid2')
assert expected_target_equivalent == actual_target_equivalent, fail_description
@@ -301,12 +447,18 @@ def test_get_mapped_target_pesid_repos(monkeypatch, mapping_data_for_find_reposi
'The get_mapped_target_pesid_repos failed to build a map mapping the target pesid '
'to the best pesid repository candidate.')
target_pesid_repos_map = handler.get_mapped_target_pesid_repos(repositories[0])
- expected_pesid_to_best_candidate_map = {'pesid2': repositories[2], 'pesid3': repositories[8]}
+ expected_pesid_to_best_candidate_map = {
+ 'pesid2': repositories[3],
+ 'pesid3': repositories[9]
+ }
assert target_pesid_repos_map == expected_pesid_to_best_candidate_map, fail_description
# The pesid3 does not have an equivalent for provided source pesid repository (due to not having any rhui repos)
target_pesid_repos_map = handler.get_mapped_target_pesid_repos(repositories[1])
- expected_pesid_to_best_candidate_map = {'pesid2': repositories[6], 'pesid3': None}
+ expected_pesid_to_best_candidate_map = {
+ 'pesid2': repositories[7],
+ 'pesid3': None
+ }
assert target_pesid_repos_map == expected_pesid_to_best_candidate_map, fail_description
@@ -315,6 +467,10 @@ def test_get_mapped_target_repoids(monkeypatch, mapping_data_for_find_repository
Test for the RepoMapDataHandler.get_mapped_target_repoids method.
Verifies that the method returns a correct list of repoids that should be present on the target system.
+
+ Note the test covers only distro 'rhel' as this part of the tested function
+ does not seem to be relevant for other distributions (RHUI and different
+ channels are relevant only for RHEL)
"""
monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4'))
@@ -326,7 +482,7 @@ def test_get_mapped_target_repoids(monkeypatch, mapping_data_for_find_repository
'to be enabled on the target system.')
# Both pesid2 and pesid3 have an equivalent for provided source pesid repository
actual_target_repoids = handler.get_mapped_target_repoids(repositories[0])
- expected_target_repoids = {repositories[2].repoid, repositories[8].repoid}
+ expected_target_repoids = {repositories[3].repoid, repositories[9].repoid}
assert len(actual_target_repoids) == len(expected_target_repoids), fail_description
assert set(actual_target_repoids) == expected_target_repoids, fail_description
@@ -401,6 +557,9 @@ def test_get_expected_target_repoids_fallback(monkeypatch):
map that maps target pesid to the best candidate pesid repository when there is a repository
on the source system that does not have exact match equivalent and some other with a fallback channel
must be found.
+
+ No need a distro specific testing - the fallback mechanism works with
+ filtered repositories and that's covered already in previous tests.
"""
monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.4'))
@@ -639,20 +798,26 @@ def test_multiple_repoids_in_repomapping(monkeypatch, rhui):
monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(arch='x86_64', src_ver='7.9', dst_ver='8.6'))
- mk_rhui_el7_pesid_repo = functools.partial(PESIDRepositoryEntry,
- pesid='rhel7-rhui',
- major_version='7',
- repoid='repoid7-rhui',
- repo_type='rpm',
- arch='x86_64',
- channel='ga')
-
- mk_rhui_el8_pesid_repo = functools.partial(PESIDRepositoryEntry,
- pesid='rhel8-rhui',
- major_version='8',
- repo_type='rpm',
- arch='x86_64',
- channel='ga')
+ mk_rhui_el7_pesid_repo = functools.partial(
+ PESIDRepositoryEntry,
+ pesid='rhel7-rhui',
+ major_version='7',
+ repoid='repoid7-rhui',
+ repo_type='rpm',
+ arch='x86_64',
+ channel='ga',
+ distro='rhel'
+ )
+
+ mk_rhui_el8_pesid_repo = functools.partial(
+ PESIDRepositoryEntry,
+ pesid='rhel8-rhui',
+ major_version='8',
+ repo_type='rpm',
+ arch='x86_64',
+ channel='ga',
+ distro='rhel',
+ )
repomap = RepositoriesMapping(
mapping=[RepoMapEntry(source='rhel7-rhui', target=['rhel8-rhui'])],
diff --git a/repos/system_upgrade/common/actors/setuptargetrepos/tests/test_setuptargetrepos.py b/repos/system_upgrade/common/actors/setuptargetrepos/tests/test_setuptargetrepos.py
index ac7f49ec..91d9cd24 100644
--- a/repos/system_upgrade/common/actors/setuptargetrepos/tests/test_setuptargetrepos.py
+++ b/repos/system_upgrade/common/actors/setuptargetrepos/tests/test_setuptargetrepos.py
@@ -127,7 +127,8 @@ def test_repos_mapping(monkeypatch):
arch='x86_64',
repo_type='rpm',
channel='ga',
- rhui=''
+ rhui='',
+ distro='rhel',
),
PESIDRepositoryEntry(
pesid='rhel8-baseos',
@@ -136,7 +137,8 @@ def test_repos_mapping(monkeypatch):
arch='x86_64',
repo_type='rpm',
channel='ga',
- rhui=''
+ rhui='',
+ distro='rhel',
),
PESIDRepositoryEntry(
pesid='rhel8-appstream',
@@ -145,7 +147,8 @@ def test_repos_mapping(monkeypatch):
arch='x86_64',
repo_type='rpm',
channel='ga',
- rhui=''
+ rhui='',
+ distro='rhel',
),
PESIDRepositoryEntry(
pesid='rhel8-blacklist',
@@ -154,7 +157,8 @@ def test_repos_mapping(monkeypatch):
arch='x86_64',
repo_type='rpm',
channel='ga',
- rhui=''
+ rhui='',
+ distro='rhel',
),
PESIDRepositoryEntry(
pesid='rhel7-satellite-extras',
@@ -163,7 +167,8 @@ def test_repos_mapping(monkeypatch):
arch='x86_64',
repo_type='rpm',
channel='ga',
- rhui=''
+ rhui='',
+ distro='rhel',
),
PESIDRepositoryEntry(
pesid='rhel8-satellite-extras',
@@ -172,7 +177,8 @@ def test_repos_mapping(monkeypatch):
arch='x86_64',
repo_type='rpm',
channel='ga',
- rhui=''
+ rhui='',
+ distro='rhel',
),
]
)
diff --git a/repos/system_upgrade/common/models/repositoriesmap.py b/repos/system_upgrade/common/models/repositoriesmap.py
index 7192a60d..842cd807 100644
--- a/repos/system_upgrade/common/models/repositoriesmap.py
+++ b/repos/system_upgrade/common/models/repositoriesmap.py
@@ -71,13 +71,18 @@ class PESIDRepositoryEntry(Model):
purposes. The other channels indicate premium repositories.
"""
- rhui = fields.StringEnum(['', 'aws', 'azure', 'google', 'alibaba'])
+ rhui = fields.String(default='')
"""
Specifies what cloud provider (RHUI) is the repository specific to.
Empty string denotes that the repository is not specific to any cloud provider.
"""
+ distro = fields.String()
+ """
+ Specifies what distribution is the repository specific to.
+ """
+
class RepositoriesMapping(Model):
"""
--
2.49.0