From e9f899c27688007d2e87144ccfd038b8b0a655d1 Mon Sep 17 00:00:00 2001 From: PeterMocary Date: Wed, 12 Jul 2023 22:24:48 +0200 Subject: [PATCH 31/38] add the posibility to upgrade with a local repository Upgrade with a local repository required to host the repository locally for it to be visible from target user-space container during the upgrade. The added actor ensures that the local repository will be visible from the container by adjusting the path to it simply by prefixing a host root mount bind '/installroot' to it. The local_repos_inhibit actor is no longer needed, thus was removed. --- .../common/actors/adjustlocalrepos/actor.py | 48 ++++++ .../libraries/adjustlocalrepos.py | 100 ++++++++++++ .../tests/test_adjustlocalrepos.py | 151 ++++++++++++++++++ .../common/actors/localreposinhibit/actor.py | 89 ----------- .../tests/test_unit_localreposinhibit.py | 81 ---------- .../common/libraries/dnfplugin.py | 5 +- 6 files changed, 302 insertions(+), 172 deletions(-) create mode 100644 repos/system_upgrade/common/actors/adjustlocalrepos/actor.py create mode 100644 repos/system_upgrade/common/actors/adjustlocalrepos/libraries/adjustlocalrepos.py create mode 100644 repos/system_upgrade/common/actors/adjustlocalrepos/tests/test_adjustlocalrepos.py delete mode 100644 repos/system_upgrade/common/actors/localreposinhibit/actor.py delete mode 100644 repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py diff --git a/repos/system_upgrade/common/actors/adjustlocalrepos/actor.py b/repos/system_upgrade/common/actors/adjustlocalrepos/actor.py new file mode 100644 index 00000000..064e7f3e --- /dev/null +++ b/repos/system_upgrade/common/actors/adjustlocalrepos/actor.py @@ -0,0 +1,48 @@ +from leapp.actors import Actor +from leapp.libraries.actor import adjustlocalrepos +from leapp.libraries.common import mounting +from leapp.libraries.stdlib import api +from leapp.models import ( + TargetOSInstallationImage, + TargetUserSpaceInfo, + TMPTargetRepositoriesFacts, + UsedTargetRepositories +) +from leapp.tags import IPUWorkflowTag, TargetTransactionChecksPhaseTag + + +class AdjustLocalRepos(Actor): + """ + Adjust local repositories to the target user-space container. + + Changes the path of local file urls (starting with 'file://') for 'baseurl' and + 'mirrorlist' fields to the container space for the used repositories. This is + done by prefixing host root mount bind ('/installroot') to the path. It ensures + that the files will be accessible from the container and thus proper functionality + of the local repository. + """ + + name = 'adjust_local_repos' + consumes = (TargetOSInstallationImage, + TargetUserSpaceInfo, + TMPTargetRepositoriesFacts, + UsedTargetRepositories) + produces = () + tags = (IPUWorkflowTag, TargetTransactionChecksPhaseTag) + + def process(self): + target_userspace_info = next(self.consume(TargetUserSpaceInfo), None) + used_target_repos = next(self.consume(UsedTargetRepositories), None) + target_repos_facts = next(self.consume(TMPTargetRepositoriesFacts), None) + target_iso = next(self.consume(TargetOSInstallationImage), None) + + if not all([target_userspace_info, used_target_repos, target_repos_facts]): + api.current_logger().error("Missing required information to proceed!") + return + + target_repos_facts = target_repos_facts.repositories + iso_repoids = set(repo.repoid for repo in target_iso.repositories) if target_iso else set() + used_target_repoids = set(repo.repoid for repo in used_target_repos.repos) + + with mounting.NspawnActions(base_dir=target_userspace_info.path) as context: + adjustlocalrepos.process(context, target_repos_facts, iso_repoids, used_target_repoids) diff --git a/repos/system_upgrade/common/actors/adjustlocalrepos/libraries/adjustlocalrepos.py b/repos/system_upgrade/common/actors/adjustlocalrepos/libraries/adjustlocalrepos.py new file mode 100644 index 00000000..55a0d075 --- /dev/null +++ b/repos/system_upgrade/common/actors/adjustlocalrepos/libraries/adjustlocalrepos.py @@ -0,0 +1,100 @@ +import os + +from leapp.libraries.stdlib import api + +HOST_ROOT_MOUNT_BIND_PATH = '/installroot' +LOCAL_FILE_URL_PREFIX = 'file://' + + +def _adjust_local_file_url(repo_file_line): + """ + Adjusts a local file url to the target user-space container in a provided + repo file line by prefixing host root mount bind '/installroot' to it + when needed. + + :param str repo_file_line: a line from a repo file + :returns str: adjusted line or the provided line if no changes are needed + """ + adjust_fields = ['baseurl', 'mirrorlist'] + + if LOCAL_FILE_URL_PREFIX in repo_file_line and not repo_file_line.startswith('#'): + entry_field, entry_value = repo_file_line.strip().split('=', 1) + if not any(entry_field.startswith(field) for field in adjust_fields): + return repo_file_line + + entry_value = entry_value.strip('\'\"') + path = entry_value[len(LOCAL_FILE_URL_PREFIX):] + new_entry_value = LOCAL_FILE_URL_PREFIX + os.path.join(HOST_ROOT_MOUNT_BIND_PATH, path.lstrip('/')) + new_repo_file_line = entry_field + '=' + new_entry_value + return new_repo_file_line + return repo_file_line + + +def _extract_repos_from_repofile(context, repo_file): + """ + Generator function that extracts repositories from a repo file in the given context + and yields them as list of lines that belong to the repository. + + :param context: target user-space context + :param str repo_file: path to repository file (inside the provided context) + """ + with context.open(repo_file, 'r') as rf: + repo_file_lines = rf.readlines() + + # Detect repo and remove lines before first repoid + repo_found = False + for idx, line in enumerate(repo_file_lines): + if line.startswith('['): + repo_file_lines = repo_file_lines[idx:] + repo_found = True + break + + if not repo_found: + return + + current_repo = [] + for line in repo_file_lines: + line = line.strip() + + if line.startswith('[') and current_repo: + yield current_repo + current_repo = [] + + current_repo.append(line) + yield current_repo + + +def _adjust_local_repos_to_container(context, repo_file, local_repoids): + new_repo_file = [] + for repo in _extract_repos_from_repofile(context, repo_file): + repoid = repo[0].strip('[]') + adjusted_repo = repo + if repoid in local_repoids: + adjusted_repo = [_adjust_local_file_url(line) for line in repo] + new_repo_file.append(adjusted_repo) + + # Combine the repo file contents into a string and write it back to the file + new_repo_file = ['\n'.join(repo) for repo in new_repo_file] + new_repo_file = '\n'.join(new_repo_file) + with context.open(repo_file, 'w') as rf: + rf.write(new_repo_file) + + +def process(context, target_repos_facts, iso_repoids, used_target_repoids): + for repo_file_facts in target_repos_facts: + repo_file_path = repo_file_facts.file + local_repoids = set() + for repo in repo_file_facts.data: + # Skip repositories that aren't used or are provided by ISO + if repo.repoid not in used_target_repoids or repo.repoid in iso_repoids: + continue + # Note repositories that contain local file url + if repo.baseurl and LOCAL_FILE_URL_PREFIX in repo.baseurl or \ + repo.mirrorlist and LOCAL_FILE_URL_PREFIX in repo.mirrorlist: + local_repoids.add(repo.repoid) + + if local_repoids: + api.current_logger().debug( + 'Adjusting following repos in the repo file - {}: {}'.format(repo_file_path, + ', '.join(local_repoids))) + _adjust_local_repos_to_container(context, repo_file_path, local_repoids) diff --git a/repos/system_upgrade/common/actors/adjustlocalrepos/tests/test_adjustlocalrepos.py b/repos/system_upgrade/common/actors/adjustlocalrepos/tests/test_adjustlocalrepos.py new file mode 100644 index 00000000..41cff200 --- /dev/null +++ b/repos/system_upgrade/common/actors/adjustlocalrepos/tests/test_adjustlocalrepos.py @@ -0,0 +1,151 @@ +import pytest + +from leapp.libraries.actor import adjustlocalrepos + +REPO_FILE_1_LOCAL_REPOIDS = ['myrepo1'] +REPO_FILE_1 = [['[myrepo1]', + 'name=mylocalrepo', + 'baseurl=file:///home/user/.local/myrepos/repo1' + ]] +REPO_FILE_1_ADJUSTED = [['[myrepo1]', + 'name=mylocalrepo', + 'baseurl=file:///installroot/home/user/.local/myrepos/repo1' + ]] + +REPO_FILE_2_LOCAL_REPOIDS = ['myrepo3'] +REPO_FILE_2 = [['[myrepo2]', + 'name=mynotlocalrepo', + 'baseurl=https://www.notlocal.com/packages' + ], + ['[myrepo3]', + 'name=mylocalrepo', + 'baseurl=file:///home/user/.local/myrepos/repo3', + 'mirrorlist=file:///home/user/.local/mymirrors/repo3.txt' + ]] +REPO_FILE_2_ADJUSTED = [['[myrepo2]', + 'name=mynotlocalrepo', + 'baseurl=https://www.notlocal.com/packages' + ], + ['[myrepo3]', + 'name=mylocalrepo', + 'baseurl=file:///installroot/home/user/.local/myrepos/repo3', + 'mirrorlist=file:///installroot/home/user/.local/mymirrors/repo3.txt' + ]] + +REPO_FILE_3_LOCAL_REPOIDS = ['myrepo4', 'myrepo5'] +REPO_FILE_3 = [['[myrepo4]', + 'name=myrepowithlocalgpgkey', + 'baseurl="file:///home/user/.local/myrepos/repo4"', + 'gpgkey=file:///home/user/.local/pki/gpgkey', + 'gpgcheck=1' + ], + ['[myrepo5]', + 'name=myrepowithcomment', + 'baseurl=file:///home/user/.local/myrepos/repo5', + '#baseurl=file:///home/user/.local/myotherrepos/repo5', + 'enabled=1', + 'exclude=sed']] +REPO_FILE_3_ADJUSTED = [['[myrepo4]', + 'name=myrepowithlocalgpgkey', + 'baseurl=file:///installroot/home/user/.local/myrepos/repo4', + 'gpgkey=file:///home/user/.local/pki/gpgkey', + 'gpgcheck=1' + ], + ['[myrepo5]', + 'name=myrepowithcomment', + 'baseurl=file:///installroot/home/user/.local/myrepos/repo5', + '#baseurl=file:///home/user/.local/myotherrepos/repo5', + 'enabled=1', + 'exclude=sed']] +REPO_FILE_EMPTY = [] + + +@pytest.mark.parametrize('repo_file_line, expected_adjusted_repo_file_line', + [('baseurl=file:///home/user/.local/repositories/repository', + 'baseurl=file:///installroot/home/user/.local/repositories/repository'), + ('baseurl="file:///home/user/my-repo"', + 'baseurl=file:///installroot/home/user/my-repo'), + ('baseurl=https://notlocal.com/packages', + 'baseurl=https://notlocal.com/packages'), + ('mirrorlist=file:///some_mirror_list.txt', + 'mirrorlist=file:///installroot/some_mirror_list.txt'), + ('gpgkey=file:///etc/pki/some.key', + 'gpgkey=file:///etc/pki/some.key'), + ('#baseurl=file:///home/user/my-repo', + '#baseurl=file:///home/user/my-repo'), + ('', ''), + ('[repoid]', '[repoid]')]) +def test_adjust_local_file_url(repo_file_line, expected_adjusted_repo_file_line): + adjusted_repo_file_line = adjustlocalrepos._adjust_local_file_url(repo_file_line) + if 'file://' not in repo_file_line: + assert adjusted_repo_file_line == repo_file_line + return + assert adjusted_repo_file_line == expected_adjusted_repo_file_line + + +class MockedFileDescriptor(object): + + def __init__(self, repo_file, expected_new_repo_file): + self.repo_file = repo_file + self.expected_new_repo_file = expected_new_repo_file + + @staticmethod + def _create_repo_file_lines(repo_file): + repo_file_lines = [] + for repo in repo_file: + repo = [line+'\n' for line in repo] + repo_file_lines += repo + return repo_file_lines + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + return + + def readlines(self): + return self._create_repo_file_lines(self.repo_file) + + def write(self, new_contents): + assert self.expected_new_repo_file + repo_file_lines = self._create_repo_file_lines(self.expected_new_repo_file) + expected_repo_file_contents = ''.join(repo_file_lines).rstrip('\n') + assert expected_repo_file_contents == new_contents + + +class MockedContext(object): + + def __init__(self, repo_contents, expected_repo_contents): + self.repo_contents = repo_contents + self.expected_repo_contents = expected_repo_contents + + def open(self, path, mode): + return MockedFileDescriptor(self.repo_contents, self.expected_repo_contents) + + +@pytest.mark.parametrize('repo_file, local_repoids, expected_repo_file', + [(REPO_FILE_1, REPO_FILE_1_LOCAL_REPOIDS, REPO_FILE_1_ADJUSTED), + (REPO_FILE_2, REPO_FILE_2_LOCAL_REPOIDS, REPO_FILE_2_ADJUSTED), + (REPO_FILE_3, REPO_FILE_3_LOCAL_REPOIDS, REPO_FILE_3_ADJUSTED)]) +def test_adjust_local_repos_to_container(repo_file, local_repoids, expected_repo_file): + # The checks for expected_repo_file comparison to a adjusted form of the + # repo_file can be found in the MockedFileDescriptor.write(). + context = MockedContext(repo_file, expected_repo_file) + adjustlocalrepos._adjust_local_repos_to_container(context, '', local_repoids) + + +@pytest.mark.parametrize('expected_repo_file, add_empty_lines', [(REPO_FILE_EMPTY, False), + (REPO_FILE_1, False), + (REPO_FILE_2, True)]) +def test_extract_repos_from_repofile(expected_repo_file, add_empty_lines): + repo_file = expected_repo_file[:] + if add_empty_lines: # add empty lines before the first repo + repo_file[0] = ['', ''] + repo_file[0] + + context = MockedContext(repo_file, None) + repo_gen = adjustlocalrepos._extract_repos_from_repofile(context, '') + + for repo in expected_repo_file: + assert repo == next(repo_gen, None) + + assert next(repo_gen, None) is None diff --git a/repos/system_upgrade/common/actors/localreposinhibit/actor.py b/repos/system_upgrade/common/actors/localreposinhibit/actor.py deleted file mode 100644 index 2bde7f15..00000000 --- a/repos/system_upgrade/common/actors/localreposinhibit/actor.py +++ /dev/null @@ -1,89 +0,0 @@ -from leapp import reporting -from leapp.actors import Actor -from leapp.models import TargetOSInstallationImage, TMPTargetRepositoriesFacts, UsedTargetRepositories -from leapp.reporting import Report -from leapp.tags import IPUWorkflowTag, TargetTransactionChecksPhaseTag -from leapp.utils.deprecation import suppress_deprecation - - -@suppress_deprecation(TMPTargetRepositoriesFacts) -class LocalReposInhibit(Actor): - """Inhibits the upgrade if local repositories were found.""" - - name = "local_repos_inhibit" - consumes = ( - UsedTargetRepositories, - TargetOSInstallationImage, - TMPTargetRepositoriesFacts, - ) - produces = (Report,) - tags = (IPUWorkflowTag, TargetTransactionChecksPhaseTag) - - def collect_target_repoids_with_local_url(self, used_target_repos, target_repos_facts, target_iso): - """Collects all repoids that have a local (file://) URL. - - UsedTargetRepositories doesn't contain baseurl attribute. So gathering - them from model TMPTargetRepositoriesFacts. - """ - used_target_repoids = set(repo.repoid for repo in used_target_repos.repos) - iso_repoids = set(iso_repo.repoid for iso_repo in target_iso.repositories) if target_iso else set() - - target_repofile_data = (repofile.data for repofile in target_repos_facts.repositories) - - local_repoids = [] - for repo_data in target_repofile_data: - for target_repo in repo_data: - # Check only in repositories that are used and are not provided by the upgrade ISO, if any - if target_repo.repoid not in used_target_repoids or target_repo.repoid in iso_repoids: - continue - - # Repo fields potentially containing local URLs have different importance, check based on their prio - url_field_to_check = target_repo.mirrorlist or target_repo.metalink or target_repo.baseurl or '' - - if url_field_to_check.startswith("file://"): - local_repoids.append(target_repo.repoid) - return local_repoids - - def process(self): - used_target_repos = next(self.consume(UsedTargetRepositories), None) - target_repos_facts = next(self.consume(TMPTargetRepositoriesFacts), None) - target_iso = next(self.consume(TargetOSInstallationImage), None) - - if not used_target_repos or not target_repos_facts: - return - - local_repoids = self.collect_target_repoids_with_local_url(used_target_repos, target_repos_facts, target_iso) - if local_repoids: - suffix, verb = ("y", "has") if len(local_repoids) == 1 else ("ies", "have") - local_repoids_str = ", ".join(local_repoids) - - warn_msg = ("The following local repositor{suffix} {verb} been found: {local_repoids} " - "(their baseurl starts with file:///). Currently leapp does not support this option.") - warn_msg = warn_msg.format(suffix=suffix, verb=verb, local_repoids=local_repoids_str) - self.log.warning(warn_msg) - - reporting.create_report( - [ - reporting.Title("Local repositor{suffix} detected".format(suffix=suffix)), - reporting.Summary(warn_msg), - reporting.Severity(reporting.Severity.HIGH), - reporting.Groups([reporting.Groups.REPOSITORY]), - reporting.Groups([reporting.Groups.INHIBITOR]), - reporting.Remediation( - hint=( - "By using Apache HTTP Server you can expose " - "your local repository via http. See the linked " - "article for details. " - ) - ), - reporting.ExternalLink( - title=( - "Customizing your Red Hat Enterprise Linux " - "in-place upgrade" - ), - url=( - "https://red.ht/ipu-customisation-repos-known-issues" - ), - ), - ] - ) diff --git a/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py b/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py deleted file mode 100644 index 64a79e80..00000000 --- a/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py +++ /dev/null @@ -1,81 +0,0 @@ -import pytest - -from leapp.models import ( - RepositoryData, - RepositoryFile, - TargetOSInstallationImage, - TMPTargetRepositoriesFacts, - UsedTargetRepositories, - UsedTargetRepository -) -from leapp.snactor.fixture import ActorContext - - -@pytest.mark.parametrize( - ("baseurl", "mirrorlist", "metalink", "exp_msgs_len"), - [ - ("file:///root/crb", None, None, 1), - ("http://localhost/crb", None, None, 0), - (None, "file:///root/crb", None, 1), - (None, "http://localhost/crb", None, 0), - (None, None, "file:///root/crb", 1), - (None, None, "http://localhost/crb", 0), - ("http://localhost/crb", "file:///root/crb", None, 1), - ("file:///root/crb", "http://localhost/crb", None, 0), - ("http://localhost/crb", None, "file:///root/crb", 1), - ("file:///root/crb", None, "http://localhost/crb", 0), - ], -) -def test_unit_localreposinhibit(current_actor_context, baseurl, mirrorlist, metalink, exp_msgs_len): - """Ensure the Report is generated when local path is used as a baseurl. - - :type current_actor_context: ActorContext - """ - with pytest.deprecated_call(): - current_actor_context.feed( - TMPTargetRepositoriesFacts( - repositories=[ - RepositoryFile( - file="the/path/to/some/file", - data=[ - RepositoryData( - name="BASEOS", - baseurl=( - "http://example.com/path/to/repo/BaseOS/x86_64/os/" - ), - repoid="BASEOS", - ), - RepositoryData( - name="APPSTREAM", - baseurl=( - "http://example.com/path/to/repo/AppStream/x86_64/os/" - ), - repoid="APPSTREAM", - ), - RepositoryData( - name="CRB", repoid="CRB", baseurl=baseurl, - mirrorlist=mirrorlist, metalink=metalink - ), - ], - ) - ] - ) - ) - current_actor_context.feed( - UsedTargetRepositories( - repos=[ - UsedTargetRepository(repoid="BASEOS"), - UsedTargetRepository(repoid="CRB"), - ] - ) - ) - current_actor_context.run() - assert len(current_actor_context.messages()) == exp_msgs_len - - -def test_upgrade_not_inhibited_if_iso_used(current_actor_context): - repofile = RepositoryFile(file="path/to/some/file", - data=[RepositoryData(name="BASEOS", baseurl="file:///path", repoid="BASEOS")]) - current_actor_context.feed(TMPTargetRepositoriesFacts(repositories=[repofile])) - current_actor_context.feed(UsedTargetRepositories(repos=[UsedTargetRepository(repoid="BASEOS")])) - current_actor_context.feed(TargetOSInstallationImage(path='', mountpoint='', repositories=[])) diff --git a/repos/system_upgrade/common/libraries/dnfplugin.py b/repos/system_upgrade/common/libraries/dnfplugin.py index ffde211f..26810e94 100644 --- a/repos/system_upgrade/common/libraries/dnfplugin.py +++ b/repos/system_upgrade/common/libraries/dnfplugin.py @@ -334,8 +334,9 @@ def install_initramdisk_requirements(packages, target_userspace_info, used_repos """ Performs the installation of packages into the initram disk """ - with _prepare_transaction(used_repos=used_repos, - target_userspace_info=target_userspace_info) as (context, target_repoids, _unused): + mount_binds = ['/:/installroot'] + with _prepare_transaction(used_repos=used_repos, target_userspace_info=target_userspace_info, + binds=mount_binds) as (context, target_repoids, _unused): if get_target_major_version() == '9': _rebuild_rpm_db(context) repos_opt = [['--enablerepo', repo] for repo in target_repoids] -- 2.41.0