leapp-repository/0031-add-the-posibility-to-upgrade-with-a-local-repositor.patch

544 lines
24 KiB
Diff
Raw Normal View History

From e9f899c27688007d2e87144ccfd038b8b0a655d1 Mon Sep 17 00:00:00 2001
From: PeterMocary <petermocary@gmail.com>
Date: Wed, 12 Jul 2023 22:24:48 +0200
Subject: [PATCH 31/38] add the posibility to upgrade with a local repository
Upgrade with a local repository required to host the repository locally
for it to be visible from target user-space container during the
upgrade. The added actor ensures that the local repository
will be visible from the container by adjusting the path to it simply by
prefixing a host root mount bind '/installroot' to it. The
local_repos_inhibit actor is no longer needed, thus was removed.
---
.../common/actors/adjustlocalrepos/actor.py | 48 ++++++
.../libraries/adjustlocalrepos.py | 100 ++++++++++++
.../tests/test_adjustlocalrepos.py | 151 ++++++++++++++++++
.../common/actors/localreposinhibit/actor.py | 89 -----------
.../tests/test_unit_localreposinhibit.py | 81 ----------
.../common/libraries/dnfplugin.py | 5 +-
6 files changed, 302 insertions(+), 172 deletions(-)
create mode 100644 repos/system_upgrade/common/actors/adjustlocalrepos/actor.py
create mode 100644 repos/system_upgrade/common/actors/adjustlocalrepos/libraries/adjustlocalrepos.py
create mode 100644 repos/system_upgrade/common/actors/adjustlocalrepos/tests/test_adjustlocalrepos.py
delete mode 100644 repos/system_upgrade/common/actors/localreposinhibit/actor.py
delete mode 100644 repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py
diff --git a/repos/system_upgrade/common/actors/adjustlocalrepos/actor.py b/repos/system_upgrade/common/actors/adjustlocalrepos/actor.py
new file mode 100644
index 00000000..064e7f3e
--- /dev/null
+++ b/repos/system_upgrade/common/actors/adjustlocalrepos/actor.py
@@ -0,0 +1,48 @@
+from leapp.actors import Actor
+from leapp.libraries.actor import adjustlocalrepos
+from leapp.libraries.common import mounting
+from leapp.libraries.stdlib import api
+from leapp.models import (
+ TargetOSInstallationImage,
+ TargetUserSpaceInfo,
+ TMPTargetRepositoriesFacts,
+ UsedTargetRepositories
+)
+from leapp.tags import IPUWorkflowTag, TargetTransactionChecksPhaseTag
+
+
+class AdjustLocalRepos(Actor):
+ """
+ Adjust local repositories to the target user-space container.
+
+ Changes the path of local file urls (starting with 'file://') for 'baseurl' and
+ 'mirrorlist' fields to the container space for the used repositories. This is
+ done by prefixing host root mount bind ('/installroot') to the path. It ensures
+ that the files will be accessible from the container and thus proper functionality
+ of the local repository.
+ """
+
+ name = 'adjust_local_repos'
+ consumes = (TargetOSInstallationImage,
+ TargetUserSpaceInfo,
+ TMPTargetRepositoriesFacts,
+ UsedTargetRepositories)
+ produces = ()
+ tags = (IPUWorkflowTag, TargetTransactionChecksPhaseTag)
+
+ def process(self):
+ target_userspace_info = next(self.consume(TargetUserSpaceInfo), None)
+ used_target_repos = next(self.consume(UsedTargetRepositories), None)
+ target_repos_facts = next(self.consume(TMPTargetRepositoriesFacts), None)
+ target_iso = next(self.consume(TargetOSInstallationImage), None)
+
+ if not all([target_userspace_info, used_target_repos, target_repos_facts]):
+ api.current_logger().error("Missing required information to proceed!")
+ return
+
+ target_repos_facts = target_repos_facts.repositories
+ iso_repoids = set(repo.repoid for repo in target_iso.repositories) if target_iso else set()
+ used_target_repoids = set(repo.repoid for repo in used_target_repos.repos)
+
+ with mounting.NspawnActions(base_dir=target_userspace_info.path) as context:
+ adjustlocalrepos.process(context, target_repos_facts, iso_repoids, used_target_repoids)
diff --git a/repos/system_upgrade/common/actors/adjustlocalrepos/libraries/adjustlocalrepos.py b/repos/system_upgrade/common/actors/adjustlocalrepos/libraries/adjustlocalrepos.py
new file mode 100644
index 00000000..55a0d075
--- /dev/null
+++ b/repos/system_upgrade/common/actors/adjustlocalrepos/libraries/adjustlocalrepos.py
@@ -0,0 +1,100 @@
+import os
+
+from leapp.libraries.stdlib import api
+
+HOST_ROOT_MOUNT_BIND_PATH = '/installroot'
+LOCAL_FILE_URL_PREFIX = 'file://'
+
+
+def _adjust_local_file_url(repo_file_line):
+ """
+ Adjusts a local file url to the target user-space container in a provided
+ repo file line by prefixing host root mount bind '/installroot' to it
+ when needed.
+
+ :param str repo_file_line: a line from a repo file
+ :returns str: adjusted line or the provided line if no changes are needed
+ """
+ adjust_fields = ['baseurl', 'mirrorlist']
+
+ if LOCAL_FILE_URL_PREFIX in repo_file_line and not repo_file_line.startswith('#'):
+ entry_field, entry_value = repo_file_line.strip().split('=', 1)
+ if not any(entry_field.startswith(field) for field in adjust_fields):
+ return repo_file_line
+
+ entry_value = entry_value.strip('\'\"')
+ path = entry_value[len(LOCAL_FILE_URL_PREFIX):]
+ new_entry_value = LOCAL_FILE_URL_PREFIX + os.path.join(HOST_ROOT_MOUNT_BIND_PATH, path.lstrip('/'))
+ new_repo_file_line = entry_field + '=' + new_entry_value
+ return new_repo_file_line
+ return repo_file_line
+
+
+def _extract_repos_from_repofile(context, repo_file):
+ """
+ Generator function that extracts repositories from a repo file in the given context
+ and yields them as list of lines that belong to the repository.
+
+ :param context: target user-space context
+ :param str repo_file: path to repository file (inside the provided context)
+ """
+ with context.open(repo_file, 'r') as rf:
+ repo_file_lines = rf.readlines()
+
+ # Detect repo and remove lines before first repoid
+ repo_found = False
+ for idx, line in enumerate(repo_file_lines):
+ if line.startswith('['):
+ repo_file_lines = repo_file_lines[idx:]
+ repo_found = True
+ break
+
+ if not repo_found:
+ return
+
+ current_repo = []
+ for line in repo_file_lines:
+ line = line.strip()
+
+ if line.startswith('[') and current_repo:
+ yield current_repo
+ current_repo = []
+
+ current_repo.append(line)
+ yield current_repo
+
+
+def _adjust_local_repos_to_container(context, repo_file, local_repoids):
+ new_repo_file = []
+ for repo in _extract_repos_from_repofile(context, repo_file):
+ repoid = repo[0].strip('[]')
+ adjusted_repo = repo
+ if repoid in local_repoids:
+ adjusted_repo = [_adjust_local_file_url(line) for line in repo]
+ new_repo_file.append(adjusted_repo)
+
+ # Combine the repo file contents into a string and write it back to the file
+ new_repo_file = ['\n'.join(repo) for repo in new_repo_file]
+ new_repo_file = '\n'.join(new_repo_file)
+ with context.open(repo_file, 'w') as rf:
+ rf.write(new_repo_file)
+
+
+def process(context, target_repos_facts, iso_repoids, used_target_repoids):
+ for repo_file_facts in target_repos_facts:
+ repo_file_path = repo_file_facts.file
+ local_repoids = set()
+ for repo in repo_file_facts.data:
+ # Skip repositories that aren't used or are provided by ISO
+ if repo.repoid not in used_target_repoids or repo.repoid in iso_repoids:
+ continue
+ # Note repositories that contain local file url
+ if repo.baseurl and LOCAL_FILE_URL_PREFIX in repo.baseurl or \
+ repo.mirrorlist and LOCAL_FILE_URL_PREFIX in repo.mirrorlist:
+ local_repoids.add(repo.repoid)
+
+ if local_repoids:
+ api.current_logger().debug(
+ 'Adjusting following repos in the repo file - {}: {}'.format(repo_file_path,
+ ', '.join(local_repoids)))
+ _adjust_local_repos_to_container(context, repo_file_path, local_repoids)
diff --git a/repos/system_upgrade/common/actors/adjustlocalrepos/tests/test_adjustlocalrepos.py b/repos/system_upgrade/common/actors/adjustlocalrepos/tests/test_adjustlocalrepos.py
new file mode 100644
index 00000000..41cff200
--- /dev/null
+++ b/repos/system_upgrade/common/actors/adjustlocalrepos/tests/test_adjustlocalrepos.py
@@ -0,0 +1,151 @@
+import pytest
+
+from leapp.libraries.actor import adjustlocalrepos
+
+REPO_FILE_1_LOCAL_REPOIDS = ['myrepo1']
+REPO_FILE_1 = [['[myrepo1]',
+ 'name=mylocalrepo',
+ 'baseurl=file:///home/user/.local/myrepos/repo1'
+ ]]
+REPO_FILE_1_ADJUSTED = [['[myrepo1]',
+ 'name=mylocalrepo',
+ 'baseurl=file:///installroot/home/user/.local/myrepos/repo1'
+ ]]
+
+REPO_FILE_2_LOCAL_REPOIDS = ['myrepo3']
+REPO_FILE_2 = [['[myrepo2]',
+ 'name=mynotlocalrepo',
+ 'baseurl=https://www.notlocal.com/packages'
+ ],
+ ['[myrepo3]',
+ 'name=mylocalrepo',
+ 'baseurl=file:///home/user/.local/myrepos/repo3',
+ 'mirrorlist=file:///home/user/.local/mymirrors/repo3.txt'
+ ]]
+REPO_FILE_2_ADJUSTED = [['[myrepo2]',
+ 'name=mynotlocalrepo',
+ 'baseurl=https://www.notlocal.com/packages'
+ ],
+ ['[myrepo3]',
+ 'name=mylocalrepo',
+ 'baseurl=file:///installroot/home/user/.local/myrepos/repo3',
+ 'mirrorlist=file:///installroot/home/user/.local/mymirrors/repo3.txt'
+ ]]
+
+REPO_FILE_3_LOCAL_REPOIDS = ['myrepo4', 'myrepo5']
+REPO_FILE_3 = [['[myrepo4]',
+ 'name=myrepowithlocalgpgkey',
+ 'baseurl="file:///home/user/.local/myrepos/repo4"',
+ 'gpgkey=file:///home/user/.local/pki/gpgkey',
+ 'gpgcheck=1'
+ ],
+ ['[myrepo5]',
+ 'name=myrepowithcomment',
+ 'baseurl=file:///home/user/.local/myrepos/repo5',
+ '#baseurl=file:///home/user/.local/myotherrepos/repo5',
+ 'enabled=1',
+ 'exclude=sed']]
+REPO_FILE_3_ADJUSTED = [['[myrepo4]',
+ 'name=myrepowithlocalgpgkey',
+ 'baseurl=file:///installroot/home/user/.local/myrepos/repo4',
+ 'gpgkey=file:///home/user/.local/pki/gpgkey',
+ 'gpgcheck=1'
+ ],
+ ['[myrepo5]',
+ 'name=myrepowithcomment',
+ 'baseurl=file:///installroot/home/user/.local/myrepos/repo5',
+ '#baseurl=file:///home/user/.local/myotherrepos/repo5',
+ 'enabled=1',
+ 'exclude=sed']]
+REPO_FILE_EMPTY = []
+
+
+@pytest.mark.parametrize('repo_file_line, expected_adjusted_repo_file_line',
+ [('baseurl=file:///home/user/.local/repositories/repository',
+ 'baseurl=file:///installroot/home/user/.local/repositories/repository'),
+ ('baseurl="file:///home/user/my-repo"',
+ 'baseurl=file:///installroot/home/user/my-repo'),
+ ('baseurl=https://notlocal.com/packages',
+ 'baseurl=https://notlocal.com/packages'),
+ ('mirrorlist=file:///some_mirror_list.txt',
+ 'mirrorlist=file:///installroot/some_mirror_list.txt'),
+ ('gpgkey=file:///etc/pki/some.key',
+ 'gpgkey=file:///etc/pki/some.key'),
+ ('#baseurl=file:///home/user/my-repo',
+ '#baseurl=file:///home/user/my-repo'),
+ ('', ''),
+ ('[repoid]', '[repoid]')])
+def test_adjust_local_file_url(repo_file_line, expected_adjusted_repo_file_line):
+ adjusted_repo_file_line = adjustlocalrepos._adjust_local_file_url(repo_file_line)
+ if 'file://' not in repo_file_line:
+ assert adjusted_repo_file_line == repo_file_line
+ return
+ assert adjusted_repo_file_line == expected_adjusted_repo_file_line
+
+
+class MockedFileDescriptor(object):
+
+ def __init__(self, repo_file, expected_new_repo_file):
+ self.repo_file = repo_file
+ self.expected_new_repo_file = expected_new_repo_file
+
+ @staticmethod
+ def _create_repo_file_lines(repo_file):
+ repo_file_lines = []
+ for repo in repo_file:
+ repo = [line+'\n' for line in repo]
+ repo_file_lines += repo
+ return repo_file_lines
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args, **kwargs):
+ return
+
+ def readlines(self):
+ return self._create_repo_file_lines(self.repo_file)
+
+ def write(self, new_contents):
+ assert self.expected_new_repo_file
+ repo_file_lines = self._create_repo_file_lines(self.expected_new_repo_file)
+ expected_repo_file_contents = ''.join(repo_file_lines).rstrip('\n')
+ assert expected_repo_file_contents == new_contents
+
+
+class MockedContext(object):
+
+ def __init__(self, repo_contents, expected_repo_contents):
+ self.repo_contents = repo_contents
+ self.expected_repo_contents = expected_repo_contents
+
+ def open(self, path, mode):
+ return MockedFileDescriptor(self.repo_contents, self.expected_repo_contents)
+
+
+@pytest.mark.parametrize('repo_file, local_repoids, expected_repo_file',
+ [(REPO_FILE_1, REPO_FILE_1_LOCAL_REPOIDS, REPO_FILE_1_ADJUSTED),
+ (REPO_FILE_2, REPO_FILE_2_LOCAL_REPOIDS, REPO_FILE_2_ADJUSTED),
+ (REPO_FILE_3, REPO_FILE_3_LOCAL_REPOIDS, REPO_FILE_3_ADJUSTED)])
+def test_adjust_local_repos_to_container(repo_file, local_repoids, expected_repo_file):
+ # The checks for expected_repo_file comparison to a adjusted form of the
+ # repo_file can be found in the MockedFileDescriptor.write().
+ context = MockedContext(repo_file, expected_repo_file)
+ adjustlocalrepos._adjust_local_repos_to_container(context, '<some_repo_file_path>', local_repoids)
+
+
+@pytest.mark.parametrize('expected_repo_file, add_empty_lines', [(REPO_FILE_EMPTY, False),
+ (REPO_FILE_1, False),
+ (REPO_FILE_2, True)])
+def test_extract_repos_from_repofile(expected_repo_file, add_empty_lines):
+ repo_file = expected_repo_file[:]
+ if add_empty_lines: # add empty lines before the first repo
+ repo_file[0] = ['', ''] + repo_file[0]
+
+ context = MockedContext(repo_file, None)
+ repo_gen = adjustlocalrepos._extract_repos_from_repofile(context, '<some_repo_file_path>')
+
+ for repo in expected_repo_file:
+ assert repo == next(repo_gen, None)
+
+ assert next(repo_gen, None) is None
diff --git a/repos/system_upgrade/common/actors/localreposinhibit/actor.py b/repos/system_upgrade/common/actors/localreposinhibit/actor.py
deleted file mode 100644
index 2bde7f15..00000000
--- a/repos/system_upgrade/common/actors/localreposinhibit/actor.py
+++ /dev/null
@@ -1,89 +0,0 @@
-from leapp import reporting
-from leapp.actors import Actor
-from leapp.models import TargetOSInstallationImage, TMPTargetRepositoriesFacts, UsedTargetRepositories
-from leapp.reporting import Report
-from leapp.tags import IPUWorkflowTag, TargetTransactionChecksPhaseTag
-from leapp.utils.deprecation import suppress_deprecation
-
-
-@suppress_deprecation(TMPTargetRepositoriesFacts)
-class LocalReposInhibit(Actor):
- """Inhibits the upgrade if local repositories were found."""
-
- name = "local_repos_inhibit"
- consumes = (
- UsedTargetRepositories,
- TargetOSInstallationImage,
- TMPTargetRepositoriesFacts,
- )
- produces = (Report,)
- tags = (IPUWorkflowTag, TargetTransactionChecksPhaseTag)
-
- def collect_target_repoids_with_local_url(self, used_target_repos, target_repos_facts, target_iso):
- """Collects all repoids that have a local (file://) URL.
-
- UsedTargetRepositories doesn't contain baseurl attribute. So gathering
- them from model TMPTargetRepositoriesFacts.
- """
- used_target_repoids = set(repo.repoid for repo in used_target_repos.repos)
- iso_repoids = set(iso_repo.repoid for iso_repo in target_iso.repositories) if target_iso else set()
-
- target_repofile_data = (repofile.data for repofile in target_repos_facts.repositories)
-
- local_repoids = []
- for repo_data in target_repofile_data:
- for target_repo in repo_data:
- # Check only in repositories that are used and are not provided by the upgrade ISO, if any
- if target_repo.repoid not in used_target_repoids or target_repo.repoid in iso_repoids:
- continue
-
- # Repo fields potentially containing local URLs have different importance, check based on their prio
- url_field_to_check = target_repo.mirrorlist or target_repo.metalink or target_repo.baseurl or ''
-
- if url_field_to_check.startswith("file://"):
- local_repoids.append(target_repo.repoid)
- return local_repoids
-
- def process(self):
- used_target_repos = next(self.consume(UsedTargetRepositories), None)
- target_repos_facts = next(self.consume(TMPTargetRepositoriesFacts), None)
- target_iso = next(self.consume(TargetOSInstallationImage), None)
-
- if not used_target_repos or not target_repos_facts:
- return
-
- local_repoids = self.collect_target_repoids_with_local_url(used_target_repos, target_repos_facts, target_iso)
- if local_repoids:
- suffix, verb = ("y", "has") if len(local_repoids) == 1 else ("ies", "have")
- local_repoids_str = ", ".join(local_repoids)
-
- warn_msg = ("The following local repositor{suffix} {verb} been found: {local_repoids} "
- "(their baseurl starts with file:///). Currently leapp does not support this option.")
- warn_msg = warn_msg.format(suffix=suffix, verb=verb, local_repoids=local_repoids_str)
- self.log.warning(warn_msg)
-
- reporting.create_report(
- [
- reporting.Title("Local repositor{suffix} detected".format(suffix=suffix)),
- reporting.Summary(warn_msg),
- reporting.Severity(reporting.Severity.HIGH),
- reporting.Groups([reporting.Groups.REPOSITORY]),
- reporting.Groups([reporting.Groups.INHIBITOR]),
- reporting.Remediation(
- hint=(
- "By using Apache HTTP Server you can expose "
- "your local repository via http. See the linked "
- "article for details. "
- )
- ),
- reporting.ExternalLink(
- title=(
- "Customizing your Red Hat Enterprise Linux "
- "in-place upgrade"
- ),
- url=(
- "https://red.ht/ipu-customisation-repos-known-issues"
- ),
- ),
- ]
- )
diff --git a/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py b/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py
deleted file mode 100644
index 64a79e80..00000000
--- a/repos/system_upgrade/common/actors/localreposinhibit/tests/test_unit_localreposinhibit.py
+++ /dev/null
@@ -1,81 +0,0 @@
-import pytest
-
-from leapp.models import (
- RepositoryData,
- RepositoryFile,
- TargetOSInstallationImage,
- TMPTargetRepositoriesFacts,
- UsedTargetRepositories,
- UsedTargetRepository
-)
-from leapp.snactor.fixture import ActorContext
-
-
-@pytest.mark.parametrize(
- ("baseurl", "mirrorlist", "metalink", "exp_msgs_len"),
- [
- ("file:///root/crb", None, None, 1),
- ("http://localhost/crb", None, None, 0),
- (None, "file:///root/crb", None, 1),
- (None, "http://localhost/crb", None, 0),
- (None, None, "file:///root/crb", 1),
- (None, None, "http://localhost/crb", 0),
- ("http://localhost/crb", "file:///root/crb", None, 1),
- ("file:///root/crb", "http://localhost/crb", None, 0),
- ("http://localhost/crb", None, "file:///root/crb", 1),
- ("file:///root/crb", None, "http://localhost/crb", 0),
- ],
-)
-def test_unit_localreposinhibit(current_actor_context, baseurl, mirrorlist, metalink, exp_msgs_len):
- """Ensure the Report is generated when local path is used as a baseurl.
-
- :type current_actor_context: ActorContext
- """
- with pytest.deprecated_call():
- current_actor_context.feed(
- TMPTargetRepositoriesFacts(
- repositories=[
- RepositoryFile(
- file="the/path/to/some/file",
- data=[
- RepositoryData(
- name="BASEOS",
- baseurl=(
- "http://example.com/path/to/repo/BaseOS/x86_64/os/"
- ),
- repoid="BASEOS",
- ),
- RepositoryData(
- name="APPSTREAM",
- baseurl=(
- "http://example.com/path/to/repo/AppStream/x86_64/os/"
- ),
- repoid="APPSTREAM",
- ),
- RepositoryData(
- name="CRB", repoid="CRB", baseurl=baseurl,
- mirrorlist=mirrorlist, metalink=metalink
- ),
- ],
- )
- ]
- )
- )
- current_actor_context.feed(
- UsedTargetRepositories(
- repos=[
- UsedTargetRepository(repoid="BASEOS"),
- UsedTargetRepository(repoid="CRB"),
- ]
- )
- )
- current_actor_context.run()
- assert len(current_actor_context.messages()) == exp_msgs_len
-
-
-def test_upgrade_not_inhibited_if_iso_used(current_actor_context):
- repofile = RepositoryFile(file="path/to/some/file",
- data=[RepositoryData(name="BASEOS", baseurl="file:///path", repoid="BASEOS")])
- current_actor_context.feed(TMPTargetRepositoriesFacts(repositories=[repofile]))
- current_actor_context.feed(UsedTargetRepositories(repos=[UsedTargetRepository(repoid="BASEOS")]))
- current_actor_context.feed(TargetOSInstallationImage(path='', mountpoint='', repositories=[]))
diff --git a/repos/system_upgrade/common/libraries/dnfplugin.py b/repos/system_upgrade/common/libraries/dnfplugin.py
index ffde211f..26810e94 100644
--- a/repos/system_upgrade/common/libraries/dnfplugin.py
+++ b/repos/system_upgrade/common/libraries/dnfplugin.py
@@ -334,8 +334,9 @@ def install_initramdisk_requirements(packages, target_userspace_info, used_repos
"""
Performs the installation of packages into the initram disk
"""
- with _prepare_transaction(used_repos=used_repos,
- target_userspace_info=target_userspace_info) as (context, target_repoids, _unused):
+ mount_binds = ['/:/installroot']
+ with _prepare_transaction(used_repos=used_repos, target_userspace_info=target_userspace_info,
+ binds=mount_binds) as (context, target_repoids, _unused):
if get_target_major_version() == '9':
_rebuild_rpm_db(context)
repos_opt = [['--enablerepo', repo] for repo in target_repoids]
--
2.41.0