From ade25f34b3b5d6b0186cac0186e9c65631c9ac92 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Thu, 24 Jan 2019 15:22:22 -0800 Subject: [PATCH] Remove duplicate repositories from the sources list In some cases when the host has, for whatever reason, multiple copies of the same repo listed the build may fail with an error about running out of space. So this commit removes duplicate entries after the host's repos have been loaded. It also adjusts some of the test repos to use different temporary repo names for the tests. Resolves: rhbz#1664128 --- src/pylorax/api/yumbase.py | 26 ++++++++++++++++++++++++++ tests/pylorax/repos/multiple.repo | 8 ++++---- tests/pylorax/repos/other.repo | 2 +- tests/pylorax/repos/single-dupe.repo | 11 +++++++++++ tests/pylorax/test_server.py | 21 ++++++++++++++------- 5 files changed, 56 insertions(+), 12 deletions(-) create mode 100644 tests/pylorax/repos/single-dupe.repo diff --git a/src/pylorax/api/yumbase.py b/src/pylorax/api/yumbase.py index 6c5b15db..f8c096b2 100644 --- a/src/pylorax/api/yumbase.py +++ b/src/pylorax/api/yumbase.py @@ -173,6 +173,32 @@ def get_base_object(conf): if any(map(lambda pattern: fnmatchcase(name, pattern), enabled_repos)): # pylint: disable=cell-var-from-loop yb.getReposFromConfigFile(repo_file) + # Remove any duplicate repo entries. These can cause problems with Anaconda, which will fail + # with space problems. + repos = list(r.id for r in yb.repos.listEnabled()) + seen = {"baseurl": [], "mirrorlist": [], "metalink": []} + for source_name in repos: + remove = False + repo = yb.repos.getRepo(source_name) + if repo.baseurl: + if repo.baseurl[0] in seen["baseurl"]: + log.info("Removing duplicate repo: %s baseurl=%s", source_name, repo.baseurl[0]) + remove = True + else: + seen["baseurl"].append(repo.baseurl[0]) + elif repo.mirrorlist: + if repo.mirrorlist in seen["mirrorlist"]: + log.info("Removing duplicate repo: %s mirrorlist=%s", source_name, repo.mirrorlist) + remove = True + else: + seen["mirrorlist"].append(repo.mirrorlist) + + if remove: + yb.repos.delete(source_name) + + # delete doesn't remove it from the cache used by listEnabled so we have to force it + yb.repos._cache_enabled_repos = None + # Update the metadata from the enabled repos to speed up later operations log.info("Updating yum repository metadata") update_metadata(yb) diff --git a/tests/pylorax/repos/multiple.repo b/tests/pylorax/repos/multiple.repo index d68d3e2a..29c24d5e 100644 --- a/tests/pylorax/repos/multiple.repo +++ b/tests/pylorax/repos/multiple.repo @@ -1,7 +1,7 @@ [lorax-1] name=Lorax test repo 1 failovermethod=priority -baseurl=file:///tmp/lorax-empty-repo/ +baseurl=file:///tmp/lorax-empty-repo-1/ enabled=1 metadata_expire=7d repo_gpgcheck=0 @@ -13,7 +13,7 @@ skip_if_unavailable=False [lorax-2] name=Lorax test repo 2 failovermethod=priority -baseurl=file:///tmp/lorax-empty-repo/ +baseurl=file:///tmp/lorax-empty-repo-2/ enabled=1 metadata_expire=7d repo_gpgcheck=0 @@ -25,7 +25,7 @@ skip_if_unavailable=False [lorax-3] name=Lorax test repo 3 failovermethod=priority -baseurl=file:///tmp/lorax-empty-repo/ +baseurl=file:///tmp/lorax-empty-repo-3/ enabled=1 metadata_expire=7d repo_gpgcheck=0 @@ -37,7 +37,7 @@ skip_if_unavailable=False [lorax-4] name=Lorax test repo 4 failovermethod=priority -baseurl=file:///tmp/lorax-empty-repo/ +baseurl=file:///tmp/lorax-empty-repo-4/ enabled=1 metadata_expire=7d repo_gpgcheck=0 diff --git a/tests/pylorax/repos/other.repo b/tests/pylorax/repos/other.repo index 0a3da20f..e23d8722 100644 --- a/tests/pylorax/repos/other.repo +++ b/tests/pylorax/repos/other.repo @@ -1,7 +1,7 @@ [other-repo] name=Other repo failovermethod=priority -baseurl=file:///tmp/lorax-empty-repo/ +baseurl=file:///tmp/lorax-other-empty-repo/ enabled=1 metadata_expire=7d repo_gpgcheck=0 diff --git a/tests/pylorax/repos/single-dupe.repo b/tests/pylorax/repos/single-dupe.repo new file mode 100644 index 00000000..551b0861 --- /dev/null +++ b/tests/pylorax/repos/single-dupe.repo @@ -0,0 +1,11 @@ +[single-repo-duplicate] +name=single-repo-duplicate +failovermethod=priority +baseurl=file:///tmp/lorax-empty-repo/ +enabled=1 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False diff --git a/tests/pylorax/test_server.py b/tests/pylorax/test_server.py index b199afb8..8288a645 100644 --- a/tests/pylorax/test_server.py +++ b/tests/pylorax/test_server.py @@ -84,12 +84,14 @@ class ServerTestCase(unittest.TestCase): shutil.copy2(f, yum_repo_dir) # yum repo baseurl has to point to an absolute directory, so we use /tmp/lorax-empty-repo/ in the files - # and create an empty repository - os.makedirs("/tmp/lorax-empty-repo/") - rc = os.system("createrepo_c /tmp/lorax-empty-repo/") - if rc != 0: - shutil.rmtree("/tmp/lorax-empty-repo/") - raise RuntimeError("Problem running createrepo_c, is it installed") + # and create an empty repository. We now remove duplicate repo entries so we need a number of them. + for d in ["/tmp/lorax-empty-repo/", "/tmp/lorax-other-empty-repo/", "/tmp/lorax-empty-repo-1/", + "/tmp/lorax-empty-repo-2/", "/tmp/lorax-empty-repo-3/", "/tmp/lorax-empty-repo-4/"]: + os.makedirs(d) + rc = os.system("createrepo_c %s" % d) + if rc != 0: + shutil.rmtree(d) + raise RuntimeError("Problem running createrepo_c, is it installed") server.config["YUMLOCK"] = YumLock(server.config["COMPOSER_CFG"]) @@ -118,7 +120,9 @@ class ServerTestCase(unittest.TestCase): @classmethod def tearDownClass(self): shutil.rmtree(server.config["REPO_DIR"]) - shutil.rmtree("/tmp/lorax-empty-repo/") + # Clean up the empty repos + for repo_dir in glob("/tmp/lorax-*empty-repo*"): + shutil.rmtree(repo_dir) def test_01_status(self): """Test the /api/status route""" @@ -559,6 +563,9 @@ class ServerTestCase(unittest.TestCase): for r in ["lorax-1", "lorax-2", "lorax-3", "lorax-4", "other-repo", "single-repo"]: self.assertTrue(r in data["sources"] ) + # Make sure the duplicate repo is not listed + self.assertFalse("single-repo-duplicate" in data["sources"]) + def test_projects_source_00_info(self): """Test /api/v0/projects/source/info""" resp = self.server.get("/api/v0/projects/source/info/single-repo")