Remove duplicate repositories from the sources list

In some cases when the host has, for whatever reason, multiple copies of
the same repo listed the build may fail with an error about running out
of space.

So this commit removes duplicate entries after the host's repos have been
loaded. It also adjusts some of the test repos to use different
temporary repo names for the tests.

Resolves: rhbz#1664128

src/pylorax/api/yumbase.py

@@ -173,6 +173,32 @@ def get_base_object(conf):
         if any(map(lambda pattern: fnmatchcase(name, pattern), enabled_repos)):     # pylint: disable=cell-var-from-loop
             yb.getReposFromConfigFile(repo_file)
 
+    # Remove any duplicate repo entries. These can cause problems with Anaconda, which will fail
+    # with space problems.
+    repos = list(r.id for r in yb.repos.listEnabled())
+    seen = {"baseurl": [], "mirrorlist": [], "metalink": []}
+    for source_name in repos:
+        remove = False
+        repo = yb.repos.getRepo(source_name)
+        if repo.baseurl:
+            if repo.baseurl[0] in seen["baseurl"]:
+                log.info("Removing duplicate repo: %s baseurl=%s", source_name, repo.baseurl[0])
+                remove = True
+            else:
+                seen["baseurl"].append(repo.baseurl[0])
+        elif repo.mirrorlist:
+            if repo.mirrorlist in seen["mirrorlist"]:
+                log.info("Removing duplicate repo: %s mirrorlist=%s", source_name, repo.mirrorlist)
+                remove = True
+            else:
+                seen["mirrorlist"].append(repo.mirrorlist)
+
+        if remove:
+            yb.repos.delete(source_name)
+
+    # delete doesn't remove it from the cache used by listEnabled so we have to force it
+    yb.repos._cache_enabled_repos = None
+
     # Update the metadata from the enabled repos to speed up later operations
     log.info("Updating yum repository metadata")
     update_metadata(yb)

tests/pylorax/repos/multiple.repo

@@ -1,7 +1,7 @@
 [lorax-1]
 name=Lorax test repo 1
 failovermethod=priority
-baseurl=file:///tmp/lorax-empty-repo/
+baseurl=file:///tmp/lorax-empty-repo-1/
 enabled=1
 metadata_expire=7d
 repo_gpgcheck=0
@@ -13,7 +13,7 @@ skip_if_unavailable=False
 [lorax-2]
 name=Lorax test repo 2
 failovermethod=priority
-baseurl=file:///tmp/lorax-empty-repo/
+baseurl=file:///tmp/lorax-empty-repo-2/
 enabled=1
 metadata_expire=7d
 repo_gpgcheck=0
@@ -25,7 +25,7 @@ skip_if_unavailable=False
 [lorax-3]
 name=Lorax test repo 3
 failovermethod=priority
-baseurl=file:///tmp/lorax-empty-repo/
+baseurl=file:///tmp/lorax-empty-repo-3/
 enabled=1
 metadata_expire=7d
 repo_gpgcheck=0
@@ -37,7 +37,7 @@ skip_if_unavailable=False
 [lorax-4]
 name=Lorax test repo 4
 failovermethod=priority
-baseurl=file:///tmp/lorax-empty-repo/
+baseurl=file:///tmp/lorax-empty-repo-4/
 enabled=1
 metadata_expire=7d
 repo_gpgcheck=0

tests/pylorax/repos/other.repo

@@ -1,7 +1,7 @@
 [other-repo]
 name=Other repo
 failovermethod=priority
-baseurl=file:///tmp/lorax-empty-repo/
+baseurl=file:///tmp/lorax-other-empty-repo/
 enabled=1
 metadata_expire=7d
 repo_gpgcheck=0

tests/pylorax/repos/single-dupe.repo

@@ -0,0 +1,11 @@
+[single-repo-duplicate]
+name=single-repo-duplicate
+failovermethod=priority
+baseurl=file:///tmp/lorax-empty-repo/
+enabled=1
+metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
+skip_if_unavailable=False

tests/pylorax/test_server.py

@@ -84,12 +84,14 @@ class ServerTestCase(unittest.TestCase):
             shutil.copy2(f, yum_repo_dir)
 
         # yum repo baseurl has to point to an absolute directory, so we use /tmp/lorax-empty-repo/ in the files
-        # and create an empty repository
+        # and create an empty repository. We now remove duplicate repo entries so we need a number of them.
-        os.makedirs("/tmp/lorax-empty-repo/")
+        for d in ["/tmp/lorax-empty-repo/", "/tmp/lorax-other-empty-repo/", "/tmp/lorax-empty-repo-1/",
-        rc = os.system("createrepo_c /tmp/lorax-empty-repo/")
+                         "/tmp/lorax-empty-repo-2/", "/tmp/lorax-empty-repo-3/", "/tmp/lorax-empty-repo-4/"]:
-        if rc != 0:
+            os.makedirs(d)
-            shutil.rmtree("/tmp/lorax-empty-repo/")
+            rc = os.system("createrepo_c %s" % d)
-            raise RuntimeError("Problem running createrepo_c, is it installed")
+            if rc != 0:
+                shutil.rmtree(d)
+                raise RuntimeError("Problem running createrepo_c, is it installed")
 
         server.config["YUMLOCK"] = YumLock(server.config["COMPOSER_CFG"])
 
@@ -118,7 +120,9 @@ class ServerTestCase(unittest.TestCase):
     @classmethod
     def tearDownClass(self):
         shutil.rmtree(server.config["REPO_DIR"])
-        shutil.rmtree("/tmp/lorax-empty-repo/")
+        # Clean up the empty repos
+        for repo_dir in glob("/tmp/lorax-*empty-repo*"):
+            shutil.rmtree(repo_dir)
 
     def test_01_status(self):
         """Test the /api/status route"""
@@ -559,6 +563,9 @@ class ServerTestCase(unittest.TestCase):
         for r in ["lorax-1", "lorax-2", "lorax-3", "lorax-4", "other-repo", "single-repo"]:
             self.assertTrue(r in data["sources"] )
 
+        # Make sure the duplicate repo is not listed
+        self.assertFalse("single-repo-duplicate" in data["sources"])
+
     def test_projects_source_00_info(self):
         """Test /api/v0/projects/source/info"""
         resp = self.server.get("/api/v0/projects/source/info/single-repo")