Remove duplicate repositories from the sources list

In some cases when the host has, for whatever reason, multiple copies of
the same repo listed the build may fail with an error about running out
of space.

So this commit removes duplicate entries after the host's repos have been
loaded. It also adjusts some of the test repos to use different
temporary repo names for the tests.

(cherry picked from commit 98482e444d)
This commit is contained in:
Brian C. Lane 2019-01-30 08:30:40 -08:00
parent 80e35d8d0e
commit 47a909209f
5 changed files with 64 additions and 9 deletions

View File

@ -136,6 +136,38 @@ def get_base_object(conf):
shutil.copy2(repo_file, repodir)
dbo.read_all_repos()
# Remove any duplicate repo entries. These can cause problems with Anaconda, which will fail
# with space problems.
repos = sorted(list(r.id for r in dbo.repos.iter_enabled()))
seen = {"baseurl": [], "mirrorlist": [], "metalink": []}
for source_name in repos:
remove = False
repo = dbo.repos.get(source_name, None)
if repo is None:
log.warning("repo %s vanished while removing duplicates", source_name)
continue
if repo.baseurl:
if repo.baseurl[0] in seen["baseurl"]:
log.info("Removing duplicate repo: %s baseurl=%s", source_name, repo.baseurl[0])
remove = True
else:
seen["baseurl"].append(repo.baseurl[0])
elif repo.mirrorlist:
if repo.mirrorlist in seen["mirrorlist"]:
log.info("Removing duplicate repo: %s mirrorlist=%s", source_name, repo.mirrorlist)
remove = True
else:
seen["mirrorlist"].append(repo.mirrorlist)
elif repo.metalink:
if repo.metalink in seen["metalink"]:
log.info("Removing duplicate repo: %s metalink=%s", source_name, repo.metalink)
remove = True
else:
seen["metalink"].append(repo.metalink)
if remove:
del dbo.repos[source_name]
# Update the metadata from the enabled repos to speed up later operations
log.info("Updating repository metadata")
try:

View File

@ -1,7 +1,7 @@
[lorax-1]
name=Lorax test repo 1
failovermethod=priority
baseurl=file:///tmp/lorax-empty-repo/
baseurl=file:///tmp/lorax-empty-repo-1/
enabled=1
metadata_expire=7d
repo_gpgcheck=0
@ -13,7 +13,7 @@ skip_if_unavailable=False
[lorax-2]
name=Lorax test repo 2
failovermethod=priority
baseurl=file:///tmp/lorax-empty-repo/
baseurl=file:///tmp/lorax-empty-repo-2/
enabled=1
metadata_expire=7d
repo_gpgcheck=0
@ -25,7 +25,7 @@ skip_if_unavailable=False
[lorax-3]
name=Lorax test repo 3
failovermethod=priority
baseurl=file:///tmp/lorax-empty-repo/
baseurl=file:///tmp/lorax-empty-repo-3/
enabled=1
metadata_expire=7d
repo_gpgcheck=0
@ -37,7 +37,7 @@ skip_if_unavailable=False
[lorax-4]
name=Lorax test repo 4
failovermethod=priority
baseurl=file:///tmp/lorax-empty-repo/
baseurl=file:///tmp/lorax-empty-repo-4/
enabled=1
metadata_expire=7d
repo_gpgcheck=0

View File

@ -1,7 +1,7 @@
[other-repo]
name=Other repo
failovermethod=priority
baseurl=file:///tmp/lorax-empty-repo/
baseurl=file:///tmp/lorax-other-empty-repo/
enabled=1
metadata_expire=7d
repo_gpgcheck=0

View File

@ -0,0 +1,11 @@
[single-repo-duplicate]
name=single-repo-duplicate
failovermethod=priority
baseurl=file:///tmp/lorax-empty-repo/
enabled=1
metadata_expire=7d
repo_gpgcheck=0
type=rpm
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
skip_if_unavailable=False

View File

@ -87,9 +87,14 @@ class ServerTestCase(unittest.TestCase):
self.rawhide = True
# dnf repo baseurl has to point to an absolute directory, so we use /tmp/lorax-empty-repo/ in the files
# and create an empty repository
os.makedirs("/tmp/lorax-empty-repo/")
os.system("createrepo_c /tmp/lorax-empty-repo/")
# and create an empty repository. We now remove duplicate repo entries so we need a number of them.
for d in ["/tmp/lorax-empty-repo/", "/tmp/lorax-other-empty-repo/", "/tmp/lorax-empty-repo-1/",
"/tmp/lorax-empty-repo-2/", "/tmp/lorax-empty-repo-3/", "/tmp/lorax-empty-repo-4/"]:
os.makedirs(d)
rc = os.system("createrepo_c %s" % d)
if rc != 0:
shutil.rmtree(d)
raise RuntimeError("Problem running createrepo_c, is it installed")
server.config["DNFLOCK"] = DNFLock(server.config["COMPOSER_CFG"])
@ -118,7 +123,9 @@ class ServerTestCase(unittest.TestCase):
@classmethod
def tearDownClass(self):
shutil.rmtree(server.config["REPO_DIR"])
shutil.rmtree("/tmp/lorax-empty-repo/")
# Clean up the empty repos
for repo_dir in glob("/tmp/lorax-*empty-repo*"):
shutil.rmtree(repo_dir)
def test_01_status(self):
"""Test the /api/status route"""
@ -560,15 +567,20 @@ class ServerTestCase(unittest.TestCase):
resp = self.server.get("/api/v0/projects/source/list")
data = json.loads(resp.data)
self.assertNotEqual(data, None)
print(data["sources"])
# Make sure it lists some common sources
for r in ["lorax-1", "lorax-2", "lorax-3", "lorax-4", "other-repo", "single-repo"]:
self.assertTrue(r in data["sources"] )
# Make sure the duplicate repo is not listed
self.assertFalse("single-repo-duplicate" in data["sources"])
def test_projects_source_00_info(self):
"""Test /api/v0/projects/source/info"""
resp = self.server.get("/api/v0/projects/source/info/single-repo")
data = json.loads(resp.data)
self.assertNotEqual(data, None)
print(data["sources"])
sources = data["sources"]
self.assertTrue("single-repo" in sources)