From c87d299a2043cb41530d3a60bafc8080bf50e158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Tue, 19 Nov 2019 08:51:18 +0100 Subject: [PATCH] hybrid: Download remote files when getting platform MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When probing lookasides for platform definition, we need to make sure it works for repos specified as HTTP urls. Createrepo doesn't seem to automatically download the repodata, so we have to help it. JIRA: COMPOSE-3958 Signed-off-by: Lubomír Sedlář --- pungi/phases/gather/methods/method_hybrid.py | 12 ++++++---- pungi/util.py | 17 ++++++++++++++ tests/test_util.py | 24 ++++++++++++++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index aec4017a..0deedd77 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -27,7 +27,7 @@ from pungi import multilib_dnf from pungi.module_util import Modulemd from pungi.arch import get_valid_arches, tree_arch_to_yum_arch from pungi.phases.gather import _mk_pkg_map -from pungi.util import get_arch_variant_data, pkg_is_debug, temp_dir +from pungi.util import get_arch_variant_data, pkg_is_debug, temp_dir, as_local_file from pungi.wrappers import fus from pungi.wrappers.comps import CompsWrapper @@ -412,14 +412,18 @@ def iter_platforms_in_repo(url): """Find all platform streams that any module in give repo requires at runtime. Yields lists of stream names (possible empty). """ - repomd = cr.Repomd(os.path.join(url, "repodata/repomd.xml")) + repomd = os.path.join(url, "repodata/repomd.xml") + with as_local_file(repomd) as url_: + repomd = cr.Repomd(url_) for rec in repomd.records: if rec.type != "modules": continue # No with statement on Python 2.6 for GzipFile... - gzipped_file = gzip.GzipFile(os.path.join(url, rec.location_href), "r") + record_url = os.path.join(url, rec.location_href) + with as_local_file(record_url) as url_: + gzipped_file = gzip.GzipFile(url_, "rb") mod_index = Modulemd.ModuleIndex.new() - mod_index.update_from_string(gzipped_file.read(), False) + mod_index.update_from_string(gzipped_file.read().decode("utf-8"), False) gzipped_file.close() for module_name in mod_index.get_module_names(): module = mod_index.get_module(module_name) diff --git a/pungi/util.py b/pungi/util.py index 8bafec48..23b68050 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -936,3 +936,20 @@ def load_config(file_path, defaults={}): conf.load_from_file(file_path) return conf + + +@contextlib.contextmanager +def as_local_file(url): + """If URL points to a file over HTTP, the file will be downloaded locally + and a path to the local copy is yielded. For local files the original path + is returned. + """ + if url.startswith("http://") or url.startswith("https://"): + local_filename, _ = urllib.request.urlretrieve(url) + try: + yield local_filename + finally: + os.remove(local_filename) + else: + # Not a remote url, return unchanged. + yield url diff --git a/tests/test_util.py b/tests/test_util.py index 6a746156..46d8a588 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -873,3 +873,27 @@ class TestCopyAll(PungiTestCase): self.assertTrue(os.path.islink(os.path.join(self.dst, "symlink"))) self.assertEqual(os.readlink(os.path.join(self.dst, "symlink")), "broken") + + +@mock.patch("six.moves.urllib.request.urlretrieve") +class TestAsLocalFile(PungiTestCase): + def test_local_file(self, urlretrieve): + with util.as_local_file("/tmp/foo") as fn: + self.assertEqual(fn, "/tmp/foo") + self.assertEqual(urlretrieve.call_args_list, []) + + def test_http(self, urlretrieve): + url = "http://example.com/repodata/repomd.xml" + + def my_mock(url_): + self.assertEqual(url, url_) + self.filename = os.path.join(self.topdir, "my-file") + touch(self.filename) + return self.filename, {} + + urlretrieve.side_effect = my_mock + + with util.as_local_file(url) as fn: + self.assertEqual(fn, self.filename) + self.assertTrue(os.path.exists(self.filename)) + self.assertFalse(os.path.exists(self.filename))