diff --git a/pungi/phases/gather/methods/method_hybrid.py b/pungi/phases/gather/methods/method_hybrid.py index aec4017a..0deedd77 100644 --- a/pungi/phases/gather/methods/method_hybrid.py +++ b/pungi/phases/gather/methods/method_hybrid.py @@ -27,7 +27,7 @@ from pungi import multilib_dnf from pungi.module_util import Modulemd from pungi.arch import get_valid_arches, tree_arch_to_yum_arch from pungi.phases.gather import _mk_pkg_map -from pungi.util import get_arch_variant_data, pkg_is_debug, temp_dir +from pungi.util import get_arch_variant_data, pkg_is_debug, temp_dir, as_local_file from pungi.wrappers import fus from pungi.wrappers.comps import CompsWrapper @@ -412,14 +412,18 @@ def iter_platforms_in_repo(url): """Find all platform streams that any module in give repo requires at runtime. Yields lists of stream names (possible empty). """ - repomd = cr.Repomd(os.path.join(url, "repodata/repomd.xml")) + repomd = os.path.join(url, "repodata/repomd.xml") + with as_local_file(repomd) as url_: + repomd = cr.Repomd(url_) for rec in repomd.records: if rec.type != "modules": continue # No with statement on Python 2.6 for GzipFile... - gzipped_file = gzip.GzipFile(os.path.join(url, rec.location_href), "r") + record_url = os.path.join(url, rec.location_href) + with as_local_file(record_url) as url_: + gzipped_file = gzip.GzipFile(url_, "rb") mod_index = Modulemd.ModuleIndex.new() - mod_index.update_from_string(gzipped_file.read(), False) + mod_index.update_from_string(gzipped_file.read().decode("utf-8"), False) gzipped_file.close() for module_name in mod_index.get_module_names(): module = mod_index.get_module(module_name) diff --git a/pungi/util.py b/pungi/util.py index 8bafec48..23b68050 100644 --- a/pungi/util.py +++ b/pungi/util.py @@ -936,3 +936,20 @@ def load_config(file_path, defaults={}): conf.load_from_file(file_path) return conf + + +@contextlib.contextmanager +def as_local_file(url): + """If URL points to a file over HTTP, the file will be downloaded locally + and a path to the local copy is yielded. For local files the original path + is returned. + """ + if url.startswith("http://") or url.startswith("https://"): + local_filename, _ = urllib.request.urlretrieve(url) + try: + yield local_filename + finally: + os.remove(local_filename) + else: + # Not a remote url, return unchanged. + yield url diff --git a/tests/test_util.py b/tests/test_util.py index 6a746156..46d8a588 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -873,3 +873,27 @@ class TestCopyAll(PungiTestCase): self.assertTrue(os.path.islink(os.path.join(self.dst, "symlink"))) self.assertEqual(os.readlink(os.path.join(self.dst, "symlink")), "broken") + + +@mock.patch("six.moves.urllib.request.urlretrieve") +class TestAsLocalFile(PungiTestCase): + def test_local_file(self, urlretrieve): + with util.as_local_file("/tmp/foo") as fn: + self.assertEqual(fn, "/tmp/foo") + self.assertEqual(urlretrieve.call_args_list, []) + + def test_http(self, urlretrieve): + url = "http://example.com/repodata/repomd.xml" + + def my_mock(url_): + self.assertEqual(url, url_) + self.filename = os.path.join(self.topdir, "my-file") + touch(self.filename) + return self.filename, {} + + urlretrieve.side_effect = my_mock + + with util.as_local_file(url) as fn: + self.assertEqual(fn, self.filename) + self.assertTrue(os.path.exists(self.filename)) + self.assertFalse(os.path.exists(self.filename))