hybrid: Download remote files when getting platform

When probing lookasides for platform definition, we need to make sure it
works for repos specified as HTTP urls. Createrepo doesn't seem to
automatically download the repodata, so we have to help it.

JIRA: COMPOSE-3958
Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>

pungi/phases/gather/methods/method_hybrid.py

@@ -27,7 +27,7 @@ from pungi import multilib_dnf
 from pungi.module_util import Modulemd
 from pungi.arch import get_valid_arches, tree_arch_to_yum_arch
 from pungi.phases.gather import _mk_pkg_map
-from pungi.util import get_arch_variant_data, pkg_is_debug, temp_dir
+from pungi.util import get_arch_variant_data, pkg_is_debug, temp_dir, as_local_file
 from pungi.wrappers import fus
 from pungi.wrappers.comps import CompsWrapper
 
@@ -412,14 +412,18 @@ def iter_platforms_in_repo(url):
     """Find all platform streams that any module in give repo requires at runtime.
     Yields lists of stream names (possible empty).
     """
-    repomd = cr.Repomd(os.path.join(url, "repodata/repomd.xml"))
+    repomd = os.path.join(url, "repodata/repomd.xml")
+    with as_local_file(repomd) as url_:
+        repomd = cr.Repomd(url_)
     for rec in repomd.records:
         if rec.type != "modules":
             continue
         # No with statement on Python 2.6 for GzipFile...
-        gzipped_file = gzip.GzipFile(os.path.join(url, rec.location_href), "r")
+        record_url = os.path.join(url, rec.location_href)
+        with as_local_file(record_url) as url_:
+            gzipped_file = gzip.GzipFile(url_, "rb")
         mod_index = Modulemd.ModuleIndex.new()
-        mod_index.update_from_string(gzipped_file.read(), False)
+        mod_index.update_from_string(gzipped_file.read().decode("utf-8"), False)
         gzipped_file.close()
         for module_name in mod_index.get_module_names():
             module = mod_index.get_module(module_name)

pungi/util.py

@@ -936,3 +936,20 @@ def load_config(file_path, defaults={}):
         conf.load_from_file(file_path)
 
     return conf
+
+
+@contextlib.contextmanager
+def as_local_file(url):
+    """If URL points to a file over HTTP, the file will be downloaded locally
+    and a path to the local copy is yielded. For local files the original path
+    is returned.
+    """
+    if url.startswith("http://") or url.startswith("https://"):
+        local_filename, _ = urllib.request.urlretrieve(url)
+        try:
+            yield local_filename
+        finally:
+            os.remove(local_filename)
+    else:
+        # Not a remote url, return unchanged.
+        yield url

tests/test_util.py

@@ -873,3 +873,27 @@ class TestCopyAll(PungiTestCase):
 
         self.assertTrue(os.path.islink(os.path.join(self.dst, "symlink")))
         self.assertEqual(os.readlink(os.path.join(self.dst, "symlink")), "broken")
+
+
+@mock.patch("six.moves.urllib.request.urlretrieve")
+class TestAsLocalFile(PungiTestCase):
+    def test_local_file(self, urlretrieve):
+        with util.as_local_file("/tmp/foo") as fn:
+            self.assertEqual(fn, "/tmp/foo")
+        self.assertEqual(urlretrieve.call_args_list, [])
+
+    def test_http(self, urlretrieve):
+        url = "http://example.com/repodata/repomd.xml"
+
+        def my_mock(url_):
+            self.assertEqual(url, url_)
+            self.filename = os.path.join(self.topdir, "my-file")
+            touch(self.filename)
+            return self.filename, {}
+
+        urlretrieve.side_effect = my_mock
+
+        with util.as_local_file(url) as fn:
+            self.assertEqual(fn, self.filename)
+            self.assertTrue(os.path.exists(self.filename))
+        self.assertFalse(os.path.exists(self.filename))