pungi/tests/test_pkgset_source_koji.py

# -*- coding: utf-8 -*-

import json

import os
import re
import six
from ddt import ddt, data, unpack
from typing import AnyStr, List, Set, Dict, Tuple

from tests.test_gather_method_hybrid import MockModule

try:
    import unittest2 as unittest
    from unittest2 import mock
except ImportError:
    import unittest
    from unittest import mock

from pungi.phases.pkgset.sources import source_koji, source_kojimock
from tests import helpers
from pungi.module_util import Modulemd
from pungi.util import read_single_module_stream_from_file

MMDS_DIR = os.path.join(helpers.FIXTURE_DIR, "mmds")
EVENT_INFO = {"id": 15681980, "ts": 1460956382.81936}
TAG_INFO = {
    "maven_support": False,
    "locked": False,
    "name": "f25",
    "extra": {"mock.package_manager": "dnf"},
    "perm": None,
    "id": 335,
    "arches": None,
    "maven_include_all": None,
    "perm_id": None,
}


class TestGetKojiEvent(helpers.PungiTestCase):
    def setUp(self):
        super(TestGetKojiEvent, self).setUp()
        self.compose = helpers.DummyCompose(self.topdir, {})

        self.event_file = self.topdir + "/work/global/koji-event"

    def test_use_preconfigured_event(self):
        koji_wrapper = mock.Mock()
        self.compose.koji_event = 123456

        koji_wrapper.koji_proxy.getEvent.return_value = EVENT_INFO

        event = source_koji.get_koji_event_info(self.compose, koji_wrapper)

        self.assertEqual(event, EVENT_INFO)
        six.assertCountEqual(
            self, koji_wrapper.mock_calls, [mock.call.koji_proxy.getEvent(123456)]
        )
        with open(self.event_file) as f:
            self.assertEqual(json.load(f), EVENT_INFO)

    def test_gets_last_event(self):
        self.compose.koji_event = None
        koji_wrapper = mock.Mock()

        koji_wrapper.koji_proxy.getLastEvent.return_value = EVENT_INFO

        event = source_koji.get_koji_event_info(self.compose, koji_wrapper)

        self.assertEqual(event, EVENT_INFO)
        six.assertCountEqual(
            self, koji_wrapper.mock_calls, [mock.call.koji_proxy.getLastEvent()]
        )
        with open(self.event_file) as f:
            self.assertEqual(json.load(f), EVENT_INFO)


class TestPopulateGlobalPkgset(helpers.PungiTestCase):
    def setUp(self):
        super(TestPopulateGlobalPkgset, self).setUp()
        self.compose = helpers.DummyCompose(
            self.topdir,
            {"pkgset_koji_tag": "f25", "sigkeys": ["foo", "bar"], "koji_cache": "/tmp"},
        )
        self.koji_wrapper = mock.Mock()
        self.pkgset_path = os.path.join(
            self.topdir, "work", "global", "pkgset_global.pickle"
        )
        self.koji_module_path = os.path.join(
            self.topdir, "work", "global", "koji-module-Server.yaml"
        )

    @mock.patch("pungi.phases.pkgset.sources.source_koji.MaterializedPackageSet.create")
    @mock.patch("pungi.phases.pkgset.pkgsets.KojiPackageSet")
    def test_populate(self, KojiPackageSet, materialize):
        materialize.side_effect = self.mock_materialize

        KojiPackageSet.return_value.reuse = None
        orig_pkgset = KojiPackageSet.return_value

        pkgsets = source_koji.populate_global_pkgset(
            self.compose, self.koji_wrapper, 123456
        )

        self.assertEqual(len(pkgsets), 1)
        self.assertIs(pkgsets[0], orig_pkgset)
        pkgsets[0].assert_has_calls(
            [mock.call.populate("f25", 123456, inherit=True, include_packages=set())],
        )

    def mock_materialize(self, compose, pkgset, prefix, mmd):
        self.assertEqual(prefix, "/prefix")
        self.assertEqual(compose, self.compose)
        return pkgset

    @mock.patch("pungi.phases.pkgset.sources.source_koji.MaterializedPackageSet.create")
    @mock.patch("pungi.phases.pkgset.pkgsets.KojiPackageSet")
    def test_populate_with_multiple_koji_tags(self, KojiPackageSet, materialize):
        self.compose = helpers.DummyCompose(
            self.topdir,
            {
                "pkgset_koji_tag": ["f25", "f25-extra"],
                "sigkeys": ["foo", "bar"],
                "koji_cache": "/tmp",
            },
        )

        materialize.side_effect = self.mock_materialize

        KojiPackageSet.return_value.reuse = None

        pkgsets = source_koji.populate_global_pkgset(
            self.compose, self.koji_wrapper, 123456
        )

        self.assertEqual(len(pkgsets), 2)
        init_calls = KojiPackageSet.call_args_list
        six.assertCountEqual(
            self, [call[0][0] for call in init_calls], ["f25", "f25-extra"]
        )
        six.assertCountEqual(
            self, [call[0][1] for call in init_calls], [self.koji_wrapper] * 2
        )
        six.assertCountEqual(
            self, [call[0][2] for call in init_calls], [["foo", "bar"]] * 2
        )

        pkgsets[0].assert_has_calls(
            [mock.call.populate("f25", 123456, inherit=True, include_packages=set())]
        )
        pkgsets[1].assert_has_calls(
            [
                mock.call.populate(
                    "f25-extra", 123456, inherit=True, include_packages=set()
                ),
            ]
        )

    @mock.patch("pungi.phases.pkgset.sources.source_koji.MaterializedPackageSet.create")
    @mock.patch("pungi.phases.pkgset.pkgsets.KojiPackageSet.populate")
    @mock.patch("pungi.phases.pkgset.pkgsets.KojiPackageSet.save_file_list")
    def test_populate_packages_to_gather(self, save_file_list, popuplate, materialize):
        self.compose = helpers.DummyCompose(
            self.topdir,
            {
                "gather_method": "nodeps",
                "pkgset_koji_tag": "f25",
                "koji_cache": "/tmp",
                "sigkeys": ["foo", "bar"],
                "additional_packages": [(".*", {"*": ["pkg", "foo.x86_64"]})],
            },
        )

        materialize.side_effect = self.mock_materialize

        pkgsets = source_koji.populate_global_pkgset(
            self.compose, self.koji_wrapper, 123456
        )
        self.assertEqual(len(pkgsets), 1)
        six.assertCountEqual(self, pkgsets[0].packages, ["pkg", "foo"])


class TestGetPackageSetFromKoji(helpers.PungiTestCase):
    def setUp(self):
        super(TestGetPackageSetFromKoji, self).setUp()
        self.compose = helpers.DummyCompose(
            self.topdir, {"pkgset_koji_tag": "f25", "koji_cache": "/tmp"}
        )
        self.compose.koji_event = None
        self.koji_wrapper = mock.Mock()
        self.koji_wrapper.koji_proxy.getLastEvent.return_value = EVENT_INFO
        self.koji_wrapper.koji_proxy.getTag.return_value = TAG_INFO

    @mock.patch("pungi.phases.pkgset.sources.source_koji.populate_global_pkgset")
    def test_get_package_sets(self, pgp):
        pkgsets = source_koji.get_pkgset_from_koji(self.compose, self.koji_wrapper)

        six.assertCountEqual(
            self, self.koji_wrapper.koji_proxy.mock_calls, [mock.call.getLastEvent()]
        )
        self.assertEqual(pkgsets, pgp.return_value)

        self.assertEqual(
            pgp.call_args_list,
            [mock.call(self.compose, self.koji_wrapper, EVENT_INFO)],
        )

    def test_get_koji_modules(self):
        mock_build_ids = [
            {"id": 1065873, "name": "testmodule2-master_dash-20180406051653.96c371af"}
        ]
        mock_extra = {
            "typeinfo": {
                "module": {
                    "content_koji_tag": "module-b62270b82443edde",
                    "modulemd_str": mock.Mock(),
                    "name": "testmodule2",
                    "stream": "master",
                    "version": "20180406051653",
                    "context": "96c371af",
                }
            }
        }
        mock_build_md = [
            {
                "id": 1065873,
                "epoch": None,
                "extra": mock_extra,
                "name": "testmodule2",
                "nvr": "testmodule2-master_dash-20180406051653.2e6f5e0a",
                "release": "20180406051653.2e6f5e0a",
                "state": 1,
                "version": "master_dash",
                "completion_ts": 1433473124.0,
            }
        ]

        self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids
        self.koji_wrapper.koji_proxy.getBuild.return_value = mock_build_md[0]
        event = {"id": 12345, "ts": 1533473124.0}

        module_info_str = "testmodule2:master-dash:20180406051653:96c371af"
        result = source_koji.get_koji_modules(
            self.compose, self.koji_wrapper, event, module_info_str
        )

        assert type(result) is list
        assert len(result) == 1
        module = result[0]
        assert type(module) is dict
        self.assertIn("module_stream", module)
        self.assertIn("module_version", module)
        self.assertIn("module_context", module)
        self.assertIn("tag", module)

        expected_query = "testmodule2-master_dash-20180406051653.96c371af"
        self.koji_wrapper.koji_proxy.search.assert_called_once_with(
            expected_query, "build", "glob"
        )
        self.koji_wrapper.koji_proxy.getBuild.assert_called_once_with(
            mock_build_ids[0]["id"]
        )

    def test_get_koji_modules_filter_by_event(self):
        mock_build_ids = [
            {"id": 1065873, "name": "testmodule2-master_dash-20180406051653.96c371af"}
        ]
        mock_extra = {
            "typeinfo": {
                "module": {
                    "content_koji_tag": "module-b62270b82443edde",
                    "modulemd_str": mock.Mock(),
                }
            }
        }
        mock_build_md = [
            {
                "id": 1065873,
                "epoch": None,
                "extra": mock_extra,
                "name": "testmodule2",
                "nvr": "testmodule2-master_dash-20180406051653.2e6f5e0a",
                "release": "20180406051653.2e6f5e0a",
                "state": 1,
                "version": "master_dash",
                "completion_ts": 1633473124.0,
            }
        ]

        self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids
        self.koji_wrapper.koji_proxy.getBuild.return_value = mock_build_md[0]
        event = {"id": 12345, "ts": 1533473124.0}

        with self.assertRaises(ValueError) as ctx:
            source_koji.get_koji_modules(
                self.compose, self.koji_wrapper, event, "testmodule2:master-dash"
            )

        self.assertIn("No module build found", str(ctx.exception))

        self.koji_wrapper.koji_proxy.search.assert_called_once_with(
            "testmodule2-master_dash-*", "build", "glob"
        )
        self.koji_wrapper.koji_proxy.getBuild.assert_called_once_with(
            mock_build_ids[0]["id"]
        )
        self.koji_wrapper.koji_proxy.listArchives.assert_not_called()
        self.koji_wrapper.koji_proxy.listRPMs.assert_not_called()

    def test_get_koji_modules_no_version(self):
        mock_build_ids = [
            {"id": 1065873, "name": "testmodule2-master-20180406051653.2e6f5e0a"},
            {"id": 1065874, "name": "testmodule2-master-20180406051653.96c371af"},
        ]
        mock_extra = [
            {
                "typeinfo": {
                    "module": {
                        "content_koji_tag": "module-b62270b82443edde",
                        "modulemd_str": mock.Mock(),
                        "name": "testmodule2",
                        "stream": "master",
                        "version": "20180406051653",
                        "context": "2e6f5e0a",
                    }
                }
            },
            {
                "typeinfo": {
                    "module": {
                        "content_koji_tag": "module-52e40b9cdd3c0f7d",
                        "modulemd_str": mock.Mock(),
                        "name": "testmodule2",
                        "stream": "master",
                        "version": "20180406051653",
                        "context": "96c371af",
                    }
                }
            },
        ]
        mock_build_md = [
            {
                "id": 1065873,
                "epoch": None,
                "extra": mock_extra[0],
                "name": "testmodule2",
                "nvr": "testmodule2-master-20180406051653.2e6f5e0a",
                "release": "20180406051653.2e6f5e0a",
                "state": 1,
                "version": "master",
                "completion_ts": 1433473124.0,
            },
            {
                "id": 1065874,
                "epoch": None,
                "extra": mock_extra[1],
                "name": "testmodule2",
                "nvr": "testmodule2-master-20180406051653.96c371af",
                "release": "20180406051653.96c371af",
                "state": 1,
                "version": "master",
                "completion_ts": 1433473124.0,
            },
        ]

        self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids
        self.koji_wrapper.koji_proxy.getBuild.side_effect = mock_build_md

        event = {"id": 12345, "ts": 1533473124.0}

        module_info_str = "testmodule2:master"
        result = source_koji.get_koji_modules(
            self.compose, self.koji_wrapper, event, module_info_str
        )

        assert type(result) is list
        assert len(result) == 2
        module = result[0]
        for module in result:
            assert type(module) is dict
            self.assertIn("module_stream", module)
            self.assertIn("module_version", module)
            self.assertIn("module_context", module)

        expected_query = "testmodule2-master-*"
        self.koji_wrapper.koji_proxy.search.assert_called_once_with(
            expected_query, "build", "glob"
        )

        expected_calls = [
            mock.call(mock_build_ids[0]["id"]),
            mock.call(mock_build_ids[1]["id"]),
        ]
        self.koji_wrapper.koji_proxy.getBuild.mock_calls == expected_calls

    def test_get_koji_modules_ignore_deleted(self):
        mock_build_ids = [
            {"id": 1065873, "name": "testmodule2-master_dash-20180406051653.96c371af"}
        ]
        mock_extra = {
            "typeinfo": {
                "module": {
                    "content_koji_tag": "module-b62270b82443edde",
                    "modulemd_str": mock.Mock(),
                    "name": "testmodule2",
                    "stream": "master",
                    "version": "20180406051653",
                    "context": "96c371af",
                }
            }
        }
        mock_build_md = [
            {
                "id": 1065873,
                "epoch": None,
                "extra": mock_extra,
                "name": "testmodule2",
                "nvr": "testmodule2-master_dash-20180406051653.96c371af",
                "release": "20180406051653.96c371af",
                "state": 2,
                "version": "master_dash",
                "completion_ts": 1433473124.0,
            }
        ]

        self.koji_wrapper.koji_proxy.search.return_value = mock_build_ids
        self.koji_wrapper.koji_proxy.getBuild.return_value = mock_build_md[0]
        event = {"id": 12345, "ts": 1533473124.0}

        with self.assertRaises(ValueError) as ctx:
            source_koji.get_koji_modules(
                self.compose, self.koji_wrapper, event, "testmodule2:master-dash"
            )

        self.compose.log_debug.assert_called_once_with(
            "Module build %s has been deleted, ignoring it." % mock_build_ids[0]["name"]
        )

        self.assertIn("No module build found", str(ctx.exception))

        self.koji_wrapper.koji_proxy.search.assert_called_once_with(
            "testmodule2-master_dash-*", "build", "glob"
        )
        self.koji_wrapper.koji_proxy.getBuild.assert_called_once_with(
            mock_build_ids[0]["id"]
        )
        self.koji_wrapper.koji_proxy.listArchives.assert_not_called()
        self.koji_wrapper.koji_proxy.listRPMs.assert_not_called()


class TestSourceKoji(helpers.PungiTestCase):
    @mock.patch("pungi.phases.pkgset.sources.source_koji.get_pkgset_from_koji")
    @mock.patch("pungi.wrappers.kojiwrapper.KojiWrapper")
    def test_run(self, KojiWrapper, gpfk):
        compose = helpers.DummyCompose(
            self.topdir, {"koji_profile": "koji", "koji_cache": "/prefix"}
        )
        KojiWrapper.return_value.koji_module.config.topdir = "/prefix"

        phase = source_koji.PkgsetSourceKoji(compose)
        pkgsets, path_prefix = phase()

        self.assertEqual(pkgsets, gpfk.return_value)
        self.assertEqual(path_prefix, "/prefix")
        self.assertEqual(KojiWrapper.mock_calls, [mock.call(compose)])


class TestCorrectNVR(helpers.PungiTestCase):
    def setUp(self):
        super(TestCorrectNVR, self).setUp()
        self.compose = helpers.DummyCompose(self.topdir, {})
        self.nv = "base-runtime-f26"
        self.nvr = "base-runtime-f26-20170502134116"
        self.release_regex = re.compile(r"^(\d){14}$")
        self.new_nv = "base-runtime:f26"
        self.new_nvr = "base-runtime:f26:20170502134116"
        self.new_nvrc = "base-runtime:f26:20170502134116:0123abcd"

    def test_nv(self):
        module_info = source_koji.variant_dict_from_str(self.compose, self.nv)
        expectedKeys = ["stream", "name"]
        six.assertCountEqual(self, module_info.keys(), expectedKeys)

    def test_nvr(self):
        module_info = source_koji.variant_dict_from_str(self.compose, self.nvr)
        expectedKeys = ["stream", "name", "version"]
        six.assertCountEqual(self, module_info.keys(), expectedKeys)

    def test_correct_release(self):
        module_info = source_koji.variant_dict_from_str(self.compose, self.nvr)
        self.assertIsNotNone(self.release_regex.match(module_info["version"]))

    def test_new_nv(self):
        module_info = source_koji.variant_dict_from_str(self.compose, self.new_nv)
        expected = {"name": "base-runtime", "stream": "f26"}

        self.assertEqual(module_info, expected)

    def test_new_nvr(self):
        module_info = source_koji.variant_dict_from_str(self.compose, self.new_nvr)
        expected = {
            "name": "base-runtime",
            "stream": "f26",
            "version": "20170502134116",
        }
        self.assertEqual(module_info, expected)

    def test_new_nvrc(self):
        module_info = source_koji.variant_dict_from_str(self.compose, self.new_nvrc)
        expected = {
            "name": "base-runtime",
            "stream": "f26",
            "version": "20170502134116",
            "context": "0123abcd",
        }
        self.assertEqual(module_info, expected)

    def test_new_garbage_value(self):
        self.assertRaises(
            ValueError,
            source_koji.variant_dict_from_str,
            self.compose,
            "foo:bar:baz:quux:qaar",
        )


class TestFilterInherited(unittest.TestCase):
    def test_empty_module_list(self):
        event = {"id": 123456}
        koji_proxy = mock.Mock()
        module_builds = []
        top_tag = "top-tag"

        koji_proxy.getFullInheritance.return_value = [
            {"name": "middle-tag"},
            {"name": "bottom-tag"},
        ]

        result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)

        self.assertEqual(result, [])
        self.assertEqual(
            koji_proxy.mock_calls,
            [mock.call.getFullInheritance("top-tag", event=123456)],
        )

    def test_exclude_middle_and_bottom_tag(self):
        event = {"id": 123456}
        koji_proxy = mock.Mock()
        top_tag = "top-tag"

        koji_proxy.getFullInheritance.return_value = [
            {"name": "middle-tag"},
            {"name": "bottom-tag"},
        ]
        module_builds = [
            {"name": "foo", "version": "1", "release": "1", "tag_name": "top-tag"},
            {"name": "foo", "version": "1", "release": "2", "tag_name": "bottom-tag"},
            {"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"},
        ]

        result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)

        six.assertCountEqual(
            self,
            result,
            [{"name": "foo", "version": "1", "release": "1", "tag_name": "top-tag"}],
        )
        self.assertEqual(
            koji_proxy.mock_calls,
            [mock.call.getFullInheritance("top-tag", event=123456)],
        )

    def test_missing_from_top_tag(self):
        event = {"id": 123456}
        koji_proxy = mock.Mock()
        top_tag = "top-tag"

        koji_proxy.getFullInheritance.return_value = [
            {"name": "middle-tag"},
            {"name": "bottom-tag"},
        ]
        module_builds = [
            {"name": "foo", "version": "1", "release": "2", "tag_name": "bottom-tag"},
            {"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"},
        ]

        result = source_koji.filter_inherited(koji_proxy, event, module_builds, top_tag)

        six.assertCountEqual(
            self,
            result,
            [{"name": "foo", "version": "1", "release": "3", "tag_name": "middle-tag"}],
        )
        self.assertEqual(
            koji_proxy.mock_calls,
            [mock.call.getFullInheritance("top-tag", event=123456)],
        )


class TestFilterByWhitelist(unittest.TestCase):
    def _build(self, n, s, v, c):
        s = s.replace("-", "_")
        return {
            "nvr": "%s-%s-%s.%s" % (n, s, v, c),
            "name": n,
            "version": s,
            "release": "%s.%s" % (v, c),
        }

    def test_no_modules(self):
        compose = mock.Mock()
        module_builds = []
        input_modules = [{"name": "foo:1"}]
        expected = set(["foo:1"])

        source_koji.filter_by_whitelist(compose, module_builds, input_modules, expected)

        self.assertEqual(expected, set(["foo:1"]))

    def test_filter_by_NS(self):
        compose = mock.Mock()
        module_builds = [
            self._build("foo", "1", "201809031048", "cafebabe"),
            self._build("foo", "1", "201809031047", "deadbeef"),
            self._build("foo", "2", "201809031047", "deadbeef"),
        ]
        input_modules = [{"name": "foo:1"}]
        expected = set(["foo:1"])

        result = source_koji.filter_by_whitelist(
            compose, module_builds, input_modules, expected
        )

        six.assertCountEqual(self, result, [module_builds[0], module_builds[1]])
        self.assertEqual(expected, set())

    def test_filter_by_NSV(self):
        compose = mock.Mock()
        module_builds = [
            self._build("foo", "1", "201809031048", "cafebabe"),
            self._build("foo", "1", "201809031047", "deadbeef"),
            self._build("foo", "2", "201809031047", "deadbeef"),
        ]
        input_modules = [{"name": "foo:1:201809031047"}]
        expected = set(["foo:1:201809031047"])

        result = source_koji.filter_by_whitelist(
            compose, module_builds, input_modules, expected
        )

        self.assertEqual(result, [module_builds[1]])
        self.assertEqual(expected, set())

    def test_filter_by_NSVC(self):
        compose = mock.Mock()
        module_builds = [
            self._build("foo", "1", "201809031048", "cafebabe"),
            self._build("foo", "1", "201809031047", "deadbeef"),
            self._build("foo", "1", "201809031047", "cafebabe"),
            self._build("foo", "2", "201809031047", "deadbeef"),
        ]
        input_modules = [{"name": "foo:1:201809031047:deadbeef"}]
        expected = set()

        result = source_koji.filter_by_whitelist(
            compose, module_builds, input_modules, expected
        )

        self.assertEqual(result, [module_builds[1]])
        self.assertEqual(expected, set())

    def test_filter_by_wildcard(self):
        compose = mock.Mock()
        module_builds = [
            self._build("foo", "1", "201809031048", "cafebabe"),
            self._build("foo", "1", "201809031047", "deadbeef"),
            self._build("foo", "2", "201809031047", "deadbeef"),
        ]
        input_modules = [{"name": "*"}]
        expected = set(["*"])

        result = source_koji.filter_by_whitelist(
            compose, module_builds, input_modules, expected
        )

        six.assertCountEqual(self, result, module_builds)
        self.assertEqual(expected, set())


@unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
class TestAddModuleToVariant(helpers.PungiTestCase):
    def setUp(self):
        super(TestAddModuleToVariant, self).setUp()
        self.koji = mock.Mock()
        self.compose = helpers.DummyCompose(self.topdir, {})
        self.koji.koji_module.pathinfo.typedir.return_value = MMDS_DIR
        files = ["modulemd.x86_64.txt", "modulemd.armv7hl.txt", "modulemd.txt"]
        self.koji.koji_proxy.listArchives.return_value = [
            {"btype": "module", "filename": fname} for fname in files
        ] + [{"btype": "foo"}]
        self.buildinfo = {
            "id": 1234,
            "extra": {
                "typeinfo": {
                    "module": {
                        "name": "module",
                        "stream": "master",
                        "version": "20190318",
                        "context": "abcdef",
                    },
                },
            },
        }

    def test_adding_module(self):
        variant = mock.Mock(arches=["armhfp", "x86_64"], arch_mmds={}, modules=[])

        source_koji._add_module_to_variant(
            self.koji, variant, self.buildinfo, compose=self.compose
        )

        mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
        self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
        mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
        self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
        self.assertEqual(len(variant.arch_mmds), 2)
        self.assertEqual(variant.modules, [])

    def test_adding_module_to_existing(self):
        variant = mock.Mock(
            arches=["armhfp", "x86_64"],
            arch_mmds={
                "x86_64": {
                    "m1:latest:20190101:cafe": read_single_module_stream_from_file(
                        os.path.join(MMDS_DIR, "m1.x86_64.txt")
                    )
                }
            },
            modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
        )

        source_koji._add_module_to_variant(
            self.koji, variant, self.buildinfo, compose=self.compose
        )

        mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
        self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
        mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
        self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
        mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
        self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")

        self.assertEqual(
            variant.modules, [{"name": "m1:latest-20190101:cafe", "glob": False}]
        )

    def test_adding_module_with_add_module(self):
        variant = mock.Mock(arches=["armhfp", "x86_64"], arch_mmds={}, modules=[])

        source_koji._add_module_to_variant(
            self.koji,
            variant,
            self.buildinfo,
            compose=self.compose,
            add_to_variant_modules=True,
        )

        mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
        self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
        mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
        self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")

        self.assertEqual(
            variant.modules, [{"name": "module:master:20190318:abcdef", "glob": False}]
        )

    def test_adding_module_to_existing_with_add_module(self):
        variant = mock.Mock(
            arches=["armhfp", "x86_64"],
            arch_mmds={
                "x86_64": {
                    "m1:latest:20190101:cafe": read_single_module_stream_from_file(
                        os.path.join(MMDS_DIR, "m1.x86_64.txt")
                    )
                }
            },
            modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
        )

        source_koji._add_module_to_variant(
            self.koji,
            variant,
            self.buildinfo,
            compose=self.compose,
            add_to_variant_modules=True,
        )

        mod1 = variant.arch_mmds["armhfp"]["module:master:20190318:abcdef"]
        self.assertEqual(mod1.get_NSVCA(), "module:master:20190318:abcdef:armhfp")
        mod2 = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
        self.assertEqual(mod2.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
        mod3 = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
        self.assertEqual(mod3.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")

        self.assertEqual(
            variant.modules,
            [
                {"name": "m1:latest-20190101:cafe", "glob": False},
                {"name": "module:master:20190318:abcdef", "glob": False},
            ],
        )

    def test_adding_module_but_filtered(self):
        compose = helpers.DummyCompose(
            self.topdir, {"filter_modules": [(".*", {"*": ["module:*"]})]}
        )
        variant = mock.Mock(
            arches=["armhfp", "x86_64"], arch_mmds={}, modules=[], uid="Variant"
        )

        nsvc = source_koji._add_module_to_variant(
            self.koji,
            variant,
            self.buildinfo,
            add_to_variant_modules=True,
            compose=compose,
        )

        self.assertIsNone(nsvc)
        self.assertEqual(variant.arch_mmds, {})
        self.assertEqual(variant.modules, [])


@mock.patch("pungi.module_util.Modulemd.ModuleStream.read_file", new=MockModule)
@unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
class TestAddModuleToVariantForKojiMock(helpers.PungiTestCase):
    def setUp(self):
        super(TestAddModuleToVariantForKojiMock, self).setUp()
        self.koji = mock.Mock()
        self.koji.koji_module.pathinfo.typedir.return_value = "/koji"
        self.compose = helpers.DummyCompose(self.topdir, {})
        self.koji.koji_module.pathinfo.topdir = MMDS_DIR
        files = [
            "modulemd.x86_64.txt",
            "scratch-module.x86_64.txt",
        ]
        self.koji.koji_proxy.listArchives.return_value = [
            {"btype": "module", "filename": fname} for fname in files
        ]
        self.buildinfo = {
            "id": 1234,
            "arch": "x86_64",
            "extra": {
                "typeinfo": {
                    "module": {
                        "name": "module",
                        "stream": "master",
                        "version": "20190318",
                        "context": "abcdef",
                        'content_koji_tag': 'module:master-20190318-abcdef'
                    },
                },
            },
        }

    def test_adding_module(self):
        variant = mock.Mock(
            arches=[
                "x86_64"
            ],
            arch_mmds={},
            modules=[],
        )

        source_kojimock._add_module_to_variant(
            self.koji,
            variant,
            self.buildinfo,
        )

        mod = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
        self.assertEqual(mod.get_NSVCA(), "module:master:20190318:abcdef:x86_64")
        self.assertEqual(len(variant.arch_mmds), 1)
        self.assertEqual(variant.modules, [])

    def test_adding_module_to_existing(self):
        variant = mock.Mock(
            arches=[
                "x86_64"
            ],
            arch_mmds={
                "x86_64": {
                    "m1:latest:20190101:cafe": read_single_module_stream_from_file(
                        os.path.join(MMDS_DIR, "m1.x86_64.txt")
                )}
            },
            modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
        )


        source_koji._add_module_to_variant(
            self.koji, variant, self.buildinfo, compose=self.compose
        )

        mod = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
        self.assertEqual(mod.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")

        self.assertEqual(
            variant.modules,
            [{"name": "m1:latest-20190101:cafe", "glob": False}]
        )

    def test_adding_module_with_add_module(self):
        variant = mock.Mock(arches=[
            "x86_64"
        ], arch_mmds={}, modules=[])

        source_kojimock._add_module_to_variant(
            self.koji, variant, self.buildinfo, add_to_variant_modules=True
        )

        mod = variant.arch_mmds["x86_64"]["module:master:20190318:abcdef"]
        self.assertEqual(mod.get_NSVCA(), "module:master:20190318:abcdef:x86_64")

        self.assertEqual(
            variant.modules, [{"name": "module:master:20190318:abcdef", "glob": False}]
        )

    def test_adding_module_to_existing_with_add_module(self):
        variant = mock.Mock(
            arches=[
                "x86_64"
            ],
            arch_mmds={
                "x86_64": {"m1:latest:20190101:cafe": read_single_module_stream_from_file(
                        os.path.join(MMDS_DIR, "m1.x86_64.txt")
                    )
                }
            },
            modules=[{"name": "m1:latest-20190101:cafe", "glob": False}],
        )

        source_kojimock._add_module_to_variant(
            self.koji, variant, self.buildinfo, add_to_variant_modules=True
        )

        mod = variant.arch_mmds["x86_64"]["m1:latest:20190101:cafe"]
        self.assertEqual(mod.get_NSVCA(), "m1:latest:20190101:cafe:x86_64")

        self.assertEqual(
            variant.modules,
            [
                {"name": "m1:latest-20190101:cafe", "glob": False},
                {"name": "module:master:20190318:abcdef", "glob": False},
            ],
        )

    def test_adding_module_but_filtered(self):
        compose = helpers.DummyCompose(
            self.topdir, {"filter_modules": [(".*", {"*": ["module:*"]})]}
        )
        variant = mock.Mock(
            arches=[
                "x86_64"
            ], arch_mmds={}, modules=[], uid="Variant"
        )

        nsvc = source_kojimock._add_module_to_variant(
            self.koji,
            variant,
            self.buildinfo,
            add_to_variant_modules=True,
            compose=compose,
        )

        self.assertIsNone(nsvc)
        self.assertEqual(variant.arch_mmds, {})
        self.assertEqual(variant.modules, [])


class TestIsModuleFiltered(helpers.PungiTestCase):
    def assertIsFiltered(self, name, stream):
        self.assertTrue(
            source_koji._is_filtered_out(
                self.compose, self.compose.variants["Server"], "x86_64", name, stream
            )
        )

    def assertIsNotFiltered(self, name, stream):
        self.assertFalse(
            source_koji._is_filtered_out(
                self.compose, self.compose.variants["Server"], "x86_64", name, stream
            )
        )

    def test_no_filters(self):
        self.compose = helpers.DummyCompose(self.topdir, {})

        self.assertIsNotFiltered("foo", "master")

    def test_filter_by_name(self):
        self.compose = helpers.DummyCompose(
            self.topdir, {"filter_modules": [(".*", {"*": ["foo"]})]}
        )

        self.assertIsFiltered("foo", "master")
        self.assertIsNotFiltered("bar", "master")

    def test_filter_by_stream(self):
        self.compose = helpers.DummyCompose(
            self.topdir, {"filter_modules": [(".*", {"*": ["foo:master"]})]}
        )

        self.assertIsFiltered("foo", "master")
        self.assertIsNotFiltered("bar", "master")
        self.assertIsNotFiltered("foo", "stable")


class MockMBS(object):
    def __init__(self, api_url):
        self.api_url = api_url

    def get_module_build_by_nsvc(self, nsvc):
        return {"id": 1, "koji_tag": "scratch-module-tag", "name": "scratch-module"}

    def final_modulemd(self, module_build_id):
        with open(os.path.join(MMDS_DIR, "scratch-module.x86_64.txt")) as f:
            return {"x86_64": f.read()}


@mock.patch("pungi.phases.pkgset.sources.source_koji.MBSWrapper", new=MockMBS)
@unittest.skipIf(Modulemd is None, "Skipping tests, no module support")
class TestAddScratchModuleToVariant(helpers.PungiTestCase):
    def setUp(self):
        super(TestAddScratchModuleToVariant, self).setUp()
        self.compose = helpers.DummyCompose(
            self.topdir, {"mbs_api_url": "http://mbs.local/module-build-service/2"}
        )
        self.nsvc = "scratch-module:master:20200710:abcdef"

    def test_adding_scratch_module(self):
        variant = mock.Mock(
            arches=[
                # "armhfp",
                "x86_64"
            ],
            arch_mmds={},
            modules=[],
            module_uid_to_koji_tag={},
        )
        variant_tags = {variant: []}
        tag_to_mmd = {}
        scratch_modules = [self.nsvc]

        source_koji._add_scratch_modules_to_variant(
            self.compose, variant, scratch_modules, variant_tags, tag_to_mmd
        )
        self.assertEqual(variant_tags, {variant: ["scratch-module-tag"]})

        self.assertEqual(
            variant.arch_mmds["x86_64"][self.nsvc].get_NSVCA(),
            "scratch-module:master:20200710:abcdef:x86_64",
        )

        self.assertTrue(isinstance(tag_to_mmd["scratch-module-tag"]["x86_64"], set))
        self.assertEqual(
            list(tag_to_mmd["scratch-module-tag"]["x86_64"])[0].get_NSVCA(),
            "scratch-module:master:20200710:abcdef:x86_64",
        )

        self.assertEqual(variant.modules, [])

    def test_adding_scratch_module_nontest_compose(self):
        self.compose.compose_type = "production"
        scratch_modules = [self.nsvc]

        source_koji._add_scratch_modules_to_variant(
            self.compose, mock.Mock(), scratch_modules, {}, {}
        )
        self.compose.log_warning.assert_called_once_with(
            "Only test composes could include scratch module builds"
        )


@ddt
class TestSourceKoji(unittest.TestCase):

    @unpack
    @data(
        (
            'AppStream',
            [
                'x86_64',
                'i386'
            ],
            {
                'python39-devel:3.9',
                'python39:3.9',
            },
            [
                (
                    '^(BaseOS|AppStream|PowerTools)$',
                    {
                        'x86_64': [
                            'python39:3.9',
                        ],
                        'aarch64': [
                            'python39-devel:3.9',
                        ]
                    }
                )
            ],
            {
                'python39-devel:3.9',
            }
        ),
        (
            'AppStream',
            [
                'x86_64',
                'i386'
            ],
            {
                'python39-devel:3.9',
                'python39:3.9',
                'python38-devel:3.8',
                'python38:3.8',
            },
            [
                (
                    '^(BaseOS|AppStream|PowerTools)$',
                    {
                        'x86_64': [
                            'python39:3.9',
                        ],
                        '*': [
                            'python38-devel:3.8',
                        ]
                    }
                )
            ],
            {
                'python39-devel:3.9',
                'python38:3.8',
            }
        ),
        (
            'AppStream',
            [
                'x86_64',
                'i386'
            ],
            {
                'python39-devel:3.9',
                'python39:3.9',
                'python38-devel:3.8',
                'python38:3.8',
            },
            [
                (
                    '^(BaseOS|AppStream|PowerTools)$',
                    {
                        'x86_64': [
                            'python39:3.9',
                        ],
                        'aarch64': [
                            'python38-devel:3.8',
                        ]
                    }
                ),
                (
                    '*',
                    {
                        '*': [
                            'python38-devel:3.8',
                        ]
                    }
                ),
            ],
            {
                'python39-devel:3.9',
                'python38:3.8',
            }
        ),
    )
    def test__filter_expected_modules(
            self,
            variant_name: AnyStr,
            variant_arches: List[AnyStr],
            expected_modules: Set[AnyStr],
            filtered_modules: List[Tuple[AnyStr, Dict[AnyStr, List[AnyStr]]]],
            expected_result: Set[AnyStr],
    ) -> None:
        real_result = source_kojimock._filter_expected_modules(
            variant_name=variant_name,
            variant_arches=variant_arches,
            expected_modules=expected_modules,
            filtered_modules=filtered_modules,
        )
        self.assertSetEqual(
            real_result,
            expected_result,
        )