pungi/tests/test_initphase.py
Haibo Lin f518c1bb7c Stop copying .git directory with module defaults
JIRA: RHELCMP-3016
Fixes: https://pagure.io/pungi/issue/1464

Signed-off-by: Haibo Lin <hlin@redhat.com>
2021-01-14 10:36:38 +08:00

685 lines
24 KiB
Python

# -*- coding: utf-8 -*-
try:
import unittest2 as unittest
except ImportError:
import unittest
import mock
import six
import os
from pungi.module_util import Modulemd
from pungi.phases import init
from tests.helpers import (
DummyCompose,
PungiTestCase,
touch,
mk_boom,
fake_run_in_threads,
)
@mock.patch("pungi.phases.init.run_in_threads", new=fake_run_in_threads)
@mock.patch("pungi.phases.init.validate_comps")
@mock.patch("pungi.phases.init.validate_module_defaults")
@mock.patch("pungi.phases.init.write_module_defaults")
@mock.patch("pungi.phases.init.write_global_comps")
@mock.patch("pungi.phases.init.write_arch_comps")
@mock.patch("pungi.phases.init.create_comps_repo")
@mock.patch("pungi.phases.init.write_variant_comps")
@mock.patch("pungi.phases.init.write_prepopulate_file")
class TestInitPhase(PungiTestCase):
def test_run(
self,
write_prepopulate,
write_variant,
create_comps,
write_arch,
write_global,
write_defaults,
validate_defaults,
validate_comps,
):
compose = DummyCompose(self.topdir, {})
compose.has_comps = True
compose.has_module_defaults = False
compose.setup_optional()
phase = init.InitPhase(compose)
phase.run()
self.assertEqual(write_global.mock_calls, [mock.call(compose)])
self.assertEqual(write_prepopulate.mock_calls, [mock.call(compose)])
six.assertCountEqual(
self,
write_arch.mock_calls,
[mock.call(compose, "x86_64"), mock.call(compose, "amd64")],
)
six.assertCountEqual(
self,
create_comps.mock_calls,
[
mock.call(compose, "x86_64", None),
mock.call(compose, "amd64", None),
mock.call(compose, "x86_64", compose.variants["Server"]),
mock.call(compose, "amd64", compose.variants["Server"]),
mock.call(compose, "amd64", compose.variants["Client"]),
mock.call(compose, "x86_64", compose.variants["Everything"]),
mock.call(compose, "amd64", compose.variants["Everything"]),
mock.call(compose, "x86_64", compose.all_variants["Server-optional"]),
],
)
six.assertCountEqual(
self,
write_variant.mock_calls,
[
mock.call(compose, "x86_64", compose.variants["Server"]),
mock.call(compose, "amd64", compose.variants["Server"]),
mock.call(compose, "amd64", compose.variants["Client"]),
mock.call(compose, "x86_64", compose.variants["Everything"]),
mock.call(compose, "amd64", compose.variants["Everything"]),
mock.call(compose, "x86_64", compose.all_variants["Server-optional"]),
],
)
self.assertEqual(write_defaults.call_args_list, [])
self.assertEqual(validate_defaults.call_args_list, [])
def test_run_with_preserve(
self,
write_prepopulate,
write_variant,
create_comps,
write_arch,
write_global,
write_defaults,
validate_defaults,
validate_comps,
):
compose = DummyCompose(self.topdir, {})
compose.has_comps = True
compose.has_module_defaults = False
compose.variants["Everything"].groups = []
compose.variants["Everything"].modules = []
phase = init.InitPhase(compose)
phase.run()
self.assertEqual(write_global.mock_calls, [mock.call(compose)])
self.assertEqual(
validate_comps.call_args_list, [mock.call(write_global.return_value)]
)
self.assertEqual(write_prepopulate.mock_calls, [mock.call(compose)])
six.assertCountEqual(
self,
write_arch.mock_calls,
[mock.call(compose, "x86_64"), mock.call(compose, "amd64")],
)
six.assertCountEqual(
self,
create_comps.mock_calls,
[
mock.call(compose, "x86_64", None),
mock.call(compose, "amd64", None),
mock.call(compose, "x86_64", compose.variants["Server"]),
mock.call(compose, "amd64", compose.variants["Server"]),
mock.call(compose, "amd64", compose.variants["Client"]),
mock.call(compose, "x86_64", compose.variants["Everything"]),
mock.call(compose, "amd64", compose.variants["Everything"]),
],
)
six.assertCountEqual(
self,
write_variant.mock_calls,
[
mock.call(compose, "x86_64", compose.variants["Server"]),
mock.call(compose, "amd64", compose.variants["Server"]),
mock.call(compose, "amd64", compose.variants["Client"]),
mock.call(compose, "x86_64", compose.variants["Everything"]),
mock.call(compose, "amd64", compose.variants["Everything"]),
],
)
self.assertEqual(write_defaults.call_args_list, [])
self.assertEqual(validate_defaults.call_args_list, [])
def test_run_without_comps(
self,
write_prepopulate,
write_variant,
create_comps,
write_arch,
write_global,
write_defaults,
validate_defaults,
validate_comps,
):
compose = DummyCompose(self.topdir, {})
compose.has_comps = False
compose.has_module_defaults = False
phase = init.InitPhase(compose)
phase.run()
self.assertEqual(write_global.mock_calls, [])
self.assertEqual(validate_comps.call_args_list, [])
self.assertEqual(write_prepopulate.mock_calls, [mock.call(compose)])
self.assertEqual(write_arch.mock_calls, [])
self.assertEqual(create_comps.mock_calls, [])
self.assertEqual(write_variant.mock_calls, [])
self.assertEqual(write_defaults.call_args_list, [])
self.assertEqual(validate_defaults.call_args_list, [])
def test_with_module_defaults(
self,
write_prepopulate,
write_variant,
create_comps,
write_arch,
write_global,
write_defaults,
validate_defaults,
validate_comps,
):
compose = DummyCompose(self.topdir, {})
compose.has_comps = False
compose.has_module_defaults = True
phase = init.InitPhase(compose)
phase.run()
self.assertEqual(write_global.mock_calls, [])
self.assertEqual(validate_comps.call_args_list, [])
self.assertEqual(write_prepopulate.mock_calls, [mock.call(compose)])
self.assertEqual(write_arch.mock_calls, [])
self.assertEqual(create_comps.mock_calls, [])
self.assertEqual(write_variant.mock_calls, [])
self.assertEqual(write_defaults.call_args_list, [mock.call(compose)])
self.assertEqual(
validate_defaults.call_args_list,
[mock.call(compose.paths.work.module_defaults_dir())],
)
class TestWriteArchComps(PungiTestCase):
@mock.patch("pungi.phases.init.run")
def test_run(self, run):
compose = DummyCompose(self.topdir, {})
init.write_arch_comps(compose, "x86_64")
self.assertEqual(
run.mock_calls,
[
mock.call(
[
"comps_filter",
"--arch=x86_64",
"--no-cleanup",
"--output=%s/work/x86_64/comps/comps-x86_64.xml" % self.topdir,
self.topdir + "/work/global/comps/comps-global.xml",
]
)
],
)
class TestCreateCompsRepo(PungiTestCase):
@mock.patch("pungi.phases.init.run")
def test_run(self, run):
compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"})
init.create_comps_repo(compose, "x86_64", None)
self.assertEqual(
run.mock_calls,
[
mock.call(
[
"createrepo_c",
self.topdir + "/work/x86_64/comps_repo",
"--outputdir=%s/work/x86_64/comps_repo" % self.topdir,
"--groupfile=%s/work/x86_64/comps/comps-x86_64.xml"
% self.topdir,
"--update",
"--no-database",
"--checksum=sha256",
"--unique-md-filenames",
],
logfile=self.topdir + "/logs/x86_64/comps_repo.x86_64.log",
show_cmd=True,
)
],
)
@mock.patch("pungi.phases.init.run")
def test_run_with_variant(self, run):
compose = DummyCompose(self.topdir, {"createrepo_checksum": "sha256"})
init.create_comps_repo(compose, "x86_64", compose.variants["Server"])
self.assertEqual(
run.mock_calls,
[
mock.call(
[
"createrepo_c",
self.topdir + "/work/x86_64/comps_repo_Server",
"--outputdir=%s/work/x86_64/comps_repo_Server" % self.topdir,
"--groupfile=%s/work/x86_64/comps/comps-Server.x86_64.xml"
% self.topdir,
"--update",
"--no-database",
"--checksum=sha256",
"--unique-md-filenames",
],
logfile=self.topdir + "/logs/x86_64/comps_repo-Server.x86_64.log",
show_cmd=True,
)
],
)
class TestWriteGlobalComps(PungiTestCase):
@mock.patch("pungi.phases.init.get_file_from_scm")
def test_run_local_file(self, get_file):
compose = DummyCompose(self.topdir, {"comps_file": "some-file.xml"})
def gen_file(src, dest, compose=None):
self.assertEqual(src, "/home/releng/config/some-file.xml")
touch(os.path.join(dest, "some-file.xml"))
get_file.side_effect = gen_file
init.write_global_comps(compose)
self.assertTrue(
os.path.isfile(self.topdir + "/work/global/comps/comps-global.xml")
)
class TestWriteVariantComps(PungiTestCase):
@mock.patch("pungi.phases.init.run")
@mock.patch("pungi.phases.init.CompsWrapper")
def test_run(self, CompsWrapper, run):
compose = DummyCompose(self.topdir, {})
variant = compose.variants["Server"]
comps = CompsWrapper.return_value
comps.filter_groups.return_value = []
init.write_variant_comps(compose, "x86_64", variant)
self.assertEqual(
run.mock_calls,
[
mock.call(
[
"comps_filter",
"--arch=x86_64",
"--keep-empty-group=conflicts",
"--keep-empty-group=conflicts-server",
"--variant=Server",
"--output=%s/work/x86_64/comps/comps-Server.x86_64.xml"
% self.topdir,
self.topdir + "/work/global/comps/comps-global.xml",
]
)
],
)
self.assertEqual(
CompsWrapper.call_args_list,
[mock.call(self.topdir + "/work/x86_64/comps/comps-Server.x86_64.xml")],
)
self.assertEqual(
comps.filter_groups.call_args_list, [mock.call(variant.groups)]
)
self.assertEqual(
comps.filter_environments.mock_calls, [mock.call(variant.environments)]
)
self.assertEqual(comps.write_comps.mock_calls, [mock.call()])
@mock.patch("pungi.phases.init.get_lookaside_groups")
@mock.patch("pungi.phases.init.run")
@mock.patch("pungi.phases.init.CompsWrapper")
def test_run_with_lookaside_groups(self, CompsWrapper, run, glg):
compose = DummyCompose(self.topdir, {})
variant = compose.variants["Server"]
comps = CompsWrapper.return_value
comps.filter_groups.return_value = []
glg.return_value = ["foo", "bar"]
init.write_variant_comps(compose, "x86_64", variant)
self.assertEqual(
run.mock_calls,
[
mock.call(
[
"comps_filter",
"--arch=x86_64",
"--keep-empty-group=conflicts",
"--keep-empty-group=conflicts-server",
"--variant=Server",
"--output=%s/work/x86_64/comps/comps-Server.x86_64.xml"
% self.topdir,
self.topdir + "/work/global/comps/comps-global.xml",
"--lookaside-group=foo",
"--lookaside-group=bar",
]
),
],
)
self.assertEqual(
CompsWrapper.call_args_list,
[mock.call(self.topdir + "/work/x86_64/comps/comps-Server.x86_64.xml")],
)
self.assertEqual(
comps.filter_groups.call_args_list, [mock.call(variant.groups)]
)
self.assertEqual(
comps.filter_environments.mock_calls, [mock.call(variant.environments)]
)
self.assertEqual(comps.write_comps.mock_calls, [mock.call()])
@mock.patch("pungi.phases.init.run")
@mock.patch("pungi.phases.init.CompsWrapper")
def test_run_no_filter_without_groups(self, CompsWrapper, run):
compose = DummyCompose(self.topdir, {})
variant = compose.variants["Server"]
variant.groups = []
comps = CompsWrapper.return_value
comps.filter_groups.return_value = []
init.write_variant_comps(compose, "x86_64", variant)
self.assertEqual(
run.mock_calls,
[
mock.call(
[
"comps_filter",
"--arch=x86_64",
"--keep-empty-group=conflicts",
"--keep-empty-group=conflicts-server",
"--variant=Server",
"--output=%s/work/x86_64/comps/comps-Server.x86_64.xml"
% self.topdir,
self.topdir + "/work/global/comps/comps-global.xml",
]
)
],
)
self.assertEqual(
CompsWrapper.call_args_list,
[mock.call(self.topdir + "/work/x86_64/comps/comps-Server.x86_64.xml")],
)
self.assertEqual(comps.filter_groups.call_args_list, [])
self.assertEqual(
comps.filter_environments.mock_calls, [mock.call(variant.environments)]
)
self.assertEqual(comps.write_comps.mock_calls, [mock.call()])
@mock.patch("pungi.phases.init.run")
@mock.patch("pungi.phases.init.CompsWrapper")
def test_run_filter_for_modular(self, CompsWrapper, run):
compose = DummyCompose(self.topdir, {})
variant = compose.variants["Server"]
variant.groups = []
variant.modules = ["testmodule:2.0"]
comps = CompsWrapper.return_value
comps.filter_groups.return_value = []
init.write_variant_comps(compose, "x86_64", variant)
self.assertEqual(
run.mock_calls,
[
mock.call(
[
"comps_filter",
"--arch=x86_64",
"--keep-empty-group=conflicts",
"--keep-empty-group=conflicts-server",
"--variant=Server",
"--output=%s/work/x86_64/comps/comps-Server.x86_64.xml"
% self.topdir,
self.topdir + "/work/global/comps/comps-global.xml",
]
)
],
)
self.assertEqual(
CompsWrapper.call_args_list,
[mock.call(self.topdir + "/work/x86_64/comps/comps-Server.x86_64.xml")],
)
self.assertEqual(comps.filter_groups.call_args_list, [mock.call([])])
self.assertEqual(
comps.filter_environments.mock_calls, [mock.call(variant.environments)]
)
self.assertEqual(comps.write_comps.mock_calls, [mock.call()])
@mock.patch("pungi.phases.init.run")
@mock.patch("pungi.phases.init.CompsWrapper")
def test_run_report_unmatched(self, CompsWrapper, run):
compose = DummyCompose(self.topdir, {})
variant = compose.variants["Server"]
comps = CompsWrapper.return_value
comps.filter_groups.return_value = ["foo", "bar"]
init.write_variant_comps(compose, "x86_64", variant)
self.assertEqual(
run.mock_calls,
[
mock.call(
[
"comps_filter",
"--arch=x86_64",
"--keep-empty-group=conflicts",
"--keep-empty-group=conflicts-server",
"--variant=Server",
"--output=%s/work/x86_64/comps/comps-Server.x86_64.xml"
% self.topdir,
self.topdir + "/work/global/comps/comps-global.xml",
]
)
],
)
self.assertEqual(
CompsWrapper.call_args_list,
[mock.call(self.topdir + "/work/x86_64/comps/comps-Server.x86_64.xml")],
)
self.assertEqual(
comps.filter_groups.call_args_list, [mock.call(variant.groups)]
)
self.assertEqual(
comps.filter_environments.mock_calls, [mock.call(variant.environments)]
)
self.assertEqual(comps.write_comps.mock_calls, [mock.call()])
self.assertEqual(
compose.log_warning.call_args_list,
[
mock.call(init.UNMATCHED_GROUP_MSG % ("Server", "x86_64", "foo")),
mock.call(init.UNMATCHED_GROUP_MSG % ("Server", "x86_64", "bar")),
],
)
class TestGetLookasideGroups(PungiTestCase):
def test_toplevel_variant(self):
compose = DummyCompose(self.topdir, {})
self.assertEqual(
init.get_lookaside_groups(compose, compose.variants["Server"]), set()
)
def test_classic_addon(self):
compose = DummyCompose(self.topdir, {})
compose.setup_addon()
compose.variants["Server"].groups = [{"name": "foo"}]
self.assertEqual(
init.get_lookaside_groups(compose, compose.all_variants["Server-HA"]),
set(["foo"]),
)
def test_variant_as_lookaside(self):
compose = DummyCompose(
self.topdir, {"variant_as_lookaside": [("Server", "Client")]}
)
compose.variants["Client"].groups = [{"name": "foo"}]
self.assertEqual(
init.get_lookaside_groups(compose, compose.variants["Server"]),
set(["foo"]),
)
@mock.patch("shutil.ignore_patterns")
@mock.patch("shutil.copytree")
@mock.patch("pungi.phases.init.get_dir_from_scm")
class TestWriteModuleDefaults(PungiTestCase):
def test_clone_git(self, gdfs, ct, igp):
conf = {"scm": "git", "repo": "https://pagure.io/pungi.git", "dir": "."}
compose = DummyCompose(self.topdir, {"module_defaults_dir": conf})
init.write_module_defaults(compose)
self.assertEqual(
gdfs.call_args_list, [mock.call(conf, mock.ANY, compose=compose)]
)
self.assertEqual(
ct.call_args_list,
[
mock.call(
gdfs.call_args_list[0][0][1],
os.path.join(self.topdir, "work/global/module_defaults"),
ignore=igp(".git"),
)
],
)
def test_clone_file_scm(self, gdfs, ct, igp):
conf = {"scm": "file", "dir": "defaults"}
compose = DummyCompose(self.topdir, {"module_defaults_dir": conf})
compose.config_dir = "/home/releng/configs"
init.write_module_defaults(compose)
self.assertEqual(
gdfs.call_args_list,
[
mock.call(
{"scm": "file", "dir": "/home/releng/configs/defaults"},
mock.ANY,
compose=compose,
)
],
)
self.assertEqual(
ct.call_args_list,
[
mock.call(
gdfs.call_args_list[0][0][1],
os.path.join(self.topdir, "work/global/module_defaults"),
ignore=igp(".git"),
)
],
)
def test_clone_file_str(self, gdfs, ct, igp):
conf = "defaults"
compose = DummyCompose(self.topdir, {"module_defaults_dir": conf})
compose.config_dir = "/home/releng/configs"
init.write_module_defaults(compose)
self.assertEqual(
gdfs.call_args_list,
[mock.call("/home/releng/configs/defaults", mock.ANY, compose=compose)],
)
self.assertEqual(
ct.call_args_list,
[
mock.call(
gdfs.call_args_list[0][0][1],
os.path.join(self.topdir, "work/global/module_defaults"),
ignore=igp(".git"),
)
],
)
@unittest.skipUnless(Modulemd, "Skipped test, no module support.")
class TestValidateModuleDefaults(PungiTestCase):
def _write_defaults(self, defs):
for mod_name, streams in defs.items():
for stream in streams:
mod_index = Modulemd.ModuleIndex.new()
mmddef = Modulemd.DefaultsV1.new(mod_name)
mmddef.set_default_stream(stream)
mod_index.add_defaults(mmddef)
filename = "%s-%s.yaml" % (mod_name, stream)
with open(os.path.join(self.topdir, filename), "w") as f:
f.write(mod_index.dump_to_string())
def test_valid_files(self):
self._write_defaults({"httpd": ["1"], "python": ["3.6"]})
init.validate_module_defaults(self.topdir)
def test_duplicated_stream(self):
self._write_defaults({"httpd": ["1"], "python": ["3.6", "3.5"]})
with self.assertRaises(RuntimeError) as ctx:
init.validate_module_defaults(self.topdir)
self.assertIn(
"Module python has multiple defaults: 3.5, 3.6", str(ctx.exception)
)
def test_reports_all(self):
self._write_defaults({"httpd": ["1", "2"], "python": ["3.6", "3.5"]})
with self.assertRaises(RuntimeError) as ctx:
init.validate_module_defaults(self.topdir)
self.assertIn("Module httpd has multiple defaults: 1, 2", str(ctx.exception))
self.assertIn(
"Module python has multiple defaults: 3.5, 3.6", str(ctx.exception)
)
def test_handles_non_defaults_file(self):
self._write_defaults({"httpd": ["1"], "python": ["3.6"]})
touch(
os.path.join(self.topdir, "boom.yaml"),
"\n".join(
[
"document: modulemd",
"version: 2",
"data:",
" summary: dummy module",
" description: dummy module",
" license:",
" module: [GPL]",
" content: [GPL]",
]
),
)
init.validate_module_defaults(self.topdir)
@mock.patch("pungi.phases.init.CompsWrapper")
class TestValidateComps(unittest.TestCase):
def test_ok(self, CompsWrapper):
init.validate_comps("/path")
self.assertEqual(
CompsWrapper.mock_calls, [mock.call("/path"), mock.call().validate()]
)
def test_fail(self, CompsWrapper):
CompsWrapper.return_value.validate.side_effect = mk_boom()
with self.assertRaises(Exception):
init.validate_comps("/path")
self.assertEqual(
CompsWrapper.mock_calls, [mock.call("/path"), mock.call().validate()]
)