Add support for customizations and repos.git to /blueprints/diff/

This also includes extensive tests for each of the currently supported
customizations. It should be generic enough to continue working as long
as the list of dicts includes a 'name' or 'user' field in the dict.
Otherwise support for a new dict key will need to be added to the
customizations_diff function.

src/pylorax/api/recipes.py

@@ -919,25 +919,78 @@ def is_parent_diff(repo, filename, tree, parent):
     diff = Git.Diff.new_tree_to_tree(repo, parent.get_tree(), tree, diff_opts)
     return diff.get_num_deltas() > 0
 
+def find_field_value(field, value, lst):
+    """Find a field matching value in the list of dicts.
+
+    :param field: field to search for
+    :type field: str
+    :param value: value to match in the field
+    :type value: str
+    :param lst: List of dict's with field
+    :type lst: list of dict
+    :returns: First dict with matching field:value, or None
+    :rtype: dict or None
+
+    Used to return a specific entry from a list that looks like this:
+
+    [{"name": "one", "attr": "green"}, ...]
+
+    find_field_value("name", "one", lst) will return the matching dict.
+    """
+    for d in lst:
+        if d.get(field) and d.get(field) == value:
+            return d
+    return None
+
 def find_name(name, lst):
     """Find the dict matching the name in a list and return it.
 
     :param name: Name to search for
     :type name: str
     :param lst: List of dict's with "name" field
+    :type lst: list of dict
     :returns: First dict with matching name, or None
     :rtype: dict or None
-    """
-    for e in lst:
-        if e["name"] == name:
-            return e
-    return None
 
-def diff_items(title, old_items, new_items):
+    This is just a wrapper for find_field_value with field set to "name"
+    """
+    return find_field_value("name", name, lst)
+
+def find_recipe_obj(path, recipe, default=None):
+    """Find a recipe object
+
+    :param path: A list of dict field names
+    :type path: list of str
+    :param recipe: The recipe to search
+    :type recipe: Recipe
+    :param default: The value to return if it is not found
+    :type default: Any
+
+    Return the object found by applying the path to the dicts in the recipe, or
+    return the default if it doesn't exist.
+
+    eg. {"customizations": {"hostname": "foo", "users": [...]}}
+
+    find_recipe_obj(["customizations", "hostname"], recipe, "")
+    """
+    o = recipe
+    try:
+        for p in path:
+            if not o.get(p):
+                return default
+            o = o.get(p)
+    except AttributeError:
+        return default
+
+    return o
+
+def diff_lists(title, field, old_items, new_items):
     """Return the differences between two lists of dicts.
 
     :param title: Title of the entry
     :type title: str
+    :param field: Field to use as the key for comparisons
+    :type field: str
     :param old_items: List of item dicts with "name" field
     :type old_items: list(dict)
     :param new_items: List of item dicts with "name" field
@@ -946,35 +999,80 @@ def diff_items(title, old_items, new_items):
     :rtype: list(dict)
     """
     diffs = []
-    old_names = set(m["name"] for m in old_items)
+    old_fields= set(m[field] for m in old_items)
-    new_names = set(m["name"] for m in new_items)
+    new_fields= set(m[field] for m in new_items)
 
-    added_items = new_names.difference(old_names)
+    added_items = new_fields.difference(old_fields)
     added_items = sorted(added_items, key=lambda n: n.lower())
 
-    removed_items = old_names.difference(new_names)
+    removed_items = old_fields.difference(new_fields)
     removed_items = sorted(removed_items, key=lambda n: n.lower())
 
-    same_items = old_names.intersection(new_names)
+    same_items = old_fields.intersection(new_fields)
     same_items = sorted(same_items, key=lambda n: n.lower())
 
-    for name in added_items:
+    for v in added_items:
         diffs.append({"old":None,
-                      "new":{title:find_name(name, new_items)}})
+                      "new":{title:find_field_value(field, v, new_items)}})
 
-    for name in removed_items:
+    for v in removed_items:
-        diffs.append({"old":{title:find_name(name, old_items)},
+        diffs.append({"old":{title:find_field_value(field, v, old_items)},
                       "new":None})
 
-    for name in same_items:
+    for v in same_items:
-        old_item = find_name(name, old_items)
+        old_item = find_field_value(field, v, old_items)
-        new_item = find_name(name, new_items)
+        new_item = find_field_value(field, v, new_items)
         if old_item != new_item:
             diffs.append({"old":{title:old_item},
                           "new":{title:new_item}})
 
     return diffs
 
+def customizations_diff(old_recipe, new_recipe):
+    """Diff the customizations sections from two versions of a recipe
+    """
+    diffs = []
+    old_keys = set(old_recipe.get("customizations", {}).keys())
+    new_keys = set(new_recipe.get("customizations", {}).keys())
+
+    added_keys = new_keys.difference(old_keys)
+    added_keys = sorted(added_keys, key=lambda n: n.lower())
+
+    removed_keys = old_keys.difference(new_keys)
+    removed_keys = sorted(removed_keys, key=lambda n: n.lower())
+
+    same_keys = old_keys.intersection(new_keys)
+    same_keys = sorted(same_keys, key=lambda n: n.lower())
+
+    for v in added_keys:
+        diffs.append({"old": None,
+                      "new": {"Customizations."+v: new_recipe["customizations"][v]}})
+
+    for v in removed_keys:
+        diffs.append({"old": {"Customizations."+v: old_recipe["customizations"][v]},
+                      "new": None})
+
+    for v in same_keys:
+        if new_recipe["customizations"][v] == old_recipe["customizations"][v]:
+            continue
+
+        if type(new_recipe["customizations"][v]) == type([]):
+            # Lists of dicts need to use diff_lists
+            # sshkey uses 'user', user and group use 'name'
+            if "user" in new_recipe["customizations"][v][0]:
+                field_name = "user"
+            elif "name" in new_recipe["customizations"][v][0]:
+                field_name = "name"
+            else:
+                raise RuntimeError("%s list has unrecognized key, not 'name' or 'user'" % "customizations."+v)
+
+            diffs.extend(diff_lists("Customizations."+v, field_name, old_recipe["customizations"][v], new_recipe["customizations"][v]))
+        else:
+            diffs.append({"old": {"Customizations."+v: old_recipe["customizations"][v]},
+                          "new": {"Customizations."+v: new_recipe["customizations"][v]}})
+
+    return diffs
+
 
 def recipe_diff(old_recipe, new_recipe):
     """Diff two versions of a recipe
@@ -994,9 +1092,18 @@ def recipe_diff(old_recipe, new_recipe):
             diffs.append({"old":{element.title():old_recipe[element]},
                           "new":{element.title():new_recipe[element]}})
 
-    diffs.extend(diff_items("Module", old_recipe["modules"], new_recipe["modules"]))
+    # These lists always exist
-    diffs.extend(diff_items("Package", old_recipe["packages"], new_recipe["packages"]))
+    diffs.extend(diff_lists("Module", "name", old_recipe["modules"], new_recipe["modules"]))
-    diffs.extend(diff_items("Group", old_recipe["groups"], new_recipe["groups"]))
+    diffs.extend(diff_lists("Package", "name", old_recipe["packages"], new_recipe["packages"]))
+    diffs.extend(diff_lists("Group", "name", old_recipe["groups"], new_recipe["groups"]))
+
+    # The customizations section can contain a number of different types
+    diffs.extend(customizations_diff(old_recipe, new_recipe))
+
+    # repos contains keys that are lists (eg. [[repos.git]])
+    diffs.extend(diff_lists("Repos.git", "rpmname",
+                            find_recipe_obj(["repos", "git"], old_recipe, []),
+                            find_recipe_obj(["repos", "git"], new_recipe, [])))
 
     return diffs
 

tests/pylorax/test_recipes.py

@@ -48,31 +48,116 @@ class BasicRecipeTest(unittest.TestCase):
                     result_dict = eval(f_dict.read())
                 self.input_toml[recipe_toml] = (f_toml.read(), result_dict)
 
+        # Used by diff tests
         self.old_modules = [recipes.RecipeModule("toml", "2.1"),
                             recipes.RecipeModule("bash", "5.*"),
                             recipes.RecipeModule("httpd", "3.7.*")]
-        self.old_packages = [recipes.RecipePackage("python", "2.7.*"),
-                             recipes.RecipePackage("parted", "3.2")]
-        self.old_groups = [recipes.RecipeGroup("backup-client"),
-                           recipes.RecipeGroup("standard")]
         self.new_modules = [recipes.RecipeModule("toml", "2.1"),
                             recipes.RecipeModule("httpd", "3.8.*"),
                             recipes.RecipeModule("openssh", "2.8.1")]
-        self.new_packages = [recipes.RecipePackage("python", "2.7.*"),
-                             recipes.RecipePackage("parted", "3.2"),
-                             recipes.RecipePackage("git", "2.13.*")]
-        self.new_groups = [recipes.RecipeGroup("console-internet"),
-                           recipes.RecipeGroup("standard")]
         self.modules_result = [{"new": {"Modules": {"version": "2.8.1", "name": "openssh"}},
                                 "old": None},
                                {"new": None,
                                 "old": {"Modules": {"name": "bash", "version": "5.*"}}},
                                {"new": {"Modules": {"version": "3.8.*", "name": "httpd"}},
                                 "old": {"Modules": {"version": "3.7.*", "name": "httpd"}}}]
+
+        self.old_packages = [recipes.RecipePackage("python", "2.7.*"),
+                             recipes.RecipePackage("parted", "3.2")]
+        self.new_packages = [recipes.RecipePackage("python", "2.7.*"),
+                             recipes.RecipePackage("parted", "3.2"),
+                             recipes.RecipePackage("git", "2.13.*")]
         self.packages_result = [{"new": {"Packages": {"name": "git", "version": "2.13.*"}}, "old": None}]
+
+        self.old_groups = [recipes.RecipeGroup("backup-client"),
+                           recipes.RecipeGroup("standard")]
+        self.new_groups = [recipes.RecipeGroup("console-internet"),
+                           recipes.RecipeGroup("standard")]
         self.groups_result = [{'new': {'Groups': {'name': 'console-internet'}}, 'old': None},
                               {'new': None, 'old': {'Groups': {'name': 'backup-client'}}}]
 
+        # customizations test data and results.
+        self.old_custom = {'hostname': 'custombase'}
+        self.custom_sshkey1 = {'sshkey': [{'user': 'root', 'key': 'A SSH KEY FOR ROOT'}]}
+        self.custom_sshkey2 = {'sshkey': [{'user': 'root', 'key': 'A DIFFERENT SSH KEY FOR ROOT'}]}
+        self.custom_sshkey3 = {'sshkey': [{'user': 'root', 'key': 'A SSH KEY FOR ROOT'}, {'user': 'cliff', 'key': 'A SSH KEY FOR CLIFF'}]}
+        self.custom_kernel = {'kernel': {'append': 'nosmt=force'}}
+        self.custom_user1 = {'user': [{'name': 'admin', 'description': 'Administrator account', 'password': '$6$CHO2$3rN8eviE2t50lmVyBYihTgVRHcaecmeCk31L...', 'key': 'PUBLIC SSH KEY', 'home': '/srv/widget/', 'shell': '/usr/bin/bash', 'groups': ['widget', 'users', 'wheel'], 'uid': 1200, 'gid': 1200}]}
+        self.custom_user2 = {'user': [{'name': 'admin', 'description': 'Administrator account', 'password': '$6$CHO2$3rN8eviE2t50lmVyBYihTgVRHcaecmeCk31L...', 'key': 'PUBLIC SSH KEY', 'home': '/root/', 'shell': '/usr/bin/bash', 'groups': ['widget', 'users', 'wheel'], 'uid': 1200, 'gid': 1200}]}
+        self.custom_user3 = {'user': [{'name': 'admin', 'description': 'Administrator account', 'password': '$6$CHO2$3rN8eviE2t50lmVyBYihTgVRHcaecmeCk31L...', 'key': 'PUBLIC SSH KEY', 'home': '/srv/widget/', 'shell': '/usr/bin/bash', 'groups': ['widget', 'users', 'wheel'], 'uid': 1200, 'gid': 1200}, {'name': 'norman', 'key': 'PUBLIC SSH KEY'}]}
+        self.custom_group = {'group': [{'name': 'widget', 'gid': 1130}]}
+        self.custom_timezone1 = {'timezone': {'timezone': 'US/Eastern', 'ntpservers': ['0.north-america.pool.ntp.org', '1.north-america.pool.ntp.org']}}
+        self.custom_timezone2 = {'timezone': {'timezone': 'US/Eastern'}}
+        self.custom_timezone3 = {'timezone': {'ntpservers': ['0.north-america.pool.ntp.org', '1.north-america.pool.ntp.org']}}
+        self.custom_locale1 = {'locale': {'languages': ['en_US.UTF-8'], 'keyboard': 'us'}}
+        self.custom_locale2 = {'locale': {'languages': ['en_US.UTF-8']}}
+        self.custom_locale3 = {'locale': {'keyboard': 'us'}}
+        self.custom_firewall1 = {'firewall': {'ports': ['22:tcp', '80:tcp', 'imap:tcp', '53:tcp', '53:udp'], 'services': {'enabled': ['ftp', 'ntp', 'dhcp'], 'disabled': ['telnet']}}}
+        self.custom_firewall2 = {'firewall': {'ports': ['22:tcp', '80:tcp', 'imap:tcp', '53:tcp', '53:udp']}}
+        self.custom_firewall3 = {'firewall': {'services': {'enabled': ['ftp', 'ntp', 'dhcp'], 'disabled': ['telnet']}}}
+        self.custom_firewall4 = {'firewall': {'services': {'enabled': ['ftp', 'ntp', 'dhcp']}}}
+        self.custom_firewall5 = {'firewall': {'services': {'disabled': ['telnet']}}}
+        self.custom_services1 = {'services': {'enabled': ['sshd', 'cockpit.socket', 'httpd'], 'disabled': ['postfix', 'telnetd']}}
+        self.custom_services2 = {'services': {'enabled': ['sshd', 'cockpit.socket', 'httpd']}}
+        self.custom_services3 = {'services': {'disabled': ['postfix', 'telnetd']}}
+
+        self.old_custom.update(self.custom_sshkey1)
+        # Build the new custom from these pieces
+        self.new_custom = self.old_custom.copy()
+        for d in [self.custom_kernel, self.custom_user1, self.custom_group, self.custom_timezone1,
+                  self.custom_locale1, self.custom_firewall1, self.custom_services1]:
+            self.new_custom.update(d)
+        self.custom_result = [{'new': {'Customizations.firewall': {'ports': ['22:tcp', '80:tcp', 'imap:tcp', '53:tcp', '53:udp'],
+                                                                   'services': {'disabled': ['telnet'], 'enabled': ['ftp', 'ntp', 'dhcp']}}},
+                               'old': None},
+                              {'new': {'Customizations.group': [{'gid': 1130, 'name': 'widget'}]},
+                               'old': None},
+                              {'new': {'Customizations.kernel': {'append': 'nosmt=force'}},
+                               'old': None},
+                              {'new': {'Customizations.locale': {'keyboard': 'us', 'languages': ['en_US.UTF-8']}},
+                               'old': None},
+                              {'new': {'Customizations.services': {'disabled': ['postfix', 'telnetd'], 'enabled': ['sshd', 'cockpit.socket', 'httpd']}},
+                               'old': None},
+                              {'new': {'Customizations.timezone': {'ntpservers': ['0.north-america.pool.ntp.org', '1.north-america.pool.ntp.org'],
+                                                                   'timezone': 'US/Eastern'}},
+                               'old': None},
+                              {'new': {'Customizations.user': [{'description': 'Administrator account', 'gid': 1200,
+                                                                'groups': ['widget', 'users', 'wheel'], 'home': '/srv/widget/',
+                                                                'key': 'PUBLIC SSH KEY', 'name': 'admin', 
+                                                                'password': '$6$CHO2$3rN8eviE2t50lmVyBYihTgVRHcaecmeCk31L...', 'shell': '/usr/bin/bash', 'uid': 1200}]},
+                               'old': None}]
+
+        # repos.git test data and results
+        self.old_git = [{'rpmname': 'server-config-files',
+                         'rpmversion': '1.0',
+                         'rpmrelease': '1',
+                         'summary': 'Setup files for server deployment',
+                         'repo': 'https://github.com/weldr/server-config-files',
+                         'ref': 'v3.0',
+                         'destination': '/srv/config/'}]
+        self.new_git = [{'rpmname': 'bart-files',
+                         'rpmversion': '1.1',
+                         'rpmrelease': '1',
+                         'summary': 'Files needed for Bart',
+                         'repo': 'https://github.com/weldr/not-a-real-repo',
+                         'ref': 'v1.0',
+                         'destination': '/home/bart/Documents/'},
+                         {'rpmname': 'server-config-files',
+                         'rpmversion': '1.0',
+                         'rpmrelease': '1',
+                         'summary': 'Setup files for server deployment',
+                         'repo': 'https://github.com/weldr/server-config-files',
+                         'ref': 'v3.0',
+                         'destination': '/srv/config/'}]
+        self.git_result = [{'old': None,
+                            'new': {'Repos.git': {'rpmname': 'bart-files',
+                                                  'rpmversion': '1.1',
+                                                  'rpmrelease': '1',
+                                                  'summary': 'Files needed for Bart',
+                                                  'repo': 'https://github.com/weldr/not-a-real-repo',
+                                                  'ref': 'v1.0',
+                                                  'destination': '/home/bart/Documents/'}}}]
+        self.maxDiff = None
 
     @classmethod
     def tearDownClass(self):
@@ -132,32 +217,379 @@ class BasicRecipeTest(unittest.TestCase):
         new_version = recipe.bump_version("0.0.1")
         self.assertEqual(new_version, "0.1.1")
 
+    def find_field_test(self):
+        """Test the find_field_value function"""
+        test_list = [{"name":"dog"}, {"name":"cat"}, {"name":"squirrel"}]
+
+        self.assertEqual(recipes.find_field_value("name", "cat", test_list), {"name":"cat"})
+        self.assertIsNone(recipes.find_field_value("name", "alien", test_list))
+        self.assertIsNone(recipes.find_field_value("color", "green", test_list))
+        self.assertIsNone(recipes.find_field_value("color", "green", []))
+
     def find_name_test(self):
         """Test the find_name function"""
         test_list = [{"name":"dog"}, {"name":"cat"}, {"name":"squirrel"}]
 
-        self.assertEqual(recipes.find_name("dog", test_list), {"name":"dog"})
         self.assertEqual(recipes.find_name("cat", test_list), {"name":"cat"})
-        self.assertEqual(recipes.find_name("squirrel", test_list), {"name":"squirrel"})
-
         self.assertIsNone(recipes.find_name("alien", test_list))
+        self.assertIsNone(recipes.find_name("alien", []))
+
+    def find_obj_test(self):
+        """Test the find_recipe_obj function"""
+        test_recipe = {"customizations": {"hostname": "foo", "users": ["root"]}, "repos": {"git": ["git-repos"]}}
+
+        self.assertEqual(recipes.find_recipe_obj(["customizations", "hostname"], test_recipe, ""), "foo")
+        self.assertEqual(recipes.find_recipe_obj(["customizations", "locale"], test_recipe, {}), {})
+        self.assertEqual(recipes.find_recipe_obj(["repos", "git"], test_recipe, ""), ["git-repos"])
+        self.assertEqual(recipes.find_recipe_obj(["repos", "git", "oak"], test_recipe, ""), "")
+        self.assertIsNone(recipes.find_recipe_obj(["pine"], test_recipe))
+
+    def diff_lists_test(self):
+        """Test the diff_lists function"""
+        self.assertEqual(recipes.diff_lists("Modules", "name", self.old_modules, self.old_modules), [])
+        self.assertEqual(recipes.diff_lists("Modules", "name", self.old_modules, self.new_modules), self.modules_result)
+        self.assertEqual(recipes.diff_lists("Packages", "name", self.old_packages, self.new_packages), self.packages_result)
+        self.assertEqual(recipes.diff_lists("Groups", "name", self.old_groups, self.new_groups), self.groups_result)
+        self.assertEqual(recipes.diff_lists("Repos.git", "rpmname", self.old_git, self.new_git), self.git_result)
+        self.assertEqual(recipes.diff_lists("Repos.git", "rpmname", self.old_git, sorted(self.new_git, reverse=True, key=lambda o: o["rpmname"].lower())), self.git_result)
+
+    def customizations_diff_test(self):
+        """Test the customizations_diff function"""
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=self.old_custom)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=self.new_custom)
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), self.custom_result)
+
+    def customizations_diff_services_test(self):
+        """Test the customizations_diff function with services variations"""
+        # Test adding the services customization
+        old_custom = self.old_custom.copy()
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_services1)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': {'Customizations.services': {'disabled': ['postfix', 'telnetd'], 'enabled': ['sshd', 'cockpit.socket', 'httpd']}},
+                   'old': None}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing disabled
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_services1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_services2)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.services': {'disabled': ['postfix', 'telnetd'], 'enabled': ['sshd', 'cockpit.socket', 'httpd']}},
+                   'new': {'Customizations.services': {'enabled': ['sshd', 'cockpit.socket', 'httpd']}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing enabled
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_services1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_services3)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.services': {'disabled': ['postfix', 'telnetd'], 'enabled': ['sshd', 'cockpit.socket', 'httpd']}},
+                   'new': {'Customizations.services': {'disabled': ['postfix', 'telnetd']}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+    def customizations_diff_firewall_test(self):
+        """Test the customizations_diff function with firewall variations"""
+        # Test adding the firewall customization
+        old_custom = self.old_custom.copy()
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_firewall1)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': {'Customizations.firewall': {'ports': ['22:tcp', '80:tcp', 'imap:tcp', '53:tcp', '53:udp'],
+                                                       'services': {'disabled': ['telnet'], 'enabled': ['ftp', 'ntp', 'dhcp']}}},
+                   'old': None}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing services
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_firewall1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_firewall2)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.firewall': {'ports': ['22:tcp', '80:tcp', 'imap:tcp', '53:tcp', '53:udp'],
+                                                       'services': {'disabled': ['telnet'], 'enabled': ['ftp', 'ntp', 'dhcp']}}},
+                   'new': {'Customizations.firewall': {'ports': ['22:tcp', '80:tcp', 'imap:tcp', '53:tcp', '53:udp']}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing ports
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_firewall1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_firewall3)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.firewall': {'ports': ['22:tcp', '80:tcp', 'imap:tcp', '53:tcp', '53:udp'],
+                                                       'services': {'disabled': ['telnet'], 'enabled': ['ftp', 'ntp', 'dhcp']}}},
+                   'new': {'Customizations.firewall': {'services': {'disabled': ['telnet'], 'enabled': ['ftp', 'ntp', 'dhcp']}}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing disabled services
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_firewall3)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_firewall4)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.firewall': {'services': {'disabled': ['telnet'], 'enabled': ['ftp', 'ntp', 'dhcp']}}},
+                   'new': {'Customizations.firewall': {'services': {'enabled': ['ftp', 'ntp', 'dhcp']}}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing enabled services
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_firewall3)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_firewall5)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.firewall': {'services': {'disabled': ['telnet'], 'enabled': ['ftp', 'ntp', 'dhcp']}}},
+                   'new': {'Customizations.firewall': {'services': {'disabled': ['telnet']}}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+    def customizations_diff_locale_test(self):
+        """Test the customizations_diff function with locale variations"""
+        # Test adding the locale customization
+        old_custom = self.old_custom.copy()
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_locale1)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': {'Customizations.locale': {'keyboard': 'us', 'languages': ['en_US.UTF-8']}},
+                   'old': None}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing keyboard
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_locale1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_locale2)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.locale': {'keyboard': 'us', 'languages': ['en_US.UTF-8']}},
+                   'new': {'Customizations.locale': {'languages': ['en_US.UTF-8']}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing languages
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_locale1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_locale3)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.locale': {'keyboard': 'us', 'languages': ['en_US.UTF-8']}},
+                   'new': {'Customizations.locale': {'keyboard': 'us'}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+    def customizations_diff_timezone_test(self):
+        """Test the customizations_diff function with timezone variations"""
+        # Test adding the timezone customization
+        old_custom = self.old_custom.copy()
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_timezone1)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': {'Customizations.timezone': {'ntpservers': ['0.north-america.pool.ntp.org', '1.north-america.pool.ntp.org'], 'timezone': 'US/Eastern'}},
+                   'old': None}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing ntpservers
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_timezone1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_timezone2)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.timezone': {'ntpservers': ['0.north-america.pool.ntp.org', '1.north-america.pool.ntp.org'], 'timezone': 'US/Eastern'}},
+                   'new': {'Customizations.timezone': {'timezone': 'US/Eastern'}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing timezone
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_timezone1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_custom.update(self.custom_timezone3)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.timezone': {'ntpservers': ['0.north-america.pool.ntp.org', '1.north-america.pool.ntp.org'], 'timezone': 'US/Eastern'}},
+                   'new': {'Customizations.timezone': {'ntpservers': ['0.north-america.pool.ntp.org', '1.north-america.pool.ntp.org']}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+
+    def customizations_diff_sshkey_test(self):
+        """Test the customizations_diff function with sshkey variations"""
+        # Test changed root ssh key
+        old_custom = self.old_custom.copy()
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_sshkey2)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': {'Customizations.sshkey': {'key': 'A DIFFERENT SSH KEY FOR ROOT', 'user': 'root'}},
+                   'old': {'Customizations.sshkey': {'key': 'A SSH KEY FOR ROOT', 'user': 'root'}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test adding a user's ssh key
+        old_custom = self.old_custom.copy()
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_sshkey3)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': {'Customizations.sshkey': {'key': 'A SSH KEY FOR CLIFF', 'user': 'cliff'}},
+                   'old': None}]
+
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing a user's ssh key
+        old_custom = old_custom.copy()
+        old_custom.update(self.custom_sshkey3)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = self.old_custom.copy()
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'old': {'Customizations.sshkey': {'key': 'A SSH KEY FOR CLIFF', 'user': 'cliff'}},
+                   'new': None}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+    def customizations_diff_user_test(self):
+        """Test the customizations_diff function with user variations"""
+        # Test changed admin user
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_user1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_user2)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': {'Customizations.user': {'description': 'Administrator account',
+                                                   'gid': 1200,
+                                                   'groups': ['widget', 'users', 'wheel'],
+                                                   'home': '/root/',
+                                                   'key': 'PUBLIC SSH KEY',
+                                                   'name': 'admin',
+                                                   'password': '$6$CHO2$3rN8eviE2t50lmVyBYihTgVRHcaecmeCk31L...',
+                                                   'shell': '/usr/bin/bash',
+                                                   'uid': 1200}},
+                   'old': {'Customizations.user': {'description': 'Administrator account',
+                                                   'gid': 1200,
+                                                   'groups': ['widget', 'users', 'wheel'],
+                                                   'home': '/srv/widget/',
+                                                   'key': 'PUBLIC SSH KEY',
+                                                   'name': 'admin',
+                                                   'password': '$6$CHO2$3rN8eviE2t50lmVyBYihTgVRHcaecmeCk31L...',
+                                                   'shell': '/usr/bin/bash',
+                                                   'uid': 1200}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test adding a user
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_user1)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_user3)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': {'Customizations.user': {'key': 'PUBLIC SSH KEY', 'name': 'norman'}}, 'old': None}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
+        # Test removing a user
+        old_custom = self.old_custom.copy()
+        old_custom.update(self.custom_user3)
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], [], [], customizations=old_custom)
+
+        new_custom = old_custom.copy()
+        new_custom.update(self.custom_user1)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", [], [], [], customizations=new_custom)
+        result = [{'new': None, 'old': {'Customizations.user': {'key': 'PUBLIC SSH KEY', 'name': 'norman'}}}]
+        self.assertEqual(recipes.customizations_diff(old_recipe, new_recipe), result)
+
 
-    def diff_items_test(self):
-        """Test the diff_items function"""
-        self.assertEqual(recipes.diff_items("Modules", self.old_modules, self.new_modules), self.modules_result)
-        self.assertEqual(recipes.diff_items("Packages", self.old_packages, self.new_packages), self.packages_result)
-        self.assertEqual(recipes.diff_items("Groups", self.old_groups, self.new_groups), self.groups_result)
 
     def recipe_diff_test(self):
         """Test the recipe_diff function"""
-        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", self.old_modules, self.old_packages, [])
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", self.old_modules, self.old_packages, [], gitrepos=self.old_git)
-        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", self.new_modules, self.new_packages, [])
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", self.new_modules, self.new_packages, [], gitrepos=self.new_git)
         result = [{'new': {'Version': '0.3.1'}, 'old': {'Version': '0.1.1'}},
                   {'new': {'Module': {'name': 'openssh', 'version': '2.8.1'}}, 'old': None},
                   {'new': None, 'old': {'Module': {'name': 'bash', 'version': '5.*'}}},
                   {'new': {'Module': {'name': 'httpd', 'version': '3.8.*'}},
                    'old': {'Module': {'name': 'httpd', 'version': '3.7.*'}}},
-                  {'new': {'Package': {'name': 'git', 'version': '2.13.*'}}, 'old': None}]
+                  {'new': {'Package': {'name': 'git', 'version': '2.13.*'}}, 'old': None},
+                  {'new': {'Repos.git': {'destination': '/home/bart/Documents/',
+                                         'ref': 'v1.0',
+                                         'repo': 'https://github.com/weldr/not-a-real-repo',
+                                         'rpmname': 'bart-files',
+                                         'rpmrelease': '1',
+                                         'rpmversion': '1.1',
+                                         'summary': 'Files needed for Bart'}},
+                   'old': None}]
+        self.assertEqual(recipes.recipe_diff(old_recipe, new_recipe), result)
+
+        # Empty starting recipe
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", [], self.old_packages, [], gitrepos=self.old_git)
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", self.new_modules, self.new_packages, [], gitrepos=self.new_git)
+        result = [{'new': {'Version': '0.3.1'}, 'old': {'Version': '0.1.1'}},
+                  {'new': {'Module': {'name': 'httpd', 'version': '3.8.*'}}, 'old': None},
+                  {'new': {'Module': {'name': 'openssh', 'version': '2.8.1'}}, 'old': None},
+                  {'new': {'Module': {'name': 'toml', 'version': '2.1'}}, 'old': None},
+                  {'new': {'Package': {'name': 'git', 'version': '2.13.*'}}, 'old': None},
+                  {'new': {'Repos.git': {'destination': '/home/bart/Documents/',
+                                         'ref': 'v1.0',
+                                         'repo': 'https://github.com/weldr/not-a-real-repo',
+                                         'rpmname': 'bart-files',
+                                         'rpmrelease': '1',
+                                         'rpmversion': '1.1',
+                                         'summary': 'Files needed for Bart'}},
+                   'old': None}]
+        self.assertEqual(recipes.recipe_diff(old_recipe, new_recipe), result)
+
+        # All new git repos
+        old_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.1.1", self.old_modules, self.old_packages, [])
+        new_recipe = recipes.Recipe("test-recipe", "A recipe used for testing", "0.3.1", self.new_modules, self.new_packages, [], gitrepos=self.new_git)
+        result = [{'new': {'Version': '0.3.1'}, 'old': {'Version': '0.1.1'}},
+                  {'new': {'Module': {'name': 'openssh', 'version': '2.8.1'}}, 'old': None},
+                  {'new': None, 'old': {'Module': {'name': 'bash', 'version': '5.*'}}},
+                  {'new': {'Module': {'name': 'httpd', 'version': '3.8.*'}},
+                   'old': {'Module': {'name': 'httpd', 'version': '3.7.*'}}},
+                  {'new': {'Package': {'name': 'git', 'version': '2.13.*'}}, 'old': None},
+                  {'new': {'Repos.git': {'destination': '/home/bart/Documents/',
+                                         'ref': 'v1.0',
+                                         'repo': 'https://github.com/weldr/not-a-real-repo',
+                                         'rpmname': 'bart-files',
+                                         'rpmrelease': '1',
+                                         'rpmversion': '1.1',
+                                         'summary': 'Files needed for Bart'}},
+                   'old': None},
+                  {'new': {'Repos.git': {'destination': '/srv/config/',
+                                         'ref': 'v3.0',
+                                         'repo': 'https://github.com/weldr/server-config-files',
+                                         'rpmname': 'server-config-files',
+                                         'rpmrelease': '1',
+                                         'rpmversion': '1.0',
+                                         'summary': 'Setup files for server deployment'}},
+                   'old': None}]
+
+
         self.assertEqual(recipes.recipe_diff(old_recipe, new_recipe), result)
 
     def recipe_freeze_test(self):