Create a new YumBase object when repodata changes

The problem this solves is that yum really isn't designed to be part of\
a long running daemon. So when repodata changes upstream, even when
you force it to download the new metadata, it doesn't change in memory
so you end up with lorax-composer depsolving against old versions, and
anaconda depsolving against new versions (because it sets up its own
YumBase and cache) and then the kickstart is no longer valid.

To solve this I have
 - Added a 6h timeout to the metadata check (because yum's doesn't work
   in this situation).
 - Added a metadata check to the YumLock .lock property, but only when
   the timeout expires.
 - Added a new .lock_check property to YumLock that always checks the
   metadata and resets the timeout.

If it has changed it does its best to tear down the existing YumBase,
deleting as much as it can in hopes it doesn't leak memory. And then it
sets up a totally new YumBase with the new repodata.

Resolves: rhbz#1632962

src/pylorax/api/compose.py

@@ -296,7 +296,8 @@ def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_m
     projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower())
     deps = []
     try:
-        with yumlock.lock:
+        # This can possibly update repodata and reset the YumBase object.
+        with yumlock.lock_check:
             (installed_size, deps) = projects_depsolve_with_size(yumlock.yb, projects, recipe.group_names, with_core=False)
     except ProjectsError as e:
         log.error("start_build depsolve: %s", str(e))

src/pylorax/api/server.py

@@ -28,7 +28,6 @@ from pylorax.api.v0 import v0_api
 from pylorax.sysutils import joinpaths
 
 GitLock = namedtuple("GitLock", ["repo", "lock", "dir"])
-YumLock = namedtuple("YumLock", ["yb", "lock"])
 
 server = Flask(__name__)
 

src/pylorax/api/yumbase.py

@@ -23,6 +23,8 @@ import ConfigParser
 from fnmatch import fnmatchcase
 from glob import glob
 import os
+from threading import Lock
+import time
 import yum
 from yum.Errors import YumBaseError
 
@@ -31,6 +33,74 @@ yum.logginglevels._added_handlers = True
 
 from pylorax.sysutils import joinpaths
 
+class YumLock(object):
+    """Hold the YumBase object and a Lock to control access to it.
+
+    self.yb is a property that returns the YumBase object, but it *may* change
+    from one call to the next if the upstream repositories have changed.
+    """
+    def __init__(self, conf, expire_secs=6*60*60):
+        self._conf = conf
+        self._lock = Lock()
+        self.yb = get_base_object(self._conf)
+        self._expire_secs = expire_secs
+        self._expire_time = time.time() + self._expire_secs
+
+    @property
+    def lock(self):
+        """Check for repo updates (using expiration time) and return the lock
+
+        If the repository has been updated, tear down the old YumBase and
+        create a new one. This is the only way to force yum to use the new
+        metadata.
+        """
+        if time.time() > self._expire_time:
+            return self.lock_check
+        return self._lock
+
+    @property
+    def lock_check(self):
+        """Force a check for repo updates and return the lock
+
+        If the repository has been updated, tear down the old YumBase and
+        create a new one. This is the only way to force yum to use the new
+        metadata.
+
+        Use this method sparingly, it removes the repodata and downloads a new copy every time.
+        """
+        self._expire_time = time.time() + self._expire_secs
+        if self._haveReposChanged():
+            self._destroyYb()
+            self.yb = get_base_object(self._conf)
+        return self._lock
+
+    def _destroyYb(self):
+        # Do our best to get yum to let go of all the things...
+        self.yb.pkgSack.dropCachedData()
+        for s in self.yb.pkgSack.sacks.values():
+            s.close()
+            del s
+        del self.yb.pkgSack
+        self.yb.closeRpmDB()
+        del self.yb.tsInfo
+        del self.yb.ts
+        self.yb.close()
+        del self.yb
+
+    def _haveReposChanged(self):
+        """Return True if the repo has new metadata"""
+        # This is a total kludge, yum doesn't really expect to deal with things changing while the
+        # object is is use.
+        try:
+            before = [(r.id, r.repoXML.checksums["sha256"]) for r in sorted(self.yb.repos.listEnabled())]
+            for r in sorted(self.yb.repos.listEnabled()):
+                r.metadata_expire = 0
+                del r.repoXML
+            after = [(r.id, r.repoXML.checksums["sha256"]) for r in sorted(self.yb.repos.listEnabled())]
+            return before != after
+        except Exception:
+            return False
+
 def get_base_object(conf):
     """Get the Yum object with settings from the config file
 

src/sbin/lorax-composer

@@ -40,8 +40,8 @@ from pylorax.api.config import configure, make_yum_dirs, make_queue_dirs
 from pylorax.api.compose import test_templates
 from pylorax.api.queue import start_queue_monitor
 from pylorax.api.recipes import open_or_create_repo, commit_recipe_directory
-from pylorax.api.server import server, GitLock, YumLock
-from pylorax.api.yumbase import get_base_object
+from pylorax.api.server import server, GitLock
+from pylorax.api.yumbase import YumLock
 
 VERSION = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum)
 
@@ -283,14 +283,14 @@ if __name__ == '__main__':
 
     # Get a YumBase to share with the requests
     try:
-        yb = get_base_object(server.config["COMPOSER_CFG"])
+        server.config["YUMLOCK"] = YumLock(server.config["COMPOSER_CFG"])
     except RuntimeError:
         # Error has already been logged. Just exit cleanly.
         sys.exit(1)
-    server.config["YUMLOCK"] = YumLock(yb=yb, lock=Lock())
 
     # Depsolve the templates and make a note of the failures for /api/status to report
-    server.config["TEMPLATE_ERRORS"] = test_templates(yb, server.config["COMPOSER_CFG"].get("composer", "share_dir"))
+    with server.config["YUMLOCK"].lock:
+        server.config["TEMPLATE_ERRORS"] = test_templates(server.config["YUMLOCK"].yb, server.config["COMPOSER_CFG"].get("composer", "share_dir"))
 
     # Setup access to the git repo
     server.config["REPO_DIR"] = opts.BLUEPRINTS