Add support for yum to lorax-composer

This includes a new configuration file at /etc/lorax/composer.conf with
built-in defaults. It also adds a YUMLOCK server config object so that
request handlers can access the yum base object without interfering
with each other.

etc/composer.conf

@@ -0,0 +1 @@
+# lorax-composer configuration file

setup.py

@@ -7,6 +7,7 @@ import sys
 
 # config file
 data_files = [("/etc/lorax", ["etc/lorax.conf"]),
+              ("/etc/lorax", ["etc/configure.conf"]),
               ("/usr/lib/systemd/system", ["systemd/lorax-composer.service"])]
 
 # shared files

src/pylorax/api/server.py

@@ -27,6 +27,7 @@ from pylorax.api.v0 import v0_api
 from pylorax.sysutils import joinpaths
 
 GitLock = namedtuple("GitLock", ["repo", "lock", "dir"])
+YumLock = namedtuple("YumLock", ["yb", "lock"])
 
 server = Flask(__name__)
 

src/pylorax/api/yumbase.py

@@ -0,0 +1,92 @@
+#
+# Copyright (C) 2017  Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import logging
+log = logging.getLogger("lorax-composer")
+
+import ConfigParser
+from fnmatch import fnmatchcase
+from glob import glob
+import os
+import yum
+
+from pylorax.sysutils import joinpaths
+
+def get_base_object(conf):
+    """Get the Yum object with settings from the config file
+
+    :param conf: configuration object
+    :type conf: ComposerParser
+    :returns: A Yum base object
+    :rtype: YumBase
+    """
+    cachedir = os.path.abspath(conf.get("composer", "cache_dir"))
+    yumconf = os.path.abspath(conf.get("composer", "yum_conf"))
+    repodir = os.path.abspath(conf.get("composer", "repo_dir"))
+
+    c = ConfigParser.ConfigParser()
+
+    # add the main section
+    section = "main"
+    data = {"cachedir": cachedir,
+            "keepcache": 0,
+            "gpgcheck": 0,
+            "plugins": 0,
+            "assumeyes": 1,
+            "reposdir": "",
+            "tsflags": "nodocs"}
+
+    if conf.get_default("yum", "proxy", None):
+        data["proxy"] = conf.get("yum", "proxy")
+
+    if conf.get_default("yum", "sslverify", None) == False:
+        data["sslverify"] = "0"
+
+    c.add_section(section)
+    map(lambda (key, value): c.set(section, key, value), data.items())
+
+    # write the yum configuration file
+    with open(yumconf, "w") as f:
+        c.write(f)
+
+    # create the yum base object
+    yb = yum.YumBase()
+
+    yb.preconf.fn = yumconf
+
+    # TODO How to handle this?
+    yb.preconf.root = "/var/tmp/composer/yum/root"
+    if not os.path.isdir(yb.preconf.root):
+        os.makedirs(yb.preconf.root)
+
+    # Turn on as much yum logging as we can
+    yb.preconf.debuglevel = 6
+    yb.preconf.errorlevel = 6
+    yb.logger.setLevel(logging.DEBUG)
+    yb.verbose_logger.setLevel(logging.DEBUG)
+
+    # Gather up all the available repo files, add the ones matching "repos":"enabled" patterns
+    enabled_repos = conf.get("repos", "enabled").split(",")
+    repo_files = glob(joinpaths(repodir, "*.repo"))
+    if conf.get_default("repos", "use_system_repos", True):
+        repo_files.extend(glob("/etc/yum.repos.d/*.repo"))
+
+    for repo_file in repo_files:
+        name = os.path.basename(repo_file)[:-5]
+        if any(map(lambda pattern: fnmatchcase(name, pattern), enabled_repos)):     # pylint: disable=cell-var-from-loop
+            yb.getReposFromConfigFile(repo_file)
+
+    return yb

src/sbin/lorax-composer

@@ -24,6 +24,7 @@ pylorax_log = logging.getLogger("pylorax")
 server_log = logging.getLogger("server")
 
 import argparse
+import ConfigParser
 import os
 import sys
 from threading import Lock
@@ -31,7 +32,8 @@ from gevent.wsgi import WSGIServer
 
 from pylorax import vernum
 from pylorax.api.recipes import open_or_create_repo, commit_recipe_directory
-from pylorax.api.server import server, GitLock
+from pylorax.api.server import server, GitLock, YumLock
+from pylorax.api.yumbase import get_base_object
 
 VERSION = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum)
 
@@ -51,6 +53,8 @@ def get_parser():
                         help="Path to JSON files used for /api/mock/ paths (/var/tmp/bdcs-mockfiles/)")
     parser.add_argument("-V", action="store_true", dest="showver",
                         help="show program's version number and exit")
+    parser.add_argument("-c", "--config", default="/etc/lorax/composer.conf", metavar="CONFIG",
+                        help="Path to lorax-composer configuration file.")
     parser.add_argument("RECIPES", metavar="RECIPES",
                         help="Path to the recipes")
 
@@ -91,6 +95,45 @@ def setup_logging(logfile):
     server_log.addHandler(fh)
 
 
+class ComposerConfig(ConfigParser.SafeConfigParser):
+    def get_default(self, section, option, default):
+        try:
+            return self.get(section, option)
+        except ConfigParser.Error:
+            return default
+
+
+def configure(conf_file="/etc/lorax/composer.conf"):
+    """lorax-composer configuration"""
+    conf = ComposerConfig()
+
+    # set defaults
+    conf.add_section("composer")
+    conf.set("composer", "yum_conf", "/var/lib/lorax/composer/yum.conf")
+    conf.set("composer", "repo_dir", "/var/lib/lorax/composer/repos.d/")
+    conf.set("composer", "cache_dir", "/var/cache/lorax/composer/yum/")
+
+    conf.add_section("users")
+    conf.set("users", "root", "1")
+
+    # Enable all available repo files by default
+    conf.add_section("repos")
+    conf.set("repos", "use_system_repos", "1")
+    conf.set("repos", "enabled", "*")
+
+    # read the config file
+    if os.path.isfile(conf_file):
+        conf.read(conf_file)
+
+    # Create any missing directories
+    for section, key in [("composer", "yum_conf"), ("composer", "repo_dir"), ("composer", "cache_dir")]:
+        path = conf.get(section, key)
+        if not os.path.isdir(os.path.dirname(path)):
+            os.makedirs(os.path.dirname(path))
+
+    return conf
+
+
 class LogWrapper(object):
     """Wrapper for the WSGIServer which only calls write()"""
     def __init__(self, log_obj):
@@ -122,6 +165,11 @@ if __name__ == '__main__':
     server.config["REPO_DIR"] = opts.RECIPES
     repo = open_or_create_repo(server.config["REPO_DIR"])
     server.config["GITLOCK"] = GitLock(repo=repo, lock=Lock(), dir=opts.RECIPES)
+    server.config["COMPOSER_CFG"] = configure(opts.config)
+
+    # Get a YumBase to share with the requests
+    yb = get_base_object(server.config["COMPOSER_CFG"])
+    server.config["YUMLOCK"] = YumLock(yb=yb, lock=Lock())
 
     # Import example recipes
     commit_recipe_directory(server.config["GITLOCK"].repo, "master", opts.RECIPES)